casa  $Rev:20696$
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines
TiledDataStMan.h
Go to the documentation of this file.
00001 //# TiledDataStMan.h: Tiled Data Storage Manager
00002 //# Copyright (C) 1995,1996,1997,1999,2001
00003 //# Associated Universities, Inc. Washington DC, USA.
00004 //#
00005 //# This library is free software; you can redistribute it and/or modify it
00006 //# under the terms of the GNU Library General Public License as published by
00007 //# the Free Software Foundation; either version 2 of the License, or (at your
00008 //# option) any later version.
00009 //#
00010 //# This library is distributed in the hope that it will be useful, but WITHOUT
00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
00012 //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
00013 //# License for more details.
00014 //#
00015 //# You should have received a copy of the GNU Library General Public License
00016 //# along with this library; if not, write to the Free Software Foundation,
00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
00018 //#
00019 //# Correspondence concerning AIPS++ should be addressed as follows:
00020 //#        Internet email: aips2-request@nrao.edu.
00021 //#        Postal address: AIPS++ Project Office
00022 //#                        National Radio Astronomy Observatory
00023 //#                        520 Edgemont Road
00024 //#                        Charlottesville, VA 22903-2475 USA
00025 //#
00026 //# $Id: TiledDataStMan.h 20551 2009-03-25 00:11:33Z Malte.Marquarding $
00027 
00028 #ifndef TABLES_TILEDDATASTMAN_H
00029 #define TABLES_TILEDDATASTMAN_H
00030 
00031 //# Includes
00032 #include <casa/aips.h>
00033 #include <tables/Tables/TiledStMan.h>
00034 #include <casa/Containers/Block.h>
00035 #include <casa/BasicSL/String.h>
00036 
00037 namespace casa { //# NAMESPACE CASA - BEGIN
00038 
00039 //# Forward Declarations
00040 
00041 
00042 // <summary>
00043 // Tiled Data Storage Manager.
00044 // </summary>
00045 
00046 // <use visibility=export>
00047 
00048 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
00049 // </reviewed>
00050 
00051 // <prerequisite>
00052 //# Classes you should understand before using this one.
00053 //   <li> <linkto class=TiledStMan>TiledStMan</linkto>
00054 //   <li> <linkto class=TSMCube>TSMCube</linkto>
00055 //   <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
00056 //        for a discussion of the maximum cache size
00057 //   <li> <linkto class=Record>Record</linkto>
00058 // </prerequisite>
00059 
00060 // <etymology>
00061 // TiledDataStMan is the Tiled Storage Manager for general
00062 // data arrays.
00063 // </etymology>
00064 
00065 // <synopsis>
00066 // TiledDataStMan is a derivation from TiledStMan, the abstract
00067 // tiled storage manager class. A description of the basics
00068 // of tiled storage managers is given in the
00069 // <linkto module=Tables:TiledStMan>Tables module</linkto> description.
00070 // <p>
00071 // TiledDataStMan allows the user explicit control over the
00072 // definition and extension of hypercubes by means of the accessor
00073 // class <linkto class=TiledDataStManAccessor>TiledDataStManAccessor</linkto>.
00074 // The user can determine which row should be put in which hypercube,
00075 // so it is possible to put row 0-9 in hypercube A, row 10-29 in B,
00076 // row 30-39 in A again, etc.. This makes it possible to use a tiled
00077 // storage manager for a data column containing data with
00078 // different shapes (e.g. line and continuum data). Actually,
00079 // this storage manager is developed for irregularly shaped
00080 // UV-data, but can be used for any purpose.
00081 // <br>
00082 // Each extensible hypercube uses a file of its own. This means that there
00083 // shouldn't be too many of them, otherwise the number of files may
00084 // get too high.
00085 // <p>
00086 // The TiledDataStMan has the following (extra) properties:
00087 // <ul>
00088 //  <li> When multiple hypercubes are used, one or more id columns have
00089 //       to be used to differentiate between them. The id values must
00090 //       be defined when the hypercube gets added; they cannot be put
00091 //       explicitly.
00092 //  <li> A hypercube can be extensible in its last dimension by setting
00093 //       its last dimension to zero. In that case extendHypercube can
00094 //       be used to extend the hypercube when needed.
00095 //       All fixed sized hypercubes are stored in one file, while there
00096 //       is one file per extensible hypercube.
00097 //  <li> The table must be large enough to accommodate the addition
00098 //       or extension of a hypercube. This means that a sufficient
00099 //       number of rows must be added to the table before a hypercube
00100 //       can be added or extended. It is the responsibility of the user
00101 //       to "synchronize" addition of rows and hypercubes.
00102 //  <li> It is possible to define coordinates for the hypercube axes
00103 //       in several ways:
00104 //       <ul>
00105 //        <li> Use the TiledDataStMan storage manager to hold their values
00106 //             and define the coordinates when adding or extending the
00107 //             hypercube. This is the preferred way.
00108 //        <li> As above, but use explicit puts to write their values.
00109 //             This has to be used when coordinates are defined after
00110 //             the hypercube has been added or extended.
00111 //             Note that several rows may share the same value, so
00112 //             overwriting a value may affect multiple rows.
00113 //        <li> Use another storage manager to hold their values.
00114 //             This is useful when their values depend on other axes,
00115 //             because that cannot be handled by TiledDataStMan.
00116 //       </ul>
00117 //       Note that it is possible to store one coordinate column with
00118 //       TiledDataStMan and another with another storage manager.
00119 // </ul>
00120 // </synopsis> 
00121 
00122 // <motivation>
00123 // This tiled storage manager allows one to create and extend hypercubes
00124 // as needed. One has complete control over which row is stored in which
00125 // hypercube.
00126 // </motivation>
00127 
00128 // <example>
00129 // The following example shows how to create a TiledDataStMan tiled
00130 // storage manager using the hypercolumn as defined in the table description.
00131 // Furthermore it shows how to use TiledDataStManAccessor
00132 // to add a hypercube, while defining its tile shape, coordinates,
00133 // and id-value.
00134 // The example shows that reading the data back does not require any knowledge
00135 // of the data manager. It's exactly the same if another data manager was used.
00136 // <br>
00137 // The table created contains the equally shaped data columns "Data" and
00138 // "Weight".
00139 // Each cell in those columns contains a 2D array with shape [12,20]. The
00140 // coordinates of those arrays are "Pol" and "Freq".
00141 // The tiled storage manager superimposes two more axes ("Baseline"and "Time")
00142 // on the data resulting in a 4D hypercube with shape [12,20,30,42].
00143 // The table contains 42*30 rows (which has to be equal to the number of
00144 // elements in the superimposed axes).
00145 // <br>
00146 // The tile shape of the hypercube is (arbitrarily) set to [4,5,6,7].
00147 // Of course, any tile shape could be chosen. This tile shape results
00148 // in a tile size of 6720 bytes (4*5*6*7 *(4+4) bytes), which is not
00149 // that large (32768 as tile size is very reasonable). The number of tiles
00150 // is integral in each dimension, so no space is wasted.
00151 // Finally it makes access along the various axes about equally efficient.
00152 // <br>
00153 // Although in this example only one hypercube is added, multiple hypercubes
00154 // are possible, because an id column has been defined. 
00155 // <note role=caution>
00156 // The example uses the global Array function indgen to fill the data
00157 // and coordinate arrays with arbitrary values.
00158 // </note>
00159 // Note that the description of class
00160 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
00161 // contains a discussion about the effect of setting the maximum cache size.
00162 //
00163 // <srcblock>
00164 //  // Define the table description and the columns in it.
00165 //  TableDesc td ("", "1", TableDesc::Scratch);
00166 //  td.addColumn (ScalarColumnDesc<float>  ("Time"));
00167 //  td.addColumn (ScalarColumnDesc<float>  ("Baseline"));
00168 //  td.addColumn (ArrayColumnDesc<float>   ("Pol", 1));
00169 //  td.addColumn (ArrayColumnDesc<float>   ("Freq", 1));
00170 //  td.addColumn (ScalarColumnDesc<String> ("Id"));
00171 //  td.addColumn (ArrayColumnDesc<float>   ("Data", 2));
00172 //  td.addColumn (ArrayColumnDesc<float>   ("Weight", 2));
00173 //  // Define the 4-dim hypercolumn with its data, coordinate and id columns.
00174 //  td.defineHypercolumn ("TSMExample",
00175 //                        4,
00176 //                        stringToVector ("Data,Weight"),
00177 //                        stringToVector ("Pol,Freq,Baseline,Time"),
00178 //                        stringToVector ("Id"));
00179 //  
00180 //  // Now create a new table from the description.
00181 //  SetupNewTable newtab("tTiledDataStMan_tmp.data", td, Table::New);
00182 //  // Create a TiledDataStMan storage manager for the hypercolumn
00183 //  // and bind the columns to it.
00184 //  TiledDataStMan sm1 ("TSMExample");
00185 //  newtab.bindAll (sm1);
00186 //  // Create the table with 42*30 rows.
00187 //  Table table(newtab, 42*30);
00188 //  // Create the accessor to be able to add a hypercube to this
00189 //  // storage manager.
00190 //  TiledDataStManAccessor accessor(table, "TSMExample");
00191 //  // Define the values for the coordinates of the hypercube
00192 //  // and put them into the record.
00193 //  Vector<float> timeValues(42);
00194 //  Vector<float> baselineValues(30);
00195 //  Vector<float> freqValues(20);
00196 //  Vector<float> polValues(12);
00197 //  indgen (timeValues);
00198 //  indgen (baselineValues, float(100));
00199 //  indgen (freqValues, float(200));
00200 //  indgen (polValues, float(300));
00201 //  Record hyperDef;
00202 //  hyperDef.define ("Time", timeValues);
00203 //  hyperDef.define ("Baseline", baselineValues);
00204 //  hyperDef.define ("Freq", freqValues);
00205 //  hyperDef.define ("Pol", polValues);
00206 //  // Define the id value as well.
00207 //  hyperDef.define ("Id", "");
00208 //  // Now add the hypercube with the given shape, tile shape,
00209 //  // and coordinate and id values.
00210 //  accessor.addHypercube (IPosition(4,12,20,30,42),
00211 //                         IPosition(4,4,5,6,7), hyperDef);
00212 //  ArrayColumn<float> data (table, "Data");
00213 //  ArrayColumn<float> weight (table, "Weight");
00214 //  Matrix<float> array(IPosition(2,12,20));
00215 //  uInt i;
00216 //  indgen (array);
00217 //  // Write some data into the data columns.
00218 //  for (i=0; i<30*42; i++) {
00219 //      data.put (i, array);
00220 //      weight.put (i, array+float(100));
00221 //      array += float(200);
00222 //  }
00223 //  // Prepare for reading the data back.
00224 //  // Note that time and baseline are in fact scalar columns. They are
00225 //  // superimposed dimensions on the hypercube.
00226 //  ROScalarColumn<float> time (table, "Time");
00227 //  ROScalarColumn<float> baseline (table, "Baseline");
00228 //  ROArrayColumn<float> freq (table, "Freq");
00229 //  ROArrayColumn<float> pol (table, "Pol");
00230 //  ROScalarColumn<String> id (table, "Id");
00231 //  float fValue;
00232 //  String sValue;
00233 //  for (i=0; i<table.nrow(); i++) {
00234 //      data.get (i, array);
00235 //      weight.get (i, array);
00236 //      pol.get (i, polValues);
00237 //      freq.get (i, freqValues);
00238 //      baseline.get (i, fValue);
00239 //      time.get (i, fValue);
00240 //      id.get (i, sValue);
00241 //  }
00242 // </srcblock>
00243 // Note that in this example an id column was not necessary, because
00244 // there is only one hypercube.
00245 // <p>
00246 // The following example is more advanced. Two (extensible) hypercubes
00247 // are used for line and continuum data. Writing such a data set
00248 // could be done as shown. Reading it back is the same as above.
00249 // <br>
00250 // In this example the data columns contain line and continuum data.
00251 // So there are two types of data, each with their own shape and
00252 // stored in their own (extensible) hypercube. Note that the last
00253 // dimension of the hypercube shape is set to zero (to make extensible),
00254 // but the last tile shape dimension has been filled in,
00255 // because the exact tile shape must be known.
00256 // <br>
00257 // Before each put of the data the appropriate hypercube is extended.
00258 // Also the time has to be put, which is done (as an example) in
00259 // two different ways (using an explicit put and using the extendHypercube).
00260 //
00261 // <srcblock>
00262 //  // Defining TableDesc and storage manager is same as in first example.
00263 //  // Create the table.
00264 //  Table table(newtab);
00265 //  // Create the accessor to be able to add the hypercubes to this
00266 //  // storage manager.
00267 //  TiledDataStManAccessor accessor(table, "TSMExample");
00268 //  // Fill the coordinate values.
00269 //  // Note that the time axis of the hypercube will have length 0 to
00270 //  // make it extensible. Therefore the time coordinate can only be
00271 //  // filled in when the hypercube is extended.
00272 //  Vector<float> baselineValues(30);
00273 //  Vector<float> freqValuesCont(1);
00274 //  Vector<float> freqValuesLine(20);
00275 //  Vector<float> polValues(4);
00276 //  indgen (baselineValues, float(100));
00277 //  indgen (freqValuesLine, float(200));
00278 //  indgen (freqValuesCont, float(200));
00279 //  indgen (polValues, float(300));
00280 //  Record hyperDefLine;
00281 //  hyperDefLine.define ("Baseline", baselineValues);
00282 //  hyperDefLine.define ("Pol", polValues);
00283 //  // Make similar record for line data.
00284 //  // Fill the correct id and frequency values for each type.
00285 //  // Add the 2 hypercubes.
00286 //  Record hyperDefCont (hyperDefLine);
00287 //  hyperDefLine.define ("Id", "L");
00288 //  hyperDefLine.define ("Freq", freqValuesLine);
00289 //  hyperDefCont.define ("Id", "C");
00290 //  hyperDefCont.define ("Freq", freqValuesCont);
00291 //  // Add the hypercubes.
00292 //  // Define their last dimension as zero to make them extensible.
00293 //  accessor.addHypercube (IPosition(4,4,20,30,0),
00294 //                         IPosition(4,4,5,6,7), hyperDefLine);
00295 //  accessor.addHypercube (IPosition(4,4,1,30,0),
00296 //                         IPosition(4,4,1,6,7), hyperDefCont);
00297 //  ScalarColumn<float> time (table, "Time");
00298 //  ScalarColumn<float> baseline (table, "Baseline");
00299 //  ArrayColumn<float> freq (table, "Freq");
00300 //  ArrayColumn<float> pol (table, "Pol");
00301 //  ArrayColumn<float> data (table, "Data");
00302 //  ArrayColumn<float> weight (table, "Weight");
00303 //  Matrix<float> arrayLine(IPosition(2,4,20));
00304 //  Matrix<float> arrayCont(IPosition(2,4,1));
00305 //  indgen (arrayLine);
00306 //  indgen (arrayCont);
00307 //  // Write some data into the data columns.
00308 //  // Alternately line and continuum is written.
00309 //  // Each hypercube requires 30 rows to be added (i.e. nr of baselines).
00310 //  // The last dimension of each hypercube is extended with 1.
00311 //  uInt i, j;
00312 //  uInt rownr = 0;
00313 //  for (i=0; i<42; i++) {
00314 //      if (i%2 == 0) {
00315 //          table.addRow (30);
00316 //          accessor.extendHypercube (1, hyperDefLine);
00317 //          time.put (rownr, float(i));
00318 //          for (j=0; j<30; j++) {
00319 //              data.put (rownr, arrayLine);
00320 //              weight.put (rownr, arrayLine);
00321 //              rownr++;
00322 //          }
00323 //      }else{
00324 //          table.addRow (30);
00325 //          Vector<float> timeValue(1);
00326 //          timeValue(0) = float(i);
00327 //          hyperDefCont.define ("Time", timeValue);
00328 //          accessor.extendHypercube (1, hyperDefCont);
00329 //          time.put (rownr, float(i));
00330 //          for (j=0; j<30; j++) {
00331 //              data.put (rownr, arrayCont);
00332 //              weight.put (rownr, arrayCont);
00333 //              rownr++;
00334 //          }
00335 //      }
00336 //  }
00337 // </srcblock>
00338 // Note that in this example the time is defined in 2 different ways.
00339 // The first one by an explicit put, the second one as a record in
00340 // the extendHypercube call. The second way if the preferred one,
00341 // although it requires a bit more coding.
00342 // </example>
00343 
00344 //# <todo asof="$DATE:$">
00345 //# A List of bugs, limitations, extensions or planned refinements.
00346 //# </todo>
00347 
00348 
00349 class TiledDataStMan : public TiledStMan
00350 {
00351 friend class TiledDataStManAccessor;
00352 
00353 public:
00354     // Create a TiledDataStMan storage manager for the hypercolumn
00355     // with the given name.
00356     // The hypercolumn name is also the name of the storage manager.
00357     // The given maximum cache size (default is unlimited) is persistent,
00358     // thus will be reused when the table is read back. Note that the class
00359     // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
00360     // allows one to overwrite the maximum cache size temporarily.
00361     // <br>The constructor taking a Record expects fields in the record with
00362     // the name of the arguments in uppercase. If not defined, their
00363     // default value is used.
00364     // <group>
00365     TiledDataStMan (const String& hypercolumnName,
00366                     uInt maximumCacheSize = 0);
00367     TiledDataStMan (const String& hypercolumnName,
00368                     const Record& spec);
00369     // </group>
00370 
00371     ~TiledDataStMan();
00372 
00373     // Clone this object.
00374     // It does not clone TSMColumn objects possibly used.
00375     DataManager* clone() const;
00376 
00377     // Get the type name of the data manager (i.e. TiledDataStMan).
00378     String dataManagerType() const;
00379 
00380     // Make the object from the type name string.
00381     // This function gets registered in the DataManager "constructor" map.
00382     static DataManager* makeObject (const String& dataManagerType,
00383                                     const Record& spec);
00384 
00385 private:
00386     // Create a TiledDataStMan.
00387     // This constructor is private, because it should only be used
00388     // by makeObject.
00389     TiledDataStMan();
00390 
00391     // Forbid copy constructor.
00392     TiledDataStMan (const TiledDataStMan&);
00393 
00394     // Forbid assignment.
00395     TiledDataStMan& operator= (const TiledDataStMan&);
00396 
00397     // Add rows to the storage manager.
00398     // This will only increase the number of rows. When a hypercube is
00399     // added or extended, it will be checked whether the number of rows
00400     // is sufficient.
00401     void addRow (uInt nrrow);
00402 
00403     // Add a hypercube.
00404     // The number of rows in the table must be large enough to
00405     // accommodate this hypercube.
00406     // The possible id values must be given in the record, while
00407     // coordinate values are optional. The field names in the record
00408     // should match the coordinate and id column names.
00409     // The last dimension in the cube shape can be zero, indicating that
00410     // the hypercube is extensible.
00411     void addHypercube (const IPosition& cubeShape,
00412                        const IPosition& tileShape,
00413                        const Record& values);
00414 
00415     // Extend the hypercube with the given number of elements in
00416     // the last dimension.
00417     // The record should contain the id values (to get the correct
00418     // hypercube) and optionally coordinate values for the elements added.
00419     void extendHypercube (uInt incrInLastDim, const Record& values);
00420 
00421     // Get the hypercube in which the given row is stored.
00422     virtual TSMCube* getHypercube (uInt rownr);
00423 
00424     // Get the hypercube in which the given row is stored.
00425     // It also returns the position of the row in that hypercube.
00426     virtual TSMCube* getHypercube (uInt rownr, IPosition& position);
00427 
00428     // Flush and optionally fsync the data.
00429     // It returns a True status if it had to flush (i.e. if data have changed).
00430     virtual Bool flush (AipsIO&, Bool fsync);
00431 
00432     // Let the storage manager create files as needed for a new table.
00433     // This allows a column with an indirect array to create its file.
00434     virtual void create (uInt nrrow);
00435 
00436     // Read the header info.
00437     virtual void readHeader (uInt nrrow, Bool firstTime);
00438 
00439     // Update the map of row numbers to cube number plus offset.
00440     void updateRowMap (uInt cubeNr, uInt incrInLastDim);
00441 
00442     // Check if the table is large enough to hold this
00443     // hypercube extension.
00444     void checkNrrow (const IPosition& cubeShape,
00445                      uInt incrInLastDim) const;
00446 
00447 
00448     //# Declare the data members.
00449     // The map of row number to cube and position in cube.
00450     Block<uInt> rowMap_p;
00451     Block<uInt> cubeMap_p;
00452     Block<uInt> posMap_p;
00453     // The nr of elements used in the map blocks.
00454     uInt nrUsedRowMap_p;
00455     // The row number since the last hypercube extension.
00456     uInt nrrowLast_p;
00457 };
00458 
00459 
00460 
00461 
00462 } //# NAMESPACE CASA - END
00463 
00464 #endif