TiledStMan.h

Classes

TiledStMan -- Base class for Tiled Storage Manager classes (full description)

class TiledStMan : public DataManager

Interface

Public Members
TiledStMan()
TiledStMan (const String& hypercolumnName, uInt maximumCacheSize)
~TiledStMan()
String dataManagerName() const
virtual Record dataManagerSpec() const
void setDataChanged()
static IPosition makeTileShape (const IPosition& hypercubeShape, Double tolerance = 0.5, uInt maxNrPixelsPerTile = 32768)
static IPosition makeTileShape (const IPosition& hypercubeShape, const Vector<double>& weight, const Vector<double>& tolerance, uInt maxNrPixelsPerTile = 32768)
virtual void setMaximumCacheSize (uInt nbytes)
uInt maximumCacheSize() const
uInt cacheSize (uInt rownr) const
const IPosition& hypercubeShape (uInt rownr) const
const IPosition& tileShape (uInt rownr) const
uInt bucketSize (uInt rownr) const
virtual Bool canChangeShape() const
virtual Bool canAccessColumn (Bool& reask) const
uInt calcCacheSize (uInt rownr, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath) const
void setCacheSize (uInt rownr, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath, Bool forceSmaller)
void setCacheSize (uInt rownr, uInt nbuckets, Bool forceSmaller)
Bool userSetCache (uInt rownr) const
void emptyCaches()
void showCacheStatistics (ostream& os) const
uInt getLengthOffset (uInt nrPixels, Block<uInt>& dataOffset, Block<uInt>& localOffset, uInt& localTileLength) const
uInt nrCoordVector() const
uInt nrow() const
Bool canAddRow() const
virtual IPosition defaultTileShape() const
uInt nhypercubes() const
virtual TSMCube* singleHypercube()
const TSMCube* getTSMCube (uInt hypercube) const
TSMCube* getTSMCube (uInt hypercube)
const TSMCube* getHypercube (uInt rownr) const
virtual TSMCube* getHypercube (uInt rownr) = 0
virtual TSMCube* getHypercube (uInt rownr, IPosition& position) = 0
void readTile (char* local, const Block<uInt>& localOffset, const char* external, const Block<uInt>& externalOffset, uInt nrpixels)
void writeTile (char* external, const Block<uInt>& externalOffset, const char* local, const Block<uInt>& localOffset, uInt nrpixels)
TSMFile* getFile (uInt sequenceNumber)
virtual void open (uInt nrrow, AipsIO&)
virtual void resync (uInt nrrow)
virtual void reopenRW()
virtual void deleteManager()
DataManagerColumn* makeScalarColumn (const String& name, int dataType, const String& dataTypeID)
DataManagerColumn* makeDirArrColumn (const String& name, int dataType, const String& dataTypeID)
DataManagerColumn* makeIndArrColumn (const String& name, int dataType, const String& dataTypeID)
Bool canReallocateColumns() const
DataManagerColumn* reallocateColumn (DataManagerColumn* column)
virtual void setShape (uInt rownr, TSMCube* hypercube, const IPosition& shape, const IPosition& tileShape)
void checkCubeShape (const TSMCube* hypercube, const IPosition& cubeShape) const
int coordinateDataType (const String& columnName) const
void initCoordinates (TSMCube* hypercube)
Protected Members
void setPersMaxCacheSize (uInt nbytes)
uInt getBindings (const Vector<String>& columnNames, PtrBlock<TSMColumn*>& colSet, Bool mustExist) const
virtual void setupCheck (const TableDesc& tableDesc, const Vector<String>& dataNames) const
virtual const TableDesc& getDesc() const
void checkValues (const PtrBlock<TSMColumn*>& colSet, const Record& values) const
void checkCoordinates (const PtrBlock<TSMColumn*>& coordColSet, const IPosition& cubeShape, const Record& values) const
void checkShapeColumn (const IPosition& shape) const
void checkCoordinatesShapes (const TSMCube* hypercube, const IPosition& cubeShape) const
void checkAddHypercube (const IPosition& cubeShape, const Record& values) const
TSMCube* makeHypercube (const IPosition& cubeShape, const IPosition& tileShape, const Record& values)
Int getCubeIndex (const Record& idValues) const
uInt addedNrrow (const IPosition& shape, uInt incrInLastDim) const
Bool flushCaches (Bool fsync)
virtual void readHeader (uInt nrrow, Bool firstTime) = 0
AipsIO* headerFileCreate()
AipsIO* headerFileOpen()
void headerFilePut (AipsIO& headerFile, uInt nrCube)
void headerFileGet (AipsIO& headerFile, uInt tabNrrow, Bool firstTime, Int extraNdim)
void headerFileClose (AipsIO* headerFile)
void setup (Int extraNdim=-1)
void createFile (uInt index)
int arrayDataType (int dataType) const

Description

Prerequisite

Synopsis

TiledStMan is the base class for Tiled Storage Managers. A tiled storage manager is capable of storing a hypercolumn (as defined by TableDesc::defineHypercolumn) in one or more hypercubes.
It is not necessary to define a hypercolumn. If not defined, it is assumed that all columns bound to this storage manager are data columns. At least one of the columns must have a fixed dimensionality and is used to determine the hypercube dimnensionality.
The general concept of these storage managers is explained in the Tables module description.

TiledStMan contains all common functions for the different tiled storage managers. In particular, it contains functions to check if the definitions of the shapes of hypercubes, coordinates, and data cells are consistent. It also contains various data members and functions to make them persistent by writing them into an AipsIO stream.

Motivation

This base class contains the common functionality of all tiled storage managers. The base class is still abstract. Only concrete tiled storage managers derived from it can be instantiated.

Tiled storage managers make access to array data possible with more or less the same efficiency for access along different axes.

Member Description

TiledStMan()

Create a TiledStMan.

TiledStMan (const String& hypercolumnName, uInt maximumCacheSize)

Create a TiledStMan storage manager. The given maximum cache size is persistent, thus will be reused when the table is read back. Note that the class ROTiledStManAccessor allows one to overwrite the maximum cache size temporarily. Its description contains a discussion about the effects of setting a maximum cache.

~TiledStMan()

String dataManagerName() const

Get the name given to the storage manager. This is the name of the hypercolumn.

virtual Record dataManagerSpec() const

Record a record containing data manager specifications.

void setDataChanged()

Set the flag to "data has changed since last flush".

static IPosition makeTileShape (const IPosition& hypercubeShape, Double tolerance = 0.5, uInt maxNrPixelsPerTile = 32768)
static IPosition makeTileShape (const IPosition& hypercubeShape, const Vector<double>& weight, const Vector<double>& tolerance, uInt maxNrPixelsPerTile = 32768)

Derive the tile shape from the hypercube shape for the given number of pixels per tile. It is tried to get the same number of tiles for each dimension. When a weight vector is given, the number of tiles for a dimension is proportional to the weight.
After the initial guess it tries to optimize it by trying to waste as little space as possible, while trying to keep as close to the initial guess. The given tolerance (possibly per axis) gives the minimum and maximum possible length of a tile axis (minimum = initial_guess*tolerance; maximum = initial_guess/tolerance). The heuristic is such that a tile axis length dividing the cube length exactly is always favoured. The test program tTiledStMan can be used to see how the algorithm works out for a given tile size and cube shape.

virtual void setMaximumCacheSize (uInt nbytes)

Set the maximum cache size (in bytes) in a non-persistent way.

uInt maximumCacheSize() const

Get the current maximum cache size (in bytes).

uInt cacheSize (uInt rownr) const

Get the current cache size (in buckets) for the hypercube in the given row.

const IPosition& hypercubeShape (uInt rownr) const

Get the hypercube shape of the data in the given row.

const IPosition& tileShape (uInt rownr) const

Get the tile shape of the data in the given row.

uInt bucketSize (uInt rownr) const

Get the bucket size (in bytes) of the hypercube in the given row.

virtual Bool canChangeShape() const

Can the tiled storage manager handle changing array shapes? The default is no (but TiledCellStMan can).

virtual Bool canAccessColumn (Bool& reask) const

Can the tiled storage manager access an entire column. TiledColumnStMan can always do that. The others might be able to do it (for this time). The default implementation returns True if there is only 1 hypercube. reask is set to True (because next time things might be different).

uInt calcCacheSize (uInt rownr, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath) const

Calculate the cache size (in buckets) for accessing the hypercube containing the given row. It takes the maximum cache size into account (allowing an overdraft of 10%). It uses the given axisPath (i.e. traversal order) to determine the optimum size. A window can be specified to indicate that only the given subset of the hypercube will be accessed.
The length of the slice and window arguments and axisPath must be less or equal to the dimensionality of the hypercube. The non-specified windowStart parts default to 0. The non-specified windowLength parts default to the hypercube shape. The non-specified sliceShape parts default to 1.
Axispath = [2,0,1] indicates that the z-axis changes most rapidly, thereafter x and y. An axis can occur only once in the axisPath. The non-specified axisPath parts get the natural order. E.g. in the previous example axisPath=[2] defines the same path.
When forceSmaller is False, the cache is not resized when the new size is smaller.
A flag is set indicating that the TSMDataColumn access functions do not need to size the cache.

void setCacheSize (uInt rownr, const IPosition& sliceShape, const IPosition& windowStart, const IPosition& windowLength, const IPosition& axisPath, Bool forceSmaller)

Set the cache size using the calcCacheSize function mentioned above.

void setCacheSize (uInt rownr, uInt nbuckets, Bool forceSmaller)

Set the cache size for accessing the hypercube containing the given row. When the give cache size exceeds the maximum cache size with more than 10%, the maximum cache size is used instead.
When forceSmaller is False, the cache is not resized when the new size is smaller.
A flag is set indicating that the TSMDataColumn access functions do not need to size the cache.

Bool userSetCache (uInt rownr) const

Determine if the user set the cache size (using setCacheSize).

void emptyCaches()

Empty the caches used by the hypercubes in this storage manager. It will flush the caches as needed and remove all buckets from them resulting in a possibly large drop in memory used. It also clears the userSetCache flag.

void showCacheStatistics (ostream& os) const

Show the statistics of all caches used.

uInt getLengthOffset (uInt nrPixels, Block<uInt>& dataOffset, Block<uInt>& localOffset, uInt& localTileLength) const

Get the length of the data for the given number of pixels. This can be used to calculate the length of a tile.

uInt nrCoordVector() const

Get the number of coordinate vectors.

uInt nrow() const

Get the nr of rows in this storage manager.

Bool canAddRow() const

Does the storage manager allow to add rows? (yes)

virtual IPosition defaultTileShape() const

Get the default tile shape. By default it returns a zero-length IPosition.

uInt nhypercubes() const

Return the number of hypercubes.

virtual TSMCube* singleHypercube()

Test if only one hypercube is used by this storage manager. If not, throw an exception. Otherwise return the hypercube.

const TSMCube* getTSMCube (uInt hypercube) const
TSMCube* getTSMCube (uInt hypercube)

Get the given hypercube.

const TSMCube* getHypercube (uInt rownr) const
virtual TSMCube* getHypercube (uInt rownr) = 0

Get the hypercube in which the given row is stored.

virtual TSMCube* getHypercube (uInt rownr, IPosition& position) = 0

Get the hypercube in which the given row is stored. It also returns the position of the row in that hypercube.

void readTile (char* local, const Block<uInt>& localOffset, const char* external, const Block<uInt>& externalOffset, uInt nrpixels)

Read a tile and convert the data to local format.

void writeTile (char* external, const Block<uInt>& externalOffset, const char* local, const Block<uInt>& localOffset, uInt nrpixels)

Write a tile after converting the data to external format.

TSMFile* getFile (uInt sequenceNumber)

Get the TSMFile object with the given sequence number.

virtual void open (uInt nrrow, AipsIO&)

Open the storage manager for an existing table.

virtual void resync (uInt nrrow)

Resync the storage manager with the new file contents.

virtual void reopenRW()

Reopen all files used in this storage manager for read/write access.

virtual void deleteManager()

The data manager will be deleted (because all its columns are requested to be deleted). So clean up the things needed (e.g. delete files).

DataManagerColumn* makeScalarColumn (const String& name, int dataType, const String& dataTypeID)

Create a column in the storage manager on behalf of a table column.

Create a scalar column.

DataManagerColumn* makeDirArrColumn (const String& name, int dataType, const String& dataTypeID)

Create a column in the storage manager on behalf of a table column.

Create a direct array column.

DataManagerColumn* makeIndArrColumn (const String& name, int dataType, const String& dataTypeID)

Create a column in the storage manager on behalf of a table column.

Create an indirect array column.

Bool canReallocateColumns() const

The TiledStMan wants to do reallocateColumn.

DataManagerColumn* reallocateColumn (DataManagerColumn* column)

Reallocate the column object if it is part of this data manager. It returns a pointer to the new column object. It is used to remove the indirection of the TSMColumn objects resulting in only one iso. two virtual column calls to get the data.

virtual void setShape (uInt rownr, TSMCube* hypercube, const IPosition& shape, const IPosition& tileShape)

Set the shape and tile shape of a hypercube. By default it throws an "impossible" exception.

void checkCubeShape (const TSMCube* hypercube, const IPosition& cubeShape) const

Check the shape to be set for a hypercube. It checks if it matches predefined (fixed shape) columns and the shape of already defined coordinate columns.

int coordinateDataType (const String& columnName) const

Get the data type of the coordinate column with the given name. An exception is thrown when the column is unknown.

void initCoordinates (TSMCube* hypercube)

Initialize the new coordinates for the given cube.

void setPersMaxCacheSize (uInt nbytes)

Set the persistent maximum cache size.

uInt getBindings (const Vector<String>& columnNames, PtrBlock<TSMColumn*>& colSet, Bool mustExist) const

Get the bindings of the columns with the given names. If bound, the pointer to the TSMColumn object is stored in the block. If mustExist is True, an exception is thrown if the column is not bound. It returns the number of bound columns.

virtual void setupCheck (const TableDesc& tableDesc, const Vector<String>& dataNames) const

Function setup calls this function to allow the derived class to check specific information. In case of errors, an exception should be thrown. By default it does nothing.

virtual const TableDesc& getDesc() const

Get the table description needed for the hypercolumn description.

void checkValues (const PtrBlock<TSMColumn*>& colSet, const Record& values) const

Check if values are given in the record for all columns in the block. Also check if the data types are correct. An exception is thrown if something is incorrect.

void checkCoordinates (const PtrBlock<TSMColumn*>& coordColSet, const IPosition& cubeShape, const Record& values) const

Check if the coordinate values are correct. This calls checkValues and checks if their shapes match the hypercube shape. An exception is thrown if invalid.

void checkShapeColumn (const IPosition& shape) const

Check if the shapes of FixedShape data and coordinate columns match. An exception is thrown if not.

void checkCoordinatesShapes (const TSMCube* hypercube, const IPosition& cubeShape) const

Check if the cube shape matches that of defined coordinates.

void checkAddHypercube (const IPosition& cubeShape, const Record& values) const

Check if the hypercube to be added is correctly defined.

TSMCube* makeHypercube (const IPosition& cubeShape, const IPosition& tileShape, const Record& values)

Make a new TSMCube object.

Int getCubeIndex (const Record& idValues) const

Get the index of the hypercube with the given id-values. If not found, -1 is returned.

uInt addedNrrow (const IPosition& shape, uInt incrInLastDim) const

Determine how many rows need to be added for an extension (in the last dimension) of a hypercube with the given shape.

Bool flushCaches (Bool fsync)

Flush the caches of all hypercubes. If data have put and fsync is set, fsync all files.

virtual void readHeader (uInt nrrow, Bool firstTime) = 0

Let a derived class read the header info. This is used by the open and resync function.

AipsIO* headerFileCreate()

Create the TSM header file. It creates an AipsIO object for it.

AipsIO* headerFileOpen()

Open the TSM header file. It creates an AipsIO object for it.

void headerFilePut (AipsIO& headerFile, uInt nrCube)

Write the data into the header file. The given number of TSMCube objects have to be written.

void headerFileGet (AipsIO& headerFile, uInt tabNrrow, Bool firstTime, Int extraNdim)

Read the data from the header file. When done for the first time, setup() is called to initialize the various variables (using the extraNdim variable).

void headerFileClose (AipsIO* headerFile)

Close the header file. It deletes the AipsIO object.

void setup (Int extraNdim=-1)

Set up the TiledStMan variables from the table description. The argument specifies the number of extra dimensions for the hypercube compared to the data array (usually 0 or 1). It is only used if no hypercolumn definition exists. -1 means that the hypercolumn definition has to be present.

void createFile (uInt index)

Create a TSMFile object and store its pointer at the given index in the block.

int arrayDataType (int dataType) const

Convert the scalar data type to an array data type. This function is temporary and can disappear when the ColumnDesc classes use type TpArray*.