casa  $Rev:20696$
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines
BucketCache.h
Go to the documentation of this file.
00001 //# BucketCache.h: Cache for buckets in a part of a file
00002 //# Copyright (C) 1994,1995,1996,1999,2000,2001
00003 //# Associated Universities, Inc. Washington DC, USA.
00004 //#
00005 //# This library is free software; you can redistribute it and/or modify it
00006 //# under the terms of the GNU Library General Public License as published by
00007 //# the Free Software Foundation; either version 2 of the License, or (at your
00008 //# option) any later version.
00009 //#
00010 //# This library is distributed in the hope that it will be useful, but WITHOUT
00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
00012 //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
00013 //# License for more details.
00014 //#
00015 //# You should have received a copy of the GNU Library General Public License
00016 //# along with this library; if not, write to the Free Software Foundation,
00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
00018 //#
00019 //# Correspondence concerning AIPS++ should be addressed as follows:
00020 //#        Internet email: aips2-request@nrao.edu.
00021 //#        Postal address: AIPS++ Project Office
00022 //#                        National Radio Astronomy Observatory
00023 //#                        520 Edgemont Road
00024 //#                        Charlottesville, VA 22903-2475 USA
00025 //#
00026 //# $Id: BucketCache.h 20551 2009-03-25 00:11:33Z Malte.Marquarding $
00027 
00028 #ifndef CASA_BUCKETCACHE_H
00029 #define CASA_BUCKETCACHE_H
00030 
00031 //# Includes
00032 #include <casa/aips.h>
00033 #include <casa/IO/BucketFile.h>
00034 #include <casa/Containers/Block.h>
00035 #include <casa/OS/CanonicalConversion.h>
00036 
00037 //# Forward clarations
00038 #include <casa/iosfwd.h>
00039 
00040 
00041 namespace casa { //# NAMESPACE CASA - BEGIN
00042 
00043 // <summary>
00044 // Define the type of the static read and write function.
00045 // </summary>
00046 // <use visibility=export>
00047 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
00048 // </reviewed>
00049 
00050 // <synopsis>
00051 // The BucketCache class needs a way to convert its data from local
00052 // to canonical format and vice-versa. This is done by callback
00053 // functions defined at construction time.
00054 // <p>
00055 // The ToLocal callback function has to allocate a buffer of the correct
00056 // size and to copy/convert the canonical data in the input buffer to
00057 // this buffer. The pointer this newly allocated buffer has to be returned.
00058 // The BucketCache class keeps this pointer in the cache block.
00059 // <p>
00060 // The FromLocal callback function has to copy/convert the data from the
00061 // buffer in local format to the buffer in canonical format. It should
00062 // NOT delete the buffer; that has to be done by the DeleteBuffer function.
00063 // <p>
00064 // The AddBuffer callback function has to create (and initialize) a
00065 // buffer to be added to the file and cache.
00066 // When the file gets extended, BucketCache only registers the new size,
00067 // but does not werite anything. When a bucket is read between the
00068 // actual file size and the new file size, the AddBuffer callback function
00069 // is called to create a buffer and possibly initialize it.
00070 // <p>
00071 // The DeleteBuffer callback function has to delete the buffer
00072 // allocated by the ToLocal function.
00073 // <p>
00074 // The functions get a pointer to the owner object, which was provided
00075 // at construction time. The callback function has to cast this to the
00076 // correct type and can use it thereafter.
00077 // <br>
00078 // C++ supports pointers to members, but it is a bit hard. Therefore pointers
00079 // to static members are used (which are simple pointers to functions).
00080 // A pointer to the owner object is also passed to let the static function
00081 // call the correct member function (when needed).
00082 // </synopsis>
00083 //
00084 // <example>
00085 // See class <linkto class=BucketCache>BucketCache</linkto>.
00086 // </example>
00087 
00088 // <group name=BucketCache_CallBack>
00089 typedef char* (*BucketCacheToLocal) (void* ownerObject, const char* canonical);
00090 typedef void (*BucketCacheFromLocal) (void* ownerObject, char* canonical,
00091                                       const char* local);
00092 typedef char* (*BucketCacheAddBuffer) (void* ownerObject);
00093 typedef void (*BucketCacheDeleteBuffer) (void* ownerObject, char* buffer);
00094 // </group>
00095 
00096 
00097 
00098 // <summary>
00099 // Cache for buckets in a part of a file
00100 // </summary>
00101 
00102 // <use visibility=export>
00103 
00104 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
00105 // </reviewed>
00106 
00107 // <prerequisite>
00108 //# Classes you should understand before using this one.
00109 //   <li> <linkto class=BucketFile>BucketFile</linkto>
00110 // </prerequisite>
00111 
00112 // <etymology>
00113 // BucketCache implements a cache for buckets in (a part of) a file.
00114 // </etymology>
00115 
00116 // <synopsis> 
00117 // A cache may allow more efficient quasi-random IO.
00118 // It can, for instance, be used when a limited number of blocks
00119 // in a file have to be accessed again and again.
00120 // <p>
00121 // The class BucketCache provides such a cache. It can be used on a
00122 // consecutive part of a file as long as that part is not simultaneously
00123 // accessed in another way (including another BucketCache object).
00124 // <p>
00125 // BucketCache stores the data as given.
00126 // It uses <linkto group=BucketCache_CallBack>callback functions</linkto>
00127 // to allocate/delete buffers and to convert the data to/from local format.
00128 // <p>
00129 // When a new bucket is needed and all slots in the cache are used,
00130 // BucketCache will remove the least recently used bucket from the
00131 // cache. When the dirty flag is set, it will first be written.
00132 // <p>
00133 // BucketCache maintains a list of free buckets. Initially this list is
00134 // empty. When a bucket is removed, it is added to the free list.
00135 // AddBucket will take buckets from the free list before extending the file.
00136 // <p>
00137 // Since it is possible to handle only a part of a file by a BucketCache
00138 // object, it is also possible to have multiple BucketCache objects on
00139 // the same file (as long as they access disjoint parts of the file).
00140 // Each BucketCache object can have its own bucket size. This can,
00141 // for example, be used to have tiled arrays with different tile shapes
00142 // in the same file.
00143 // <p>
00144 // Statistics are kept to know how efficient the cache is working.
00145 // It is possible to initialize and show the statistics.
00146 // </synopsis> 
00147 
00148 // <motivation>
00149 // A cache may reduce IO traffix considerably.
00150 // Furthermore it is more efficient to keep a cache in local format.
00151 // In that way conversion to/from local only have to be done when
00152 // data gets read/written. It also allows for precalculations.
00153 // </motivation>
00154 
00155 // <example>
00156 // <srcblock>
00157 //  // Define the callback function for reading a bucket.
00158 //  char* bToLocal (void*, const char* data)
00159 //  {
00160 //    char* ptr = new char[32768];
00161 //    memcpy (ptr, data, 32768);
00162 //    return ptr;
00163 //  }
00164 //  // Define the callback function for writing a bucket.
00165 //  void bFromLocal (void*, char* data, const char* local)
00166 //  {
00167 //    memcpy (data, local, 32768);
00168 //  }
00169 //  // Define the callback function for initializing a new bucket.
00170 //  char* bAddBuffer (void*)
00171 //  {
00172 //    char* ptr = new char[32768];
00173 //    for (uInt i=0; i++; i<32768) {
00174 //      ptr[i] = 0;
00175 //    }
00176 //    return ptr;
00177 //  }
00178 //  // Define the callback function for deleting a bucket.
00179 //  void bDeleteBuffer (void*, char* buffer)
00180 //  {
00181 //    delete [] buffer;
00182 //  }
00183 //
00184 //  void someFunc()
00185 //  {
00186 //    // Open the filebuf.
00187 //    BucketFile file(...);
00188 //    file.open();
00189 //    uInt i;
00190 //    // Create a cache for the part of the file starting at offset 512
00191 //    // consisting of 1000 buckets. The cache consists of 10 buckets.
00192 //    // Each bucket is 32768 bytes.
00193 //    BucketCache cache (&file, 512, 32768, 1000, 10, 0,
00194 //                       bToLocal, bFromLocal, bAddBuffer, bDeleteBuffer);
00195 //    // Write all buckets into the file.
00196 //    for (i=0; i<100; i++) {
00197 //      char* buf = new char[32768];
00198 //      cache.addBucket (buf);
00199 //    }
00200 //    Flush the cache to write all buckets in it.
00201 //    cache.flush();
00202 //    // Read all buckets from the file.
00203 //    for (i=0; i<1000; i++) {
00204 //      char* buf = cache.getBucket(i);
00205 //      ...
00206 //    }
00207 //    cout << cache.nBucket() << endl;
00208 //  }
00209 // </srcblock>
00210 // </example>
00211 
00212 // <todo asof="$DATE:$">
00213 //   <li> When ready, use HashMap for the internal maps.
00214 // </todo>
00215 
00216 
00217 class BucketCache
00218 {
00219 public:
00220 
00221     // Create the cache for (a part of) a file.
00222     // The file part used starts at startOffset. Its length is
00223     // bucketSize*nrOfBuckets bytes.
00224     // When the file is smaller, the remainder is indicated as an extension
00225     // similarly to the behaviour of function extend.
00226     BucketCache (BucketFile* file, Int64 startOffset, uInt bucketSize,
00227                  uInt nrOfBuckets, uInt cacheSize,
00228                  void* ownerObject,
00229                  BucketCacheToLocal readCallBack,
00230                  BucketCacheFromLocal writeCallBack,
00231                  BucketCacheAddBuffer addCallBack,
00232                  BucketCacheDeleteBuffer deleteCallBack);
00233 
00234     ~BucketCache();
00235 
00236     // Flush the cache from the given slot on.
00237     // By default the entire cache is flushed.
00238     // When the entire cache is flushed, possible remaining uninitialized
00239     // buckets will be initialized first.
00240     // A True status is returned when buckets had to be written.
00241     Bool flush (uInt fromSlot = 0);
00242 
00243     // Clear the cache from the given slot on.
00244     // By default the entire cache is cleared.
00245     // It will remove the buckets in the cleared part.
00246     // If wanted and needed, the buckets are flushed to the file
00247     // before removing them.
00248     // It can be used to enforce rereading buckets from the file.
00249     void clear (uInt fromSlot = 0, Bool doFlush = True);
00250 
00251     // Resize the cache.
00252     // When the cache gets smaller, the latter buckets are cached out.
00253     // It does not take "least recently used" into account.
00254     void resize (uInt cacheSize);
00255 
00256     // Resynchronize the object (after another process updated the file).
00257     // It clears the cache (so all data will be reread) and sets
00258     // the new sizes.
00259     void resync (uInt nrBucket, uInt nrOfFreeBucket, Int firstFreeBucket);
00260 
00261     // Get the current nr of buckets in the file.
00262     uInt nBucket() const;
00263 
00264     // Get the current cache size (in buckets).
00265     uInt cacheSize() const;
00266 
00267     // Set the dirty bit for the current bucket.
00268     void setDirty();
00269 
00270     // Make another bucket current.
00271     // When no more cache slots are available, the one least recently
00272     // used is flushed.
00273     // The data in the bucket is converted using the ToLocal callback
00274     // function. When the bucket does not exist yet in the file, it
00275     // gets added and initialized using the AddBuffer callback function.
00276     // A pointer to the data in converted format is returned.
00277     char* getBucket (uInt bucketNr);
00278 
00279     // Extend the file with the given number of buckets.
00280     // The buckets get initialized when they are acquired
00281     // (using getBucket) for the first time.
00282     void extend (uInt nrBucket);
00283 
00284     // Add a bucket to the file and make it the current one.
00285     // When no more cache slots are available, the one least recently
00286     // used is flushed.
00287     // <br> When no free buckets are available, the file will be
00288     // extended with one bucket. It returns the new bucket number.
00289     // The buffer must have been allocated on the heap.
00290     // It will get part of the cache; its contents are not copied.
00291     // Thus the buffer should hereafter NOT be used for other purposes.
00292     // It will be deleted later via the DeleteBuffer callback function.
00293     // The data is copied into the bucket. A pointer to the data in
00294     // local format is returned.
00295     uInt addBucket (char* data);
00296 
00297     // Remove the current bucket; i.e. add it to the beginning of the
00298     // free bucket list.
00299     void removeBucket();
00300 
00301     // Get a part from the file outside the cached area.
00302     // It is checked if that part is indeed outside the cached file area.
00303     void get (char* buf, uInt length, Int64 offset);
00304 
00305     // Put a part from the file outside the cached area.
00306     // It is checked if that part is indeed outside the cached file area.
00307     void put (const char* buf, uInt length, Int64 offset);
00308 
00309     // Get the bucket number of the first free bucket.
00310     // -1 = no free buckets.
00311     Int firstFreeBucket() const;
00312 
00313     // Get the number of free buckets.
00314     uInt nFreeBucket() const;
00315 
00316     // (Re)initialize the cache statistics.
00317     void initStatistics();
00318 
00319     // Show the statistics.
00320     void showStatistics (ostream& os) const;
00321 
00322 private:
00323     // The file used.
00324     BucketFile* its_file;
00325     // The owner object.
00326     void*    its_Owner;
00327     // The read callback function.
00328     BucketCacheToLocal   its_ReadCallBack;
00329     // The write callback function.
00330     BucketCacheFromLocal its_WriteCallBack;
00331     // The add bucket callback function.
00332     BucketCacheAddBuffer its_InitCallBack;
00333     // The delete callback function.
00334     BucketCacheDeleteBuffer its_DeleteCallBack;
00335     // The starting offsets of the buckets in the file.
00336     Int64    its_StartOffset;
00337     // The bucket size.
00338     uInt     its_BucketSize;
00339     // The current nr of buckets in the file.
00340     uInt     its_CurNrOfBuckets;
00341     // The new nr of buckets in the file (after extension).
00342     uInt     its_NewNrOfBuckets;
00343     // The size of the cache (i.e. #buckets fitting in it).
00344     uInt     its_CacheSize;
00345     // The nr of slots used in the cache.
00346     uInt     its_CacheSizeUsed;
00347     // The cache itself.
00348     PtrBlock<char*> its_Cache; 
00349     // The cache slot actually used.
00350     uInt         its_ActualSlot;
00351     // The slot numbers of the buckets in the cache (-1 = not in cache).
00352     Block<Int>   its_SlotNr;
00353     // The buckets in the cache.
00354     Block<uInt>  its_BucketNr;
00355     // Determine if a block is dirty (i.e. changed) (1=dirty).
00356     Block<uInt>  its_Dirty;
00357     // Determine when a block is used for the last time.
00358     Block<uInt>  its_LRU;
00359     // The Least Recently Used counter.
00360     uInt         its_LRUCounter;
00361     // The internal buffer.
00362     char*        its_Buffer;
00363     // The number of free buckets.
00364     uInt its_NrOfFree;
00365     // The first free bucket (-1 = no free buckets).
00366     Int  its_FirstFree;
00367     // The statistics.
00368     uInt naccess_p;
00369     uInt nread_p;
00370     uInt ninit_p;
00371     uInt nwrite_p;
00372 
00373 
00374     // Copy constructor is not possible.
00375     BucketCache (const BucketCache&);
00376 
00377     // Assignment is not possible.
00378     BucketCache& operator= (const BucketCache&);
00379 
00380     // Set the LRU information for the current slot.
00381     void setLRU();
00382 
00383     // Get a cache slot for the bucket.
00384     void getSlot (uInt bucketNr);
00385 
00386     // Write a bucket.
00387     void writeBucket (uInt slotNr);
00388 
00389     // Read a bucket.
00390     void readBucket (uInt slotNr);
00391 
00392     // Initialize the bucket buffer.
00393     // The uninitialized buckets before this bucket are also initialized.
00394     // It returns a pointer to the buffer.
00395     void initializeBuckets (uInt bucketNr);
00396 
00397     // Check if the offset of a non-cached part is correct.
00398     void checkOffset (uInt length, Int64 offset) const;
00399 };
00400 
00401 
00402 
00403 inline uInt BucketCache::cacheSize() const
00404     { return its_CacheSize; }
00405 
00406 inline Int BucketCache::firstFreeBucket() const
00407     { return its_FirstFree; }
00408 
00409 inline uInt BucketCache::nFreeBucket() const
00410     { return its_NrOfFree; }
00411 
00412 
00413 
00414 
00415 } //# NAMESPACE CASA - END
00416 
00417 #endif