TSMCube.h

Go to the documentation of this file.

// # TSMCube.h: Tiled hypercube in a table
// # Copyright (C) 1995,1996,1997,1999,2000,2001,2002
// # Associated Universities, Inc. Washington DC, USA.
// #
// # This library is free software; you can redistribute it and/or modify it
// # under the terms of the GNU Library General Public License as published by
// # the Free Software Foundation; either version 2 of the License, or (at your
// # option) any later version.
// #
// # This library is distributed in the hope that it will be useful, but WITHOUT
// # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
// # License for more details.
// #
// # You should have received a copy of the GNU Library General Public License
// # along with this library; if not, write to the Free Software Foundation,
// # Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
// #
// # Correspondence concerning AIPS++ should be addressed as follows:
// #        Internet email: casa-feedback@nrao.edu.
// #        Postal address: AIPS++ Project Office
// #                        National Radio Astronomy Observatory
// #                        520 Edgemont Road
// #                        Charlottesville, VA 22903-2475 USA
 
#ifndef TABLES_TSMCUBE_H
#define TABLES_TSMCUBE_H
 
// # Includes
#include <casacore/casa/aips.h>
#include <casacore/tables/DataMan/TSMShape.h>
#include <casacore/casa/Containers/Record.h>
#include <casacore/casa/Arrays/IPosition.h>
#include <casacore/casa/OS/Conversion.h>
#include <casacore/casa/iosfwd.h>
 
namespace casacore {  // # NAMESPACE CASACORE - BEGIN
 
// # Forward declarations
class TiledStMan;
class TSMFile;
class TSMColumn;
class BucketCache;
template <class T>
class Block;
 
// <summary>
// Tiled hypercube in a table
// </summary>
 
// <use visibility=local>
 
// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
// </reviewed>
 
// <prerequisite>
// # Classes you should understand before using this one.
//   <li> <linkto class=TiledStMan>TiledStMan</linkto>
//   <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
//        for a discussion of the maximum cache size
//   <li> <linkto class=TSMFile>TSMFile</linkto>
//   <li> <linkto class=BucketCache>BucketCache</linkto>
// </prerequisite>
 
// <etymology>
// TSMCube represents a hypercube in the Tiled Storage Manager.
// </etymology>
 
// <synopsis>
// TSMCube defines a tiled hypercube. The data is stored in a TSMFile
// object and accessed using a BucketCache object. The hypercube can
// be extensible in its last dimension to support tables with a size
// which is not known in advance.
// <br>
// Normally hypercubes share the same TSMFile object, but extensible
// hypercubes have their own TSMFile object (to be extensible).
// If the hypercolumn has multiple data columns, their cells share the same
// tiles. Per tile data column A appears first, thereafter B, etc..
// <br>
// The data in the cache is held in external format and is converted
// when accessed. The alternative would be to hold it in the cache in
// local format and convert it when read/written from the file. It was
// felt that the latter approach would generate more needless conversions.
// <p>
// The possible id and coordinate values are stored in a Record
// object. They are written in the main hypercube AipsIO file.
// <p>
// TSMCube uses the maximum cache size set for a Tiled Storage manager.
// The description of class
// <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
// contains a discussion about the effect of setting the maximum cache size.
// </synopsis>
 
// <motivation>
// TSMCube encapsulates all operations on a hypercube.
// </motivation>
 
// # <todo asof="$DATE:$">
// # A List of bugs, limitations, extensions or planned refinements.
// # </todo>
 
class TSMCube {
 public:
  // Define the possible access types for TSMDataColumn.
  enum AccessType { NoAccess, CellAccess, SliceAccess, ColumnAccess, ColumnSliceAccess };
 
  // Construct the hypercube using the given file with the given shape.
  // The record contains the id and possible coordinate values.
  // <br>If the cubeshape is empty, the hypercube is still undefined and
  // can be added later with setShape. That is only used by TiledCellStMan.
  // <br> The fileOffset argument is meant for class TiledFileAccess.
  TSMCube(TiledStMan* stman, TSMFile* file, const IPosition& cubeShape, const IPosition& tileShape,
          const Record& values, Int64 fileOffset, Bool useDerived = False);
 
  // Reconstruct the hypercube by reading its data from the AipsIO stream.
  // It will link itself to the correct TSMFile. The TSMFile objects
  // must have been reconstructed in advance.
  TSMCube(TiledStMan* stman, AipsIO& ios, Bool useDerived = False);
 
  virtual ~TSMCube();
 
  // Forbid copy constructor.
  TSMCube(const TSMCube&) = delete;
 
  // Forbid assignment.
  TSMCube& operator=(const TSMCube&) = delete;
 
  // Flush the data in the cache.
  virtual void flushCache();
 
  // Clear the cache, so data will be reread.
  // If wanted, the data is flushed before the cache is cleared.
  void clearCache(Bool doFlush = True);
 
  // Empty the cache.
  // It will flush the cache as needed and remove all buckets from it
  // resulting in a possibly large drop in memory used.
  // It'll also clear the <src>userSetCache_p</src> flag.
  void emptyCache();
 
  // Show the cache statistics.
  virtual void showCacheStatistics(ostream& os) const;
 
  // Put the data of the object into the AipsIO stream.
  void putObject(AipsIO& ios);
 
  // Get the data of the object from the AipsIO stream.
  // It returns the data manager sequence number, which is -1 if
  // no file is attached to the cube (for cells without a value).
  Int getObject(AipsIO& ios);
 
  // Resync the object with the data file.
  // It reads the object, and adjusts the cache.
  virtual void resync(AipsIO& ios);
 
  // Is the hypercube extensible?
  Bool isExtensible() const;
 
  // Get the bucket size (bytes).
  // It is the length of a tile in external format.
  uInt bucketSize() const;
 
  // Get the length of a tile (in bytes) in local format.
  uInt localTileLength() const;
 
  // Set the hypercube shape.
  // This is only possible if the shape was not defined yet.
  virtual void setShape(const IPosition& cubeShape, const IPosition& tileShape);
 
  // Get the shape of the hypercube.
  const IPosition& cubeShape() const;
 
  // Get the shape of the tiles.
  const IPosition& tileShape() const;
 
  // Get the shape of the data cells in the cube.
  IPosition cellShape() const;
 
  // Get the size of a coordinate (i.e. the number of values in it).
  // If not defined, it returns zero.
  uInt coordinateSize(const String& coordinateName) const;
 
  // Get the record containing the id and coordinate values.
  // It is used by TSMIdColumn and TSMCoordColumn.
  // <group>
  const Record& valueRecord() const;
  Record& rwValueRecord();
  // </group>
 
  // Test if the id values match.
  Bool matches(const Block<TSMColumn*>& idColSet, const Record& idValues);
 
  // Extend the last dimension of the cube with the given number.
  // The record can contain the coordinates of the elements added.
  virtual void extend(uInt64 nr, const Record& coordValues, const TSMColumn* lastCoordColumn);
 
  // Extend the coordinates vector for the given coordinate
  // to the given length with the given coordValues.
  // It will be initialized to zero if no coordValues are given.
  // If the coordinate vector does not exist yet, it will be created.
  void extendCoordinates(const Record& coordValues, const String& coordName, uInt length);
 
  // Read or write a section in the cube.
  // It is assumed that the section buffer is long enough.
  virtual void accessSection(const IPosition& start, const IPosition& end, char* section,
                             uInt colnr, uInt localPixelSize, uInt externalPixelSize,
                             Bool writeFlag);
 
  // Read or write a section in a strided way.
  // It is assumed that the section buffer is long enough.
  virtual void accessStrided(const IPosition& start, const IPosition& end, const IPosition& stride,
                             char* section, uInt colnr, uInt localPixelSize, uInt externalPixelSize,
                             Bool writeFlag);
 
  // Get the current cache size (in buckets).
  uInt cacheSize() const;
 
  // Calculate the cache size (in buckets) for the given slice
  // and access path.
  // <group>
  uInt calcCacheSize(const IPosition& sliceShape, const IPosition& windowStart,
                     const IPosition& windowLength, const IPosition& axisPath) const;
  static uInt calcCacheSize(const IPosition& cubeShape, const IPosition& tileShape, Bool extensible,
                            const IPosition& sliceShape, const IPosition& windowStart,
                            const IPosition& windowLength, const IPosition& axisPath,
                            uInt maxCacheSizeMiB, uInt bucketSize);
  // </group>
 
  // Set the cache size for the given slice and access path.
  virtual void setCacheSize(const IPosition& sliceShape, const IPosition& windowStart,
                            const IPosition& windowLength, const IPosition& axisPath,
                            Bool forceSmaller, Bool userSet);
 
  // Resize the cache object.
  // If forceSmaller is False, the cache will only be resized when it grows.
  // If the given size exceeds the maximum size with more
  // than 10%, the maximum size will be used.
  // The cacheSize has to be given in buckets.
  // <br>The flag <src>userSet</src> inidicates if the cache size is set by
  // the user (by an Accessor object) or automatically (by TSMDataColumn).
  virtual void setCacheSize(uInt cacheSize, Bool forceSmaller, Bool userSet);
 
  // Validate the cache size (in buckets).
  // This means it will return the given cache size (in buckets) if
  // smaller than the maximum cache size (given in MiB).
  // Otherwise the maximum is returned.
  // <group>
  uInt validateCacheSize(uInt cacheSize) const;
  static uInt validateCacheSize(uInt cacheSize, uInt maxSizeMiB, uInt bucketSize);
  // </group>
 
  // Determine if the user set the cache size (using setCacheSize).
  Bool userSetCache() const;
 
  // Functions for TSMDataColumn to keep track of the last type of
  // access to a hypercube. It uses it to determine if the cache
  // has to be reset.
  // <group>
  AccessType getLastColAccess() const;
  const IPosition& getLastColSlice() const;
  void setLastColAccess(AccessType type);
  void setLastColSlice(const IPosition& slice);
  // </group>
 
 protected:
  // Initialize the various variables.
  // <group>
  void setup();
  void setupNrTiles();
  // </group>
 
  // Adjust the tile shape to the hypercube shape.
  // A size of 0 gets set to 1.
  // A tile size > cube size gets set to the cube size.
  IPosition adjustTileShape(const IPosition& cubeShape, const IPosition& tileShape) const;
 
  // Resize the IPosition member variables used in accessSection()
  // if nrdim_p changes value.
  void resizeTileSections();
 
 private:
  // Get the cache object.
  // This will construct the cache object if not present yet.
  BucketCache* getCache();
 
  // Construct the cache object (if not constructed yet).
  virtual void makeCache();
 
  // Resync the cache object.
  virtual void resyncCache();
 
  // Delete the cache object.
  virtual void deleteCache();
 
  // Access a line in a more optimized way.
  void accessLine(char* section, uInt pixelOffset, uInt localPixelSize, Bool writeFlag,
                  BucketCache* cachePtr, const IPosition& startTile, uInt endTile,
                  const IPosition& startPixelInFirstTile, uInt endPixelInLastTile, uInt lineIndex);
 
  // Define the callback functions for the BucketCache.
  // <group>
  static char* readCallBack(void* owner, const char* external);
  static void writeCallBack(void* owner, char* external, const char* local);
  static char* initCallBack(void* owner);
  static void deleteCallBack(void* owner, char* buffer);
  // </group>
 
  // Define the functions doing the actual read and write of the
  // data in the tile and converting it to/from local format.
  // <group>
  char* readTile(const char* external);
  void writeTile(char* external, const char* local);
  // </group>
 
 protected:
  // # Declare member variables.
 
  char* cachedTile_p;  // optimization to hold one tile chunk
 
  // Pointer to the parent storage manager.
  TiledStMan* stmanPtr_p;
  // Is the class used directly or only by a derived class only?
  Bool useDerived_p;
  // The values of the possible id and coordinate columns.
  Record values_p;
  // Is the hypercube extensible?
  Bool extensible_p;
  // Dimensionality of the hypercube.
  uInt nrdim_p;
  // Number of tiles in the hypercube.
  uInt nrTiles_p;
  // The shape of the hypercube.
  IPosition cubeShape_p;
  // The shape of the tiles in the hypercube.
  IPosition tileShape_p;
  // The number of tiles in each hypercube dimension.
  IPosition tilesPerDim_p;
  // Precomputed tileShape information.
  TSMShape expandedTileShape_p;
  // Precomputed tilesPerDim information.
  TSMShape expandedTilesPerDim_p;
  // Number of tiles in all but last dimension (used when extending).
  uInt nrTilesSubCube_p;
  // The tilesize in bytes.
  uInt tileSize_p;
  // Pointer to the TSMFile object holding the data.
  TSMFile* filePtr_p;
  // Offset in the TSMFile object where the data of this hypercube starts.
  Int64 fileOffset_p;
  // Offset for each data column in a tile (in external format).
  Block<uInt> externalOffset_p;
  // Offset for each data column in a tile (in local format).
  Block<uInt> localOffset_p;
  // The bucket size in bytes (is equal to tile size in bytes).
  uInt bucketSize_p;
  // The tile size in bytes in local format.
  uInt localTileLength_p;
  // The bucket cache.
  BucketCache* cache_p;
  // Did the user set the cache size?
  Bool userSetCache_p;
  // Was the last column access to a cell, slice, or column?
  AccessType lastColAccess_p;
  // The slice shape of the last column access to a slice.
  IPosition lastColSlice_p;
 
  // IPosition variables used in accessSection(); declared here
  // as member variables to avoid significant construction and
  // desctruction overhead if they are local to accessSection()
  // #tiles needed for the section
  IPosition nrTileSection_p;
  // First tile needed
  IPosition startTile_p;
  // Last tile needed
  IPosition endTile_p;
  // First pixel in first tile
  IPosition startPixelInFirstTile_p;
  // Last pixel in first tile
  IPosition endPixelInFirstTile_p;
  // Last pixel in last tile
  IPosition endPixelInLastTile_p;
};
 
inline BucketCache* TSMCube::getCache() {
  if (cache_p == 0) {
    makeCache();
  }
  return cache_p;
}
inline uInt TSMCube::bucketSize() const { return bucketSize_p; }
inline uInt TSMCube::localTileLength() const { return localTileLength_p; }
inline const IPosition& TSMCube::cubeShape() const { return cubeShape_p; }
inline const IPosition& TSMCube::tileShape() const { return tileShape_p; }
inline const Record& TSMCube::valueRecord() const { return values_p; }
inline Record& TSMCube::rwValueRecord() { return values_p; }
inline Bool TSMCube::userSetCache() const { return userSetCache_p; }
inline TSMCube::AccessType TSMCube::getLastColAccess() const { return lastColAccess_p; }
inline const IPosition& TSMCube::getLastColSlice() const { return lastColSlice_p; }
inline void TSMCube::setLastColAccess(TSMCube::AccessType type) { lastColAccess_p = type; }
inline void TSMCube::setLastColSlice(const IPosition& slice) {
  lastColSlice_p.resize(slice.nelements());
  lastColSlice_p = slice;
}
 
}  // namespace casacore
 
#endif