casa  $Rev:20696$
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines
FlagDataHandler.h
Go to the documentation of this file.
00001 //# FlagDataHandler.h: This file contains the interface definition of the FlagDataHandler class.
00002 //#
00003 //#  CASA - Common Astronomy Software Applications (http://casa.nrao.edu/)
00004 //#  Copyright (C) Associated Universities, Inc. Washington DC, USA 2011, All rights reserved.
00005 //#  Copyright (C) European Southern Observatory, 2011, All rights reserved.
00006 //#
00007 //#  This library is free software; you can redistribute it and/or
00008 //#  modify it under the terms of the GNU Lesser General Public
00009 //#  License as published by the Free software Foundation; either
00010 //#  version 2.1 of the License, or (at your option) any later version.
00011 //#
00012 //#  This library is distributed in the hope that it will be useful,
00013 //#  but WITHOUT ANY WARRANTY, without even the implied warranty of
00014 //#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00015 //#  Lesser General Public License for more details.
00016 //#
00017 //#  You should have received a copy of the GNU Lesser General Public
00018 //#  License along with this library; if not, write to the Free Software
00019 //#  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
00020 //#  MA 02111-1307  USA
00021 //# $Id: $
00022 
00023 #ifndef FLAGDATAHANDLER_H_
00024 #define FLAGDATAHANDLER_H_
00025 
00026 // Measurement Set selection
00027 #include <ms/MeasurementSets/MeasurementSet.h>
00028 #include <ms/MeasurementSets/MSSelection.h>
00029 #include <ms/MeasurementSets/MSAntennaColumns.h>
00030 #include <ms/MeasurementSets/MSFieldColumns.h>
00031 #include <ms/MeasurementSets/MSPolColumns.h>
00032 #include <ms/MeasurementSets/MSSpWindowColumns.h>
00033 
00034 // VI/VB infrastructure
00035 #include <synthesis/MSVis/StokesVector.h>
00036 #include <synthesis/MSVis/VisBuffer2.h>
00037 #include <synthesis/MSVis/VisibilityIterator2.h>
00038 
00039 // .casarc interface
00040 #include <casa/System/AipsrcValue.h>
00041 
00042 // Records interface
00043 #include <casa/Containers/Record.h>
00044 
00045 // System utilities (for profiling macros)
00046 #include <casa/OS/HostInfo.h>
00047 #include <sys/time.h>
00048 
00049 // Data mapping
00050 #include <algorithm>
00051 #include <map>
00052 
00053 #define STARTCLOCK timeval start,stop; double elapsedTime; if (profiling_p) gettimeofday(&start,0);
00054 #define STOPCLOCK if (profiling_p) \
00055         {\
00056                 gettimeofday(&stop,0);\
00057                 elapsedTime = (stop.tv_sec-start.tv_sec)*1000.0+(stop.tv_usec-start.tv_usec)/1000.0;\
00058                 *logger_p << LogIO::DEBUG2 << "FlagDataHandler::" << __FUNCTION__ << " Executed in: " << elapsedTime << " ms, Memory free: " << HostInfo::memoryFree( )/1024.0 << " MB" << LogIO::POST;\
00059         }
00060 
00061 namespace casa { //# NAMESPACE CASA - BEGIN
00062 
00063 // Type definitions
00064 typedef std::map< std::pair<Int,Int>,std::vector<uInt> >::iterator antennaPairMapIterator;
00065 typedef std::map< Double,std::vector<uInt> >::iterator subIntegrationMapIterator;
00066 typedef std::map< uShort,uShort >::iterator polartizationMapIterator;
00067 typedef std::map< std::pair<Int,Int>,std::vector<uInt> > antennaPairMap;
00068 typedef std::map< Double,std::vector<uInt> > subIntegrationMap;
00069 typedef std::map< uShort,uShort > polarizationMap;
00070 typedef std::map< uInt,String > polarizationIndexMap;
00071 typedef std::vector< vector<Double> > antennaPointingMap;
00072 typedef std::map< Int,vector<Double> > scanStartStopMap;
00073 typedef std::map< Int,Double > lambdaMap;
00074 
00075 const Complex ImaginaryUnit = Complex(0,1);
00076 
00077 // We need to have the CubeView definition here because its type is used by FlagDataHandler class
00078 template<class T> class CubeView
00079 {
00080 
00081 public:
00082 
00083         CubeView(Cube<T> *parentCube, std::vector<uInt> *rows = NULL, std::vector<uInt> *channels = NULL, std::vector<uInt> *polarizations = NULL)
00084         {
00085                 parentCube_p = parentCube;
00086                 IPosition baseCubeShape = parentCube_p->shape();
00087                 reducedLength_p = IPosition(3);
00088 
00089                 if (((polarizations != NULL) and (polarizations->size() > 0)) and
00090                         ((channels != NULL) and (channels->size() > 0)) and
00091                         ((rows != NULL) and (rows->size() > 0)))
00092                 {
00093                         access_p = &CubeView::accessMapped;
00094                 }
00095                 else if (((polarizations != NULL) and (polarizations->size() > 0)) and
00096                                 ((channels != NULL) and (channels->size() > 0)))
00097                 {
00098                         access_p = &CubeView::accessIndex12Mapped;
00099                 }
00100                 else if (((polarizations != NULL) and (polarizations->size() > 0)) and
00101                                 ((rows != NULL) and (rows->size() > 0)))
00102                 {
00103                         access_p = &CubeView::accessIndex13Mapped;
00104                 }
00105                 else if (((channels != NULL) and (channels->size() > 0)) and
00106                                 ((rows != NULL) and (rows->size() > 0)))
00107                 {
00108                         access_p = &CubeView::accessIndex23Mapped;
00109                 }
00110                 else if ((polarizations != NULL) and (polarizations->size() > 0))
00111                 {
00112                         access_p = &CubeView::accessIndex1Mapped;
00113                 }
00114                 else if ((channels != NULL) and (channels->size() > 0))
00115                 {
00116                         access_p = &CubeView::accessIndex2Mapped;
00117                 }
00118                 else if ((rows != NULL) and (rows->size() > 0))
00119                 {
00120                         access_p = &CubeView::accessIndex3Mapped;
00121                 }
00122                 else
00123                 {
00124                         access_p = &CubeView::accessUnmapped;
00125                 }
00126 
00127                 if ((polarizations != NULL) and (polarizations->size() > 0))
00128                 {
00129                         polarizations_p = polarizations;
00130                         reducedLength_p(0) = polarizations_p->size();
00131                 }
00132                 else
00133                 {
00134                         polarizations_p = NULL;
00135                         reducedLength_p(0) = baseCubeShape(0);
00136                 }
00137 
00138                 if ((channels != NULL) and (channels->size() > 0))
00139                 {
00140                         channels_p = channels;
00141                         reducedLength_p(1) = channels_p->size();
00142                 }
00143                 else
00144                 {
00145                         channels_p = NULL;
00146                         reducedLength_p(1) = baseCubeShape(1);
00147                 }
00148 
00149                 if ((rows != NULL) and (rows->size() > 0))
00150                 {
00151                         rows_p = rows;
00152                         reducedLength_p(2) = rows_p->size();
00153                 }
00154                 else
00155                 {
00156                         rows_p = NULL;
00157                         reducedLength_p(2) = baseCubeShape(2);
00158                 }
00159         }
00160 
00161     T &operator()(uInt i1, uInt i2, uInt i3)
00162     {
00163         return (*this.*access_p)(i1,i2,i3);
00164     }
00165 
00166     const IPosition &shape() const
00167     {
00168         return reducedLength_p;
00169     }
00170 
00171     void shape(Int &s1, Int &s2, Int &s3) const
00172     {
00173         s1 = reducedLength_p(0);
00174         s2 = reducedLength_p(1);
00175         s3 = reducedLength_p(2);
00176         return;
00177     }
00178 
00179 protected:
00180 
00181     vector<uInt> *createIndex(uInt size)
00182     {
00183         vector<uInt> *index = new vector<uInt>(size);
00184         index->clear();
00185         for (uInt i=0; i<size; i++ )
00186         {
00187                 index->push_back(i);
00188         }
00189         return index;
00190     }
00191 
00192     T &accessUnmapped(uInt i1, uInt i2, uInt i3)
00193     {
00194         return parentCube_p->at(i1,i2,i3);
00195     }
00196 
00197     T &accessMapped(uInt i1, uInt i2, uInt i3)
00198     {
00199         uInt i1_index = polarizations_p->at(i1);
00200         uInt i2_index = channels_p->at(i2);
00201         uInt i3_index = rows_p->at(i3);
00202         return parentCube_p->at(i1_index,i2_index,i3_index);
00203     }
00204 
00205     T &accessIndex1Mapped(uInt i1, uInt i2, uInt i3)
00206     {
00207         uInt i1_index = polarizations_p->at(i1);
00208         return parentCube_p->at(i1_index,i2,i3);
00209     }
00210 
00211     T &accessIndex2Mapped(uInt i1, uInt i2, uInt i3)
00212     {
00213         uInt i2_index = channels_p->at(i2);
00214         return parentCube_p->at(i1,i2_index,i3);
00215     }
00216 
00217     T &accessIndex3Mapped(uInt i1, uInt i2, uInt i3)
00218     {
00219         uInt i3_index = rows_p->at(i3);
00220         return parentCube_p->at(i1,i2,i3_index);
00221     }
00222 
00223     T &accessIndex12Mapped(uInt i1, uInt i2, uInt i3)
00224     {
00225         uInt i1_index = polarizations_p->at(i1);
00226         uInt i2_index = channels_p->at(i2);
00227         return parentCube_p->at(i1_index,i2_index,i3);
00228     }
00229 
00230     T &accessIndex13Mapped(uInt i1, uInt i2, uInt i3)
00231     {
00232         uInt i1_index = polarizations_p->at(i1);
00233         uInt i3_index = rows_p->at(i3);
00234         return parentCube_p->at(i1_index,i2,i3_index);
00235     }
00236 
00237     T &accessIndex23Mapped(uInt i1, uInt i2, uInt i3)
00238     {
00239         uInt i2_index = channels_p->at(i2);
00240         uInt i3_index = rows_p->at(i3);
00241         return parentCube_p->at(i1,i2_index,i3_index);
00242     }
00243 
00244 private:
00245     Cube<T> *parentCube_p;
00246         std::vector<uInt> *rows_p;
00247         std::vector<uInt> *channels_p;
00248         std::vector<uInt> *polarizations_p;
00249         IPosition reducedLength_p;
00250         T &(casa::CubeView<T>::*access_p)(uInt,uInt,uInt);
00251 };
00252 
00253 template<class T> class VectorView
00254 {
00255 
00256 public:
00257         VectorView(Vector<T> *parentVector, std::vector<uInt> *rows = NULL)
00258         {
00259                 IPosition parentVectorShape = parentVector->shape();
00260                 parentVector_p = parentVector;
00261                 reducedLength_p = IPosition(1);
00262 
00263                 if ((rows != NULL) and (rows->size() > 0))
00264                 {
00265                         access_p = &VectorView::accessMapped;
00266                 }
00267                 else
00268                 {
00269                         access_p = &VectorView::accessUnmapped;
00270                 }
00271 
00272                 if ((rows != NULL) and (rows->size() > 0))
00273                 {
00274                         rows_p = rows;
00275                         reducedLength_p(0) = rows_p->size();
00276                 }
00277                 else
00278                 {
00279                         rows_p = NULL;
00280                         reducedLength_p(0) = parentVectorShape(0);
00281                 }
00282         }
00283 
00284     T &operator()(uInt i1)
00285     {
00286         return (*this.*access_p)(i1);
00287     }
00288 
00289     const IPosition &shape() const
00290     {
00291         return reducedLength_p;
00292     }
00293 
00294     void shape(Int &s1) const
00295     {
00296         s1 = reducedLength_p(0);
00297         return;
00298     }
00299 
00300 protected:
00301 
00302     vector<uInt> *createIndex(uInt size)
00303     {
00304         vector<uInt> *index = new vector<uInt>(size);
00305         index->clear();
00306         for (uInt i=0; i<size; i++ )
00307         {
00308                 index->push_back(i);
00309         }
00310         return index;
00311     }
00312 
00313     T &accessUnmapped(uInt i1)
00314     {
00315         return parentVector_p->operator()(i1);
00316     }
00317 
00318     T &accessMapped(uInt i1)
00319     {
00320         uInt i1_index = rows_p->at(i1);
00321         return parentVector_p->operator()(i1_index);
00322     }
00323 
00324 private:
00325     Vector<T> *parentVector_p;
00326         std::vector<uInt> *rows_p;
00327         IPosition reducedLength_p;
00328         T &(casa::VectorView<T>::*access_p)(uInt);
00329 };
00330 
00331 class VisMapper
00332 {
00333         typedef Complex (casa::VisMapper::*corrProduct)(uInt,uInt);
00334 
00335 public:
00336 
00337         enum calsolutions {
00338 
00339                 CALSOL1=Stokes::NumberOfTypes,
00340                 CALSOL2,
00341                 CALSOL3,
00342                 CALSOL4
00343         };
00344 
00345         VisMapper(String expression,polarizationMap *polMap,CubeView<Complex> *leftVis,CubeView<Complex> *rightVis=NULL);
00346         VisMapper(String expression,polarizationMap *polMap);
00347         ~VisMapper();
00348 
00349     void setParentCubes(CubeView<Complex> *leftVis,CubeView<Complex> *rightVis=NULL);
00350 
00351     vector< vector<uInt> > getSelectedCorrelations() { return selectedCorrelations_p;}
00352     vector< string > getSelectedCorrelationStrings() { return selectedCorrelationStrings_p;}
00353 
00354         Float operator()(uInt chan, uInt row);
00355         Float operator()(uInt pol, uInt chan, uInt row);
00356 
00357         // Direct access to the complex correlation product
00358         Complex correlationProduct(uInt pol, uInt chan, uInt row);
00359 
00360     const IPosition &shape() const
00361     {
00362         return reducedLength_p;
00363     }
00364 
00365     void shape(Int &chan, Int &row) const
00366     {
00367         chan = reducedLength_p(0);
00368         row = reducedLength_p(1);
00369         return;
00370     }
00371 
00372     void shape(Int &pol, Int &chan, Int &row) const
00373     {
00374         chan = reducedLength_p(0);
00375         row = reducedLength_p(1);
00376         pol = reducedLength_p(2);
00377         return;
00378     }
00379 
00380 
00381 protected:
00382     void setExpressionMapping(String expression,polarizationMap *polMap);
00383         Float real(Complex val) {return val.real();}
00384         Float imag(Complex val) {return val.imag();}
00385         Float abs(Complex val) {return std::abs(val);}
00386         Float arg(Complex val) {return std::arg(val);}
00387         Float norm(Complex val) {return std::norm(val);}
00388         Complex leftVis(uInt pol, uInt chan, uInt row);
00389         Complex diffVis(uInt pol, uInt chan, uInt row);
00390         Complex stokes_i(uInt pol, uInt chan);
00391         Complex stokes_q(uInt pol, uInt chan);
00392         Complex stokes_u(uInt pol, uInt chan);
00393         Complex stokes_v(uInt pol, uInt chan);
00394         Complex linear_xx(uInt pol, uInt chan);
00395         Complex linear_yy(uInt pol, uInt chan);
00396         Complex linear_xy(uInt pol, uInt chan);
00397         Complex linear_yx(uInt pol, uInt chan);
00398         Complex circular_rr(uInt pol, uInt chan);
00399         Complex circular_ll(uInt pol, uInt chan);
00400         Complex circular_rl(uInt pol, uInt chan);
00401         Complex circular_lr(uInt pol, uInt chan);
00402         Complex stokes_i_from_linear(uInt chan, uInt row);
00403         Complex stokes_q_from_linear(uInt chan, uInt row);
00404         Complex stokes_u_from_linear(uInt chan, uInt row);
00405         Complex stokes_v_from_linear(uInt chan, uInt row);
00406         Complex stokes_i_from_circular(uInt chan, uInt row);
00407         Complex stokes_q_from_circular(uInt chan, uInt row);
00408         Complex stokes_u_from_circular(uInt chan, uInt row);
00409         Complex stokes_v_from_circular(uInt chan, uInt row);
00410         Complex calsol1(uInt chan, uInt row);
00411         Complex calsol2(uInt chan, uInt row);
00412         Complex calsol3(uInt chan, uInt row);
00413         Complex calsol4(uInt chan, uInt row);
00414 
00415 
00416 private:
00417         Float (casa::VisMapper::*applyVisExpr_p)(Complex);
00418         Complex (casa::VisMapper::*getVis_p)(uInt,uInt,uInt);
00419         Complex (casa::VisMapper::*getCorr_p)(uInt,uInt);
00420         vector<corrProduct> selectedCorrelationProducts_p;
00421         vector< vector<uInt> > selectedCorrelations_p;
00422         vector<string> selectedCorrelationStrings_p;
00423         CubeView<Complex> *leftVis_p;
00424         CubeView<Complex> *rightVis_p;
00425         IPosition reducedLength_p;
00426         polarizationMap *polMap_p;
00427         String expression_p;
00428 };
00429 
00430 class FlagMapper
00431 {
00432 
00433 public:
00434 
00435         FlagMapper(Bool flag,   vector < vector<uInt> > selectedCorrelations,
00436                                                         CubeView<Bool> *commonFlagsView,
00437                                                         CubeView<Bool> *originalFlagsView,
00438                                                         CubeView<Bool> *privateFlagsView=NULL,
00439                                                         VectorView<Bool> *commonFlagRowView=NULL,
00440                                                         VectorView<Bool> *originalFlagRowView=NULL,
00441                                                         VectorView<Bool> *privateFlagRowView=NULL);
00442         FlagMapper(Bool flag,vector< vector<uInt> > selectedCorrelations);
00443         ~FlagMapper();
00444 
00445         void setParentCubes(CubeView<Bool> *commonFlagsView,CubeView<Bool> *originalFlagsView,CubeView<Bool> *privateFlagsView=NULL);
00446         void setParentFlagRow(VectorView<Bool> *commonFlagRowView,VectorView<Bool> *originalFlagRowView,VectorView<Bool> *privateFlagRowView=NULL);
00447 
00448         void applyFlag(uInt chan, uInt row);
00449         void applyFlag(uInt pol, uInt channel, uInt row);
00450         void applyFlagRow(uInt row);
00451         void applyFlagInRow(uInt row);
00452 
00453         Bool getOriginalFlags(uInt chan, uInt row);
00454         Bool getModifiedFlags(uInt chan, uInt row);
00455         Bool getPrivateFlags(uInt chan, uInt row);
00456 
00457         Bool getOriginalFlags(uInt pol, uInt channel, uInt row);
00458         Bool getModifiedFlags(uInt pol, uInt channel, uInt row);
00459         Bool getPrivateFlags(uInt pol, uInt channel, uInt row);
00460 
00461         // These methods are needed for flag extension
00462         void setModifiedFlags(uInt pol, uInt channel, uInt row);
00463         void setPrivateFlags(uInt pol, uInt channel, uInt row);
00464 
00465         Bool getOriginalFlagRow(uInt row);
00466         Bool getModifiedFlagRow(uInt row);
00467         Bool getPrivateFlagRow(uInt row);
00468 
00469     const IPosition &shape() const
00470     {
00471         return reducedLength_p;
00472     }
00473 
00474     void shape(Int &chan, Int &row) const
00475     {
00476         chan = reducedLength_p(0);
00477         row = reducedLength_p(1);
00478         return;
00479     }
00480 
00481     void shape(Int &pol, Int &chan, Int &row) const
00482     {
00483         chan = reducedLength_p(0);
00484         row = reducedLength_p(1);
00485         pol = reducedLength_p(2);
00486         return;
00487     }
00488 
00489         vector< vector<uInt> > getSelectedCorrelations() {return selectedCorrelations_p;}
00490 
00491     void activateCheckMode() {applyFlag_p = &FlagMapper::checkCommonFlags;}
00492 
00493     uInt nSelectedCorrelations() {return nSelectedCorrelations_p;}
00494     uInt flagsPerRow() {return flagsPerRow_p;}
00495 
00496 protected:
00497 
00498         void setExpressionMapping(vector< vector<uInt> > selectedCorrelations);
00499 
00500         // Apply flags to common flag cube
00501         void applyCommonFlags(uInt pol, uInt channel, uInt row);
00502         // Apply flags to common and private flag cubes
00503         void applyPrivateFlags(uInt pol, uInt channel, uInt row);
00504         // Apply flags to common and private flag cubes
00505         void checkCommonFlags(uInt pol, uInt channel, uInt row);
00506 
00507         // Apply flags to common flag rows
00508         void applyCommonFlagRow(uInt row);
00509         // Apply flags to common and private flag rows
00510         void applyPrivateFlagRow(uInt row);
00511 
00512 
00513 private:
00514 
00515         Bool flag_p;
00516     IPosition reducedLength_p;
00517         CubeView<Bool> *commonFlagsView_p;
00518         CubeView<Bool> *originalFlagsView_p;
00519         CubeView<Bool> *privateFlagsView_p;
00520         VectorView<Bool> *commonFlagRowView_p;
00521         VectorView<Bool> *originalFlagRowView_p;
00522         VectorView<Bool> *privateFlagRowView_p;
00523         vector< vector<uInt> > selectedCorrelations_p;
00524         uInt nSelectedCorrelations_p;
00525         uInt flagsPerRow_p;
00526         void (casa::FlagMapper::*applyFlag_p)(uInt,uInt,uInt);
00527         void (casa::FlagMapper::*applyFlagRow_p)(uInt);
00528 };
00529 
00530 // <summary>
00531 // A top level class defining the data handling interface for the flagging module
00532 // </summary>
00533 //
00534 // <use visibility=export>
00535 //
00536 // <prerequisite>
00537 //   <li> <linkto class="VisBuffer:description">VisBuffer</linkto>
00538 //   <li> <linkto class="FlagMapper:description">FlagMapper</linkto>
00539 //   <li> <linkto class="VisMapper:description">VisMapper</linkto>
00540 // </prerequisite>
00541 //
00542 // <etymology>
00543 // FlagDataHandler stands for generic data handling (i.e. MSs, CalTables, ...) specific to the flagging module
00544 // </etymology>
00545 //
00546 // <synopsis>
00547 //
00548 // This is a top-level class defining the data handling interface for the flagging module.
00549 // There are various methods (virtual) that must be re-implemented by the specific derived
00550 // classes (e.g. FlagMSHandler, FlagCalTableHandler). These methods essentially cover:
00551 //
00552 // - Table access (i.e. open/close/iteration/rw)
00553 //
00554 // - Table selection (i.e. expression parsing, sub-table selection)
00555 //
00556 // Additionally there are public non-virtual methods to:
00557 //
00558 // - Set configuration parameters (pre-sort columns, data selection, time interval, async I/O)
00559 //
00560 // - Enable and access different kinds of mapping (baselines, correlation products, antenna pointing)
00561 //
00562 // Also at this top level there are public members which are used by the FlagAgent classes,
00563 // so that there is no dependency with the specific implementation classes (e.g. FlagMsHandler,
00564 // FlagCalTableHandler), and thus no re-implementation is required at the FlagAgent level.
00565 //
00566 // - VisBuffer (for accessing the data and meta data columns)
00567 //
00568 // - Chunk and Table flag counters (for statistics)
00569 //
00570 // </synopsis>
00571 //
00572 // <motivation>
00573 // The motivation for the FlagDataHandler class is having all the data operations encapsulated
00574 // in one single class, with a common interface for all types of tables (MSs, CalTables, SingleDish),
00575 // so that no specific specific table type implementation has to be done at the FlagAgent level.
00576 // </motivation>
00577 //
00578 // <example>
00579 // <srcblock>
00580 //
00581 // // The following code sets up a FlagDataHandler with either CalTable or MS implementation and
00582 // // iterates through the table applying a clip agent, flushing the flags, and extracting summaries.
00583 //
00584 // // IMPORTANT NOTE:
00585 // // The order of FlagDataHandler and FlagAgent initialization is critical to have everything right,
00586 // // in particular data selection must happen before initializing FlagAgents, and iterator generation
00587 // // must be done after initializing FlagAgents, so that each agent can communicate to the FlagDataHandler
00588 // // which columns have to be pre-fetched (async i/o or parallel mode), and what mapping options are necessary.
00589 //
00590 // // NOTE ON ASYNC I/O:
00591 // // Asyncnronous I/O is only enabled for MS-type tables, but not for CalTables, and it is necessary to switch
00592 // // it on before generating the iterators. Something else to take into account, is that there are 2 global
00593 // // switches at .casarc level which invalidate the application code selection:
00594 // //
00595 // // VisibilityIterator.async.enabled rules over
00596 // // |-> FlagDataHandler.asyncio, and in turns rules over
00597 // //     |-> FlagDataHandler.enableAsyncIO(True)
00598 //
00599 // // Identify table type
00600 // Table table(msname_p,TableLock(TableLock::AutoNoReadLocking));
00601 // TableInfo& info = table.tableInfo();
00602 // String type=info.type();
00603 //
00604 // // Create derived FlagDataHandler object with corresponding implementation
00605 // FlagDataHandler *fdh_p = NULL;
00606 // if (type == "Measurement Set")
00607 // {
00608 //    fdh_p = new FlagMSHandler(msname_p, FlagDataHandler::COMPLETE_SCAN_UNMAPPED, timeInterval_p);
00609 // }
00610 // else
00611 // {
00612 //    fdh_p = new FlagCalTableHandler(msname_p, FlagDataHandler::COMPLETE_SCAN_UNMAPPED, timeInterval_p);
00613 // }
00614 //
00615 // // NOTE: It is also possible to independently set the iteration approach via the setIterationApproach
00616 // //       method which accepts the following modes, defined in the FlagDataHandler iteration enumeration
00617 // //
00618 // //       COMPLETE_SCAN_MAPPED:
00619 // //       - Sort by OBSERVATION_ID, ARRAY_ID, SCAN_NUMBER, FIELD_ID, DATA_DESC_ID and TIME
00620 // //       - Group all time steps together, so that there is no sub-chunk iteration
00621 // //       - Generate baseline maps (to iterate trough rows with the same antenna1, antenna2)
00622 // //       - Generate sub-integration maps (to iterate trough rows with the same timestamp)
00623 // //       COMPLETE_SCAN_MAP_SUB_INTEGRATIONS_ONLY:
00624 // //       - Sort by OBSERVATION_ID, ARRAY_ID, SCAN_NUMBER, FIELD_ID, DATA_DESC_ID and TIME
00625 // //       - Group all time steps together, so that there is no sub-chunk iteration
00626 // //       * Don't generate baseline maps
00627 // //       - Generate sub-integration maps (to iterate trough rows with the same timestamp)
00628 // //       COMPLETE_SCAN_MAP_ANTENNA_PAIRS_ONLY:
00629 // //       - Sort by OBSERVATION_ID, ARRAY_ID, SCAN_NUMBER, FIELD_ID, DATA_DESC_ID and TIME
00630 // //       - Group all time steps together, so that there is no sub-chunk iteration
00631 // //       - Generate baseline maps (to iterate trough rows with the same antenna1, antenna2)
00632 // //       * Don't generate sub-integration maps
00633 // //       COMPLETE_SCAN_UNMAPPED:
00634 // //       - Sort by OBSERVATION_ID, ARRAY_ID, SCAN_NUMBER, FIELD_ID, DATA_DESC_ID and TIME
00635 // //       - Group all time steps together, so that there is no sub-chunk iteration
00636 // //       * Don't generate baseline maps
00637 // //       * Don't generate sub-integration maps
00638 // //       COMBINE_SCANS_MAPPED:
00639 // //       - Sort by OBSERVATION_ID, ARRAY_ID, FIELD_ID, DATA_DESC_ID and TIME
00640 // //       - Group all time steps together, so that there is no sub-chunk iteration
00641 // //       - Generate baseline maps (to iterate trough rows with the same antenna1, antenna2)
00642 // //       - Generate sub-integration maps (to iterate trough rows with the same timestamp)
00643 // //       COMBINE_SCANS_MAP_SUB_INTEGRATIONS_ONLY:
00644 // //       - Sort by OBSERVATION_ID, ARRAY_ID, FIELD_ID, DATA_DESC_ID and TIME
00645 // //       - Group all time steps together, so that there is no sub-chunk iteration
00646 // //       * Don't generate baseline maps
00647 // //       - Generate sub-integration maps (to iterate trough rows with the same timestamp)
00648 // //       COMBINE_SCANS_MAP_ANTENNA_PAIRS_ONLY:
00649 // //       - Sort by OBSERVATION_ID, ARRAY_ID, FIELD_ID, DATA_DESC_ID and TIME
00650 // //       - Group all time steps together, so that there is no sub-chunk iteration
00651 // //       - Generate baseline maps (to iterate trough rows with the same antenna1, antenna2)
00652 // //       * Don't generate sub-integration maps
00653 // //       COMBINE_SCANS_UNMAPPED:
00654 // //       - Sort by OBSERVATION_ID, ARRAY_ID, FIELD_ID, DATA_DESC_ID and TIME
00655 // //       - Group all time steps together, so that there is no sub-chunk iteration
00656 // //       * Don't generate baseline maps
00657 // //       * Don't generate sub-integration maps
00658 // //       ANTENNA_PAIR:
00659 // //       - Sort by OBSERVATION_ID, ARRAY_ID, FIELD_ID, ANTENNA1, ANTENNA2, DATA_DESC_ID and TIME
00660 // //       - Group all time steps together, so that there is no sub-chunk iteration
00661 // //       * Don't generate baseline maps (they are not necessary because the chunks have constant ANTENNA1,ANTENNA2)
00662 // //       * Don't generate sub-integration maps
00663 // //       SUB_INTEGRATION:
00664 // //       - Sort by OBSERVATION_ID, ARRAY_ID, SCAN_NUMBER, FIELD_ID, DATA_DESC_ID and TIME
00665 // //       - Don't group all time steps together, so it is necessary to add an inner sub-chunk iteration loop
00666 // //       * Don't generate baseline maps (it is not possible because not all the rows corresponding to a given baseline are available)
00667 // //       * Don't generate sub-integration maps (they are not necessary because the sub-chunks have constant TIME)
00668 // //       ARRAY_FIELD:
00669 // //       - Sort by OBSERVATION_ID, ARRAY_ID, FIELD_ID, DATA_DESC_ID and TIME
00670 // //       - Don't group all time steps together, so it is necessary to add an inner sub-chunk iteration loop
00671 // //       * Don't generate baseline maps (it is not possible because not all the rows corresponding to a given baseline are available)
00672 // //       * Don't generate sub-integration maps (they are not necessary because the sub-chunks have constant TIME)
00673 // //       * NOTE: This is the iteration approach used by the old flagger framework
00674 //
00675 // // Open table
00676 // fdh_p->open();
00677 //
00678 // // Parse data selection to Flag Data Handler
00679 // fdh_p->setDataSelection(dataSelection);
00680 //
00681 // // Select data (thus creating selected table)
00682 // fdh_p->selectData();
00683 //
00684 // // Create flagging agent and list
00685 // Record agentConfig;
00686 // agentConfig.define("mode","clip")
00687 // FlagAgentBase *agent = FlagAgentBase::create(fdh_p,agentConfig);
00688 // FlagAgentList agentList;
00689 // agentList.push_back(agent);
00690 //
00691 // // Switch on/off async i/p
00692 // fdh_p->enableAsyncIO(true);
00693 //
00694 // // Generate table iterator
00695 // fdh_p->generateIterator();
00696 //
00697 // // Start Flag Agent
00698 // agentList.start();
00699 //
00700 // // Iterates over chunks (constant column values)
00701 // while (fdh_p->nextChunk())
00702 // {
00703 //    // Iterates over buffers (time steps)
00704 //    while (fdh_p->nextBuffer())
00705 //    {
00706 //       // Apply flags
00707 //       agentList.apply();
00708 //       // Flush flag cube
00709 //       fdh_p->flushFlags();
00710 //    }
00711 //
00712 //    // Print end-of-chunk statistics
00713 //    agentList.chunkSummary();
00714 // }
00715 //
00716 // // Print total stats from each agent
00717 // agentList.msSummary();
00718 //
00719 // // Stop Flag Agent
00720 // agentList.terminate();
00721 // agentList.join();
00722 //
00723 // // Close MS
00724 // fdh_p->close();
00725 //
00726 // // Clear Flag Agent List
00727 // agentList.clear();
00728 //
00729 // </srcblock>
00730 // </example>
00731 //
00732 // <example>
00733 // <srcblock>
00734 //
00735 // // The following code shows the FlagAgent-FlagDataHandler interaction works internally:
00736 //
00737 // // First of all, at construction time, each agent has to communicate to the FlagDataHandler
00738 // // which columns have to be pre-fetched (for async i/o or parallel mode) and what mapping
00739 // // options are necessary
00740 
00741 // // ...for instance in the case of FlagAgentShadow we need Ant1,Ant2,UVW,TimeCentroid and PhaseCenter:
00742 // flagDataHandler_p->preLoadColumn(vi::Antenna1);
00743 // flagDataHandler_p->preLoadColumn(vi::Antenna2);
00744 // flagDataHandler_p->preLoadColumn(vi::uvw);
00745 // flagDataHandler_p->preLoadColumn(vi::TimeCentroid);
00746 // flagDataHandler_p->preLoadColumn(vi::PhaseCenter);
00747 //
00748 // // ...and FlagAgentElevation needs to have the antenna pointing information globally available for each chunk:
00749 // flagDataHandler_p->setMapAntennaPointing(true);
00750 //
00751 // // Then, at iteration time, the FlagAgentBase class has access to the VisBuffer held
00752 // // in the FlagDataHandler, in order to retrieve the meta-data columns needed for the
00753 // // data selection engine (agent-level row filtering).
00754 // // NOTE: The VisBuffer is actually held within an auto-pointer wrapper,
00755 // //       thus there is an additional get() involved when accessing it.
00756 //
00757 // if (spwList_p.size())
00758 // {
00759 //    if (!find(spwList_p,visibilityBuffer_p->get()->spectralWindow())) return false;
00760 // }
00761 //
00762 // // The sorting columns used for the iteration are also accessible to optimize the selection engine:
00763 // // (e.g.: If scan is constant check only 1st row)
00764 // if ( (scanList_p.size()>0) and (find(flagDataHandler_p->sortOrder_p,MS::SCAN_NUMBER)==true) )
00765 // {
00766 //    if (!find(scanList_p,visibilityBuffer_p->get()->scan()[0])) return false;
00767 // }
00768 //
00769 // // Once that chunk/rows are evaluated as eligible for the flagging process
00770 // // by the data selection engine, the previously booked maps at construction
00771 // // time can be access in order to iterate trough the data as desired:
00772 // // e.g.: Baseline (antenna pairs) iteration
00773 // for (myAntennaPairMapIterator=flagDataHandler_p->getAntennaPairMap()->begin();
00774 //      myAntennaPairMapIterator != flagDataHandler_p->getAntennaPairMap()->end();
00775 //      ++myAntennaPairMapIterator)
00776 // {
00777 //    // NOTE: The following code is also encapsulated in the FlagAgentBase::processAntennaPair(Int antenna1,Int antenna2) code
00778 //
00779 //    // From the antenna map we can retrieve the rows corresponding to the baseline defined by the antenna pair
00780 //    vector<uInt> baselineRows = (*flagDataHandler_p->getAntennaPairMap())[std::make_pair(antennaPair.first,antennaPair.second)];
00781 //
00782 //    // This rows can be now inserted in the mapper classes (VisMapper and FlagMapper using the CubeView<T> template class)
00783 //    VisMapper visibilitiesMap = VisMapper(expression_p,flagDataHandler_p->getPolarizationMap());
00784 //    FlagMapper flagsMap = FlagMapper(flag_p,visibilitiesMap.getSelectedCorrelations());
00785 //    setVisibilitiesMap(antennaRows,&visibilitiesMap);
00786 //    setFlagsMap(antennaRows,&flagsMap);
00787 // }
00788 //
00789 // // Finally, after flagging time, the FlagAgent can communicate to the FlagDataHandler
00790 // // that the modified FlagCube has to be flushed to disk, this is a small but very important
00791 // // step in order to avoid unnecessary I/O activity when a chunk is not eligible for flagging
00792 // // or the auto-flagging algorithms don't produce any flags.
00793 //
00794 // // If any row was flag, then we have to flush the flagRow
00795 // if (flagRow_p) flagDataHandler_p->flushFlagRow_p = true;
00796 // // If any flag was raised, then we have to flush the flagCube
00797 // if (visBufferFlags_p>0) flagDataHandler_p->flushFlags_p = true;
00798 //
00799 // </srcblock>
00800 // </example>
00801 
00802 class FlagDataHandler
00803 {
00804 
00805 public:
00806 
00807         enum iteration {
00808 
00809                 COMPLETE_SCAN_MAPPED=0,
00810                 COMPLETE_SCAN_MAP_SUB_INTEGRATIONS_ONLY,
00811                 COMPLETE_SCAN_MAP_ANTENNA_PAIRS_ONLY,
00812                 COMPLETE_SCAN_UNMAPPED,
00813                 COMBINE_SCANS_MAPPED,
00814                 COMBINE_SCANS_MAP_SUB_INTEGRATIONS_ONLY,
00815                 COMBINE_SCANS_MAP_ANTENNA_PAIRS_ONLY,
00816                 COMBINE_SCANS_UNMAPPED,
00817                 ANTENNA_PAIR,
00818                 SUB_INTEGRATION,
00819                 ARRAY_FIELD
00820         };
00821 
00822         enum tableType {
00823 
00824                 MEASUREMENT_SET=0,
00825                 CALIBRATION_TABLE
00826         };
00827 
00828         // Default constructor
00829         // NOTE: Time interval 0 groups all time steps together in one chunk.
00830         FlagDataHandler(string msname, uShort iterationApproach = SUB_INTEGRATION, Double timeInterval = 0);
00831 
00832         // Default destructor
00833         virtual ~FlagDataHandler();
00834 
00835         // Common MS/CalTables public interface
00836         virtual bool open() {return false;}
00837         virtual bool close() {return false;}
00838         virtual bool selectData() {return false;}
00839         virtual bool generateIterator() {return false;}
00840         virtual bool nextChunk() {return false;}
00841         virtual bool nextBuffer() {return false;}
00842         virtual bool flushFlags() {return false;}
00843         virtual String getTableName() {return String("none");}
00844         virtual bool parseExpression(MSSelection &/*parser*/) {return true;}
00845         virtual bool checkIfColumnExists(String column) {return true;}
00846         virtual bool summarySignal() {return true;}
00847 
00848         // Set the iteration approach
00849         void setIterationApproach(uShort iterationApproach);
00850 
00851         // Set Data Selection parameters
00852         bool setDataSelection(Record record);
00853 
00854         // Set time interval (also known as ntime)
00855         void setTimeInterval(Double timeInterval);
00856 
00857         // Methods to switch on/off async i/o
00858         void enableAsyncIO(Bool enable);
00859 
00860         // Pre-Load columns (in order to avoid parallelism problems when not using
00861         // async i/o, and also to know what columns to pre-fetch in async i/o mode)
00862         void preLoadColumn(uInt column);
00863         void preFetchColumns();
00864 
00865         // Stop iterating
00866         void stopIteration() {stopIteration_p = true;};
00867 
00868         // As requested by Urvashi R.V. provide access to the original and modified flag cubes
00869         Cube<Bool> * getModifiedFlagCube() {return &modifiedFlagCube_p;}
00870         Cube<Bool> * getOriginalFlagCube() {return &originalFlagCube_p;}
00871         Vector<Bool> * getModifiedFlagRow() {return &modifiedFlagRow_p;}
00872         Vector<Bool> * getOriginalFlagRow() {return &originalFlagRow_p;}
00873 
00874         // Functions to switch on/off mapping functions
00875         void setMapAntennaPairs(bool activated);
00876         void setMapSubIntegrations(bool activated);
00877         void setMapPolarizations(bool activated);
00878         void setMapAntennaPointing(bool activated);
00879         void setScanStartStopMap(bool activated);
00880         void setScanStartStopFlaggedMap(bool activated);
00881 
00882         // Accessors for the mapping functions
00883         antennaPairMap * getAntennaPairMap() {return antennaPairMap_p;}
00884         subIntegrationMap * getSubIntegrationMap() {return subIntegrationMap_p;}
00885         polarizationMap * getPolarizationMap() {return polarizationMap_p;}
00886         polarizationIndexMap * getPolarizationIndexMap() {return polarizationIndexMap_p;}
00887         antennaPointingMap * getMapAntennaPointing() {return antennaPointingMap_p;}
00888         scanStartStopMap * getMapScanStartStop() {return scanStartStopMap_p;}
00889         lambdaMap * getLambdaMap() {return lambdaMap_p;}
00890 
00891         void setProfiling(Bool value) {profiling_p=value;}
00892 
00893         // Make the logger public to that we can use it from FlagAgentBase::create
00894         casa::LogIO *logger_p;
00895 
00896         // Measurement set section
00897         String tablename_p;
00898         MSSelection *measurementSetSelection_p;
00899         Vector<String> *antennaNames_p;
00900         std::map< string, std::pair<Int,Int> > baselineToAnt1Ant2_p;
00901         std::map< std::pair<Int,Int>, string > Ant1Ant2ToBaseline_p;
00902         ROScalarMeasColumn<MPosition> *antennaPositions_p;
00903         Vector<Double> *antennaDiameters_p;
00904         Vector<String> *fieldNames_p;
00905         std::vector<String> *corrProducts_p;
00906 
00907         // RO Visibility Iterator
00908         VisBufferComponents2 *prefetchColumns_p;
00909         // Iteration counters
00910         uLong processedRows;
00911         uShort chunkNo;
00912         uShort bufferNo;
00913 
00914         // FlagDataHanler-FlagAgents interaction
00915         bool flushFlags_p;
00916         bool flushFlagRow_p;
00917         uInt64 chunkCounts_p;
00918         uInt64 progressCounts_p;
00919         uInt64 msCounts_p;
00920         uShort summaryThreshold_p;
00921         bool printChunkSummary_p;
00922         uShort tableTye_p;
00923 
00924         // Visibility Buffer
00925         // WARNING: The attach mechanism only works with pointers or
00926         // referenced variables. Otherwise the VisBuffer is created
00927         // and attached, but when it is assigned to the member it is
00928         // detached because of the dynamically called destructor
00929         vi::VisBuffer2 *visibilityBuffer_p;
00930 
00931         // Vis buffer characteristics (constant values)
00932         bool groupTimeSteps_p;
00933         Block<int> sortOrder_p;
00934 
00935 
00936 protected:
00937 
00938         // Common MS/CalTables private interface
00939         virtual void generateAntennaPairMap();
00940         virtual void generateSubIntegrationMap();
00941         virtual void generatePolarizationsMap();
00942         virtual void generateAntennaPointingMap();
00943         virtual void generateScanStartStopMap();
00944 
00945         // Data Selection ranges
00946         bool anySelection_p;
00947         bool inrowSelection_p;
00948         casa::String arraySelection_p;
00949         casa::String fieldSelection_p;
00950         casa::String scanSelection_p;
00951         casa::String timeSelection_p;
00952         casa::String spwSelection_p;
00953         casa::String baselineSelection_p;
00954         casa::String uvwSelection_p;
00955         casa::String polarizationSelection_p;
00956         casa::String scanIntentSelection_p;
00957         casa::String observationSelection_p;
00958 
00959         // Async I/O stuff
00960         bool asyncio_enabled_p;
00961         // Pre-Load columns (in order to avoid parallelism problems when not using
00962         // async i/o, and also to know what columns to pre-fetch in async i/o mode)
00963         vector<uInt> preLoadColumns_p;
00964 
00965         // Iteration parameters
00966         uShort iterationApproach_p;
00967         Double timeInterval_p;
00968         // Slurp flag
00969         bool slurp_p;
00970         // Iteration initialization parameters
00971         bool chunksInitialized_p;
00972         bool buffersInitialized_p;
00973         bool iteratorGenerated_p;
00974         bool stopIteration_p;
00975 
00976         // Flag Cubes
00977         Cube<Bool> originalFlagCube_p;
00978         Cube<Bool> modifiedFlagCube_p;
00979 
00980         // FlagRows
00981         Vector<Bool> originalFlagRow_p;
00982         Vector<Bool> modifiedFlagRow_p;
00983 
00984         // Mapping members
00985         antennaPairMap *antennaPairMap_p;
00986         subIntegrationMap *subIntegrationMap_p;
00987         polarizationMap *polarizationMap_p;
00988         polarizationIndexMap *polarizationIndexMap_p;
00989         antennaPointingMap *antennaPointingMap_p;
00990         scanStartStopMap *scanStartStopMap_p;
00991         lambdaMap *lambdaMap_p;
00992         bool mapAntennaPairs_p;
00993         bool mapSubIntegrations_p;
00994         bool mapPolarizations_p;
00995         bool mapAntennaPointing_p;
00996         bool mapScanStartStop_p;
00997         bool mapScanStartStopFlagged_p;
00998 
00999         // Stats members
01000         bool stats_p;
01001         uLong cubeAccessCounter_p;
01002         double cubeAccessTime_p;
01003         uLong cubeAccessCounterTotal_p;
01004         double cubeAccessTimeTotal_p;
01005 
01006         // Profiling
01007         bool profiling_p;
01008 
01009 };
01010 
01011 } //# NAMESPACE CASA - END
01012 
01013 #endif /* FLAGDATAHANDLER_H_ */