Line data Source code
1 : //# MSTransformDataHandler.h: This file contains the interface definition of the MSTransformDataHandler class.
2 : //#
3 : //# CASA - Common Astronomy Software Applications (http://casa.nrao.edu/)
4 : //# Copyright (C) Associated Universities, Inc. Washington DC, USA 2011, All rights reserved.
5 : //# Copyright (C) European Southern Observatory, 2011, All rights reserved.
6 : //#
7 : //# This library is free software; you can redistribute it and/or
8 : //# modify it under the terms of the GNU Lesser General Public
9 : //# License as published by the Free software Foundation; either
10 : //# version 2.1 of the License, or (at your option) any later version.
11 : //#
12 : //# This library is distributed in the hope that it will be useful,
13 : //# but WITHOUT ANY WARRANTY, without even the implied warranty of
14 : //# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : //# Lesser General Public License for more details.
16 : //#
17 : //# You should have received a copy of the GNU Lesser General Public
18 : //# License along with this library; if not, write to the Free Software
19 : //# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20 : //# MA 02111-1307 USA
21 : //# $Id: $
22 :
23 : #ifndef MSTransformDataHandler_H_
24 : #define MSTransformDataHandler_H_
25 :
26 : // Measurement Set
27 : #include <casacore/ms/MeasurementSets.h>
28 :
29 : // Measurement Set Selection
30 : #include <casacore/ms/MSSel/MSSelection.h>
31 : #include <casacore/ms/MSSel/MSSelectionTools.h>
32 :
33 : // Needed by setupMS
34 : #include <casacore/tables/Tables.h>
35 : #include <casacore/ms/MeasurementSets/MSTileLayout.h>
36 :
37 : // Needed by copyTable
38 : #include <casacore/tables/Tables/PlainTable.h>
39 :
40 : // OS methods needed by fillSubTables
41 : #include <casacore/casa/OS/Timer.h>
42 : #include <casacore/casa/OS/Path.h>
43 : #include <casacore/casa/OS/Directory.h>
44 :
45 :
46 : namespace casa { //# NAMESPACE CASA - BEGIN
47 :
48 : // MSTransformDataHandler definition
49 : class MSTransformDataHandler
50 : {
51 :
52 : public:
53 :
54 : enum asdmStManUseAlternatives
55 : {
56 : DONT,
57 : USE_FOR_DATA,
58 : USE_FOR_DATA_WEIGHT_SIGMA_FLAG
59 : };
60 :
61 : MSTransformDataHandler(const casacore::String& theMS, casacore::Table::TableOption option,
62 : bool virtualModelCol=false, bool virtualCorrectedCol=false,
63 : bool reindex=true);
64 : MSTransformDataHandler(const casacore::MeasurementSet& ms,
65 : bool virtualModelCol=false, bool virtualCorrectedCol=false,
66 : bool reindex=true);
67 : ~MSTransformDataHandler();
68 :
69 : // Declared static because it's used in setupMS().
70 : // colNameList is internally up-cased, so it is not const or passed by reference.
71 : static const casacore::Vector<casacore::MS::PredefinedColumns>& parseColumnNames(casacore::String colNameList);
72 :
73 : // This version uses the casacore::MeasurementSet to check what columns are present,
74 : // i.e. it makes col=="all" smarter, and it is not necessary to call
75 : // verifyColumns() after calling this. Unlike the other version, it knows
76 : // about FLOAT_DATA and LAG_DATA. It throws an exception if a
77 : // _specifically_ requested column is absent.
78 : static const casacore::Vector<casacore::MS::PredefinedColumns>& parseColumnNames(
79 : casacore::String colNameList, bool produceModel,
80 : const casacore::MeasurementSet& ms, bool virtualModelCol=false,
81 : bool virtualCorrectedCol=false);
82 :
83 : // Helper function for parseColumnNames(). Converts col to a list of
84 : // casacore::MS::PredefinedColumnss, and returns the # of recognized data columns.
85 : // static because parseColumnNames() is static.
86 : static casacore::uInt dataColStrToEnums(const casacore::String& col,casacore::Vector<casacore::MS::PredefinedColumns>& colvec);
87 :
88 : // Selection method using msselection syntax. casacore::Time is not handled by this method.
89 : bool setmsselect( const casacore::String& spw = "", const casacore::String& field = "",
90 : const casacore::String& baseline = "", const casacore::String& scan = "",
91 : const casacore::String& uvrange = "", const casacore::String& taql = "",
92 : const casacore::Vector<casacore::Int>& step = casacore::Vector<casacore::Int> (1, 1),
93 : const casacore::String& subarray = "", const casacore::String& correlation = "",
94 : const casacore::String& intent = "", const casacore::String& obs = "",
95 : const casacore::String& feed = "");
96 :
97 : // Select source or field
98 : bool selectSource(const casacore::Vector<casacore::Int>& fieldid);
99 :
100 : // Select spw and channels for each spw.
101 : bool selectSpw(const casacore::String& spwstr, const casacore::Vector<casacore::Int>& steps);
102 :
103 : // Returns the set (possibly empty) of spectral windows that are
104 : // in SPW but not listed in ms's DATA_DESCRIPTION sub-table.
105 : // (This happens with certain calibration/hardware setups.)
106 : static std::set<casacore::Int> findSpwsNotInDD(casacore::MeasurementSet& ms, casacore::Vector<casacore::Int> spwv);
107 :
108 : // Select Antennas to split out
109 : void selectAntenna(const casacore::Vector<casacore::Int>& antennaids,const casacore::Vector<casacore::String>& antennaSel);
110 :
111 : // Helper function for selectAntenna()
112 : static bool pickAntennas( casacore::Vector<casacore::Int>& selected_antennaids,
113 : casacore::Vector<casacore::String>& selected_antenna_strs,
114 : const casacore::Vector<casacore::Int>& antennaids,
115 : const casacore::Vector<casacore::String>& antennaSel);
116 :
117 : // Select array IDs to use.
118 : void selectArray(const casacore::String& subarray);
119 :
120 : // Setup polarization selection (for now, only from available correlations - no casacore::Stokes transformations.)
121 : bool selectCorrelations(const casacore::String& corrstr);
122 :
123 : // Fills outToIn[pol] with a map from output correlation index to input
124 : // correlation index, for each input polID pol.
125 : // It does not yet check the appropriateness of the correlation selection
126 : // string, so ignore the return value for now. outToIn[pol] defaults to
127 : // an empty casacore::Vector if no correlations are selected for pol.
128 : // That is not the same as the default "select everything in ms".
129 : static bool getCorrMaps( casacore::MSSelection& mssel,
130 : const casacore::MeasurementSet& ms,
131 : casacore::Vector<casacore::Vector<casacore::Int> >& outToIn,
132 : const bool areSelecting = false);
133 :
134 : // Select time parameters
135 : void selectTime(casacore::Double timeBin=-1.0, casacore::String timerng="");
136 :
137 : //Method to make the basic structure of the MS
138 : //
139 : //TileShape of size 1 can have 2 values [0], and [1] ...these are used in to
140 : //determine the tileshape by using MSTileLayout. Otherwise it has to be a
141 : //vector size 3 e.g [4, 15, 351] => a tile shape of 4 stokes, 15 channels 351
142 : //rows.
143 : //
144 : // combine sets combine_p. (Columns to ignore while time averaging.)
145 : //
146 : // createWeightSpectrumCols: add WEIGHT/SIGMA_SPECTRUM columns
147 : bool makeMSBasicStructure(casacore::String& msname,
148 : casacore::String& whichDataCol,
149 : bool produceModel,
150 : bool createWeightSpectrumCols,
151 : const casacore::Vector<casacore::Int>& tileShape = casacore::Vector<casacore::Int> (1, 0),
152 : const casacore::String& combine = "",
153 : casacore::Table::TableOption option=casacore::Table::New);
154 :
155 : bool isAllColumns(const casacore::Vector<casacore::MS::PredefinedColumns>& colNames);
156 :
157 : // Method that returns the selected ms (?! - but it's boolean - RR)
158 : bool makeSelection();
159 :
160 : // This sets up a default new ms
161 : // Declared static as it can be (and is) called directly from outside
162 : // Therefore it is not dependent on any member variable.
163 : static casacore::MeasurementSet* setupMS(const casacore::String& msname, const casacore::Int nchan,
164 : const casacore::Int npol, const casacore::String& telescop,
165 : const casacore::Vector<casacore::MS::PredefinedColumns>& colNamesTok,
166 : bool createWeightSpectrumCols,
167 : const casacore::Int obstype = 0, const bool compress = false,
168 : const asdmStManUseAlternatives asdmStManUse = DONT,
169 : casacore::Table::TableOption option=casacore::Table::New);
170 :
171 : // Same as above except allowing manual tileshapes
172 : static casacore::MeasurementSet* setupMS(const casacore::String& msname, const casacore::Int nchan,
173 : const casacore::Int npol, const casacore::Vector<casacore::MS::PredefinedColumns>& colNamesTok,
174 : bool createWeightSpectrumCols,
175 : const casacore::Vector<casacore::Int>& tileShape = casacore::Vector<casacore::Int> (1, 0),
176 : const bool compress = false,
177 : const asdmStManUseAlternatives asdmStManUse = DONT,
178 : casacore::Table::TableOption option=casacore::Table::New);
179 :
180 :
181 : // The output casacore::MS must have (at least?) 1 of DATA, FLOAT_DATA, or LAG_DATA.
182 : // MODEL_DATA or CORRECTED_DATA will be converted to DATA if necessary.
183 : // jagonzal (CAS-5327): The implementation has to go here because a member function cannot have static linkage
184 1860 : static bool mustConvertToData( const casacore::uInt nTok,const casacore::Vector<casacore::MS::PredefinedColumns>& datacols)
185 : {
186 1860 : return (nTok == 1) && (datacols[0] != casacore::MS::FLOAT_DATA) && (datacols[0] != casacore::MS::LAG_DATA);
187 : }
188 :
189 : // A customized version of casacore::MS::createDefaultSubtables().
190 : static void createSubtables(casacore::MeasurementSet& ms, casacore::Table::TableOption option);
191 :
192 : // Sub-table fillers.
193 : bool fillSubTables(const casacore::Vector<casacore::MS::PredefinedColumns>& colNames);
194 : bool fillFieldTable();
195 : bool fillDDTables(); // Includes spw and pol
196 :
197 : bool fillPolTable();
198 : bool fillDDITable();
199 : bool fillSPWTable();
200 :
201 : // Add optional columns to outTab if present in inTab and possColNames.
202 : // beLazy should only be true if outTab is in its default state.
203 : // Returns the number of added columns.
204 : static casacore::uInt addOptionalColumns(const casacore::Table& inTab, casacore::Table& outTab,const bool beLazy=false);
205 :
206 : // Sets up sourceRelabel_p for mapping input SourceIDs (if any) to output
207 : // ones. Must be called after fieldid_p is set and before calling
208 : // fillFieldTable() or copySource().
209 : void relabelSources();
210 :
211 : // Adds and copies inTab to msOut_p without any filtering.
212 : // tabName is the table "type", i.e. POINTING or SYSPOWER
213 : // without the preceding path.
214 : //
215 : // If noRows is true, the structure will be setup but no
216 : // rows will be copied (useful for filtering).
217 : void copySubtable(const casacore::String& tabName, const casacore::Table& inTab,const bool noRows = false);
218 :
219 : // Sets mapper to to a map from the distinct values of inv,
220 : // in increasing order, to 0, 1, 2, ..., mapper.size() - 1.
221 : static void make_map(std::map<casacore::Int, casacore::Int>& mapper, const casacore::Vector<casacore::Int>& inv);
222 :
223 : bool copyPointing();
224 : // Sets up the stub of a POINTING, enough to create an MSColumns.
225 : void setupNewPointing();
226 :
227 : bool copySource();
228 : bool copyAntenna();
229 : bool copyFeed();
230 : bool copyFlag_Cmd();
231 : bool copyHistory();
232 : bool copyObservation();
233 : bool copyProcessor();
234 : bool copyState();
235 : bool copySyscal();
236 : bool copyWeather();
237 : void copyMainTableKeywords (casacore::TableRecord& outKeys,
238 : const casacore::TableRecord& inKeys);
239 :
240 : casacore::Int getProcessorId(casacore::Int dataDescriptionId, casacore::String msname);
241 :
242 : // This falls between copyGenericSubtables() and the copiers for standard
243 : // sub-tables like copyFeed(). It is for optional sub-tables like CALDEVICE
244 : // and SYSPOWER which can be watched for by name and may need their
245 : // ANTENNA_ID and SPECTRAL_WINDOW_ID columns re-mapped.
246 : // (Technically FEED_ID, too, if split ever starts re-mapping feeds.)
247 : //
248 : // It must be called BEFORE copyGenericSubtables()!
249 : //
250 : bool filterOptSubtable(const casacore::String& subtabname);
251 :
252 : bool copyGenericSubtables();
253 :
254 : // To consolidate several sub-tables when dealing with MMS
255 : static bool mergeSpwSubTables(casacore::Vector<casacore::String> filenames);
256 : static bool mergeDDISubTables(casacore::Vector<casacore::String> filenames);
257 : static bool mergeFeedSubTables(casacore::Vector<casacore::String> filenames, casacore::Vector<casacore::uInt> mapSubmsSpwid);
258 : static bool mergeSourceSubTables(casacore::Vector<casacore::String> filenames, casacore::Vector<casacore::uInt> mapSubmsSpwid);
259 : static bool mergeSyscalSubTables(casacore::Vector<casacore::String> filenames, casacore::Vector<casacore::uInt> mapSubmsSpwid);
260 : static bool mergeFreqOffsetTables(casacore::Vector<casacore::String> filenames, casacore::Vector<casacore::uInt> mapSubmsSpwid);
261 : static bool mergeCalDeviceSubtables(casacore::Vector<casacore::String> filenames, casacore::Vector<casacore::uInt> mapSubmsSpwid);
262 : static bool mergeSysPowerSubtables(casacore::Vector<casacore::String> filenames, casacore::Vector<casacore::uInt> mapSubmsSpwid);
263 : // -----------------------------------------------------------------------
264 : //
265 : // -----------------------------------------------------------------------
266 : //template <class T> bool MSTransformDataHandler::columnOk (casacore::ArrayColumn<T> column)
267 4864 : template <class T> static bool columnOk(casacore::ArrayColumn<T> column)
268 : {
269 : bool ret;
270 : // jagonzal (CAS-6206): ndimColumn only returns >0 is there is the array column has fixed size
271 4864 : if (column.isNull()==false and column.hasContent()==true and column.ndim(0) > 0)
272 : {
273 800 : ret = true;
274 : }
275 : else
276 : {
277 4064 : ret = false;
278 : }
279 :
280 4864 : return ret;
281 : }
282 :
283 : // -----------------------------------------------------------------------
284 : //
285 : // -----------------------------------------------------------------------
286 : // template <class T> bool MSTransformDataHandler::columnOk (casacore::ScalarColumn<T> column)
287 7133 : template <class T> static bool columnOk(casacore::ScalarColumn<T> column)
288 : {
289 : bool ret;
290 7133 : if (column.isNull()==false and column.hasContent()==true)
291 : {
292 927 : ret = true;
293 : }
294 : else
295 : {
296 6206 : ret = false;
297 : }
298 :
299 7133 : return ret;
300 : }
301 :
302 :
303 :
304 : // Accesors for the casacore::MS objects
305 1988 : casacore::MeasurementSet * getInputMS() {return &ms_p;};
306 1860 : casacore::MeasurementSet * getSelectedInputMS() {return &mssel_p;};
307 1860 : casacore::MeasurementSet * getOutputMS() {return &msOut_p;};
308 1860 : casacore::MSColumns * getSelectedInputMSColumns() {return mscIn_p;};
309 1860 : casacore::MSColumns * getOutputMSColumns() {return msc_p;};
310 :
311 : // Accesors for the Re-mapper objects
312 : std::map<casacore::Int, casacore::Int> & getStateRemapper() {return stateRemapper_p;};
313 742 : casacore::Vector<casacore::Int> & getAntennaRemapper() {return antNewIndex_p;};
314 2778 : std::map<casacore::Int, std::vector<casacore::Int>> & getDroppedChannelsMap() {return spwDropChannelMap_p;};
315 0 : std::map<casacore::Int,std::map < casacore::Int, std::vector<casacore::Int> > > & getSelectedChannelsMap() {return spwSelectedChannelMap_p;};
316 :
317 : // Accesors for additional parameters
318 1886 : void setVirtualModelCol(bool virtualModelCol) {virtualModelCol_p = virtualModelCol;};
319 1886 : void setVirtualCorrectedCol(bool virtualCorrectedCol) {virtualCorrectedCol_p = virtualCorrectedCol;};
320 1886 : void setReindex(bool reindex) {reindex_p = reindex;};
321 :
322 : protected:
323 : // copy ephemeris table and reindex field table if requested
324 : bool copyEphemerisTable(casacore::MSFieldColumns & msField);
325 :
326 : // Initialized* by ctors. (Maintain order both here and in ctors.)
327 : // * not necessarily to anything useful.
328 : casacore::MeasurementSet ms_p, mssel_p;
329 : casacore::MSColumns * msc_p; // columns of msOut_p
330 : casacore::MSColumns * mscIn_p;
331 : bool keepShape_p, // Iff true, each output array has the
332 : // same shape as the corresponding input one.
333 : // sameShape_p, // Iff true, the shapes of the arrays do not
334 : // // vary with row number.
335 : antennaSel_p; // Selecting by antenna?
336 : casacore::Double timeBin_p;
337 : casacore::String scanString_p, // Selects scans by #number#. Historically named.
338 : intentString_p, // Selects scans by string. scanString_p was taken.
339 : obsString_p, // casacore::String for observationID selection.
340 : uvrangeString_p, taqlString_p, feedString_p;
341 : casacore::String timeRange_p, arrayExpr_p, corrString_p, spwString_p;
342 : casacore::String combine_p; // Should time averaging not split bins by
343 : // scan #, observation, and/or state ID?
344 : // Must be lowercase at all times.
345 : casacore::Int fitorder_p; // The polynomial order for continuum fitting.
346 : // If < 0 (default), continuum subtraction is
347 : // not done.
348 : casacore::String fitspw_p; // Selection string for line-free channels.
349 : casacore::String fitoutspw_p; // Selection string for output channels if doing
350 : // continuum subtraction.
351 :
352 : // Uninitialized by ctors.
353 : casacore::MeasurementSet msOut_p;
354 : casacore::Vector<casacore::Int> spw_p, // The input spw corresponding to each output spw.
355 : spw_uniq_p, // Uniquified version of spw_p.
356 : nchan_p, // The # of output channels for each range.
357 : totnchan_p, // The # of output channels for each output spw.
358 : chanStart_p, // 1st input channel index in a selection.
359 : chanEnd_p, // last input channel index in a selection.
360 : chanStep_p, // Increment between input chans, i.e. if 3, only every third
361 : // input channel will be used.
362 : widths_p, // # of input chans per output chan for each range.
363 : ncorr_p, // The # of output correlations for each DDID.
364 : inNumChan_p, // The # of input channels for each spw.
365 : inNumCorr_p; // The # of input correlations for each DDID.
366 :
367 : std::map<casacore::Int, std::vector<casacore::Int> > spwDropChannelMap_p;
368 : std::map<casacore::Int, std::map < casacore::Int, std::vector<casacore::Int> > > spwSelectedChannelMap_p;
369 :
370 : casacore::Vector<casacore::Int> fieldid_p;
371 : casacore::Vector<casacore::Int> spwRelabel_p, fieldRelabel_p, sourceRelabel_p;
372 : casacore::Vector<casacore::Int> oldDDSpwMatch_p;
373 : casacore::Vector<casacore::String> antennaSelStr_p;
374 : casacore::Vector<casacore::Int> antennaId_p;
375 : casacore::Vector<casacore::Int> antIndexer_p;
376 : casacore::Vector<casacore::Int> antNewIndex_p;
377 :
378 : casacore::Vector<casacore::Int> selObsId_p; // casacore::List of selected OBSERVATION_IDs.
379 : casacore::Vector<casacore::Int> polID_p; // casacore::Map from input DDID to input polID, filled in fillDDTables().
380 : casacore::Vector<casacore::Int> spw2ddid_p;
381 :
382 : // inCorrInd = outPolCorrToInCorrMap_p[polID_p[ddID]][outCorrInd]
383 : casacore::Vector<casacore::Vector<casacore::Int> > inPolOutCorrToInCorrMap_p;
384 :
385 : std::map<casacore::Int, casacore::Int> stateRemapper_p;
386 :
387 : casacore::Vector<casacore::Vector<casacore::Slice> > chanSlices_p; // Used by VisIterator::selectChannel()
388 : casacore::Vector<casacore::Slice> corrSlice_p;
389 : casacore::Vector<casacore::Vector<casacore::Slice> > corrSlices_p; // Used by VisIterator::selectCorrelation()
390 : casacore::Matrix<casacore::Double> selTimeRanges_p;
391 :
392 : bool virtualModelCol_p; // CAS-5348 (jagonzal): Make virtual MODEL data column real
393 : bool virtualCorrectedCol_p; //CAS-7286 (jagonzal): Make virtual CORRECTED data column real
394 : bool reindex_p; // jagonzal: In order not to re-index asub-tables
395 :
396 : };
397 :
398 : } //# NAMESPACE CASA - END
399 :
400 : #endif /* MSTransformDataHandler_H_ */
|
