Line data Source code
1 : //# MSTransformDataHandler.cc: This file contains the implementation of the MSTransformDataHandler class.
2 : //#
3 : //# CASA - Common Astronomy Software Applications (http://casa.nrao.edu/)
4 : //# Copyright (C) Associated Universities, Inc. Washington DC, USA 2011, All rights reserved.
5 : //# Copyright (C) European Southern Observatory, 2011, All rights reserved.
6 : //#
7 : //# This library is free software; you can redistribute it and/or
8 : //# modify it under the terms of the GNU Lesser General Public
9 : //# License as published by the Free software Foundation; either
10 : //# version 2.1 of the License, or (at your option) any later version.
11 : //#
12 : //# This library is distributed in the hope that it will be useful,
13 : //# but WITHOUT ANY WARRANTY, without even the implied warranty of
14 : //# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : //# Lesser General Public License for more details.
16 : //#
17 : //# You should have received a copy of the GNU Lesser General Public
18 : //# License along with this library; if not, write to the Free Software
19 : //# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20 : //# MA 02111-1307 USA
21 : //# $Id: $
22 :
23 : #include <mstransform/MSTransform/MSTransformDataHandler.h>
24 : #include <casacore/tables/Tables/TableProxy.h>
25 : #include <casacore/tables/TaQL/TableParse.h>
26 : #include <casacore/ms/MSOper/MSMetaData.h>
27 : #include <asdmstman/AsdmStMan.h>
28 :
29 :
30 : using namespace casacore;
31 : namespace casa { //# NAMESPACE CASA - BEGIN
32 :
33 : /////////////////////////////////////////////
34 : /// MSTransformDataHandler implementation ///
35 : /////////////////////////////////////////////
36 :
37 : // -----------------------------------------------------------------------
38 : //
39 : // -----------------------------------------------------------------------
40 1886 : MSTransformDataHandler::MSTransformDataHandler(const String& theMS, Table::TableOption option,
41 : bool virtualModelCol,bool virtualCorrectedCol,
42 1886 : bool reindex) :
43 : ms_p(MeasurementSet(theMS, option)),
44 1886 : mssel_p(ms_p),
45 : msc_p(NULL),
46 : mscIn_p(NULL),
47 : keepShape_p(true),
48 : antennaSel_p(false),
49 : timeBin_p(-1.0),
50 : scanString_p(""),
51 : intentString_p(""),
52 : obsString_p(""),
53 : uvrangeString_p(""),
54 : taqlString_p(""),
55 : timeRange_p(""),
56 : arrayExpr_p(""),
57 : combine_p(""),
58 : fitorder_p(-1),
59 : fitspw_p("*"),
60 : fitoutspw_p("*"),
61 : virtualModelCol_p(virtualModelCol),
62 : virtualCorrectedCol_p(virtualCorrectedCol),
63 1886 : reindex_p(reindex)
64 : {
65 1886 : return;
66 : }
67 :
68 : // -----------------------------------------------------------------------
69 : //
70 : // -----------------------------------------------------------------------
71 0 : MSTransformDataHandler::MSTransformDataHandler(const MeasurementSet& ms,
72 : bool virtualModelCol,bool virtualCorrectedCol,
73 0 : bool reindex) :
74 : ms_p(ms),
75 0 : mssel_p(ms_p),
76 : msc_p(NULL),
77 : mscIn_p(NULL),
78 : keepShape_p(true),
79 : antennaSel_p(false),
80 : timeBin_p(-1.0),
81 : scanString_p(""),
82 : intentString_p(""),
83 : obsString_p(""),
84 : uvrangeString_p(""),
85 : taqlString_p(""),
86 : timeRange_p(""),
87 : arrayExpr_p(""),
88 : combine_p(""),
89 : fitorder_p(-1),
90 : fitspw_p("*"),
91 : fitoutspw_p("*"),
92 : virtualModelCol_p(virtualModelCol),
93 : virtualCorrectedCol_p(virtualCorrectedCol),
94 0 : reindex_p(reindex)
95 : {
96 0 : return;
97 : }
98 :
99 : // -----------------------------------------------------------------------
100 : //
101 : // -----------------------------------------------------------------------
102 1886 : MSTransformDataHandler::~MSTransformDataHandler()
103 : {
104 1886 : if (msc_p) delete msc_p;
105 1886 : msc_p = nullptr;
106 :
107 1886 : if (mscIn_p) delete mscIn_p;
108 1886 : mscIn_p = nullptr;
109 :
110 1886 : msOut_p=MeasurementSet();
111 :
112 : // parseColumnNames unavoidably has a static String and Vector<MS::PredefinedColumns>.
113 : // Collapse them down to free most of that memory.
114 1886 : parseColumnNames("None");
115 :
116 1886 : return;
117 1886 : }
118 :
119 : // -----------------------------------------------------------------------
120 : //
121 : // -----------------------------------------------------------------------
122 1886 : const Vector<MS::PredefinedColumns>& MSTransformDataHandler::parseColumnNames(String col)
123 : {
124 : // Memorize both for efficiency and so that the info
125 : // message at the bottom isn't unnecessarily repeated.
126 1886 : static String my_colNameStr = "";
127 1886 : static Vector<MS::PredefinedColumns> my_colNameVect;
128 :
129 1886 : col.upcase();
130 1886 : if(col == my_colNameStr && col != "")
131 : {
132 0 : return my_colNameVect;
133 : }
134 1886 : else if(col == "NONE")
135 : {
136 1886 : my_colNameStr = "";
137 1886 : my_colNameVect.resize(0);
138 1886 : return my_colNameVect;
139 : }
140 :
141 : uInt nNames;
142 :
143 0 : if(col.contains("ALL"))
144 : {
145 0 : nNames = 3;
146 0 : my_colNameVect.resize(nNames);
147 0 : my_colNameVect[0] = MS::DATA;
148 0 : my_colNameVect[1] = MS::MODEL_DATA;
149 0 : my_colNameVect[2] = MS::CORRECTED_DATA;
150 : }
151 : else
152 : {
153 0 : nNames = dataColStrToEnums(col, my_colNameVect);
154 : }
155 :
156 : // Whether or not the MS has the columns is checked by verifyColumns().
157 : // Unfortunately it cannot be done here because this is a static method.
158 :
159 :
160 : /*
161 : * jagonzal: Redundant logging message (this info is already provided by MSTransformManager)
162 : *
163 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
164 : // "NONE" is used by the destructor
165 : if(col != "NONE")
166 : {
167 : os << LogIO::NORMAL << "Using ";
168 : for(uInt i = 0; i < nNames; ++i)
169 : {
170 : os << MS::columnName(my_colNameVect[i]) << " ";
171 : }
172 :
173 : os << " column" << (my_colNameVect.nelements() > 1 ? "s." : ".") << LogIO::POST;
174 : }
175 : */
176 :
177 0 : my_colNameStr = col;
178 0 : return my_colNameVect;
179 : }
180 :
181 : // -----------------------------------------------------------------------
182 : //
183 : // -----------------------------------------------------------------------
184 1878 : const Vector<MS::PredefinedColumns>& MSTransformDataHandler::parseColumnNames( String col,
185 : bool produceModel,
186 : const MeasurementSet& msref,
187 : bool virtualModelCol,
188 : bool virtualCorrectedCol)
189 : {
190 : // Memorize both for efficiency and so that the info
191 : // message at the bottom isn't unnecessarily repeated.
192 1878 : static String my_colNameStr = "";
193 1878 : static Vector<MS::PredefinedColumns> my_colNameVect;
194 :
195 : // Data columns to pick up if present.
196 3756 : Vector<MS::PredefinedColumns> wanted;
197 :
198 1878 : col.upcase();
199 :
200 : // This version of parseColumnNames does not reuse results of previous calls
201 : // but always checks the given columns because it cannot be certain that msref
202 : // refers to the same MS with every call.
203 :
204 1878 : if (col == "NONE")
205 : {
206 0 : my_colNameStr = "";
207 0 : my_colNameVect.resize(0);
208 0 : return my_colNameVect;
209 : }
210 :
211 3763 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
212 :
213 : // Are we choosy?
214 1878 : const bool doAny = col.contains("ALL") || col.contains("ANY");
215 :
216 : uInt nPoss;
217 1878 : if (doAny)
218 : {
219 366 : nPoss = 5;
220 366 : wanted.resize(nPoss);
221 366 : wanted[0] = MS::DATA;
222 366 : wanted[1] = MS::MODEL_DATA;
223 366 : wanted[2] = MS::CORRECTED_DATA;
224 366 : wanted[3] = MS::FLOAT_DATA;
225 366 : wanted[4] = MS::LAG_DATA;
226 : }
227 : // split name string into individual names
228 : else
229 : {
230 1512 : nPoss = dataColStrToEnums(col, wanted);
231 : }
232 :
233 : // Add MODEL_DATA separately from 'datacolumn' selection
234 1878 : if (produceModel) {
235 3 : const auto nelem = wanted.nelements();
236 3 : wanted.resize(nelem + 1, true);
237 3 : wanted[nelem] = MS::MODEL_DATA;
238 3 : ++nPoss;
239 : // When producing output MS with DATA,MODEL from CORRECTED, setupMS will have
240 : // to add MS::DATA to the output MS, not MS::CORRECTED_DATA.
241 : // An alternative would be to add a special case in setupMS, similar
242 : // to mustWriteOnlyToData, when wanted == CORRECTED_DATA, MODEL_DATA.
243 3 : if (MS::CORRECTED_DATA == wanted[0]) {
244 2 : wanted[0] = MS::DATA;
245 : }
246 : }
247 :
248 1878 : uInt nFound = 0;
249 1878 : my_colNameVect.resize(0);
250 5222 : for (uInt i = 0; i < nPoss; ++i)
251 : {
252 3351 : if (msref.tableDesc().isColumn(MS::columnName(wanted[i])))
253 : {
254 2231 : ++nFound;
255 2231 : my_colNameVect.resize(nFound, true);
256 2231 : my_colNameVect[nFound - 1] = wanted[i];
257 : }
258 : // CAS-5348 (jagonzal): Model parameters check is done at construction time
259 : // (produceModel comes from uvcontsub, doesn't require MODEL in input MS)
260 1120 : else if ((wanted[i] == MS::MODEL_DATA and virtualModelCol) or produceModel)
261 : {
262 3 : ++nFound;
263 3 : my_colNameVect.resize(nFound, true);
264 3 : my_colNameVect[nFound - 1] = wanted[i];
265 : }
266 1117 : else if (wanted[i] == MS::CORRECTED_DATA and virtualCorrectedCol)
267 : {
268 9 : ++nFound;
269 9 : my_colNameVect.resize(nFound, true);
270 9 : my_colNameVect[nFound - 1] = wanted[i];
271 : }
272 1108 : else if (!doAny)
273 : {
274 14 : ostringstream ostr;
275 7 : ostr << "Desired column (" << MS::columnName(wanted[i])
276 7 : << ") not found in the input MS (" << msref.tableName()
277 7 : << ").";
278 7 : throw(AipsError(ostr.str()));
279 : }
280 : }
281 1871 : if (nFound == 0) throw(AipsError("Did not find and select any data columns."));
282 :
283 1871 : my_colNameStr = col;
284 1871 : return my_colNameVect;
285 : }
286 :
287 : // -----------------------------------------------------------------------
288 : //
289 : // -----------------------------------------------------------------------
290 1512 : uInt MSTransformDataHandler::dataColStrToEnums(const String& col, Vector<MS::PredefinedColumns>& colvec)
291 : {
292 4536 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
293 3024 : String tmpNames(col);
294 1512 : Vector<String> tokens;
295 1512 : tmpNames.upcase();
296 :
297 : // split name string into individual names
298 : char * pch;
299 1512 : Int it = 0;
300 1512 : pch = strtok((char*) tmpNames.c_str(), " ,");
301 3030 : while (pch != NULL)
302 : {
303 1518 : tokens.resize(it + 1, true);
304 1518 : tokens[it] = String(pch);
305 1518 : ++it;
306 1518 : pch = strtok(NULL, " ,");
307 : }
308 :
309 1512 : uInt nNames = tokens.nelements();
310 :
311 1512 : uInt nFound = 0;
312 3030 : for (uInt i = 0; i < nNames; ++i)
313 : {
314 1518 : colvec.resize(nFound + 1, true);
315 1518 : colvec[nFound] = MS::UNDEFINED_COLUMN;
316 :
317 1518 : if ( tokens[i] == "OBSERVED"
318 1518 : || tokens[i] == "DATA"
319 3036 : || tokens[i] == MS::columnName(MS::DATA))
320 : {
321 821 : colvec[nFound++] = MS::DATA;
322 : }
323 697 : else if ( tokens[i] == "FLOAT"
324 697 : || tokens[i] == "FLOAT_DATA"
325 1394 : || tokens[i] == MS::columnName(MS::FLOAT_DATA))
326 : {
327 458 : colvec[nFound++] = MS::FLOAT_DATA;
328 : }
329 239 : else if ( tokens[i] == "LAG"
330 239 : || tokens[i] == "LAG_DATA"
331 478 : || tokens[i] == MS::columnName(MS::LAG_DATA))
332 : {
333 0 : colvec[nFound++] = MS::LAG_DATA;
334 : }
335 239 : else if ( tokens[i] == "MODEL"
336 201 : || tokens[i] == "MODEL_DATA"
337 440 : || tokens[i] == MS::columnName(MS::MODEL_DATA))
338 : {
339 38 : colvec[nFound++] = MS::MODEL_DATA;
340 : }
341 201 : else if ( tokens[i] == "CORRECTED"
342 0 : || tokens[i] == "CORRECTED_DATA"
343 201 : || tokens[i] == MS::columnName(MS::CORRECTED_DATA))
344 : {
345 201 : colvec[nFound++] = MS::CORRECTED_DATA;
346 : }
347 : // "NONE" is used by the destructor
348 0 : else if (tmpNames != "NONE")
349 : {
350 0 : os << LogIO::SEVERE;
351 :
352 0 : if (nFound == 0)
353 : {
354 0 : colvec[0] = MS::DATA;
355 0 : os << "Unrecognized data column " << tokens[i] << "...trying DATA.";
356 : }
357 : else
358 : {
359 0 : os << "Skipping unrecognized data column " << tokens[i];
360 : }
361 :
362 0 : os << LogIO::POST;
363 : }
364 : }
365 3024 : return nFound;
366 : }
367 :
368 : // -----------------------------------------------------------------------
369 : //
370 : // -----------------------------------------------------------------------
371 1878 : bool MSTransformDataHandler::setmsselect( const String& spw, const String& field,
372 : const String& baseline, const String& scan,
373 : const String& uvrange, const String& taql,
374 : const Vector<Int>& step, const String& subarray,
375 : const String& correlation, const String& intent,
376 : const String& obs, const String& feed)
377 : {
378 5634 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
379 : bool ok;
380 :
381 3756 : String myspwstr(spw == "" ? "*" : spw);
382 3756 : Record selrec = ms_p.msseltoindex(myspwstr, field);
383 :
384 1878 : ok = selectSource(selrec.asArrayInt("field"));
385 :
386 : // All of the requested selection functions will be tried, even if an
387 : // earlier one has indicated its failure. This allows all of the selection
388 : // strings to be tested, yielding more complete feedback for the user
389 : // (fewer retries). This is a matter of taste, though. If the selections
390 : // turn out to be slow, this function should return on the first false.
391 :
392 1878 : if (!selectSpw(myspwstr, step))
393 : {
394 0 : os << LogIO::SEVERE << "No channels selected." << LogIO::POST;
395 0 : ok = false;
396 : }
397 :
398 1878 : if (baseline != "")
399 : {
400 1566 : Vector<Int> antid(0);
401 1566 : Vector<String> antstr(1, baseline);
402 783 : selectAntenna(antid, antstr);
403 : }
404 :
405 1878 : scanString_p = scan;
406 1878 : intentString_p = intent;
407 1878 : obsString_p = obs;
408 1878 : uvrangeString_p = uvrange;
409 1878 : taqlString_p = taql;
410 1878 : feedString_p = feed;
411 :
412 1878 : if (subarray != "") selectArray(subarray);
413 :
414 1878 : if (!selectCorrelations(correlation))
415 : {
416 0 : os << LogIO::SEVERE << "No correlations selected." << LogIO::POST;
417 0 : ok = false;
418 : }
419 :
420 3756 : return ok;
421 : }
422 :
423 : // -----------------------------------------------------------------------
424 : //
425 : // -----------------------------------------------------------------------
426 1878 : bool MSTransformDataHandler::selectSource(const Vector<Int>& fieldid)
427 : {
428 3756 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
429 :
430 1878 : bool cando = true;
431 :
432 1878 : if (fieldid.nelements() < 1)
433 : {
434 1802 : fieldid_p = Vector<Int> (1, -1);
435 : }
436 76 : else if (fieldid.nelements() > ms_p.field().nrow())
437 : {
438 0 : os << LogIO::SEVERE << "More fields were requested than are in the input MS." << LogIO::POST;
439 0 : cando = false;
440 : }
441 76 : else if (max(fieldid) >= static_cast<Int> (ms_p.field().nrow()))
442 : {
443 : // Arriving here is very unlikely since if fieldid came from MSSelection
444 : // bad fields were presumably already quietly dropped.
445 0 : os << LogIO::SEVERE << "At least 1 field was requested that is not in the input MS." << LogIO::POST;
446 0 : cando = false;
447 : }
448 : else
449 : {
450 76 : fieldid_p = fieldid;
451 : }
452 :
453 1878 : if (fieldid_p.nelements() == 1 && fieldid_p[0] < 0)
454 : {
455 1802 : fieldid_p.resize(ms_p.field().nrow());
456 1802 : indgen(fieldid_p);
457 : }
458 :
459 3756 : return cando;
460 : }
461 :
462 : // -----------------------------------------------------------------------
463 : //
464 : // -----------------------------------------------------------------------
465 1878 : bool MSTransformDataHandler::selectSpw(const String& spwstr,const Vector<Int>& steps)
466 : {
467 5634 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
468 :
469 3756 : MSSelection mssel;
470 3756 : String myspwstr(spwstr == "" ? "*" : spwstr);
471 1878 : spwString_p = myspwstr;
472 :
473 1878 : mssel.setSpwExpr(myspwstr);
474 :
475 1878 : widths_p = steps.copy();
476 1878 : if (widths_p.nelements() < 1)
477 : {
478 0 : widths_p.resize(1);
479 0 : widths_p[0] = 1;
480 : }
481 : else
482 : {
483 5002 : for (uInt k = 0; k < widths_p.nelements(); ++k)
484 : {
485 3124 : if (widths_p[k] == 0)
486 : {
487 0 : os << LogIO::WARN << "0 cannot be used for channel width...using 1 instead." << LogIO::POST;
488 0 : widths_p[k] = 1;
489 : }
490 : }
491 : }
492 :
493 : // Each row should have spw, start, stop, step
494 : // A single width is a default, but multiple widths should be used literally.
495 3756 : Matrix<Int> chansel = mssel.getChanList(&ms_p, 1);
496 :
497 1878 : if (chansel.nrow() > 0)
498 : {
499 : // Use myspwstr if it selected anything...
500 1878 : spw_p = chansel.column(0);
501 1878 : chanStart_p = chansel.column(1);
502 1878 : chanEnd_p = chansel.column(2);
503 1878 : chanStep_p = chansel.column(3);
504 :
505 1878 : uInt nspw = chanEnd_p.nelements();
506 1878 : nchan_p.resize(nspw);
507 :
508 : // A single width is a default, but multiple widths should be used literally.
509 1878 : if (widths_p.nelements() > 1 && widths_p.nelements() != spw_p.nelements())
510 : {
511 : os << LogIO::SEVERE
512 : << "Mismatch between the # of widths specified by width and the # of spws."
513 0 : << LogIO::POST;
514 0 : return false;
515 : }
516 :
517 : // Copy the default width to all spws.
518 1878 : if (widths_p.nelements() < nspw)
519 : {
520 1043 : widths_p.resize(nspw, true);
521 10696 : for (uInt k = 1; k < nspw; ++k)
522 : {
523 9653 : widths_p[k] = widths_p[0];
524 : }
525 : }
526 :
527 14655 : for (uInt k = 0; k < nspw; ++k)
528 : {
529 : // CAS-2224, triggered by spw='0:2' (as opposed to '0:2~2').
530 12777 : if (chanStep_p[k] == 0) chanStep_p[k] = 1;
531 :
532 12777 : nchan_p[k] = 1 + (chanEnd_p[k] - chanStart_p[k]) / (chanStep_p[k]* widths_p[k]);
533 12777 : if (nchan_p[k] < 1) nchan_p[k] = 1;
534 : }
535 : }
536 : else
537 : {
538 : // Select everything and rely on widths.
539 0 : MSSpWindowColumns mySpwTab(ms_p.spectralWindow());
540 0 : uInt nspw = mySpwTab.nrow();
541 :
542 0 : nchan_p = mySpwTab.numChan().getColumn();
543 :
544 0 : spw_p.resize(nspw);
545 0 : indgen(spw_p);
546 :
547 0 : chanStart_p.resize(nspw);
548 0 : chanStep_p.resize(nspw);
549 0 : for (uInt k = 0; k < nspw; ++k)
550 : {
551 0 : chanStart_p[k] = 0;
552 0 : chanEnd_p[k] = nchan_p[k] - 1;
553 0 : chanStep_p[k] = 1;
554 : }
555 :
556 0 : if (widths_p.nelements() != spw_p.nelements())
557 : {
558 0 : if (widths_p.nelements() == 1)
559 : {
560 0 : widths_p.resize(spw_p.nelements(), true);
561 0 : for (uInt k = 1; k < spw_p.nelements(); ++k)
562 : {
563 0 : widths_p[k] = widths_p[0];
564 : }
565 :
566 : }
567 : else
568 : {
569 : os << LogIO::SEVERE
570 : << "Mismatch between the # of widths specified by width and the # of spws."
571 0 : << LogIO::POST;
572 0 : return false;
573 : }
574 : }
575 :
576 0 : for (uInt k = 0; k < nspw; ++k)
577 : {
578 0 : nchan_p[k] = 1 + (nchan_p[k] - 1) / widths_p[k];
579 : }
580 : }
581 :
582 : // Check for and filter out selected spws that aren't included in DATA_DESCRIPTION.
583 : // (See CAS-1673 for an example.)
584 1878 : std::set<Int> selectedSpwNotInDD(MSTransformDataHandler::findSpwsNotInDD(ms_p, spw_p));
585 1878 : uInt nSelectedSpwNotInDD = selectedSpwNotInDD.size();
586 1878 : if (nSelectedSpwNotInDD > 0)
587 : {
588 120 : os << LogIO::NORMAL << "The following a priori selected input spw(s)\n";
589 2308 : for (std::set<Int>::iterator spwit = selectedSpwNotInDD.begin();
590 4496 : spwit != selectedSpwNotInDD.end(); ++spwit)
591 : {
592 2188 : os << spw_p[*spwit] << " ";
593 : }
594 : os << "\nwere not found in DATA_DESCRIPTION (i. e. no rows in the main "
595 : "table reference them) and therefore "
596 120 : "are not included to the output." << LogIO::POST;
597 :
598 120 : uInt nSelSpw = spw_p.nelements();
599 120 : uInt ngoodSelSpwSlots = nSelSpw - nSelectedSpwNotInDD;
600 240 : Vector<Int> spwc(ngoodSelSpwSlots);
601 240 : Vector<Int> chanStartc(ngoodSelSpwSlots);
602 240 : Vector<Int> chanEndc(ngoodSelSpwSlots);
603 240 : Vector<Int> nchanc(ngoodSelSpwSlots);
604 240 : Vector<Int> chanStepc(ngoodSelSpwSlots);
605 120 : std::set<Int>::iterator spwNotDDEnd = selectedSpwNotInDD.end();
606 :
607 120 : uInt j = 0;
608 4780 : for (uInt k = 0; k < nSelSpw; ++k)
609 : {
610 4660 : if (selectedSpwNotInDD.find(k) == spwNotDDEnd)
611 : {
612 2472 : spwc[j] = spw_p[k];
613 2472 : chanStartc[j] = chanStart_p[k];
614 2472 : chanEndc[j] = chanEnd_p[k];
615 2472 : nchanc[j] = nchan_p[k];
616 2472 : chanStepc[j] = chanStep_p[k];
617 2472 : ++j;
618 : }
619 : }
620 120 : spw_p.resize(ngoodSelSpwSlots);
621 120 : spw_p = spwc;
622 120 : chanStart_p.resize(ngoodSelSpwSlots);
623 120 : chanStart_p = chanStartc;
624 120 : chanEnd_p.resize(ngoodSelSpwSlots);
625 120 : chanEnd_p = chanEndc;
626 120 : nchan_p.resize(ngoodSelSpwSlots);
627 120 : nchan_p = nchanc;
628 120 : chanStep_p.resize(ngoodSelSpwSlots);
629 120 : chanStep_p = chanStepc;
630 : }
631 :
632 1878 : mssel.getChanSlices(chanSlices_p, &ms_p, 1);
633 1878 : return true;
634 : }
635 :
636 : // -----------------------------------------------------------------------
637 : //
638 : // -----------------------------------------------------------------------
639 1878 : std::set<Int> MSTransformDataHandler::findSpwsNotInDD(MeasurementSet& ms,Vector<Int> spwv)
640 : {
641 1878 : ScalarColumn<Int> spws_in_dd( ms.dataDescription(),
642 3756 : MSDataDescription::columnName(MSDataDescription::SPECTRAL_WINDOW_ID));
643 3756 : std::set<Int> uniqSpwsInDD;
644 1878 : uInt nspwsInDD = spws_in_dd.nrow();
645 :
646 16901 : for (uInt ddrow = 0; ddrow < nspwsInDD; ++ddrow)
647 : {
648 15023 : uniqSpwsInDD.insert(spws_in_dd(ddrow));
649 : }
650 :
651 1878 : std::set<Int> badSelSpwSlots;
652 1878 : std::set<Int>::iterator ddend = uniqSpwsInDD.end();
653 14655 : for (uInt k = 0; k < spwv.nelements(); ++k)
654 : {
655 12777 : if (uniqSpwsInDD.find(spwv[k]) == ddend)
656 : {
657 2188 : badSelSpwSlots.insert(k);
658 : }
659 : }
660 :
661 3756 : return badSelSpwSlots;
662 : }
663 :
664 : // -----------------------------------------------------------------------
665 : //
666 : // -----------------------------------------------------------------------
667 783 : void MSTransformDataHandler::selectAntenna(const Vector<Int>& antennaids,const Vector<String>& antennaSel)
668 : {
669 783 : antennaSel_p = MSTransformDataHandler::pickAntennas(antennaId_p, antennaSelStr_p, antennaids,antennaSel);
670 783 : return;
671 : }
672 :
673 : // -----------------------------------------------------------------------
674 : //
675 : // -----------------------------------------------------------------------
676 783 : bool MSTransformDataHandler::pickAntennas( Vector<Int>& selected_antennaids,
677 : Vector<String>& selected_antenna_strs,
678 : const Vector<Int>& antennaids,
679 : const Vector<String>& antennaSel)
680 : {
681 783 : bool didSelect = true;
682 :
683 783 : if ((antennaids.nelements() == 1) && (antennaids[0] == -1))
684 : {
685 0 : if (antennaSel[0] == "")
686 : {
687 0 : didSelect = false;
688 : }
689 : else
690 : {
691 0 : selected_antennaids.resize();
692 : }
693 : }
694 : else
695 : {
696 783 : selected_antennaids = antennaids;
697 : }
698 :
699 783 : selected_antenna_strs = antennaSel;
700 :
701 783 : return didSelect;
702 : }
703 :
704 : // -----------------------------------------------------------------------
705 : //
706 : // -----------------------------------------------------------------------
707 3 : void MSTransformDataHandler::selectArray(const String& subarray)
708 : {
709 3 : arrayExpr_p = subarray;
710 3 : return;
711 : }
712 :
713 : // -----------------------------------------------------------------------
714 : //
715 : // -----------------------------------------------------------------------
716 1878 : bool MSTransformDataHandler::selectCorrelations(const String& corrstr)
717 : {
718 5634 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
719 :
720 1878 : corrString_p = corrstr;
721 1878 : const bool areSelecting = corrstr != "" && corrstr != "*";
722 :
723 : // Get correlation slices
724 3756 : MSSelection mssel1;
725 1878 : if (areSelecting) mssel1.setPolnExpr(corrstr.c_str());
726 1878 : mssel1.getCorrSlices(corrSlices_p, &ms_p);
727 :
728 : // Get correlation map
729 : // jagonzal (CAS-6951): We have to use another MSSelection because the first one corrupts the correlation
730 : // expression for instance "XX;YY" is turned into "XX" after calling getCorrSlices
731 3756 : MSSelection mssel2;
732 1878 : if (areSelecting) mssel2.setPolnExpr(corrstr.c_str());
733 3756 : return MSTransformDataHandler::getCorrMaps(mssel2, ms_p, inPolOutCorrToInCorrMap_p, areSelecting);
734 : }
735 :
736 : // -----------------------------------------------------------------------
737 : //
738 : // -----------------------------------------------------------------------
739 1878 : bool MSTransformDataHandler::getCorrMaps(MSSelection& mssel,
740 : const MeasurementSet& ms,
741 : Vector<Vector<Int> >& outToIn,
742 : const bool areSelecting)
743 : {
744 :
745 : // ?? This always returns true!!!?!!
746 1878 : bool cando = true;
747 :
748 : // The total number of polids
749 1878 : uInt npol = ms.polarization().nrow();
750 :
751 : // Nominally empty selection for all polids
752 1878 : outToIn.resize(npol);
753 1878 : outToIn.set(Vector<Int> ());
754 1878 : if (areSelecting)
755 : {
756 : // Get the corr indices as an ordered map
757 154 : std::map<Int, Vector<Vector<Int> > > corrmap(mssel.getCorrMap(&ms));
758 :
759 : // Iterate over the ordered map to fill the vector maps
760 176 : for ( auto mi = corrmap.begin( ); mi != corrmap.end( ); ++mi)
761 : {
762 99 : Int pol = mi->first;
763 99 : std::vector<int> correlations_idx = mi->second[0].tovector();
764 99 : std::sort(correlations_idx.begin(), correlations_idx.end());
765 99 : outToIn[pol] = Vector<Int>(correlations_idx);
766 : }
767 : }
768 : else
769 : { // Make outToIn an identity map.
770 3602 : ScalarColumn<Int> numCorr(ms.polarization(),MSPolarization::columnName(MSPolarization::NUM_CORR));
771 :
772 3957 : for (uInt polid = 0; polid < npol; ++polid)
773 : {
774 2156 : uInt ncorr = numCorr(polid);
775 2156 : outToIn[polid].resize(ncorr);
776 6819 : for (uInt cid = 0; cid < ncorr; ++cid)
777 : {
778 4663 : outToIn[polid][cid] = cid;
779 : }
780 : }
781 : }
782 :
783 1878 : return cando;
784 : }
785 :
786 : // -----------------------------------------------------------------------
787 : //
788 : // -----------------------------------------------------------------------
789 1878 : void MSTransformDataHandler::selectTime(Double timeBin, String timerng)
790 : {
791 1878 : timeBin_p = timeBin;
792 1878 : timeRange_p = timerng;
793 1878 : }
794 :
795 : // -----------------------------------------------------------------------
796 : //
797 : // -----------------------------------------------------------------------
798 1878 : bool MSTransformDataHandler::makeMSBasicStructure(String& msname,
799 : String& colname,
800 : bool produceModel,
801 : bool createWeightSpectrumCols,
802 : const Vector<Int>& tileShape,
803 : const String& combine,
804 : Table::TableOption option)
805 : {
806 5634 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
807 : os << LogIO::DEBUG1 << "Preparing to setup output MS with createWeightSpectrumCols: "
808 1878 : << createWeightSpectrumCols << LogIO::POST;;
809 :
810 1878 : if ((spw_p.nelements() > 0) && (max(spw_p) >= Int(ms_p.spectralWindow().nrow())))
811 : {
812 0 : os << LogIO::SEVERE << "SpectralWindow selection contains elements that do not exist in this MS" << LogIO::POST;
813 0 : ms_p = MeasurementSet();
814 0 : return false;
815 : }
816 :
817 : // Watch out! This throws an AipsError if ms_p doesn't have the requested columns.
818 : const Vector<MS::PredefinedColumns> colNamesTok =
819 3756 : parseColumnNames(colname,produceModel, ms_p,virtualModelCol_p,virtualCorrectedCol_p);
820 :
821 1871 : if (!makeSelection())
822 : {
823 9 : ms_p = MeasurementSet();
824 9 : throw(MSSelectionNullSelection("MSSelectionNullSelection : The selected table has zero rows."));
825 : return false;
826 : }
827 :
828 1860 : mscIn_p = new MSColumns(mssel_p);
829 :
830 : // Note again the parseColumnNames() a few lines back that stops setupMS()
831 : // from being called if the MS doesn't have the requested columns.
832 1860 : MeasurementSet* outpointer = 0;
833 :
834 1860 : if (tileShape.nelements() == 3)
835 : {
836 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0], colNamesTok,
837 : createWeightSpectrumCols, tileShape);
838 : }
839 :
840 : // the following calls MSTileLayout... disabled for now because it
841 : // forces tiles to be the full spw bandwidth in width (gmoellen, 2010/11/07)
842 1860 : else if ((tileShape.nelements() == 1) && (tileShape[0] == 0 || tileShape[0]== 1))
843 : {
844 1860 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0],
845 3720 : mscIn_p->observation().telescopeName()(0),
846 1860 : colNamesTok, createWeightSpectrumCols, tileShape[0]);
847 : }
848 : else {
849 : // Sweep all other cases of bad tileshape to a default one.
850 : // (this probably never happens)
851 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0],
852 0 : mscIn_p->observation().telescopeName()(0),
853 : colNamesTok, createWeightSpectrumCols, 0);
854 : }
855 :
856 1860 : combine_p = combine;
857 :
858 1860 : msOut_p = *outpointer;
859 :
860 : // handle column keywords copy for CORRECTED_DATA -> DATA
861 1860 : if (colNamesTok.nelements() == 1 && colNamesTok[0] == MS::CORRECTED_DATA && mssel_p.isColumn(MS::CORRECTED_DATA)) {
862 386 : TableColumn outCol(msOut_p, "DATA");
863 386 : TableColumn inCol(mssel_p, "CORRECTED_DATA");
864 : // Copy the keywords CORRECTED_DATA -> DATA
865 193 : copyMainTableKeywords(outCol.rwKeywordSet(), inCol.keywordSet());
866 : }
867 :
868 1860 : bool ret = true;
869 : try
870 : {
871 1860 : if (option == Table::Scratch)
872 : {
873 : // Set up pointing (has to be done in the copied MS)
874 0 : SetupNewTable pointingSetup(msOut_p.pointingTableName(),MSPointing::requiredTableDesc(), Table::New);
875 0 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::POINTING),Table(pointingSetup));
876 0 : msOut_p.initRefs();
877 :
878 : // Add additional columns to SPECTRAL_WINDOW sub-table
879 0 : addOptionalColumns(mssel_p.spectralWindow(), msOut_p.spectralWindow(), true);
880 :
881 : // Initialize output MS Columns
882 0 : msc_p = new MSColumns(msOut_p);
883 :
884 : // Write transformed SPECTRAL_WINDOW, DATA_DESCRIPTION_ID and POLARIZATION
885 0 : ret &= fillDDTables();
886 : }
887 : else
888 : {
889 1860 : ret = fillSubTables(colNamesTok);
890 : }
891 : }
892 0 : catch (AipsError ex)
893 : {
894 0 : ret = false;
895 : os << LogIO::SEVERE
896 : << "Exception filling the sub-tables: " << ex.getMesg() << endl
897 0 : << "Stack Trace: " << ex.getStackTrace()
898 0 : << LogIO::POST;
899 : }
900 :
901 1860 : if (!ret)
902 : {
903 0 : delete outpointer;
904 0 : ms_p = MeasurementSet();
905 0 : msOut_p = MeasurementSet();
906 0 : os << LogIO::SEVERE << msname << " left unfinished." << LogIO::POST;
907 0 : return false;
908 : }
909 :
910 : //Detaching the selected part
911 1860 : ms_p = MeasurementSet();
912 :
913 1860 : delete outpointer;
914 1860 : return true;
915 : }
916 :
917 : // -----------------------------------------------------------------------
918 : //
919 : // -----------------------------------------------------------------------
920 0 : bool MSTransformDataHandler::isAllColumns(const Vector<MS::PredefinedColumns>& colNames)
921 : {
922 0 : bool dCol = false, mCol = false, cCol = false;
923 0 : for (uInt i = 0; i < colNames.nelements(); i++)
924 : {
925 0 : if (colNames[i] == MS::DATA) dCol = true;
926 0 : else if (colNames[i] == MS::MODEL_DATA) mCol = true;
927 0 : else if (colNames[i] == MS::CORRECTED_DATA) cCol = true;
928 : // else turn off all?
929 : }
930 :
931 0 : return (dCol && mCol && cCol);
932 : }
933 :
934 : // -----------------------------------------------------------------------
935 : // Modified version of makeSelection that uses the new getter methods
936 : // MSS::getSPWDDIDList() and MSS::getDDIDList()
937 : // -----------------------------------------------------------------------
938 1871 : bool MSTransformDataHandler::makeSelection()
939 : {
940 :
941 5613 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
942 :
943 : // VisSet/MSIter will check if the SORTED exists
944 : // jagonzal (CAS-5327): Commenting this out, since this implies all sorts of memory leaks
945 : // Block<Int> sort;
946 : // ROVisibilityIterator(ms_p, sort);
947 :
948 : const MeasurementSet *elms;
949 1871 : elms = &ms_p;
950 3742 : MeasurementSet sorted;
951 1871 : if (ms_p.keywordSet().isDefined("SORTED_TABLE"))
952 : {
953 589 : sorted = ms_p.keywordSet().asTable("SORTED_TABLE");
954 :
955 : //If ms is not writable and sort is a subselection...use original ms
956 589 : if (ms_p.nrow() == sorted.nrow()) elms = &sorted;
957 : }
958 :
959 3742 : MSSelection thisSelection;
960 1871 : if (fieldid_p.nelements() > 0)
961 : {
962 1871 : thisSelection.setFieldExpr(MSSelection::indexExprStr(fieldid_p));
963 : }
964 :
965 1871 : if (spw_p.nelements() > 0)
966 : {
967 1871 : thisSelection.setSpwExpr(spwString_p);
968 : }
969 :
970 1871 : if (antennaSel_p)
971 : {
972 779 : if (antennaId_p.nelements() > 0)
973 : {
974 0 : thisSelection.setAntennaExpr(MSSelection::indexExprStr(antennaId_p));
975 : }
976 779 : if (antennaSelStr_p[0] != "")
977 : {
978 779 : thisSelection.setAntennaExpr(MSSelection::nameExprStr(antennaSelStr_p));
979 : }
980 :
981 : }
982 :
983 1871 : if (timeRange_p != "")
984 : {
985 39 : thisSelection.setTimeExpr(timeRange_p);
986 : }
987 :
988 :
989 1871 : thisSelection.setUvDistExpr(uvrangeString_p);
990 1871 : thisSelection.setScanExpr(scanString_p);
991 1871 : thisSelection.setStateExpr(intentString_p);
992 1871 : thisSelection.setObservationExpr(obsString_p);
993 :
994 1871 : if (arrayExpr_p != "")
995 : {
996 3 : thisSelection.setArrayExpr(arrayExpr_p);
997 : }
998 :
999 1871 : if (corrString_p != "")
1000 : {
1001 77 : thisSelection.setPolnExpr(corrString_p.c_str());
1002 : }
1003 :
1004 1871 : thisSelection.setTaQLExpr(taqlString_p);
1005 1871 : thisSelection.setFeedExpr(feedString_p);
1006 :
1007 3740 : TableExprNode exprNode = thisSelection.toTableExprNode(elms);
1008 1869 : selTimeRanges_p = thisSelection.getTimeList();
1009 1869 : selObsId_p = thisSelection.getObservationList();
1010 :
1011 : // Get the list of DDI for the selected spws
1012 1869 : spw2ddid_p = thisSelection.getSPWDDIDList(elms);
1013 :
1014 3738 : const MSDataDescription ddtable = elms->dataDescription();
1015 3738 : ScalarColumn<Int> polId(ddtable,MSDataDescription::columnName(MSDataDescription::POLARIZATION_ID));
1016 3738 : const MSPolarization poltable = elms->polarization();
1017 3738 : ArrayColumn<Int> pols(poltable,MSPolarization::columnName(MSPolarization::CORR_TYPE));
1018 :
1019 : // Get the list of DDI for the selected polarizations
1020 3738 : Vector<Int> polDDIList = thisSelection.getDDIDList(elms);
1021 :
1022 : // When polDDIList is empty, do not do an intersection
1023 1869 : bool doIntersection = true;
1024 :
1025 1869 : if (polDDIList.size() == 0){
1026 1792 : doIntersection = false;
1027 : }
1028 :
1029 : // intersection between selected DDI from spw selection and
1030 : // selected DDI from polarization selection
1031 1869 : if (doIntersection) {
1032 154 : Vector<Int> intersectedDDI = set_intersection(spw2ddid_p, polDDIList);
1033 77 : uInt nddids = intersectedDDI.size();
1034 77 : if (nddids > 0){
1035 77 : spw2ddid_p.resize(nddids);
1036 508 : for (uInt ii = 0; ii < nddids; ++ii){
1037 431 : spw2ddid_p[ii] = intersectedDDI[ii];
1038 : }
1039 : }
1040 : else {
1041 : os << LogIO::SEVERE << "None of the selected correlations are in spectral window "
1042 0 : << LogIO::POST;
1043 : }
1044 : }
1045 :
1046 :
1047 : // This is actually the number of selected DDI
1048 1869 : uInt nDDIs = spw2ddid_p.size();
1049 :
1050 1869 : inNumCorr_p.resize(nDDIs);
1051 1869 : ncorr_p.resize(nDDIs);
1052 :
1053 : // Map the correlations from input selected DDI to output
1054 12419 : for (uInt k = 0; k < nDDIs; ++k)
1055 : {
1056 10550 : Int ddid = spw2ddid_p[k];
1057 :
1058 : // Number of input correlations for each DDI
1059 : // It reads the nelements of the CORR_TYPE column cell
1060 10550 : inNumCorr_p[k] = pols(polId(ddid)).nelements();
1061 :
1062 : // Corresponding number of output correlations for each DDI
1063 10550 : ncorr_p[k] = inPolOutCorrToInCorrMap_p[polId(ddid)].nelements();
1064 10550 : if (ncorr_p[k] == 0)
1065 : {
1066 : os << LogIO::SEVERE
1067 : << "None of the selected correlations are in spectral window "
1068 0 : << spw_p[k] << LogIO::POST;
1069 0 : return false;
1070 : }
1071 : }
1072 :
1073 :
1074 : // Now remake the selected ms
1075 1869 : if (!(exprNode.isNull()))
1076 : {
1077 1869 : mssel_p = MeasurementSet((*elms)(exprNode));
1078 : }
1079 : else
1080 : {
1081 : // Null take all the ms ...setdata() blank means that
1082 0 : mssel_p = MeasurementSet((*elms));
1083 : }
1084 :
1085 1869 : if (mssel_p.nrow() == 0) return false;
1086 :
1087 : // Setup antNewIndex_p now that mssel_p is ready.
1088 1860 : if (antennaSel_p and reindex_p)
1089 : {
1090 : /*
1091 : // Watch out! getAntenna*List() and getBaselineList() return negative numbers for negated antennas!
1092 : ScalarColumn<Int> ant1c(mssel_p, MS::columnName(MS::ANTENNA1));
1093 : ScalarColumn<Int> ant2c(mssel_p, MS::columnName(MS::ANTENNA2));
1094 : Vector<Int> selAnts(ant1c.getColumn());
1095 : uInt nAnts = selAnts.nelements();
1096 :
1097 : selAnts.resize(2 * nAnts, true);
1098 : selAnts(Slice(nAnts, nAnts)) = ant2c.getColumn();
1099 : nAnts = GenSort<Int>::sort(selAnts, Sort::Ascending,Sort::NoDuplicates);
1100 : selAnts.resize(nAnts, true);
1101 : Int maxAnt = max(selAnts);
1102 : */
1103 :
1104 : // jagonzal: Scanning the main table is extremely inefficient, and depends on TaQL
1105 : // Therefore simply remove the negated antennas from getAntenna1List
1106 742 : vector<Int> antsSel;
1107 :
1108 : // Get antennas selected on position 1
1109 742 : Vector<Int> ant1List = thisSelection.getAntenna1List();
1110 1708 : for (uInt idx=0;idx<ant1List.size();idx++)
1111 : {
1112 966 : if (ant1List(idx) >= 0) antsSel.push_back(ant1List(idx));
1113 : }
1114 :
1115 : // Get antennas selected on position 2
1116 742 : Vector<Int> ant2List = thisSelection.getAntenna2List();
1117 1766 : for (uInt idx=0;idx<ant2List.size();idx++)
1118 : {
1119 1024 : if (ant2List(idx) >= 0) antsSel.push_back(ant2List(idx));
1120 : }
1121 :
1122 : // Sort and remove duplicates
1123 742 : std::sort(antsSel.begin(), antsSel.end());
1124 742 : antsSel.erase(std::unique(antsSel.begin(), antsSel.end()),antsSel.end());
1125 :
1126 742 : Vector<Int> selAnts(antsSel);
1127 742 : uInt nAnts = selAnts.size();
1128 742 : Int maxAnt = max(selAnts);
1129 :
1130 742 : if (maxAnt < 0)
1131 : {
1132 : os << LogIO::SEVERE
1133 : << "The maximum selected antenna number, " << maxAnt
1134 : << ", seems to be < 0."
1135 0 : << LogIO::POST;
1136 0 : return false;
1137 : }
1138 :
1139 742 : antNewIndex_p.resize(maxAnt + 1);
1140 : //So if you see -1 in the main, feed, or pointing tables, fix it
1141 742 : antNewIndex_p.set(-1);
1142 :
1143 1787 : for (uInt k = 0; k < nAnts; ++k)
1144 1045 : antNewIndex_p[selAnts[k]] = k;
1145 :
1146 : //If the total number of output antennas is the same as the input antennas
1147 : //this means that the selection of baselines includes at the end
1148 : //all the input antennas. Therefore setting antenna selection to false.
1149 : //See CAS-11111
1150 742 : if(nAnts == elms->antenna().nrow())
1151 1458 : antennaSel_p = false;
1152 : }
1153 : // This still gets tripped up by VLA:OUT.
1154 : else
1155 : {
1156 : // Make a default antNewIndex_p.
1157 1118 : antNewIndex_p.resize(mssel_p.antenna().nrow());
1158 1118 : indgen(antNewIndex_p);
1159 : }
1160 :
1161 1860 : if (mssel_p.nrow() < ms_p.nrow())
1162 : {
1163 : os << LogIO::NORMAL
1164 : << mssel_p.nrow() << " out of " << ms_p.nrow()
1165 : << " rows are going to be considered due to the selection criteria."
1166 764 : << LogIO::POST;
1167 : }
1168 :
1169 1860 : return true;
1170 : }
1171 :
1172 :
1173 : // -----------------------------------------------------------------------
1174 : //
1175 : // -----------------------------------------------------------------------
1176 1860 : MeasurementSet* MSTransformDataHandler::setupMS(const String& MSFileName, const Int nchan,
1177 : const Int nCorr, const String& telescop,
1178 : const Vector<MS::PredefinedColumns>& colNames,
1179 : bool createWeightSpectrumCols,
1180 : const Int obstype, const bool compress,
1181 : const asdmStManUseAlternatives asdmStManUse,
1182 : Table::TableOption option)
1183 : {
1184 : //Choose an appropriate tileshape
1185 3720 : IPosition dataShape(2, nCorr, nchan);
1186 1860 : IPosition tileShape = MSTileLayout::tileShape(dataShape, obstype, telescop);
1187 1860 : return setupMS(MSFileName, nchan, nCorr, colNames, createWeightSpectrumCols,
1188 5580 : tileShape.asVector(),compress, asdmStManUse,option);
1189 : }
1190 :
1191 : // -----------------------------------------------------------------------
1192 : //
1193 : // -----------------------------------------------------------------------
1194 1860 : MeasurementSet* MSTransformDataHandler::setupMS(const String& MSFileName, const Int nchan,
1195 : const Int nCorr,
1196 : const Vector<MS::PredefinedColumns>& colNamesTok,
1197 : bool createWeightSpectrumCols,
1198 : const Vector<Int>& tshape, const bool compress,
1199 : const asdmStManUseAlternatives asdmStManUse,
1200 : Table::TableOption option)
1201 : {
1202 1860 : if (tshape.nelements() != 3) throw(AipsError("TileShape has to have 3 elements "));
1203 :
1204 : // This is more to shush a compiler warning than to warn users.
1205 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1206 1860 : if (tshape[0] != nCorr)
1207 0 : os << LogIO::DEBUG1 << "Warning: using " << tshape[0]
1208 : << " from the tileshape instead of " << nCorr
1209 0 : << " for the number of correlations." << LogIO::POST;
1210 1860 : if (tshape[1] != nchan)
1211 1554 : os << LogIO::DEBUG1 << "Warning: using " << tshape[1]
1212 : << " from the tileshape instead of " << nchan
1213 777 : << " for the number of channels." << LogIO::POST;
1214 :
1215 : // Choose an appropriate tileshape //////////////////
1216 :
1217 3720 : IPosition tileShape(tshape);
1218 :
1219 : // Make the MS table
1220 3720 : TableDesc td = MS::requiredTableDesc();
1221 3720 : Vector<String> tiledDataNames;
1222 :
1223 1860 : if (option == Table::Scratch)
1224 : {
1225 0 : SetupNewTable newtab(MSFileName, td, option);
1226 0 : TableLock lock(TableLock::AutoLocking);
1227 0 : MeasurementSet *ms = new MeasurementSet(newtab, lock);
1228 :
1229 : // Set up default sub-tables for the MS
1230 0 : SetupNewTable dataDescSetup(ms->dataDescriptionTableName(),MSDataDescription::requiredTableDesc(), option);
1231 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::DATA_DESCRIPTION),Table(dataDescSetup));
1232 0 : SetupNewTable polarizationSetup(ms->polarizationTableName(),MSPolarization::requiredTableDesc(), option);
1233 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::POLARIZATION),Table(polarizationSetup));
1234 0 : SetupNewTable spectralWindowSetup(ms->spectralWindowTableName(),MSSpectralWindow::requiredTableDesc(), option);
1235 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::SPECTRAL_WINDOW),Table(spectralWindowSetup));
1236 0 : ms->initRefs();
1237 :
1238 0 : return ms;
1239 : }
1240 :
1241 : // Even though we know the data is going to be the same shape throughout I'll
1242 : // still create a column that has a variable shape as this will permit MS's
1243 : // with other shapes to be appended.
1244 1860 : uInt ncols = colNamesTok.nelements();
1245 1860 : const bool mustWriteOnlyToData = mustConvertToData(ncols, colNamesTok);
1246 1860 : if (mustWriteOnlyToData)
1247 : {
1248 1201 : MS::addColumnToDesc(td, MS::DATA, 2);
1249 1201 : if (asdmStManUse == DONT)
1250 : {
1251 1201 : if (compress) MS::addColumnCompression(td, MS::DATA, true);
1252 3603 : String hcolName = String("Tiled") + String("DATA");
1253 1201 : td.defineHypercolumn(hcolName, 3, stringToVector("DATA"));
1254 1201 : tiledDataNames.resize(1);
1255 1201 : tiledDataNames[0] = hcolName;
1256 : }
1257 : }
1258 : else
1259 : {
1260 659 : tiledDataNames.resize(ncols);
1261 1690 : for (uInt i = 0; i < ncols; ++i)
1262 : {
1263 : // Unfortunately MS::PredefinedColumns aren't ordered so that
1264 : // I can just check if colNamesTok[i] is in the "data range".
1265 1031 : if ( colNamesTok[i] == MS::DATA
1266 829 : || colNamesTok[i] == MS::MODEL_DATA
1267 651 : || colNamesTok[i] == MS::CORRECTED_DATA
1268 457 : || colNamesTok[i] == MS::FLOAT_DATA
1269 1860 : || colNamesTok[i] == MS::LAG_DATA)
1270 : {
1271 1031 : if (asdmStManUse == DONT || colNamesTok[i] != MS::DATA)
1272 : {
1273 1031 : MS::addColumnToDesc(td, colNamesTok[i], 2);
1274 1031 : if (compress) MS::addColumnCompression(td, colNamesTok[i], true);
1275 : }
1276 : }
1277 : else
1278 : {
1279 0 : throw(AipsError( MS::columnName(colNamesTok[i]) + " is not a recognized data column "));
1280 : }
1281 1031 : if (asdmStManUse == DONT || colNamesTok[i] != MS::DATA)
1282 : {
1283 2062 : String hcolName = String("Tiled") + MS::columnName(colNamesTok[i]);
1284 1031 : td.defineHypercolumn(hcolName, 3,stringToVector(MS::columnName(colNamesTok[i])));
1285 1031 : tiledDataNames[i] = hcolName;
1286 : }
1287 : }
1288 : }
1289 :
1290 : //other cols for compression
1291 1860 : if (compress && asdmStManUse != USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1292 : {
1293 0 : MS::addColumnCompression(td, MS::WEIGHT, true);
1294 0 : MS::addColumnCompression(td, MS::SIGMA, true);
1295 : }
1296 :
1297 1860 : if (createWeightSpectrumCols) {
1298 324 : MS::addColumnToDesc(td, MS::WEIGHT_SPECTRUM, 2);
1299 324 : MS::addColumnToDesc(td, MS::SIGMA_SPECTRUM, 2);
1300 :
1301 324 : td.defineHypercolumn("TiledWgtSpectrum", 3,stringToVector(MS::columnName(MS::WEIGHT_SPECTRUM)));
1302 324 : td.defineHypercolumn("TiledSigmaSpectrum", 3,stringToVector(MS::columnName(MS::SIGMA_SPECTRUM)));
1303 : }
1304 :
1305 1860 : td.defineHypercolumn("TiledFlagCategory", 4,stringToVector(MS::columnName(MS::FLAG_CATEGORY)));
1306 1860 : td.defineHypercolumn("TiledUVW", 2, stringToVector(MS::columnName(MS::UVW)));
1307 :
1308 1860 : if (asdmStManUse != USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1309 : {
1310 1860 : td.defineHypercolumn("TiledFlag", 3,stringToVector(MS::columnName(MS::FLAG)));
1311 1860 : td.defineHypercolumn("TiledWgt", 2,stringToVector(MS::columnName(MS::WEIGHT)));
1312 1860 : td.defineHypercolumn("TiledSigma", 2,stringToVector(MS::columnName(MS::SIGMA)));
1313 : }
1314 :
1315 3720 : SetupNewTable newtab(MSFileName, td, option);
1316 :
1317 1860 : uInt cache_val = 32768;
1318 :
1319 : // Set the default Storage Manager to be the Incr one
1320 3720 : IncrementalStMan incrStMan("ISMData", cache_val);
1321 1860 : newtab.bindAll(incrStMan, true);
1322 :
1323 : //Override the binding for specific columns
1324 3720 : IncrementalStMan incrStMan0("Array_ID", cache_val);
1325 1860 : newtab.bindColumn(MS::columnName(MS::ARRAY_ID), incrStMan0);
1326 3720 : IncrementalStMan incrStMan1("EXPOSURE", cache_val);
1327 1860 : newtab.bindColumn(MS::columnName(MS::EXPOSURE), incrStMan1);
1328 3720 : IncrementalStMan incrStMan2("FEED1", cache_val);
1329 1860 : newtab.bindColumn(MS::columnName(MS::FEED1), incrStMan2);
1330 3720 : IncrementalStMan incrStMan3("FEED2", cache_val);
1331 1860 : newtab.bindColumn(MS::columnName(MS::FEED2), incrStMan3);
1332 3720 : IncrementalStMan incrStMan4("FIELD_ID", cache_val);
1333 1860 : newtab.bindColumn(MS::columnName(MS::FIELD_ID), incrStMan4);
1334 :
1335 : // jagonzal (CAS-6746): Don't use IncrementalStMan with RW cols ///////////////////////////////////////////////////
1336 : // IncrementalStMan incrStMan5("FLAG_ROW",cache_val/4);
1337 : // newtab.bindColumn(MS::columnName(MS::FLAG_ROW), incrStMan5);
1338 3720 : StandardStMan aipsStManFlagRow("FLAG_ROW", cache_val/4);
1339 1860 : newtab.bindColumn(MS::columnName(MS::FLAG_ROW), aipsStManFlagRow);
1340 : ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
1341 :
1342 3720 : IncrementalStMan incrStMan6("INTERVAL", cache_val);
1343 1860 : newtab.bindColumn(MS::columnName(MS::INTERVAL), incrStMan6);
1344 3720 : IncrementalStMan incrStMan7("OBSERVATION_ID", cache_val);
1345 1860 : newtab.bindColumn(MS::columnName(MS::OBSERVATION_ID), incrStMan7);
1346 3720 : IncrementalStMan incrStMan8("PROCESSOR_ID", cache_val);
1347 1860 : newtab.bindColumn(MS::columnName(MS::PROCESSOR_ID), incrStMan8);
1348 3720 : IncrementalStMan incrStMan9("SCAN_NUMBER", cache_val);
1349 1860 : newtab.bindColumn(MS::columnName(MS::SCAN_NUMBER), incrStMan9);
1350 3720 : IncrementalStMan incrStMan10("STATE_ID", cache_val);
1351 1860 : newtab.bindColumn(MS::columnName(MS::STATE_ID), incrStMan10);
1352 3720 : IncrementalStMan incrStMan11("TIME", cache_val);
1353 1860 : newtab.bindColumn(MS::columnName(MS::TIME), incrStMan11);
1354 3720 : IncrementalStMan incrStMan12("TIME_CENTROID", cache_val);
1355 1860 : newtab.bindColumn(MS::columnName(MS::TIME_CENTROID), incrStMan12);
1356 :
1357 : // Bind ANTENNA1, ANTENNA2 and DATA_DESC_ID to the standardStMan
1358 : // as they may change sufficiently frequently to make the
1359 : // incremental storage manager inefficient for these columns.
1360 3720 : StandardStMan aipsStMan0("ANTENNA1", cache_val);
1361 1860 : newtab.bindColumn(MS::columnName(MS::ANTENNA1), aipsStMan0);
1362 3720 : StandardStMan aipsStMan1("ANTENNA2", cache_val);
1363 1860 : newtab.bindColumn(MS::columnName(MS::ANTENNA2), aipsStMan1);
1364 3720 : StandardStMan aipsStMan2("DATA_DESC_ID", cache_val);
1365 1860 : newtab.bindColumn(MS::columnName(MS::DATA_DESC_ID), aipsStMan2);
1366 :
1367 : // Bind the DATA, FLAG & WEIGHT/SIGMA_SPECTRUM columns to the tiled stman or
1368 : // asdmStMan
1369 3720 : AsdmStMan sm;
1370 :
1371 1860 : if (mustWriteOnlyToData)
1372 : {
1373 1201 : if (asdmStManUse == DONT)
1374 : {
1375 2402 : TiledShapeStMan tiledStMan1Data("TiledDATA", tileShape);
1376 1201 : newtab.bindColumn(MS::columnName(MS::DATA), tiledStMan1Data);
1377 : }
1378 : else
1379 : {
1380 0 : newtab.bindColumn(MS::columnName(MS::DATA), sm);
1381 : }
1382 : }
1383 : else
1384 : {
1385 1690 : for (uInt i = 0; i < ncols; ++i)
1386 : {
1387 2062 : TiledShapeStMan tiledStMan1Data(tiledDataNames[i], tileShape);
1388 1031 : newtab.bindColumn(MS::columnName(colNamesTok[i]), tiledStMan1Data);
1389 : }
1390 659 : if (asdmStManUse != DONT)
1391 : {
1392 0 : newtab.bindColumn(MS::columnName(MS::DATA), sm);
1393 : }
1394 : }
1395 :
1396 5580 : TiledShapeStMan tiledStMan1fc("TiledFlagCategory",IPosition(4, tileShape(0), tileShape(1), 1, tileShape(2)));
1397 1860 : newtab.bindColumn(MS::columnName(MS::FLAG_CATEGORY), tiledStMan1fc);
1398 :
1399 1860 : if (createWeightSpectrumCols) {
1400 648 : TiledShapeStMan tiledStMan2("TiledWgtSpectrum", tileShape);
1401 648 : TiledShapeStMan tiledStMan6("TiledSigmaSpectrum", tileShape);
1402 324 : newtab.bindColumn(MS::columnName(MS::WEIGHT_SPECTRUM), tiledStMan2);
1403 324 : newtab.bindColumn(MS::columnName(MS::SIGMA_SPECTRUM), tiledStMan6);
1404 : }
1405 :
1406 3720 : TiledColumnStMan tiledStMan3("TiledUVW",IPosition(2, 3, (tileShape(0) * tileShape(1) * tileShape(2)) / 3));
1407 1860 : newtab.bindColumn(MS::columnName(MS::UVW), tiledStMan3);
1408 1860 : if (asdmStManUse == USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1409 : {
1410 0 : newtab.bindColumn(MS::columnName(MS::FLAG), sm);
1411 0 : newtab.bindColumn(MS::columnName(MS::WEIGHT), sm);
1412 0 : newtab.bindColumn(MS::columnName(MS::SIGMA), sm);
1413 : }
1414 : else
1415 : {
1416 3720 : TiledShapeStMan tiledStMan1f("TiledFlag", tileShape);
1417 1860 : TiledShapeStMan tiledStMan4("TiledWgt", IPosition(2, tileShape(0),
1418 7440 : tileShape(1) * tileShape(2)));
1419 1860 : TiledShapeStMan tiledStMan5("TiledSigma",IPosition(2, tileShape(0),
1420 7440 : tileShape(1) * tileShape(2)));
1421 :
1422 1860 : newtab.bindColumn(MS::columnName(MS::FLAG), tiledStMan1f);
1423 1860 : newtab.bindColumn(MS::columnName(MS::WEIGHT), tiledStMan4);
1424 1860 : newtab.bindColumn(MS::columnName(MS::SIGMA), tiledStMan5);
1425 : }
1426 :
1427 : // Avoid lock overheads by locking the table permanently
1428 1860 : TableLock lock(TableLock::AutoLocking);
1429 1860 : MeasurementSet *ms = new MeasurementSet(newtab, lock);
1430 :
1431 : // Set up the sub-tables for the UVFITS MS (we make new tables with 0 rows)
1432 : // Table::TableOption option = Table::New;
1433 1860 : createSubtables(*ms, option);
1434 :
1435 : // Set the TableInfo
1436 1860 : TableInfo& info(ms->tableInfo());
1437 1860 : info.setType(TableInfo::type(TableInfo::MEASUREMENTSET));
1438 1860 : info.setSubType(String("UVFITS"));
1439 1860 : info.readmeAddLine("This is a measurement set Table holding astronomical observations");
1440 :
1441 1860 : return ms;
1442 : }
1443 :
1444 : // -----------------------------------------------------------------------
1445 : //
1446 : // -----------------------------------------------------------------------
1447 1860 : void MSTransformDataHandler::createSubtables(MeasurementSet& ms, Table::TableOption option)
1448 : {
1449 3720 : SetupNewTable antennaSetup(ms.antennaTableName(),MSAntenna::requiredTableDesc(), option);
1450 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::ANTENNA),Table(antennaSetup));
1451 3720 : SetupNewTable dataDescSetup(ms.dataDescriptionTableName(),MSDataDescription::requiredTableDesc(), option);
1452 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::DATA_DESCRIPTION),Table(dataDescSetup));
1453 3720 : SetupNewTable feedSetup(ms.feedTableName(), MSFeed::requiredTableDesc(),option);
1454 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FEED), Table(feedSetup));
1455 3720 : SetupNewTable flagCmdSetup(ms.flagCmdTableName(),MSFlagCmd::requiredTableDesc(), option);
1456 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FLAG_CMD),Table(flagCmdSetup));
1457 3720 : SetupNewTable fieldSetup(ms.fieldTableName(), MSField::requiredTableDesc(),option);
1458 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FIELD), Table(fieldSetup));
1459 3720 : SetupNewTable historySetup(ms.historyTableName(),MSHistory::requiredTableDesc(), option);
1460 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::HISTORY),Table(historySetup));
1461 3720 : SetupNewTable observationSetup(ms.observationTableName(),MSObservation::requiredTableDesc(), option);
1462 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::OBSERVATION),Table(observationSetup));
1463 3720 : SetupNewTable polarizationSetup(ms.polarizationTableName(),MSPolarization::requiredTableDesc(), option);
1464 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::POLARIZATION),Table(polarizationSetup));
1465 3720 : SetupNewTable processorSetup(ms.processorTableName(),MSProcessor::requiredTableDesc(), option);
1466 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::PROCESSOR),Table(processorSetup));
1467 3720 : SetupNewTable spectralWindowSetup(ms.spectralWindowTableName(),MSSpectralWindow::requiredTableDesc(), option);
1468 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::SPECTRAL_WINDOW),Table(spectralWindowSetup));
1469 3720 : SetupNewTable stateSetup(ms.stateTableName(), MSState::requiredTableDesc(),option);
1470 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::STATE), Table(stateSetup));
1471 :
1472 : // Add the optional Source sub table to allow for specification of the rest frequency
1473 3720 : SetupNewTable sourceSetup(ms.sourceTableName(),MSSource::requiredTableDesc(), option);
1474 1860 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::SOURCE),Table(sourceSetup, 0));
1475 :
1476 : // Update the references to the sub-table keywords
1477 1860 : ms.initRefs();
1478 :
1479 3720 : return;
1480 : }
1481 :
1482 : // -----------------------------------------------------------------------
1483 : //
1484 : // -----------------------------------------------------------------------
1485 1860 : bool MSTransformDataHandler::fillSubTables(const Vector<MS::PredefinedColumns>&)
1486 : {
1487 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1488 1860 : bool success = true;
1489 :
1490 : // Copy the sub-tables before doing anything with the main table.
1491 : // Otherwise MSColumns won't work.
1492 :
1493 : // fill or update
1494 1860 : Timer timer;
1495 :
1496 1860 : timer.mark();
1497 1860 : success &= copyPointing();
1498 1860 : os << LogIO::DEBUG1 << "copyPointing took " << timer.real() << "s." << LogIO::POST;
1499 :
1500 : // Optional columns should be set up before msc_p.
1501 1860 : addOptionalColumns(mssel_p.spectralWindow(), msOut_p.spectralWindow(), true);
1502 :
1503 : // Force the Measures frames for all the time type columns to
1504 : // have the same reference as the TIME column of the main table.
1505 : // Disable the empty table check (with false) because some of the
1506 : // sub-tables (like POINTING) might already have been written.
1507 : // However, empty tables are still empty after setting up the reference codes here.
1508 1860 : msc_p = new MSColumns(msOut_p);
1509 1860 : msc_p->setEpochRef(MEpoch::castType(mscIn_p->timeMeas().getMeasRef().getType()), false);
1510 :
1511 : // UVW is the only other Measures column in the main table.
1512 1860 : msc_p->uvwMeas().setDescRefCode(Muvw::castType(mscIn_p->uvwMeas().getMeasRef().getType()));
1513 :
1514 1860 : if (!mscIn_p->flagCategory().isNull() && mscIn_p->flagCategory().isDefined(0))
1515 : {
1516 415 : msc_p->setFlagCategories(mscIn_p->flagCategories());
1517 : }
1518 :
1519 :
1520 1860 : timer.mark();
1521 1860 : if (!fillDDTables()) return false;
1522 1860 : os << LogIO::DEBUG1 << "fillDDTables took " << timer.real() << "s." << LogIO::POST;
1523 :
1524 : // SourceIDs need to be re-mapped around here
1525 : // (It cannot not be done in selectSource() because mssel_p is not set up yet)
1526 1860 : timer.mark();
1527 1860 : relabelSources();
1528 1860 : os << LogIO::DEBUG1 << "relabelSources took " << timer.real() << "s." << LogIO::POST;
1529 :
1530 1860 : success &= fillFieldTable();
1531 1860 : success &= copySource();
1532 :
1533 1860 : success &= copyAntenna();
1534 : // Feed table writing has to be after antenna
1535 1860 : if (!copyFeed()) return false;
1536 :
1537 1860 : success &= copyFlag_Cmd();
1538 1860 : success &= copyHistory();
1539 1860 : success &= copyObservation();
1540 1860 : success &= copyProcessor();
1541 1860 : success &= copyState();
1542 :
1543 1860 : timer.mark();
1544 1860 : success &= copySyscal();
1545 1860 : os << LogIO::DEBUG1 << "copySyscal took " << timer.real() << "s." << LogIO::POST;
1546 :
1547 1860 : timer.mark();
1548 1860 : success &= copyWeather();
1549 1860 : os << LogIO::DEBUG1 << "copyWeather took " << timer.real() << "s." << LogIO::POST;
1550 :
1551 1860 : timer.mark();
1552 1860 : success &= filterOptSubtable("CALDEVICE");
1553 1860 : os << LogIO::DEBUG1 << "CALDEVICE took " << timer.real() << "s." << LogIO::POST;
1554 :
1555 1860 : timer.mark();
1556 1860 : success &= filterOptSubtable("SYSPOWER");
1557 1860 : os << LogIO::DEBUG1 << "SYSPOWER took " << timer.real() << "s." << LogIO::POST;
1558 :
1559 : // Run this after running the other copy*()s.
1560 : // Maybe there should be an option to *not* run it.
1561 1860 : success &= copyGenericSubtables();
1562 1860 : return success;
1563 : }
1564 :
1565 : // -----------------------------------------------------------------------
1566 : //
1567 : // -----------------------------------------------------------------------
1568 1860 : bool MSTransformDataHandler::fillFieldTable()
1569 : {
1570 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1571 :
1572 1860 : uInt nAddedCols = addOptionalColumns(mssel_p.field(), msOut_p.field(), true);
1573 :
1574 3720 : MSFieldColumns msField(msOut_p.field());
1575 :
1576 1860 : const MSFieldColumns& fieldIn = mscIn_p->field();
1577 3720 : ScalarColumn<String> code(fieldIn.code());
1578 3720 : ArrayColumn<Double> delayDir(fieldIn.delayDir());
1579 3720 : ScalarColumn<bool> flagRow(fieldIn.flagRow());
1580 3720 : ScalarColumn<String> name(fieldIn.name());
1581 3720 : ScalarColumn<Int> numPoly(fieldIn.numPoly());
1582 3720 : ArrayColumn<Double> phaseDir(fieldIn.phaseDir());
1583 3720 : ArrayColumn<Double> refDir(fieldIn.referenceDir());
1584 3720 : ScalarColumn<Int> sourceId(fieldIn.sourceId());
1585 3720 : ScalarColumn<Double> time(fieldIn.time());
1586 :
1587 3720 : String refstr;
1588 3720 : String nameVarRefColDelayDir, nameVarRefColPhaseDir, nameVarRefColRefDir;
1589 :
1590 : // Need to correctly define the direction measures.
1591 :
1592 : // DelayDir
1593 1860 : if (delayDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1594 : {
1595 1477 : delayDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1596 1477 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("Ref",refstr);
1597 : }
1598 :
1599 : // it's a variable ref. column
1600 1860 : if (delayDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1601 : {
1602 383 : delayDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1603 383 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1604 383 : nameVarRefColDelayDir = refstr;
1605 :
1606 766 : Vector<String> refTypeV;
1607 383 : delayDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1608 383 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1609 :
1610 766 : Vector<uInt> refCodeV;
1611 383 : delayDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1612 383 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1613 383 : Int refid = msField.delayDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1614 :
1615 : // Erase the redundant Ref keyword
1616 383 : if (refid >= 0)
1617 : {
1618 383 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1619 : }
1620 : }
1621 :
1622 : // PhaseDir
1623 1860 : if (phaseDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1624 : {
1625 1477 : phaseDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1626 1477 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("Ref",refstr);
1627 : }
1628 :
1629 : // It's a variable ref. column
1630 1860 : if (phaseDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1631 : {
1632 383 : phaseDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1633 383 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1634 383 : nameVarRefColPhaseDir = refstr;
1635 :
1636 766 : Vector<String> refTypeV;
1637 383 : phaseDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1638 383 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1639 :
1640 766 : Vector<uInt> refCodeV;
1641 383 : phaseDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1642 383 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1643 :
1644 383 : Int refid = msField.phaseDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1645 383 : if (refid >= 0)
1646 : {
1647 383 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1648 : }
1649 : }
1650 :
1651 : // ReferenceDir
1652 1860 : if (refDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1653 : {
1654 1477 : refDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1655 1477 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define(
1656 : "Ref", refstr);
1657 : }
1658 :
1659 : // It's a variable ref. column
1660 1860 : if (refDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1661 : {
1662 383 : refDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1663 383 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1664 383 : nameVarRefColRefDir = refstr;
1665 :
1666 766 : Vector<String> refTypeV;
1667 383 : refDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1668 383 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1669 :
1670 766 : Vector<uInt> refCodeV;
1671 383 : refDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1672 383 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1673 :
1674 383 : Int refid = msField.referenceDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1675 383 : if (refid >= 0)
1676 : {
1677 383 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1678 : }
1679 : }
1680 :
1681 : // ...and the time measure...
1682 1860 : time.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1683 1860 : msField.time().rwKeywordSet().asrwRecord("MEASINFO").define("Ref", refstr);
1684 :
1685 1860 : if (!reindex_p)
1686 : {
1687 94 : const MSField &inputField = mssel_p.field();
1688 94 : MSField &outputField = msOut_p.field();
1689 94 : TableCopy::copyRows(outputField, inputField);
1690 94 : copyEphemerisTable(msField);
1691 94 : return true;
1692 : }
1693 :
1694 : // fieldRelabel_p size: nrow of a input MS, -1 for unselected field ids
1695 1766 : fieldRelabel_p.resize(mscIn_p->field().nrow());
1696 1766 : fieldRelabel_p.set(-1);
1697 :
1698 : os << LogIO::DEBUG1 << fieldid_p.nelements()
1699 1766 : << " fields selected out of " << mscIn_p->field().nrow()
1700 1766 : << LogIO::POST;
1701 :
1702 : try {
1703 :
1704 1766 : msOut_p.field().addRow(fieldid_p.nelements());
1705 :
1706 6339 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1707 : {
1708 4573 : fieldRelabel_p[fieldid_p[k]] = k;
1709 :
1710 4573 : msField.code().put(k, code(fieldid_p[k]));
1711 4573 : msField.delayDir().put(k, delayDir(fieldid_p[k]));
1712 4573 : msField.flagRow().put(k, flagRow(fieldid_p[k]));
1713 4573 : msField.name().put(k, name(fieldid_p[k]));
1714 4573 : msField.numPoly().put(k, numPoly(fieldid_p[k]));
1715 4573 : msField.phaseDir().put(k, phaseDir(fieldid_p[k]));
1716 4573 : msField.referenceDir().put(k, refDir(fieldid_p[k]));
1717 4573 : msField.time().put(k, time(fieldid_p[k]));
1718 :
1719 4573 : Int inSrcID = sourceId(fieldid_p[k]);
1720 4573 : if (inSrcID < 0)
1721 : {
1722 517 : msField.sourceId().put(k, -1);
1723 : }
1724 : else
1725 : {
1726 4056 : msField.sourceId().put(k, sourceRelabel_p[inSrcID]);
1727 : }
1728 : }
1729 :
1730 1766 : if (nAddedCols > 0)
1731 : {
1732 339 : copyEphemerisTable(msField);
1733 :
1734 : // need to copy the reference column
1735 339 : if (!nameVarRefColDelayDir.empty())
1736 : {
1737 670 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColDelayDir);
1738 670 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColDelayDir);
1739 1658 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1740 : {
1741 1323 : cdMDirRef.put(k, dM(fieldid_p[k]));
1742 : }
1743 : }
1744 :
1745 : // need to copy the reference column
1746 339 : if (!nameVarRefColPhaseDir.empty())
1747 : {
1748 670 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColPhaseDir);
1749 670 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColPhaseDir);
1750 1658 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1751 : {
1752 1323 : cdMDirRef.put(k, dM(fieldid_p[k]));
1753 : }
1754 : }
1755 :
1756 : // need to copy the reference column
1757 339 : if (!nameVarRefColRefDir.empty())
1758 : {
1759 670 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColRefDir);
1760 670 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColRefDir);
1761 1658 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1762 : {
1763 1323 : cdMDirRef.put(k, dM(fieldid_p[k]));
1764 : }
1765 : }
1766 : }
1767 :
1768 : }
1769 0 : catch (AipsError x)
1770 : {
1771 0 : os << LogIO::EXCEPTION << "Error " << x.getMesg() << " setting up the output FIELD table." << LogIO::POST;
1772 : }
1773 0 : catch (...)
1774 : {
1775 0 : throw(AipsError("Unknown exception caught and released in fillFieldTable()"));
1776 : }
1777 :
1778 1766 : return true;
1779 : }
1780 :
1781 433 : bool MSTransformDataHandler::copyEphemerisTable(MSFieldColumns & msField)
1782 : {
1783 1299 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1784 433 : const MSFieldColumns& fieldIn = mscIn_p->field();
1785 433 : ScalarColumn<Int> eID(fieldIn.ephemerisId());
1786 :
1787 433 : if (eID.hasContent())
1788 : {
1789 : uInt nField;
1790 320 : if (reindex_p)
1791 : {
1792 274 : nField = fieldid_p.nelements();
1793 : }
1794 : else
1795 : {
1796 46 : nField = eID.nrow();
1797 : }
1798 :
1799 640 : String destPathName = Path(msOut_p.field().tableName()).absoluteName();
1800 640 : ScalarColumn<String> name(fieldIn.name());
1801 :
1802 1643 : for (uInt k = 0; k < nField; ++k)
1803 : {
1804 : uInt fieldId;
1805 1323 : if (reindex_p)
1806 : {
1807 1188 : fieldId = fieldid_p[k];
1808 : }
1809 : else
1810 : {
1811 135 : fieldId = k;
1812 : }
1813 :
1814 1323 : Int theEphId = eID(fieldId);
1815 :
1816 : // There is an ephemeris attached to this field
1817 1323 : if (theEphId > -1)
1818 : {
1819 100 : Path ephPath = Path(fieldIn.ephemPath(fieldId));
1820 :
1821 : // Copy the ephemeris table over to the output FIELD table
1822 50 : if (ephPath.length() > 0)
1823 : {
1824 50 : Directory origEphemDir(ephPath);
1825 50 : origEphemDir.copy(destPathName + "/" + ephPath.baseName());
1826 :
1827 : os << LogIO::NORMAL
1828 50 : << "Transferring ephemeris " << ephPath.baseName()
1829 50 : << " for output field " << name(fieldId)
1830 50 : << LogIO::POST;
1831 : }
1832 : }
1833 :
1834 1323 : if (reindex_p)
1835 : {
1836 1188 : msField.ephemerisId().put(k, theEphId);
1837 : }
1838 : }
1839 : }
1840 :
1841 866 : return true;
1842 : }
1843 :
1844 : // -----------------------------------------------------------------------
1845 : // Modified version of fillDDTables
1846 : // -----------------------------------------------------------------------
1847 1860 : bool MSTransformDataHandler::fillDDTables()
1848 : {
1849 1860 : fillPolTable();
1850 1860 : fillDDITable();
1851 1860 : fillSPWTable();
1852 :
1853 1860 : return true;
1854 : }
1855 :
1856 : // -----------------------------------------------------------------------
1857 : //
1858 : // -----------------------------------------------------------------------
1859 1860 : bool MSTransformDataHandler::fillPolTable()
1860 : {
1861 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1862 :
1863 : // Input polarization table
1864 1860 : const MSPolarization &poltable = mssel_p.polarization();
1865 3720 : ScalarColumn<Int> numCorr(poltable,MSPolarization::columnName(MSPolarization::NUM_CORR));
1866 3720 : ArrayColumn<Int> corrType(poltable,MSPolarization::columnName(MSPolarization::CORR_TYPE));
1867 3720 : ArrayColumn<Int> corrProd(poltable,MSPolarization::columnName(MSPolarization::CORR_PRODUCT));
1868 3720 : ScalarColumn<bool> flagRow(poltable,MSPolarization::columnName(MSPolarization::FLAG_ROW));
1869 :
1870 : // Output polarization table
1871 1860 : MSPolarizationColumns& msPol(msc_p->polarization());
1872 :
1873 : // Fill output polarization table
1874 1860 : uInt nPol = poltable.nrow(); // nOutputPol = nInputPol (no PolId re-index)
1875 1860 : corrSlice_p.resize(nPol);
1876 4109 : for (uInt polId = 0; polId < nPol; polId++)
1877 : {
1878 2249 : uInt ncorr = inPolOutCorrToInCorrMap_p[polId].nelements();
1879 2249 : const Vector<Int> inCT(corrType(polId));
1880 :
1881 : // Add row
1882 2249 : msOut_p.polarization().addRow();
1883 2249 : msPol.numCorr().put(polId, ncorr);
1884 2249 : msPol.flagRow().put(polId, flagRow(polId));
1885 :
1886 : // Setup correlation slices
1887 2249 : if (ncorr > 0 && ncorr < inCT.nelements())
1888 : {
1889 78 : keepShape_p = false;
1890 :
1891 : // Check whether the requested correlations can be accessed by slicing.
1892 : // That means there must be a constant stride. The most likely (only?)
1893 : // way to violate that is to ask for 3 out of 4 correlations.
1894 78 : if (ncorr > 2)
1895 : {
1896 : os << LogIO::SEVERE
1897 : << "Sorry, the requested correlation selection is not unsupported.\n"
1898 : << "Try selecting fewer or all of the correlations."
1899 0 : << LogIO::POST;
1900 0 : return false;
1901 : }
1902 :
1903 78 : size_t increment = 2;
1904 78 : if (ncorr > 1)
1905 : {
1906 18 : increment = inPolOutCorrToInCorrMap_p[polId][1] - inPolOutCorrToInCorrMap_p[polId][0];
1907 : }
1908 78 : corrSlice_p[polId] = Slice(inPolOutCorrToInCorrMap_p[polId][0],ncorr,increment);
1909 : }
1910 : else
1911 : {
1912 2171 : corrSlice_p[polId] = Slice(0, ncorr);
1913 : }
1914 :
1915 : // Apply slices to correlation type and product
1916 4498 : Vector<Int> outCT;
1917 4498 : const Matrix<Int> inCP(corrProd(polId));
1918 4498 : Matrix<Int> outCP;
1919 2249 : outCT.resize(ncorr);
1920 2249 : outCP.resize(2, ncorr);
1921 6984 : for (uInt k = 0; k < ncorr; ++k)
1922 : {
1923 4735 : Int inCorrInd = inPolOutCorrToInCorrMap_p[polId][k];
1924 :
1925 4735 : outCT[k] = inCT[inCorrInd];
1926 14205 : for (uInt feedind = 0; feedind < 2; ++feedind)
1927 : {
1928 9470 : outCP(feedind, k) = inCP(feedind, inCorrInd);
1929 : }
1930 :
1931 : }
1932 :
1933 : // Fill correlation type and product
1934 2249 : msPol.corrType().put(polId, outCT);
1935 2249 : msPol.corrProduct().put(polId, outCP);
1936 : }
1937 :
1938 1860 : return true;
1939 : }
1940 :
1941 : // -----------------------------------------------------------------------
1942 : //
1943 : // -----------------------------------------------------------------------
1944 1860 : bool MSTransformDataHandler::fillDDITable()
1945 : {
1946 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1947 :
1948 : // Input selected DDIs based on joint SPW-Pol selection
1949 1860 : uInt nddid = spw2ddid_p.size();
1950 :
1951 : // Input ddi table
1952 1860 : const MSDataDescription &inputDDI = mssel_p.dataDescription();
1953 :
1954 3720 : ScalarColumn<Int> polIdCol(inputDDI, MSDataDescription::columnName(MSDataDescription::POLARIZATION_ID));
1955 3720 : ScalarColumn<Int> spwIdCol(inputDDI,MSDataDescription::columnName(MSDataDescription::SPECTRAL_WINDOW_ID));
1956 :
1957 : // Get selected SPWs
1958 3720 : Vector<Int> selectedSpwIds(nddid);
1959 12397 : for (uInt row=0; row<spw2ddid_p.size(); ++row)
1960 : {
1961 10537 : selectedSpwIds[row] = spwIdCol(spw2ddid_p[row]);
1962 : }
1963 :
1964 : // Get list of unique selected SPWs
1965 1860 : bool option(false);
1966 3720 : Sort sortSpws(spw_p.getStorage(option), sizeof(Int));
1967 1860 : sortSpws.sortKey((uInt) 0, TpInt);
1968 3720 : Vector<uInt> spwsortindex, spwuniqinds;
1969 1860 : sortSpws.sort(spwsortindex, spw_p.nelements());
1970 1860 : uInt nuniqSpws = sortSpws.unique(spwuniqinds, spwsortindex);
1971 1860 : spw_uniq_p.resize(nuniqSpws);
1972 :
1973 : // Make map from input to output SPWs
1974 1860 : spwRelabel_p.resize(mscIn_p->spectralWindow().nrow());
1975 1860 : spwRelabel_p.set(-1);
1976 12368 : for (uInt k = 0; k < nuniqSpws; ++k)
1977 : {
1978 10508 : spw_uniq_p[k] = spw_p[spwuniqinds[k]];
1979 10508 : spwRelabel_p[spw_uniq_p[k]] = k;
1980 : }
1981 :
1982 1860 : if (!reindex_p)
1983 : {
1984 94 : MSDataDescription &outputDDI = msOut_p.dataDescription();
1985 94 : TableCopy::copyRows(outputDDI, inputDDI);
1986 94 : return true;
1987 : }
1988 :
1989 : // Output SPECTRAL_WINDOW_ID column
1990 3532 : Vector<Int> newSPWId(nddid);
1991 11151 : for (uInt ddi=0; ddi<nddid; ddi++)
1992 : {
1993 9385 : newSPWId[ddi] = spwRelabel_p[spwIdCol(spw2ddid_p[ddi])];
1994 : }
1995 :
1996 : // Output POLARIZATION_ID column
1997 1766 : Vector<Int> newPolId(nddid);
1998 11151 : for (uInt ddi=0; ddi<nddid; ddi++)
1999 : {
2000 9385 : newPolId[ddi] = polIdCol(spw2ddid_p[ddi]);
2001 : }
2002 :
2003 : // Fill output DDI table
2004 1766 : MSDataDescColumns& outputDDI(msc_p->dataDescription());
2005 11151 : for (uInt ddi=0; ddi<nddid; ddi++)
2006 : {
2007 9385 : msOut_p.dataDescription().addRow();
2008 9385 : outputDDI.flagRow().put(ddi, false);
2009 9385 : outputDDI.polarizationId().put(ddi, newPolId[ddi]);
2010 9385 : outputDDI.spectralWindowId().put(ddi, newSPWId[ddi]);
2011 : }
2012 :
2013 :
2014 1766 : return true;
2015 : }
2016 :
2017 : // -----------------------------------------------------------------------
2018 : //
2019 : // -----------------------------------------------------------------------
2020 1860 : bool MSTransformDataHandler::fillSPWTable()
2021 : {
2022 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2023 :
2024 : // Selected input MS SPW Table Columns (read-only)
2025 3720 : MSSpWindowColumns inSpWCols(mssel_p.spectralWindow());
2026 :
2027 : // SPW Table Columns of output MS (read-write)
2028 1860 : MSSpWindowColumns& msSpW(msc_p->spectralWindow());
2029 :
2030 : // Detect which optional columns of SPECTRAL_WINDOW are present.
2031 : // inSpWCols and msSpW should agree because addOptionalColumns() was done for
2032 : // SPECTRAL_WINDOW in fillAllTables() before making msc_p or calling fillDDTables
2033 1860 : bool haveSpwAN = columnOk(inSpWCols.assocNature());
2034 1860 : bool haveSpwASI = columnOk(inSpWCols.assocSpwId());
2035 1860 : bool haveSpwBN = columnOk(inSpWCols.bbcNo());
2036 1860 : bool haveSpwBS = columnOk(inSpWCols.bbcSideband());
2037 1860 : bool haveSpwDI = columnOk(inSpWCols.dopplerId());
2038 3855 : bool haveSpwSWF = mssel_p.spectralWindow().tableDesc().isColumn("SDM_WINDOW_FUNCTION") &&
2039 1995 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_WINDOW_FUNCTION");
2040 3855 : bool haveSpwSNB = mssel_p.spectralWindow().tableDesc().isColumn("SDM_NUM_BIN") &&
2041 1995 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_NUM_BIN");
2042 3815 : bool haveSpwCorrBit = mssel_p.spectralWindow().tableDesc().isColumn("SDM_CORR_BIT") &&
2043 1955 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_CORR_BIT");
2044 :
2045 1860 : uInt nuniqSpws = spw_uniq_p.size();
2046 :
2047 : // This sets the number of input channels for each spw. But
2048 : // it considers that a SPW ID contains only one set of channels.
2049 : // I hope this is true!!
2050 : // Write to SPECTRAL_WINDOW table
2051 1860 : inNumChan_p.resize(spw_p.nelements());
2052 12375 : for (uInt k = 0; k < spw_p.nelements(); ++k)
2053 : {
2054 10515 : inNumChan_p[k] = inSpWCols.numChan()(spw_p[k]);
2055 : }
2056 :
2057 1860 : if (reindex_p)
2058 : {
2059 1766 : msOut_p.spectralWindow().addRow(nuniqSpws);
2060 : }
2061 : else
2062 : {
2063 94 : const MSSpectralWindow &inputSPW = mssel_p.spectralWindow();
2064 94 : MSSpectralWindow &outputSPW = msOut_p.spectralWindow();
2065 94 : TableCopy::copyRows(outputSPW, inputSPW);
2066 : }
2067 :
2068 :
2069 3720 : Vector<Vector<Int> > spwinds_of_uniq_spws(nuniqSpws);
2070 1860 : totnchan_p.resize(nuniqSpws);
2071 12368 : for (uInt k = 0; k < nuniqSpws; ++k)
2072 : {
2073 10508 : Int maxchan = 0;
2074 10508 : uInt j = 0;
2075 :
2076 10508 : totnchan_p[k] = 0;
2077 10508 : spwinds_of_uniq_spws[k].resize();
2078 182407 : for (uInt spwind = 0; spwind < spw_p.nelements(); ++spwind)
2079 : {
2080 171899 : if (spw_p[spwind] == spw_uniq_p[k])
2081 : {
2082 10515 : Int highchan = nchan_p[spwind] * chanStep_p[spwind] + chanStart_p[spwind];
2083 :
2084 10515 : if (highchan > maxchan) maxchan = highchan;
2085 :
2086 10515 : totnchan_p[k] += nchan_p[spwind];
2087 :
2088 : // The true is necessary to avoid scrambling previously assigned values.
2089 10515 : spwinds_of_uniq_spws[k].resize(j + 1, true);
2090 :
2091 : // Warning! spwinds_of_uniq_spws[k][j] will compile without warning, but dump core at runtime.
2092 10515 : (spwinds_of_uniq_spws[k])[j] = spwind;
2093 :
2094 10515 : ++j;
2095 : }
2096 : }
2097 10508 : if (maxchan > inSpWCols.numChan()(spw_uniq_p[k]))
2098 : {
2099 : os << LogIO::SEVERE
2100 : << " Channel settings wrong; exceeding number of channels in spw "
2101 0 : << spw_uniq_p[k] << LogIO::POST;
2102 0 : return false;
2103 : }
2104 : }
2105 :
2106 : // min_k is an index for getting an spw index via spw_uniq_p[min_k].
2107 : // k is an index for getting an spw index via spw_p[k].
2108 12368 : for (uInt min_k = 0; min_k < nuniqSpws; ++min_k)
2109 : {
2110 10508 : uInt k = spwinds_of_uniq_spws[min_k][0];
2111 10508 : uInt outSPWId = reindex_p? min_k : spw_p[k];
2112 :
2113 10508 : if (spwinds_of_uniq_spws[min_k].nelements() > 1 || nchan_p[k] != inSpWCols.numChan()(spw_p[k]))
2114 : {
2115 2906 : Vector<Double> effBWIn = inSpWCols.effectiveBW()(spw_uniq_p[min_k]);
2116 1453 : Int nOutChan = totnchan_p[min_k];
2117 2906 : Vector<Double> chanFreqOut(nOutChan);
2118 2906 : Vector<Double> chanFreqIn = inSpWCols.chanFreq()(spw_uniq_p[min_k]);
2119 2906 : Vector<Double> chanWidthOut(nOutChan);
2120 2906 : Vector<Double> chanWidthIn = inSpWCols.chanWidth()(spw_uniq_p[min_k]);
2121 2906 : Vector<Double> spwResolOut(nOutChan);
2122 2906 : Vector<Double> spwResolIn = inSpWCols.resolution()(spw_uniq_p[min_k]);
2123 2906 : Vector<Double> effBWOut(nOutChan);
2124 1453 : Int outChan = 0;
2125 1453 : Int outChanNotDropped = 0;
2126 :
2127 1453 : keepShape_p = false;
2128 :
2129 : // The sign of CHAN_WIDTH defaults to +. Its determination assumes that
2130 : // chanFreqIn is monotonic, but not that the sign of the chanWidthIn is correct.
2131 1453 : bool neginc = chanFreqIn[chanFreqIn.nelements() - 1] < chanFreqIn[0];
2132 :
2133 1453 : effBWOut.set(0.0);
2134 1453 : Double totalBW = 0.0;
2135 2913 : for (uInt rangeNum = 0; rangeNum < spwinds_of_uniq_spws[min_k].nelements(); ++rangeNum)
2136 : {
2137 1460 : k = spwinds_of_uniq_spws[min_k][rangeNum];
2138 :
2139 1460 : Int span = chanStep_p[k] * widths_p[k];
2140 :
2141 44468 : for (Int j = 0; j < nchan_p[k]; ++j)
2142 : {
2143 43008 : Int inpChan = chanStart_p[k] + j * span;
2144 :
2145 43008 : if (span >= 1)
2146 : {
2147 43008 : Int lastChan = inpChan + span - 1;
2148 :
2149 43008 : if (lastChan > chanEnd_p[k])
2150 : {
2151 : // The averaging width is not a factor of the number of
2152 : // selected input channels, so the last output bin receives
2153 : // fewer input channels than the other bins.
2154 14 : lastChan = chanEnd_p[k];
2155 :
2156 14 : Int nchan = lastChan - inpChan + 1;
2157 : os << LogIO::NORMAL
2158 : << "The last output channel of spw "
2159 28 : << spw_p[k] << " has only " << nchan
2160 14 : << " input channel";
2161 14 : if (nchan > 1) os << "s.";
2162 14 : os << LogIO::POST;
2163 :
2164 : // jagonzal (CAS-8618): We may have more than one channel dropped per SPW
2165 : // due to multiple channel selections
2166 14 : spwDropChannelMap_p[spw_p[k]].push_back(outChan);
2167 : }
2168 : else
2169 : {
2170 132819 : for (Int inputChan = inpChan;inputChan<=lastChan;inputChan++)
2171 : {
2172 89825 : spwSelectedChannelMap_p[spw_p[k]][outChanNotDropped].push_back(inputChan);
2173 : }
2174 42994 : outChanNotDropped++;
2175 : }
2176 :
2177 43008 : chanFreqOut[outChan] = (chanFreqIn[inpChan]
2178 43008 : + chanFreqIn[lastChan]) / 2;
2179 :
2180 43008 : Double sep = chanFreqIn[lastChan] - chanFreqIn[inpChan];
2181 :
2182 43008 : if (neginc) sep = -sep;
2183 :
2184 : // The internal abs is necessary because the sign of chanWidthIn may be wrong.
2185 43008 : chanWidthOut[outChan] = sep + 0.5 * abs(chanWidthIn[inpChan] + chanWidthIn[lastChan]);
2186 43008 : if (neginc) chanWidthOut[outChan] = -chanWidthOut[outChan];
2187 :
2188 43008 : spwResolOut[outChan] = 0.5 * (spwResolIn[inpChan] + spwResolIn[lastChan]) + sep;
2189 :
2190 132888 : for (Int avgChan = inpChan; avgChan <= lastChan; avgChan += chanStep_p[k])
2191 : {
2192 89880 : effBWOut[outChan] += effBWIn[avgChan];
2193 : }
2194 :
2195 : }
2196 : else
2197 : {
2198 0 : chanFreqOut[outChan] = chanFreqIn[inpChan];
2199 0 : spwResolOut[outChan] = spwResolIn[inpChan];
2200 0 : chanWidthOut[outChan] = chanWidthIn[inpChan];
2201 0 : effBWOut[outChan] = effBWIn[inpChan];
2202 : }
2203 : // Use CHAN_WIDTH instead of EFFECTIVE_BW
2204 43008 : totalBW += abs(chanWidthOut[outChan]);
2205 43008 : ++outChan;
2206 : }
2207 : }
2208 1453 : --outChan;
2209 :
2210 1453 : msSpW.chanFreq().put(outSPWId, chanFreqOut);
2211 1453 : msSpW.refFrequency().put(outSPWId,min(chanFreqOut[0], chanFreqOut[chanFreqOut.size() - 1]));
2212 1453 : msSpW.resolution().put(outSPWId, spwResolOut);
2213 1453 : msSpW.numChan().put(outSPWId, nOutChan);
2214 1453 : msSpW.chanWidth().put(outSPWId, chanWidthOut);
2215 1453 : msSpW.effectiveBW().put(outSPWId, spwResolOut);
2216 1453 : msSpW.totalBandwidth().put(outSPWId, totalBW);
2217 : }
2218 : else
2219 : {
2220 9055 : msSpW.chanFreq().put(outSPWId, inSpWCols.chanFreq()(spw_p[k]));
2221 9055 : msSpW.refFrequency().put(outSPWId, inSpWCols.refFrequency()(spw_p[k]));
2222 9055 : msSpW.resolution().put(outSPWId, inSpWCols.resolution()(spw_p[k]));
2223 9055 : msSpW.numChan().put(outSPWId, inSpWCols.numChan()(spw_p[k]));
2224 9055 : msSpW.chanWidth().put(outSPWId, inSpWCols.chanWidth()(spw_p[k]));
2225 9055 : msSpW.effectiveBW().put(outSPWId, inSpWCols.effectiveBW()(spw_p[k]));
2226 9055 : msSpW.totalBandwidth().put(outSPWId,inSpWCols.totalBandwidth()(spw_p[k]));
2227 : }
2228 :
2229 10508 : msSpW.flagRow().put(outSPWId, inSpWCols.flagRow()(spw_p[k]));
2230 10508 : msSpW.freqGroup().put(outSPWId, inSpWCols.freqGroup()(spw_p[k]));
2231 10508 : msSpW.freqGroupName().put(outSPWId, inSpWCols.freqGroupName()(spw_p[k]));
2232 10508 : msSpW.ifConvChain().put(outSPWId, inSpWCols.ifConvChain()(spw_p[k]));
2233 10508 : msSpW.measFreqRef().put(outSPWId, inSpWCols.measFreqRef()(spw_p[k]));
2234 10508 : msSpW.name().put(outSPWId, inSpWCols.name()(spw_p[k]));
2235 10508 : msSpW.netSideband().put(outSPWId, inSpWCols.netSideband()(spw_p[k]));
2236 :
2237 :
2238 10508 : if (haveSpwBN) msSpW.bbcNo().put(outSPWId, inSpWCols.bbcNo()(spw_p[k]));
2239 10508 : if (haveSpwBS) msSpW.bbcSideband().put(outSPWId, inSpWCols.bbcSideband()(spw_p[k]));
2240 10508 : if (haveSpwDI) msSpW.dopplerId().put(outSPWId, inSpWCols.dopplerId()(spw_p[k]));
2241 10508 : if (haveSpwSWF)
2242 : {
2243 5676 : ScalarColumn<String> inSwfCol(mssel_p.spectralWindow(), "SDM_WINDOW_FUNCTION");
2244 2838 : ScalarColumn<String> outSwfCol(msOut_p.spectralWindow(), "SDM_WINDOW_FUNCTION");
2245 2838 : outSwfCol.put(outSPWId, inSwfCol(spw_p[k]));
2246 : }
2247 10508 : if (haveSpwSNB)
2248 : {
2249 5676 : ScalarColumn<Int> inSnbCol(mssel_p.spectralWindow(), "SDM_NUM_BIN");
2250 2838 : ScalarColumn<Int> outSnbCol(msOut_p.spectralWindow(), "SDM_NUM_BIN");
2251 2838 : outSnbCol.put(outSPWId, inSnbCol(spw_p[k]));
2252 : }
2253 10508 : if (haveSpwCorrBit)
2254 : {
2255 3654 : ScalarColumn<String> inCorrBitCol(mssel_p.spectralWindow(), "SDM_CORR_BIT");
2256 1827 : ScalarColumn<String> outCorrBitCol(msOut_p.spectralWindow(), "SDM_CORR_BIT");
2257 1827 : outCorrBitCol.put(outSPWId, inCorrBitCol(spw_p[k]));
2258 : }
2259 :
2260 10508 : if (haveSpwASI)
2261 : {
2262 3985 : if (reindex_p)
2263 : {
2264 : // Get list of SPWs associated to his one
2265 6584 : std::vector<Int> selectedSPWs = spw_p.tovector();
2266 :
2267 : // Get the list of selected SPWs and association nature
2268 6584 : Array<Int> assocSpwId = inSpWCols.assocSpwId()(spw_p[k]);
2269 6584 : Array<String> assocNature = inSpWCols.assocNature()(spw_p[k]);
2270 :
2271 : // Find which associated SPWs are selected, and store the transformed Id
2272 3292 : std::vector<Int>::iterator findIt;
2273 6584 : std::vector<Int> selectedAssocSpwId;
2274 6584 : std::vector<String> selectedAssocNature;
2275 53784 : for (uInt idx=0;idx<assocSpwId.size();idx++)
2276 : {
2277 100984 : IPosition pos(1,idx);
2278 50492 : Int spw = assocSpwId(pos);
2279 50492 : findIt = find (selectedSPWs.begin(), selectedSPWs.end(), spw);
2280 50492 : if (findIt != selectedSPWs.end())
2281 : {
2282 5656 : selectedAssocSpwId.push_back(spwRelabel_p[spw]);
2283 5656 : if (haveSpwAN) selectedAssocNature.push_back(assocNature(pos));
2284 : }
2285 : }
2286 :
2287 : // Store selected associated SPW Ids
2288 6584 : Vector<Int> selectedAssocSpwIdVector(selectedAssocSpwId);
2289 3292 : msSpW.assocSpwId().put(min_k, selectedAssocSpwIdVector);
2290 :
2291 : // Store selected association nature
2292 3292 : if (haveSpwAN)
2293 : {
2294 6584 : Vector<String> selectedAssocNatureVector(selectedAssocNature);
2295 3292 : msSpW.assocNature().put(min_k, selectedAssocNatureVector);
2296 : }
2297 : }
2298 : else
2299 : {
2300 693 : msSpW.assocNature().put(outSPWId, inSpWCols.assocNature()(outSPWId));
2301 : }
2302 : }
2303 : }
2304 :
2305 :
2306 1860 : return true;
2307 : }
2308 :
2309 : // -----------------------------------------------------------------------
2310 : //
2311 : // -----------------------------------------------------------------------
2312 9635 : uInt MSTransformDataHandler::addOptionalColumns(const Table& inTab, Table& outTab,const bool beLazy)
2313 : {
2314 9635 : uInt nAdded = 0;
2315 9635 : const TableDesc& inTD = inTab.actualTableDesc();
2316 :
2317 : // Only rely on the # of columns if you are sure that inTab and outTab
2318 : // can't have the same # of columns without having _different_ columns,
2319 : // i.e. use beLazy if outTab.actualTableDesc() is in its default state.
2320 9635 : uInt nInCol = inTD.ncolumn();
2321 9635 : if (!beLazy || nInCol > outTab.actualTableDesc().ncolumn())
2322 : {
2323 13632 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2324 :
2325 9088 : Vector<String> oldColNames = inTD.columnNames();
2326 :
2327 67484 : for (uInt k = 0; k < nInCol; ++k)
2328 : {
2329 62940 : if (!outTab.actualTableDesc().isColumn(oldColNames[k]))
2330 : {
2331 : try
2332 : {
2333 28148 : outTab.addColumn(inTD.columnDesc(k), false);
2334 28148 : ++nAdded;
2335 : }
2336 : // NOT AipsError x
2337 0 : catch (...)
2338 : {
2339 : os << LogIO::WARN << "Could not add column "
2340 0 : << oldColNames[k] << " to " << outTab.tableName()
2341 0 : << LogIO::POST;
2342 : }
2343 : }
2344 : }
2345 : }
2346 :
2347 19270 : return nAdded;
2348 : }
2349 :
2350 : // -----------------------------------------------------------------------
2351 : //
2352 : // -----------------------------------------------------------------------
2353 1860 : void MSTransformDataHandler::relabelSources()
2354 : {
2355 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2356 :
2357 : //Source is an optional table, so it may not exist
2358 1860 : if (Table::isReadable(mssel_p.sourceTableName()))
2359 : {
2360 : // Note that mscIn_p->field().sourceId() has ALL of the
2361 : // sourceIDs in the input MS, not just the selected ones.
2362 3720 : const Vector<Int>& inSrcIDs = mscIn_p->field().sourceId().getColumn();
2363 :
2364 1860 : Int highestInpSrc = max(inSrcIDs);
2365 :
2366 : // Ensure space for -1.
2367 1860 : if (highestInpSrc < 0) highestInpSrc = 0;
2368 :
2369 1860 : sourceRelabel_p.resize(highestInpSrc + 1);
2370 : // Default to "any".
2371 1860 : sourceRelabel_p.set(-1);
2372 :
2373 : // Enable sourceIDs that are actually referred
2374 : // by selected fields, and remap them using j.
2375 1860 : uInt j = 0;
2376 6663 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
2377 : {
2378 4803 : Int fldInSrcID = inSrcIDs[fieldid_p[k]];
2379 :
2380 4803 : if (fldInSrcID > -1)
2381 : {
2382 : // Multiple fields can use the same
2383 4286 : if (sourceRelabel_p[fldInSrcID] == -1)
2384 : {
2385 : // source in a mosaic.
2386 3804 : sourceRelabel_p[fldInSrcID] = j;
2387 3804 : ++j;
2388 : }
2389 : }
2390 : }
2391 : }
2392 : else
2393 : {
2394 : // Default to "any".
2395 : os << LogIO::NORMAL
2396 : << "The input MS does not have the optional SOURCE table.\n"
2397 0 : << "-1 will be used as a generic source ID." << LogIO::POST;
2398 0 : sourceRelabel_p.resize(1);
2399 0 : sourceRelabel_p.set(-1);
2400 : }
2401 :
2402 3720 : return;
2403 : }
2404 :
2405 : // -----------------------------------------------------------------------
2406 : //
2407 : // -----------------------------------------------------------------------
2408 2212 : void MSTransformDataHandler::copySubtable(const String& tabName, const Table& inTab,const bool doFilter)
2409 : {
2410 4424 : String outName(msOut_p.tableName() + '/' + tabName);
2411 :
2412 2212 : if (PlainTable::tableCache()(outName)) PlainTable::tableCache().remove(outName);
2413 2212 : inTab.deepCopy(outName, Table::New, false, Table::AipsrcEndian, doFilter);
2414 4424 : Table outTab(outName, Table::Update);
2415 2212 : msOut_p.rwKeywordSet().defineTable(tabName, outTab);
2416 2212 : msOut_p.initRefs();
2417 :
2418 4424 : return;
2419 : }
2420 :
2421 : // -----------------------------------------------------------------------
2422 : //
2423 : // -----------------------------------------------------------------------
2424 0 : void MSTransformDataHandler::make_map(std::map<Int, Int>& mapper, const Vector<Int>& inv)
2425 : {
2426 0 : std::set<Int> valSet;
2427 :
2428 : // Strange, but slightly more efficient than going forward.
2429 0 : for (Int i = inv.nelements(); i--;)
2430 : {
2431 0 : valSet.insert(inv[i]);
2432 : }
2433 :
2434 0 : uInt remaval = 0;
2435 0 : for (std::set<Int>::const_iterator vs_iter = valSet.begin(); vs_iter != valSet.end(); ++vs_iter)
2436 : {
2437 0 : mapper[*vs_iter] = remaval;
2438 0 : ++remaval;
2439 : }
2440 :
2441 0 : return;
2442 : }
2443 :
2444 : // -----------------------------------------------------------------------
2445 : //
2446 : // -----------------------------------------------------------------------
2447 1860 : bool MSTransformDataHandler::copyPointing()
2448 : {
2449 3720 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2450 :
2451 : //Pointing is allowed to not exist
2452 1860 : if (Table::isReadable(mssel_p.pointingTableName()))
2453 : {
2454 1860 : const MSPointing& oldPoint = mssel_p.pointing();
2455 :
2456 1860 : if ((!antennaSel_p && timeRange_p == "") or !reindex_p)
2457 : {
2458 : // jagonzal: copySubtables works with POINTING because it
2459 : // is not created by default in the method createSubtables
2460 1812 : copySubtable(MS::keywordName(MS::POINTING), oldPoint);
2461 : }
2462 : else
2463 : {
2464 48 : setupNewPointing();
2465 :
2466 48 : if (oldPoint.nrow() > 0)
2467 : {
2468 : // Could be declared as Table&
2469 22 : MSPointing& newPoint = msOut_p.pointing();
2470 :
2471 : // Add optional columns if present in oldPoint.
2472 22 : uInt nAddedCols = addOptionalColumns(oldPoint, newPoint, true);
2473 22 : os << LogIO::DEBUG1 << "POINTING has " << nAddedCols << " optional columns." << LogIO::POST;
2474 :
2475 44 : const MSPointingColumns oldPCs(oldPoint);
2476 44 : MSPointingColumns newPCs(newPoint);
2477 22 : newPCs.setEpochRef(MEpoch::castType(oldPCs.timeMeas().getMeasRef().getType()));
2478 22 : newPCs.setDirectionRef(MDirection::castType(oldPCs.directionMeasCol().getMeasRef().getType()));
2479 22 : newPCs.setEncoderDirectionRef(MDirection::castType(oldPCs.encoderMeas().getMeasRef().getType()));
2480 :
2481 22 : const ScalarColumn<Int>& antIds = oldPCs.antennaId();
2482 22 : const ScalarColumn<Double>& time = oldPCs.time();
2483 22 : ScalarColumn<Int>& outants = newPCs.antennaId();
2484 :
2485 22 : uInt nTRanges = selTimeRanges_p.ncolumn();
2486 :
2487 22 : uInt outRow = 0;
2488 :
2489 : // Int for comparison
2490 22 : Int maxSelAntp1 = antNewIndex_p.nelements();
2491 : // with newAntInd.
2492 122172 : for (uInt inRow = 0; inRow < antIds.nrow(); ++inRow)
2493 : {
2494 122150 : Int newAntInd = antIds(inRow);
2495 122150 : if (antennaSel_p)
2496 : {
2497 0 : newAntInd = newAntInd < maxSelAntp1 ? antNewIndex_p[newAntInd] : -1;
2498 : }
2499 :
2500 122150 : Double t = time(inRow);
2501 :
2502 122150 : if (newAntInd > -1)
2503 : {
2504 122150 : bool matchT = false;
2505 122150 : if (nTRanges == 0)
2506 : {
2507 0 : matchT = true;
2508 : }
2509 : else
2510 : {
2511 242222 : for (uInt tr = 0; tr < nTRanges; ++tr)
2512 : {
2513 122150 : if (t >= selTimeRanges_p(0, tr) && t <= selTimeRanges_p(1, tr))
2514 : {
2515 2078 : matchT = true;
2516 2078 : break;
2517 : }
2518 : }
2519 : }
2520 :
2521 122150 : if (matchT)
2522 : {
2523 2078 : TableCopy::copyRows(newPoint, oldPoint, outRow,inRow, 1, false);
2524 2078 : outants.put(outRow, newAntInd);
2525 2078 : ++outRow;
2526 : }
2527 : }
2528 : }
2529 : }
2530 : }
2531 : }
2532 : else
2533 : {
2534 : // Make an empty stub for MSColumns.
2535 0 : setupNewPointing();
2536 : }
2537 :
2538 :
2539 3720 : return true;
2540 : }
2541 :
2542 : // -----------------------------------------------------------------------
2543 : //
2544 : // -----------------------------------------------------------------------
2545 48 : void MSTransformDataHandler::setupNewPointing()
2546 : {
2547 0 : SetupNewTable pointingSetup(msOut_p.pointingTableName(),
2548 96 : MSPointing::requiredTableDesc(), Table::New);
2549 :
2550 : // POINTING can be large, set some sensible defaults for storageMgrs
2551 96 : IncrementalStMan ismPointing("ISMPointing");
2552 96 : StandardStMan ssmPointing("SSMPointing", 32768);
2553 48 : pointingSetup.bindAll(ismPointing, true);
2554 48 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::DIRECTION),ssmPointing);
2555 48 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::TARGET),ssmPointing);
2556 48 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::TIME),ssmPointing);
2557 48 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::POINTING),Table(pointingSetup));
2558 48 : msOut_p.initRefs();
2559 :
2560 96 : return;
2561 : }
2562 :
2563 : // -----------------------------------------------------------------------
2564 : //
2565 : // -----------------------------------------------------------------------
2566 1860 : bool MSTransformDataHandler::copySource()
2567 : {
2568 : //Source is an optional table, so it may not exist
2569 1860 : if (Table::isReadable(mssel_p.sourceTableName()))
2570 : {
2571 3720 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2572 :
2573 1860 : const MSSource& oldSource = mssel_p.source();
2574 1860 : MSSource& newSource = msOut_p.source();
2575 :
2576 : // Add optional columns if present in oldSource.
2577 1860 : uInt nAddedCols = addOptionalColumns(oldSource, newSource, true);
2578 1860 : os << LogIO::DEBUG1 << "SOURCE has " << nAddedCols << " optional columns." << LogIO::POST;
2579 :
2580 1860 : const MSSourceColumns incols(oldSource);
2581 1860 : MSSourceColumns outcols(newSource);
2582 :
2583 : // Copy the Measures frame info. This has to be done before filling the rows.
2584 1860 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2585 1860 : outcols.setDirectionRef(MDirection::castType(incols.directionMeas().getMeasRef().getType()));
2586 1860 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2587 1860 : outcols.setFrequencyRef(MFrequency::castType(incols.restFrequencyMeas().getMeasRef().getType()));
2588 1860 : outcols.setRadialVelocityRef(MRadialVelocity::castType(incols.sysvelMeas().getMeasRef().getType()));
2589 :
2590 1860 : if (!reindex_p)
2591 : {
2592 94 : TableCopy::copyRows(newSource, oldSource);
2593 94 : return true;
2594 : }
2595 :
2596 1766 : const ScalarColumn<Int>& inSId = incols.sourceId();
2597 1766 : ScalarColumn<Int>& outSId = outcols.sourceId();
2598 1766 : const ScalarColumn<Int>& inSPW = incols.spectralWindowId();
2599 1766 : ScalarColumn<Int>& outSPW = outcols.spectralWindowId();
2600 :
2601 : // row number in output.
2602 1766 : uInt outrn = 0;
2603 1766 : uInt nInputRows = inSId.nrow();
2604 : // inSidVal is Int.
2605 1766 : Int maxSId = sourceRelabel_p.nelements();
2606 1766 : Int maxSPWId = spwRelabel_p.nelements();
2607 50852 : for (uInt inrn = 0; inrn < nInputRows; ++inrn)
2608 : {
2609 49086 : Int inSidVal = inSId(inrn);
2610 : // -1 means the source is valid for any SPW.
2611 49086 : Int inSPWVal = inSPW(inrn);
2612 49086 : if (inSidVal >= maxSId)
2613 : {
2614 47 : os << LogIO::WARN << "Invalid SOURCE ID in SOURCE table row " << inrn << LogIO::POST;
2615 : }
2616 49086 : if (inSPWVal >= maxSPWId)
2617 : {
2618 3 : os << LogIO::WARN << "Invalid SPW ID in SOURCE table row " << inrn << LogIO::POST;
2619 : }
2620 :
2621 49085 : if ( (inSidVal > -1) &&
2622 49038 : (inSidVal < maxSId) &&
2623 144699 : (sourceRelabel_p[inSidVal] > -1) &&
2624 46528 : ((inSPWVal == -1) || (inSPWVal < maxSPWId && spwRelabel_p[inSPWVal] > -1)))
2625 : {
2626 : // Copy inrn to outrn.
2627 19887 : TableCopy::copyRows(newSource, oldSource, outrn, inrn, 1);
2628 19887 : outSId.put(outrn, sourceRelabel_p[inSidVal]);
2629 19887 : outSPW.put(outrn, inSPWVal > -1 ? spwRelabel_p[inSPWVal] : -1);
2630 19887 : ++outrn;
2631 : }
2632 : }
2633 :
2634 : }
2635 :
2636 1766 : return true;
2637 : }
2638 :
2639 : // -----------------------------------------------------------------------
2640 : //
2641 : // -----------------------------------------------------------------------
2642 1860 : bool MSTransformDataHandler::copyAntenna()
2643 : {
2644 1860 : const MSAntenna& oldAnt = mssel_p.antenna();
2645 1860 : MSAntenna& newAnt = msOut_p.antenna();
2646 3720 : const MSAntennaColumns incols(oldAnt);
2647 1860 : MSAntennaColumns outcols(newAnt);
2648 1860 : bool retval = false;
2649 :
2650 1860 : outcols.setOffsetRef(MPosition::castType(incols.offsetMeas().getMeasRef().getType()));
2651 1860 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2652 :
2653 : //TableCopy::copyRows(newAnt, oldAnt);
2654 : //retval = true;
2655 :
2656 1860 : if (!antennaSel_p or !reindex_p)
2657 : {
2658 1834 : TableCopy::copyRows(newAnt, oldAnt);
2659 1834 : retval = true;
2660 : }
2661 : else
2662 : {
2663 26 : uInt nAnt = antNewIndex_p.nelements();
2664 : // Don't use min() here, it's too overloaded.
2665 26 : if (nAnt > oldAnt.nrow()) nAnt = oldAnt.nrow();
2666 :
2667 99 : for (uInt k = 0; k < nAnt; ++k)
2668 : {
2669 73 : if (antNewIndex_p[k] > -1) TableCopy::copyRows(newAnt, oldAnt, antNewIndex_p[k], k, 1, false);
2670 : }
2671 26 : retval = true;
2672 : }
2673 :
2674 3720 : return retval;
2675 : }
2676 :
2677 : // -----------------------------------------------------------------------
2678 : //
2679 : // -----------------------------------------------------------------------
2680 1860 : bool MSTransformDataHandler::copyFeed()
2681 : {
2682 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2683 :
2684 1860 : const MSFeed& oldFeed = mssel_p.feed();
2685 1860 : MSFeed& newFeed = msOut_p.feed();
2686 3720 : const MSFeedColumns incols(oldFeed);
2687 3720 : MSFeedColumns outcols(newFeed);
2688 :
2689 1860 : outcols.setDirectionRef(MDirection::castType(incols.beamOffsetMeas().getMeasRef().getType()));
2690 1860 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2691 1860 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2692 :
2693 1860 : if ((!antennaSel_p && allEQ(spwRelabel_p, spw_p)) or !reindex_p)
2694 : {
2695 1403 : TableCopy::copyRows(newFeed, oldFeed);
2696 : }
2697 : else
2698 : {
2699 914 : const Vector<Int>& antIds = incols.antennaId().getColumn();
2700 914 : const Vector<Int>& spwIds = incols.spectralWindowId().getColumn();
2701 :
2702 : // Copy selected rows.
2703 457 : uInt totNFeeds = antIds.nelements();
2704 457 : uInt totalSelFeeds = 0;
2705 457 : Int maxSelAntp1 = antNewIndex_p.nelements();
2706 146111 : for (uInt k = 0; k < totNFeeds; ++k)
2707 : {
2708 : // antenna must be selected, and spwId must be -1 (any) or selected.
2709 145654 : if ( antIds[k] < maxSelAntp1 &&
2710 290733 : antNewIndex_p[antIds[k]] > -1 &&
2711 145079 : (spwIds[k] < 0 || spwRelabel_p[spwIds[k]] > -1)
2712 : )
2713 : {
2714 44441 : TableCopy::copyRows(newFeed, oldFeed, totalSelFeeds, k, 1,false);
2715 44441 : ++totalSelFeeds;
2716 : }
2717 : }
2718 :
2719 : // Remap antenna and spw #s.
2720 457 : ScalarColumn<Int>& antCol = outcols.antennaId();
2721 457 : ScalarColumn<Int>& spwCol = outcols.spectralWindowId();
2722 :
2723 44898 : for (uInt k = 0; k < totalSelFeeds; ++k)
2724 : {
2725 :
2726 44441 : antCol.put(k, antNewIndex_p[antCol(k)]);
2727 44441 : if (spwCol(k) > -1) spwCol.put(k, spwRelabel_p[spwCol(k)]);
2728 : }
2729 : }
2730 :
2731 : // Check if selected spw is WVR data. WVR data does not have FEED data
2732 : // so it is not a failure if newFeed.nrow == 0
2733 1860 : if (newFeed.nrow() < 1 and spw_p.size() == 1){
2734 : // Using the MSMetaData class
2735 0 : MSMetaData msmeta(&mssel_p, 0);
2736 0 : std::set<uInt> wvrspw = msmeta.getWVRSpw();
2737 0 : for (std::set<uInt>::iterator bbit = wvrspw.begin(); bbit != wvrspw.end(); ++bbit){
2738 0 : if ((uInt)spw_p[0] == *bbit){
2739 0 : os << LogIO::DEBUG1 << "Skipping spw="<<*bbit<<" because it is WVR and has no FEED content" << LogIO::POST;
2740 0 : return true;
2741 : }
2742 : }
2743 : }
2744 :
2745 1860 : if (newFeed.nrow() < 1)
2746 : {
2747 : // LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2748 0 : os << LogIO::SEVERE << "No feeds were selected." << LogIO::POST;
2749 0 : return false;
2750 : }
2751 :
2752 1860 : return true;
2753 : }
2754 :
2755 : // -----------------------------------------------------------------------
2756 : // Get the processorId corresponding to a given DDI
2757 : // -----------------------------------------------------------------------
2758 0 : Int MSTransformDataHandler::getProcessorId(Int dataDescriptionId, String msname)
2759 : {
2760 0 : ostringstream taql;
2761 0 : taql << "SELECT PROCESSOR_ID from " << msname;
2762 0 : taql << " WHERE DATA_DESC_ID ==" << dataDescriptionId;
2763 0 : taql << " LIMIT 1";
2764 :
2765 0 : casacore::TableProxy *firstSelectedRow = new TableProxy(tableCommand(taql.str()).table());
2766 0 : Record colWrapper = firstSelectedRow->getVarColumn(String("PROCESSOR_ID"),0,1,1);
2767 0 : casacore::Vector<Int> processorId = colWrapper.asArrayInt("r1");
2768 :
2769 0 : delete firstSelectedRow;
2770 0 : return processorId[0];
2771 : }
2772 :
2773 :
2774 : // -----------------------------------------------------------------------
2775 : //
2776 : // -----------------------------------------------------------------------
2777 1860 : bool MSTransformDataHandler::copyFlag_Cmd()
2778 : {
2779 :
2780 : // Like POINTING, FLAG_CMD is supposed to exist but is allowed not to.
2781 1860 : if (Table::isReadable(mssel_p.flagCmdTableName()))
2782 : {
2783 1860 : const MSFlagCmd& oldFlag_Cmd = mssel_p.flagCmd();
2784 :
2785 1860 : if (oldFlag_Cmd.nrow() > 0)
2786 : {
2787 :
2788 : // Could be declared as Table&
2789 296 : MSFlagCmd& newFlag_Cmd = msOut_p.flagCmd();
2790 :
2791 888 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2792 :
2793 : // Add optional columns if present in oldFlag_Cmd.
2794 296 : uInt nAddedCols = addOptionalColumns(oldFlag_Cmd, newFlag_Cmd, true);
2795 296 : os << LogIO::DEBUG1 << "FLAG_CMD has " << nAddedCols << " optional columns." << LogIO::POST;
2796 :
2797 592 : const MSFlagCmdColumns oldFCs(oldFlag_Cmd);
2798 592 : MSFlagCmdColumns newFCs(newFlag_Cmd);
2799 296 : newFCs.setEpochRef(MEpoch::castType(oldFCs.timeMeas().getMeasRef().getType()));
2800 :
2801 296 : TableCopy::copyRows(newFlag_Cmd, oldFlag_Cmd);
2802 :
2803 : }
2804 : }
2805 :
2806 1860 : return true;
2807 : }
2808 :
2809 : // -----------------------------------------------------------------------
2810 : //
2811 : // -----------------------------------------------------------------------
2812 1860 : bool MSTransformDataHandler::copyHistory()
2813 : {
2814 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2815 :
2816 1860 : const MSHistory& oldHistory = mssel_p.history();
2817 :
2818 : // Could be declared as Table&
2819 1860 : MSHistory& newHistory = msOut_p.history();
2820 :
2821 : // Add optional columns if present in oldHistory.
2822 1860 : uInt nAddedCols = addOptionalColumns(oldHistory, newHistory, true);
2823 1860 : os << LogIO::DEBUG1 << "HISTORY has " << nAddedCols << " optional columns." << LogIO::POST;
2824 :
2825 3720 : const MSHistoryColumns oldHCs(oldHistory);
2826 1860 : MSHistoryColumns newHCs(newHistory);
2827 1860 : newHCs.setEpochRef(MEpoch::castType(oldHCs.timeMeas().getMeasRef().getType()));
2828 :
2829 1860 : TableCopy::copyRows(newHistory, oldHistory);
2830 :
2831 3720 : return true;
2832 : }
2833 :
2834 : // -----------------------------------------------------------------------
2835 : //
2836 : // -----------------------------------------------------------------------
2837 1860 : bool MSTransformDataHandler::copyObservation()
2838 : {
2839 1860 : const MSObservation& oldObs = mssel_p.observation();
2840 1860 : MSObservation& newObs = msOut_p.observation();
2841 3720 : const MSObservationColumns oldObsCols(oldObs);
2842 1860 : MSObservationColumns newObsCols(newObs);
2843 1860 : newObsCols.setEpochRef(MEpoch::castType(oldObsCols.releaseDateMeas().getMeasRef().getType()));
2844 :
2845 1860 : uInt nObs = selObsId_p.nelements();
2846 1860 : if (nObs > 0 and reindex_p)
2847 : {
2848 21 : for (uInt outrn = 0; outrn < nObs; ++outrn)
2849 : {
2850 12 : TableCopy::copyRows(newObs, oldObs, outrn, selObsId_p[outrn], 1);
2851 9 : }
2852 :
2853 : }
2854 : else
2855 : {
2856 1851 : TableCopy::copyRows(newObs, oldObs);
2857 : }
2858 :
2859 3720 : return true;
2860 : }
2861 :
2862 : // -----------------------------------------------------------------------
2863 : //
2864 : // -----------------------------------------------------------------------
2865 1860 : bool MSTransformDataHandler::copyProcessor()
2866 : {
2867 1860 : const MSProcessor& oldProc = mssel_p.processor();
2868 1860 : MSProcessor& newProc = msOut_p.processor();
2869 1860 : TableCopy::copyRows(newProc, oldProc);
2870 :
2871 1860 : return true;
2872 : }
2873 :
2874 : // -----------------------------------------------------------------------
2875 : //
2876 : // -----------------------------------------------------------------------
2877 1860 : bool MSTransformDataHandler::copyState()
2878 : {
2879 : // STATE is allowed to not exist, even though it is not optional in
2880 : // the MS def'n. For one thing, split dropped it for quite a while.
2881 1860 : if (Table::isReadable(mssel_p.stateTableName()))
2882 : {
2883 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2884 1860 : const MSState& oldState = mssel_p.state();
2885 :
2886 1860 : if (oldState.nrow() > 0)
2887 : {
2888 1650 : if (!intentString_p.empty() and reindex_p)
2889 : {
2890 102 : MSState& newState = msOut_p.state();
2891 204 : const MSStateColumns oldStateCols(oldState);
2892 204 : MSStateColumns newStateCols(newState);
2893 :
2894 : // Initialize stateRemapper_p if necessary.
2895 : // if (stateRemapper_p.size() < 1) make_map(stateRemapper_p, mscIn_p->stateId().getColumn());
2896 :
2897 : // jagonzal (CAS-6351): Do not apply implicit re-indexing //////////////////////////////////////////////////
2898 : //
2899 : // Get list of selected scan intent indexes
2900 204 : MSSelection mssel;
2901 102 : mssel.setStateExpr(intentString_p);
2902 204 : Vector<Int> scanIntentList = mssel.getStateObsModeList(getInputMS());
2903 : //
2904 : // Populate state re-mapper using all selected indexes (not only the implicit ones)
2905 102 : stateRemapper_p.clear();
2906 342 : for (uInt index=0; index < scanIntentList.size(); index++)
2907 : {
2908 240 : stateRemapper_p[scanIntentList(index)] = index;
2909 : }
2910 : ///////////////////////////////////////////////////////////////////////////////////////////////////////////
2911 :
2912 102 : uInt nStates = stateRemapper_p.size();
2913 :
2914 : // stateRemapper_p goes from input to output, as is wanted in most
2915 : // places. Here we need a map going the other way, so make one.
2916 204 : Vector<Int> outStateToInState(nStates);
2917 102 : std::map<Int, Int>::iterator mit;
2918 :
2919 342 : for (mit = stateRemapper_p.begin(); mit != stateRemapper_p.end(); ++mit)
2920 : {
2921 240 : outStateToInState[(*mit).second] = (*mit).first;
2922 : }
2923 :
2924 :
2925 342 : for (uInt outrn = 0; outrn < nStates; ++outrn)
2926 : {
2927 240 : TableCopy::copyRows(newState, oldState, outrn,outStateToInState[outrn], 1);
2928 : }
2929 : }
2930 : // jagonzal (CAS-6351): Do not apply implicit re-indexing
2931 : // Therefore just copy the input state sub-table to the output state sub-table
2932 : else
2933 : {
2934 1548 : MSState& newState = msOut_p.state();
2935 1548 : TableCopy::copyRows(newState, oldState);
2936 : }
2937 :
2938 : }
2939 : }
2940 1860 : return true;
2941 : }
2942 :
2943 : // -----------------------------------------------------------------------
2944 : //
2945 : // -----------------------------------------------------------------------
2946 1860 : bool MSTransformDataHandler::copySyscal()
2947 : {
2948 : // SYSCAL is allowed to not exist.
2949 1860 : if (Table::isReadable(mssel_p.sysCalTableName()))
2950 : {
2951 1264 : const MSSysCal& oldSysc = mssel_p.sysCal();
2952 :
2953 1264 : if (oldSysc.nrow() > 0)
2954 : {
2955 : // Add a SYSCAL subtable to msOut_p with 0 rows for now.
2956 622 : Table::TableOption option = Table::New;
2957 1244 : TableDesc sysCalTD = MSSysCal::requiredTableDesc();
2958 1244 : SetupNewTable sysCalSetup(msOut_p.sysCalTableName(), sysCalTD,option);
2959 622 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::SYSCAL),Table(sysCalSetup, 0));
2960 :
2961 : // update the references to the subtable keywords
2962 622 : msOut_p.initRefs();
2963 :
2964 : // Could be declared as Table&.
2965 622 : MSSysCal& newSysc = msOut_p.sysCal();
2966 :
2967 1866 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2968 :
2969 622 : uInt nAddedCols = addOptionalColumns(oldSysc, newSysc, true);
2970 622 : os << LogIO::DEBUG1 << "SYSCAL has " << nAddedCols << " optional columns." << LogIO::POST;
2971 :
2972 1244 : const MSSysCalColumns incols(oldSysc);
2973 1244 : MSSysCalColumns outcols(newSysc);
2974 622 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2975 :
2976 622 : if ((!antennaSel_p && allEQ(spwRelabel_p, spw_p)) or !reindex_p)
2977 : {
2978 461 : TableCopy::copyRows(newSysc, oldSysc);
2979 : }
2980 : else
2981 : {
2982 322 : const Vector<Int>& antIds = incols.antennaId().getColumn();
2983 322 : const Vector<Int>& spwIds = incols.spectralWindowId().getColumn();
2984 :
2985 : // Copy selected rows.
2986 161 : uInt totNSyscals = antIds.nelements();
2987 161 : uInt totalSelSyscals = 0;
2988 161 : Int maxSelAntp1 = antNewIndex_p.nelements(); // Int for comparison with antIds.
2989 29846 : for (uInt k = 0; k < totNSyscals; ++k)
2990 : {
2991 : // antenna must be selected, and spwId must be -1 (any) or selected.
2992 29685 : if ( antIds[k] < maxSelAntp1 &&
2993 59370 : antNewIndex_p[antIds[k]] > -1 &&
2994 29685 : (spwIds[k] < 0 || spwRelabel_p[spwIds[k]] > -1)
2995 : )
2996 : {
2997 22738 : TableCopy::copyRows(newSysc, oldSysc, totalSelSyscals,k, 1, false);
2998 22738 : ++totalSelSyscals;
2999 : }
3000 : }
3001 :
3002 : // Remap antenna and spw #s.
3003 161 : ScalarColumn<Int>& antCol = outcols.antennaId();
3004 161 : ScalarColumn<Int>& spwCol = outcols.spectralWindowId();
3005 :
3006 22899 : for (uInt k = 0; k < totalSelSyscals; ++k)
3007 : {
3008 22738 : antCol.put(k, antNewIndex_p[antCol(k)]);
3009 22738 : if (spwCol(k) > -1) spwCol.put(k, spwRelabel_p[spwCol(k)]);
3010 : }
3011 : }
3012 : }
3013 : }
3014 :
3015 1860 : return true;
3016 : }
3017 :
3018 1860 : bool MSTransformDataHandler::copyWeather()
3019 : {
3020 : // Weather is allowed to not exist.
3021 1860 : if (Table::isReadable(mssel_p.weatherTableName()))
3022 : {
3023 1255 : const MSWeather& oldWeath = mssel_p.weather();
3024 :
3025 1255 : if (oldWeath.nrow() > 0)
3026 : {
3027 : // Add a WEATHER subtable to msOut_p with 0 rows for now.
3028 1255 : Table::TableOption option = Table::New;
3029 2510 : TableDesc weatherTD = MSWeather::requiredTableDesc();
3030 2510 : SetupNewTable weatherSetup(msOut_p.weatherTableName(), weatherTD,option);
3031 1255 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::WEATHER),Table(weatherSetup, 0));
3032 :
3033 : // update the references to the subtable keywords
3034 1255 : msOut_p.initRefs();
3035 :
3036 1255 : MSWeather& newWeath = msOut_p.weather(); // Could be declared as Table&
3037 :
3038 3765 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3039 :
3040 1255 : uInt nAddedCols = addOptionalColumns(oldWeath, newWeath, true);
3041 1255 : os << LogIO::DEBUG1 << "WEATHER has " << nAddedCols << " optional columns." << LogIO::POST;
3042 :
3043 2510 : const MSWeatherColumns oldWCs(oldWeath);
3044 2510 : MSWeatherColumns newWCs(newWeath);
3045 1255 : newWCs.setEpochRef(MEpoch::castType(oldWCs.timeMeas().getMeasRef().getType()));
3046 :
3047 1255 : if (!antennaSel_p or !reindex_p)
3048 : {
3049 1232 : TableCopy::copyRows(newWeath, oldWeath);
3050 : }
3051 : else
3052 : {
3053 46 : const Vector<Int>& antIds(oldWCs.antennaId().getColumn());
3054 23 : ScalarColumn<Int>& outants = newWCs.antennaId();
3055 :
3056 23 : uInt selRow = 0;
3057 23 : Int maxSelAntp1 = antNewIndex_p.nelements();
3058 :
3059 1638 : for (uInt k = 0; k < antIds.nelements(); ++k)
3060 : {
3061 : // Weather station is on antenna?
3062 1615 : if (antIds[k] > -1)
3063 : {
3064 : // remap ant num
3065 0 : if (antIds[k] < maxSelAntp1)
3066 : {
3067 0 : Int newAntInd = antNewIndex_p[antIds[k]];
3068 :
3069 : // Ant selected?
3070 0 : if (newAntInd > -1)
3071 : {
3072 0 : TableCopy::copyRows(newWeath, oldWeath, selRow,k, 1);
3073 0 : outants.put(selRow, newAntInd);
3074 0 : ++selRow;
3075 : }
3076 : }
3077 : }
3078 : else
3079 : {
3080 : // means valid for all antennas.
3081 1615 : TableCopy::copyRows(newWeath, oldWeath, selRow, k, 1);
3082 1615 : outants.put(selRow, -1);
3083 1615 : ++selRow;
3084 : }
3085 : }
3086 : }
3087 : }
3088 : }
3089 :
3090 1860 : return true;
3091 : }
3092 :
3093 : // -----------------------------------------------------------------------
3094 : //
3095 : // -----------------------------------------------------------------------
3096 3720 : bool MSTransformDataHandler::filterOptSubtable(const String& subtabname)
3097 : {
3098 7440 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3099 :
3100 : // subtabname is allowed to not exist. Use ms_p instead of mssel_p, because
3101 : // the latter has a random name which does NOT necessarily lead to
3102 : // subtabname. (And if selection took care of subtables, we probably
3103 : // wouldn't have to do it here.)
3104 3720 : if (Table::isReadable(ms_p.tableName() + '/' + subtabname))
3105 : {
3106 2298 : const Table intab(ms_p.tableName() + '/' + subtabname);
3107 :
3108 766 : if (intab.nrow() > 0) {
3109 :
3110 : // Add feed if selecting by it is ever added.
3111 400 : bool doFilter = (antennaSel_p || !allEQ(spwRelabel_p, spw_p)) && reindex_p;
3112 :
3113 400 : copySubtable(subtabname, intab, doFilter);
3114 :
3115 400 : if (doFilter) {
3116 :
3117 : // At this point msOut_p has subtab with 0 rows.
3118 570 : Table outtab(msOut_p.tableName() + '/' + subtabname,Table::Update);
3119 380 : ScalarColumn<Int> inAntIdCol(intab, "ANTENNA_ID");
3120 380 : ScalarColumn<Int> inSpwIdCol(intab, "SPECTRAL_WINDOW_ID");
3121 380 : const Vector<Int>& antIds = inAntIdCol.getColumn();
3122 380 : const Vector<Int>& spwIds = inSpwIdCol.getColumn();
3123 :
3124 : // Copy selected rows.
3125 190 : uInt totNOuttabs = antIds.nelements();
3126 190 : uInt totalSelOuttabs = 0;
3127 :
3128 : // Int for comparison with antIds.
3129 190 : Int maxSelAntp1 = antNewIndex_p.nelements();
3130 :
3131 190 : bool haveRemappingProblem = false;
3132 4423741 : for (uInt inrow = 0; inrow < totNOuttabs; ++inrow)
3133 : {
3134 : // antenna must be selected, and spwId must be -1 (any) or selected.
3135 : // Extra care must be taken because for a while MSes had CALDEVICE
3136 : // and SYSPOWER, but the ANTENNA_ID and SPECTRAL_WINDOW_ID of those
3137 : // sub-tables were not being re-mapped by split.
3138 4423551 : if (antIds[inrow] < maxSelAntp1 && antNewIndex_p[antIds[inrow]] > -1)
3139 : {
3140 :
3141 8807826 : if (spwIds[inrow] < 0 ||
3142 4403913 : (spwIds[inrow] < static_cast<Int> (spwRelabel_p.nelements()) &&
3143 4401458 : spwRelabel_p[spwIds[inrow]] > -1))
3144 : {
3145 831136 : TableCopy::copyRows(outtab, intab, totalSelOuttabs,inrow, 1, false);
3146 831136 : ++totalSelOuttabs;
3147 : }
3148 :
3149 : // Ideally we'd like to catch antenna errors too. They are
3150 : // avoided, but because of the way antNewIndex_p is made,
3151 : // antIds[inrow] >= maxSelAntp1
3152 : // is not necessarily an error. It's not even possible to catch
3153 : // all the spw errors, so we settle for catching the ones we can
3154 : // and reliably avoiding segfaults.
3155 3572777 : else if (spwIds[inrow] >= static_cast<Int> (spwRelabel_p.nelements()))
3156 : {
3157 2455 : haveRemappingProblem = true;
3158 : }
3159 :
3160 : }
3161 : }
3162 :
3163 190 : if (haveRemappingProblem)
3164 : {
3165 : os << LogIO::WARN << "At least one row of "
3166 : << intab.tableName()
3167 : << " has an antenna or spw mismatch.\n"
3168 : << "(Presumably from an older split, sorry.)\n"
3169 : << "If " << subtabname
3170 : << " is important, it should be fixed with tb or browsetable,\n"
3171 : << "or by redoing the split that made "
3172 : << ms_p.tableName() << " (check its history)."
3173 2 : << LogIO::POST;
3174 : }
3175 :
3176 :
3177 : // Remap antenna and spw #s.
3178 380 : ScalarColumn<Int> outAntCol(outtab, "ANTENNA_ID");
3179 380 : ScalarColumn<Int> outSpwCol(outtab, "SPECTRAL_WINDOW_ID");
3180 :
3181 831326 : for (uInt k = 0; k < totalSelOuttabs; ++k)
3182 : {
3183 831136 : outAntCol.put(k, antNewIndex_p[outAntCol(k)]);
3184 831136 : if (outSpwCol(k) > -1) outSpwCol.put(k, spwRelabel_p[outSpwCol(k)]);
3185 : }
3186 : }
3187 : }
3188 : }
3189 :
3190 7440 : return true;
3191 : }
3192 :
3193 : // -----------------------------------------------------------------------
3194 : //
3195 : // -----------------------------------------------------------------------
3196 1860 : bool MSTransformDataHandler::copyGenericSubtables()
3197 : {
3198 5580 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3199 :
3200 : // Already handled subtables will be removed from this,
3201 : // so a modifiable copy is needed.
3202 3720 : TableRecord inkws(mssel_p.keywordSet());
3203 :
3204 : // Some of the standard subtables need special handling,
3205 : // e.g. DATA_DESCRIPTION, SPECTRAL_WINDOW, and ANTENNA, so they are already
3206 : // defined in msOut_p. Several more (e.g. FLAG_CMD) were also already
3207 : // created by MS::createDefaultSubtables(). Don't try to write over them - a
3208 : // locking error will result.
3209 1860 : const TableRecord& outkws = msOut_p.keywordSet();
3210 30177 : for (uInt i = 0; i < outkws.nfields(); ++i)
3211 : {
3212 28317 : if (outkws.type(i) == TpTable && inkws.isDefined(outkws.name(i)))
3213 : {
3214 26457 : inkws.removeField(outkws.name(i));
3215 : }
3216 : }
3217 :
3218 : // Find ephemerides files
3219 1860 : std::vector<String> ephemerides;
3220 7337 : for (uInt i = 0; i < inkws.nfields(); ++i)
3221 : {
3222 5477 : if (inkws.type(i) == TpTable && inkws.name(i).contains("EPHEM"))
3223 : {
3224 25 : ephemerides.push_back(inkws.name(i));
3225 : }
3226 : }
3227 :
3228 : // Remove ephemerides files
3229 1885 : for (uInt i = 0; i < ephemerides.size(); ++i)
3230 : {
3231 25 : inkws.removeField(ephemerides.at(i));
3232 : }
3233 :
3234 :
3235 : // msOut_p.rwKeywordSet() (pass a reference, not a copy) will put a lock on msOut_p.
3236 1860 : TableCopy::copySubTables(msOut_p.rwKeywordSet(), inkws, msOut_p.tableName(), msOut_p.tableType(), mssel_p);
3237 :
3238 : // TableCopy::copySubTables(Table, Table, bool) includes this other code,
3239 : // which seems to be copying subtables at one level deeper, but not recursively?
3240 1860 : const TableDesc& inDesc = mssel_p.tableDesc();
3241 1860 : const TableDesc& outDesc = msOut_p.tableDesc();
3242 43800 : for (uInt i = 0; i < outDesc.ncolumn(); ++i)
3243 : {
3244 : // Only writable cols can have keywords (and thus subtables) defined.
3245 41940 : if (msOut_p.isColumnWritable(i))
3246 : {
3247 41940 : const String& name = outDesc[i].name();
3248 :
3249 41940 : if (inDesc.isColumn(name))
3250 : {
3251 83216 : TableColumn outCol(msOut_p, name);
3252 83216 : TableColumn inCol(mssel_p, name);
3253 :
3254 41608 : TableCopy::copySubTables(outCol.rwKeywordSet(),
3255 : inCol.keywordSet(),
3256 : msOut_p.tableName(),
3257 : msOut_p.tableType(),
3258 : mssel_p);
3259 : // Copy the keywords if column is [FLOAT_|CORRECTED_]DATA
3260 : // KS NOTE CORRECTED_DATA -> DATA case should be handled separately.
3261 41608 : if (name == "FLOAT_DATA" || name == "DATA" || name == "CORRECTED_DATA")
3262 2016 : copyMainTableKeywords(outCol.rwKeywordSet(), inCol.keywordSet());
3263 :
3264 : }
3265 : }
3266 : }
3267 :
3268 3720 : return true;
3269 : }
3270 :
3271 : // -----------------------------------------------------------------------
3272 : // Method to merge SPW sub-tables from SubMSs to create the MMS level SPW sub-table
3273 : // -----------------------------------------------------------------------
3274 28 : bool MSTransformDataHandler::mergeSpwSubTables(Vector<String> filenames)
3275 : {
3276 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3277 :
3278 56 : String filename_0 = filenames(0);
3279 56 : MeasurementSet ms_0(filename_0,Table::Update);
3280 :
3281 28 : if (Table::isReadable(ms_0.spectralWindowTableName()) and !ms_0.spectralWindow().isNull())
3282 : {
3283 28 : MSSpectralWindow spwTable_0 = ms_0.spectralWindow();
3284 :
3285 28 : if (spwTable_0.nrow() > 0)
3286 : {
3287 56 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3288 56 : << "Merging SPECTRAL_WINDOW sub-tables from all sub-MSs to form MMS-level SPECTRAL_WINDOW sub-table" << LogIO::POST;
3289 :
3290 56 : MSSpWindowColumns spwCols_0(spwTable_0);
3291 :
3292 : // Map subMS with spw_id to merge the FEED table later
3293 28 : Vector<uInt> mapSubmsSpwid;
3294 :
3295 : // subMS_0000 starts with spw 0
3296 28 : uInt spwStart = 0;
3297 28 : mapSubmsSpwid.resize(filenames.size());
3298 28 : mapSubmsSpwid[0] = spwStart;
3299 :
3300 : // for next subMS
3301 28 : uInt rowIndex = spwTable_0.nrow();
3302 28 : spwStart = spwStart + rowIndex;
3303 100 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3304 : {
3305 144 : String filename_i = filenames(subms_index);
3306 144 : MeasurementSet ms_i(filename_i);
3307 144 : MSSpectralWindow spwTable_i = ms_i.spectralWindow();
3308 :
3309 72 : if (spwTable_i.nrow() > 0)
3310 : {
3311 144 : MSSpWindowColumns spwCols_i(spwTable_i);
3312 :
3313 72 : uInt nrow = spwTable_i.nrow();
3314 72 : spwTable_0.addRow(nrow);
3315 :
3316 : // Map of this subMS to spw ID
3317 72 : mapSubmsSpwid[subms_index] = spwStart;
3318 :
3319 : // for next subMS
3320 72 : spwStart = spwStart + nrow;
3321 :
3322 160 : for (uInt subms_row_index=0;subms_row_index<spwTable_i.nrow();subms_row_index++)
3323 : {
3324 88 : spwCols_0.measFreqRef().put(rowIndex,spwCols_i.measFreqRef()(subms_row_index));
3325 88 : spwCols_0.chanFreq().put(rowIndex,spwCols_i.chanFreq()(subms_row_index));
3326 88 : spwCols_0.refFrequency().put(rowIndex,spwCols_i.refFrequency()(subms_row_index));
3327 88 : spwCols_0.chanWidth().put(rowIndex,spwCols_i.chanWidth()(subms_row_index));
3328 88 : spwCols_0.effectiveBW().put(rowIndex,spwCols_i.effectiveBW()(subms_row_index));
3329 88 : spwCols_0.resolution().put(rowIndex,spwCols_i.resolution()(subms_row_index));
3330 88 : spwCols_0.flagRow().put(rowIndex,spwCols_i.flagRow()(subms_row_index));
3331 88 : spwCols_0.freqGroup().put(rowIndex,spwCols_i.freqGroup()(subms_row_index));
3332 88 : spwCols_0.freqGroupName().put(rowIndex,spwCols_i.freqGroupName()(subms_row_index));
3333 88 : spwCols_0.ifConvChain().put(rowIndex,spwCols_i.ifConvChain()(subms_row_index));
3334 88 : spwCols_0.name().put(rowIndex,spwCols_i.name()(subms_row_index));
3335 88 : spwCols_0.netSideband().put(rowIndex,spwCols_i.netSideband()(subms_row_index));
3336 88 : spwCols_0.numChan().put(rowIndex,spwCols_i.numChan()(subms_row_index));
3337 88 : spwCols_0.totalBandwidth().put(rowIndex,spwCols_i.totalBandwidth()(subms_row_index));
3338 :
3339 : // Optional columns
3340 88 : if (columnOk(spwCols_i.bbcNo()))
3341 26 : spwCols_0.bbcNo().put(rowIndex,spwCols_i.bbcNo()(subms_row_index));
3342 :
3343 88 : if (columnOk(spwCols_i.assocSpwId()))
3344 6 : spwCols_0.assocSpwId().put(rowIndex,spwCols_i.assocSpwId()(subms_row_index));
3345 :
3346 88 : if(columnOk(spwCols_i.assocNature()))
3347 6 : spwCols_0.assocNature().put(rowIndex,spwCols_i.assocNature()(subms_row_index));
3348 :
3349 88 : if (columnOk(spwCols_i.bbcSideband()))
3350 0 : spwCols_0.bbcSideband().put(rowIndex,spwCols_i.bbcSideband()(subms_row_index));
3351 :
3352 88 : if (columnOk(spwCols_i.dopplerId()))
3353 4 : spwCols_0.dopplerId().put(rowIndex,spwCols_i.dopplerId()(subms_row_index));
3354 :
3355 88 : if (columnOk(spwCols_i.receiverId()))
3356 0 : spwCols_0.receiverId().put(rowIndex,spwCols_i.receiverId()(subms_row_index));
3357 :
3358 191 : if (spwTable_i.tableDesc().isColumn("SDM_WINDOW_FUNCTION") &&
3359 103 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_WINDOW_FUNCTION"))
3360 : {
3361 30 : ScalarColumn<String> swfCol_i(spwTable_i, "SDM_WINDOW_FUNCTION");
3362 15 : ScalarColumn<String> swfCol_0(spwTable_0, "SDM_WINDOW_FUNCTION");
3363 15 : swfCol_0.put(rowIndex, swfCol_i(subms_row_index));
3364 : }
3365 :
3366 191 : if (spwTable_i.tableDesc().isColumn("SDM_NUM_BIN") &&
3367 103 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_NUM_BIN"))
3368 : {
3369 30 : ScalarColumn<Int> snbCol_i(spwTable_i, "SDM_NUM_BIN");
3370 15 : ScalarColumn<Int> snbCol_0(spwTable_0, "SDM_NUM_BIN");
3371 15 : snbCol_0.put(rowIndex, snbCol_i(subms_row_index));
3372 : }
3373 191 : if (spwTable_i.tableDesc().isColumn("SDM_CORR_BIT") &&
3374 103 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_CORR_BIT"))
3375 : {
3376 30 : ScalarColumn<String> corrBitCol_i(spwTable_i, "SDM_CORR_BIT");
3377 15 : ScalarColumn<String> corrBitCol_0(spwTable_0, "SDM_CORR_BIT");
3378 15 : corrBitCol_0.put(rowIndex, corrBitCol_i(subms_row_index));
3379 : }
3380 :
3381 88 : rowIndex += 1;
3382 : }
3383 : }
3384 : }
3385 :
3386 : // Merge the other sub-tables using SPW map generated here
3387 28 : mergeDDISubTables(filenames);
3388 28 : mergeFeedSubTables(filenames, mapSubmsSpwid);
3389 28 : mergeSourceSubTables(filenames, mapSubmsSpwid);
3390 28 : mergeFreqOffsetTables(filenames, mapSubmsSpwid);
3391 28 : mergeCalDeviceSubtables(filenames, mapSubmsSpwid);
3392 28 : mergeSysPowerSubtables(filenames, mapSubmsSpwid);
3393 28 : mergeSyscalSubTables(filenames, mapSubmsSpwid);
3394 : }
3395 : else
3396 : {
3397 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3398 0 : << "SPECTRAL_WINDOW sub-table found but has no valid content" << LogIO::POST;
3399 0 : return false;
3400 : }
3401 : }
3402 : else
3403 : {
3404 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3405 0 : << "SPECTRAL_WINDOW sub-table not found " << LogIO::POST;
3406 0 : return false;
3407 : }
3408 :
3409 28 : return true;
3410 : }
3411 :
3412 : // -----------------------------------------------------------------------
3413 : // Method to merge DDI sub-tables from SubMSs to create the MMS-level DDI sub-table
3414 : // -----------------------------------------------------------------------
3415 28 : bool MSTransformDataHandler::mergeDDISubTables(Vector<String> filenames)
3416 : {
3417 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3418 :
3419 56 : String filename_0 = filenames(0);
3420 56 : MeasurementSet ms_0(filename_0,Table::Update);
3421 :
3422 28 : if (Table::isReadable(ms_0.dataDescriptionTableName()) and !ms_0.dataDescription().isNull())
3423 : {
3424 28 : MSDataDescription ddiTable_0 = ms_0.dataDescription();
3425 :
3426 28 : if (ddiTable_0.nrow() > 0)
3427 : {
3428 56 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3429 56 : << "Merging DDI sub-tables from all sub-MSs to form MMS-level DDI sub-table" << LogIO::POST;
3430 :
3431 56 : MSDataDescColumns ddiCols_0(ddiTable_0);
3432 :
3433 28 : uInt rowIndex = ddiTable_0.nrow();
3434 100 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3435 : {
3436 144 : String filename_i = filenames(subms_index);
3437 144 : MeasurementSet ms_i(filename_i);
3438 144 : MSDataDescription dditable_i = ms_i.dataDescription();
3439 :
3440 72 : if (dditable_i.nrow() > 0)
3441 : {
3442 144 : MSDataDescColumns ddicols_i(dditable_i);
3443 :
3444 72 : ddiTable_0.addRow(dditable_i.nrow());
3445 :
3446 162 : for (uInt subms_row_index=0;subms_row_index<dditable_i.nrow();subms_row_index++)
3447 : {
3448 : // get the last spw id entered in the 0th DD table
3449 90 : uInt spwid = ddiCols_0.spectralWindowId().get(rowIndex-1);
3450 :
3451 90 : ddiCols_0.flagRow().put(rowIndex,ddicols_i.flagRow()(subms_row_index));
3452 90 : ddiCols_0.polarizationId().put(rowIndex,ddicols_i.polarizationId()(subms_row_index));
3453 :
3454 : // Take into account that some SPW may be pointed by several DDIs
3455 90 : uInt deltaDDI = 1;
3456 90 : if (subms_row_index>0)
3457 : {
3458 18 : deltaDDI = ddicols_i.spectralWindowId()(subms_row_index) - ddicols_i.spectralWindowId()(subms_row_index-1);
3459 : }
3460 :
3461 90 : ddiCols_0.spectralWindowId().put(rowIndex,spwid+deltaDDI);
3462 90 : rowIndex += 1;
3463 : }
3464 : }
3465 : }
3466 : }
3467 : else
3468 : {
3469 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3470 0 : << "DDI sub-table found but has no valid content" << LogIO::POST;
3471 0 : return false;
3472 : }
3473 : }
3474 : else
3475 : {
3476 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3477 0 : << "DDI sub-table not found " << LogIO::POST;
3478 0 : return false;
3479 : }
3480 :
3481 28 : return true;
3482 : }
3483 :
3484 :
3485 : // -----------------------------------------------------------------------
3486 : // Method to merge FEED sub-tables from SubMSs to create the MMS-level FEED sub-table
3487 : // -----------------------------------------------------------------------
3488 28 : bool MSTransformDataHandler::mergeFeedSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3489 : {
3490 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3491 :
3492 28 : if (filenames.size() != mapSubmsSpwid.size())
3493 : {
3494 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3495 0 : return false;
3496 : }
3497 :
3498 56 : String filename_0 = filenames(0);
3499 56 : MeasurementSet ms_0(filename_0,Table::Update);
3500 :
3501 28 : if (Table::isReadable(ms_0.feedTableName()) and !ms_0.feed().isNull())
3502 : {
3503 56 : MSFeed feedTable_0 = ms_0.feed();
3504 :
3505 : // CAS-7167. The WVR spw has no FEED content.
3506 : // if (feedTable_0.nrow() >= 0)
3507 : // {
3508 56 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3509 56 : << "Merging FEED sub-tables from all sub-MSs to form MMS-level FEED sub-table" << LogIO::POST;
3510 :
3511 56 : MSFeedColumns feedCols_0(feedTable_0);
3512 :
3513 28 : uInt rowIndex = feedTable_0.nrow();
3514 100 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3515 : {
3516 144 : String filename_i = filenames(subms_index);
3517 144 : MeasurementSet ms_i(filename_i);
3518 144 : MSFeed feedtable_i = ms_i.feed();
3519 :
3520 72 : if (feedtable_i.nrow() > 0)
3521 : {
3522 144 : MSFeedColumns feedcols_i(feedtable_i);
3523 :
3524 72 : feedTable_0.addRow(feedtable_i.nrow());
3525 :
3526 : // Prepare row reference object
3527 144 : RefRows refRow(rowIndex,rowIndex+feedtable_i.nrow()-1);
3528 :
3529 : // Re-index SPW col
3530 72 : Vector<Int> spectralWindowId_output(feedtable_i.nrow(),mapSubmsSpwid[subms_index]);
3531 72 : spectralWindowId_output += feedcols_i.spectralWindowId().getColumn();
3532 72 : feedCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3533 :
3534 : // Columns that can be just copied
3535 72 : feedCols_0.position().putColumnCells(refRow,feedcols_i.position().getColumn());
3536 72 : feedCols_0.beamOffset().putColumnCells(refRow,feedcols_i.beamOffset().getColumn());
3537 72 : feedCols_0.polarizationType().putColumnCells(refRow,feedcols_i.polarizationType().getColumn());
3538 72 : feedCols_0.polResponse().putColumnCells(refRow,feedcols_i.polResponse().getColumn());
3539 72 : feedCols_0.receptorAngle().putColumnCells(refRow,feedcols_i.receptorAngle().getColumn());
3540 72 : feedCols_0.antennaId().putColumnCells(refRow,feedcols_i.antennaId().getColumn());
3541 72 : feedCols_0.beamId().putColumnCells(refRow,feedcols_i.beamId().getColumn());
3542 72 : feedCols_0.feedId().putColumnCells(refRow,feedcols_i.feedId().getColumn());
3543 72 : feedCols_0.interval().putColumnCells(refRow,feedcols_i.interval().getColumn());
3544 72 : feedCols_0.numReceptors().putColumnCells(refRow,feedcols_i.numReceptors().getColumn());
3545 72 : feedCols_0.time().putColumnCells(refRow,feedcols_i.time().getColumn());
3546 :
3547 : // optional columns
3548 72 : if (columnOk(feedcols_i.focusLength()))
3549 : {
3550 0 : feedCols_0.focusLength().putColumnCells(refRow,feedcols_i.focusLength().getColumn());
3551 : }
3552 :
3553 72 : if (columnOk(feedcols_i.phasedFeedId()))
3554 : {
3555 0 : feedCols_0.phasedFeedId().putColumnCells(refRow,feedcols_i.phasedFeedId().getColumn());
3556 : }
3557 :
3558 : // Increment row offset
3559 72 : rowIndex += feedtable_i.nrow();
3560 : }
3561 : }
3562 :
3563 : // }
3564 : /*
3565 : else
3566 : {
3567 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3568 : << "FEED sub-table found but has no valid content" << LogIO::POST;
3569 : return false;
3570 : }
3571 : */
3572 : }
3573 : else
3574 : {
3575 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__) <<
3576 0 : "FEED sub-table not found " << LogIO::POST;
3577 0 : return false;
3578 : }
3579 :
3580 28 : return true;
3581 : }
3582 :
3583 : // -----------------------------------------------------------------------
3584 : // Method to merge Source sub-tables from SubMSs to create the MMS-level FEED sub-table
3585 : // -----------------------------------------------------------------------
3586 28 : bool MSTransformDataHandler::mergeSourceSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3587 : {
3588 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3589 :
3590 28 : if (filenames.size() != mapSubmsSpwid.size())
3591 : {
3592 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3593 0 : return false;
3594 : }
3595 :
3596 56 : String filename_0 = filenames(0);
3597 56 : MeasurementSet ms_0(filename_0,Table::Update);
3598 :
3599 28 : if (Table::isReadable(ms_0.sourceTableName()) and !ms_0.source().isNull())
3600 : {
3601 28 : MSSource sourceTable_0 = ms_0.source();
3602 :
3603 28 : if (sourceTable_0.nrow() > 0)
3604 : {
3605 54 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3606 54 : << "Merging SOURCE sub-tables from all sub-MSs to form MMS-level SOURCE sub-table" << LogIO::POST;
3607 :
3608 54 : MSSourceColumns sourceCols_0(sourceTable_0);
3609 :
3610 27 : uInt rowIndex = sourceTable_0.nrow();
3611 98 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3612 : {
3613 142 : String filename_i = filenames(subms_index);
3614 142 : MeasurementSet ms_i(filename_i);
3615 142 : MSSource sourcetable_i = ms_i.source();
3616 :
3617 71 : if (sourcetable_i.nrow() > 0)
3618 : {
3619 142 : MSSourceColumns sourcecols_i(sourcetable_i);
3620 :
3621 71 : sourceTable_0.addRow(sourcetable_i.nrow());
3622 :
3623 : // Prepare row reference object
3624 142 : RefRows refRow(rowIndex,rowIndex+sourcetable_i.nrow()-1);
3625 :
3626 : // Re-index SPW col
3627 71 : Vector<Int> spectralWindowId_output(sourcetable_i.nrow(),mapSubmsSpwid[subms_index]);
3628 71 : spectralWindowId_output += sourcecols_i.spectralWindowId().getColumn();
3629 71 : sourceCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3630 :
3631 : // Columns that can be just copied
3632 71 : sourceCols_0.direction().putColumnCells(refRow,sourcecols_i.direction().getColumn());
3633 71 : sourceCols_0.properMotion().putColumnCells(refRow,sourcecols_i.properMotion().getColumn());
3634 71 : sourceCols_0.calibrationGroup().putColumnCells(refRow,sourcecols_i.calibrationGroup().getColumn());
3635 71 : sourceCols_0.code().putColumnCells(refRow,sourcecols_i.code().getColumn());
3636 71 : sourceCols_0.interval().putColumnCells(refRow,sourcecols_i.interval().getColumn());
3637 71 : sourceCols_0.name().putColumnCells(refRow,sourcecols_i.name().getColumn());
3638 71 : sourceCols_0.numLines().putColumnCells(refRow,sourcecols_i.numLines().getColumn());
3639 71 : sourceCols_0.sourceId().putColumnCells(refRow,sourcecols_i.sourceId().getColumn());
3640 71 : sourceCols_0.time().putColumnCells(refRow,sourcecols_i.time().getColumn());
3641 :
3642 : // Optional columns
3643 71 : if (columnOk(sourcecols_i.position()))
3644 : {
3645 4 : sourceCols_0.position().putColumnCells(refRow,sourcecols_i.position().getColumn());
3646 : }
3647 :
3648 71 : if (columnOk(sourcecols_i.transition()))
3649 : {
3650 6 : sourceCols_0.transition().putColumnCells(refRow,sourcecols_i.transition().getColumn());
3651 : }
3652 :
3653 71 : if (columnOk(sourcecols_i.restFrequency()))
3654 : {
3655 6 : sourceCols_0.restFrequency().putColumnCells(refRow,sourcecols_i.restFrequency().getColumn());
3656 : }
3657 :
3658 71 : if (columnOk(sourcecols_i.sysvel()))
3659 : {
3660 6 : sourceCols_0.sysvel().putColumnCells(refRow,sourcecols_i.sysvel().getColumn());
3661 : }
3662 :
3663 71 : if (columnOk(sourcecols_i.pulsarId()))
3664 : {
3665 0 : sourceCols_0.pulsarId().putColumnCells(refRow,sourcecols_i.pulsarId().getColumn());
3666 : }
3667 :
3668 71 : if (columnOk(sourcecols_i.sourceModel()))
3669 : {
3670 0 : sourceCols_0.sourceModel().putColumnCells(refRow,sourcecols_i.sourceModel().getColumn());
3671 : }
3672 :
3673 : // Increment row offset
3674 71 : rowIndex += sourcetable_i.nrow();
3675 : }
3676 : }
3677 :
3678 : }
3679 : else
3680 : {
3681 2 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3682 2 : << "SOURCE sub-table found but has no valid content" << LogIO::POST;
3683 1 : return false;
3684 : }
3685 : }
3686 : else
3687 : {
3688 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3689 0 : << "SOURCE sub-table not found " << LogIO::POST;
3690 0 : return false;
3691 : }
3692 :
3693 27 : return true;
3694 : }
3695 :
3696 :
3697 : // -----------------------------------------------------------------------
3698 : // Method to merge Syscal sub-tables from SubMSs to create the MMS-level Syscal sub-table
3699 : // -----------------------------------------------------------------------
3700 28 : bool MSTransformDataHandler::mergeSyscalSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3701 : {
3702 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3703 :
3704 28 : if (filenames.size() != mapSubmsSpwid.size())
3705 : {
3706 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3707 0 : return false;
3708 : }
3709 :
3710 56 : String filename_0 = filenames(0);
3711 56 : MeasurementSet ms_0(filename_0,Table::Update);
3712 :
3713 28 : if(Table::isReadable(ms_0.sysCalTableName()) and !ms_0.sysCal().isNull())
3714 : {
3715 50 : MSSysCal syscalTable_0 = ms_0.sysCal();
3716 :
3717 : // CAS-7167. The WVR spw has no FEED content.
3718 : // if (syscalTable_0.nrow() >= 0)
3719 : // {
3720 50 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3721 50 : << "Merging SYSCAL sub-tables from all sub-MSs to form MMS-level SYSCAL sub-table" << LogIO::POST;
3722 :
3723 50 : MSSysCalColumns syscalCols_0(syscalTable_0);
3724 :
3725 25 : uInt rowIndex = syscalTable_0.nrow();
3726 93 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3727 : {
3728 136 : String filename_i = filenames(subms_index);
3729 136 : MeasurementSet ms_i(filename_i);
3730 136 : MSSysCal syscaltable_i = ms_i.sysCal();
3731 :
3732 68 : if (syscaltable_i.nrow() > 0)
3733 : {
3734 6 : MSSysCalColumns syscalcols_i(syscaltable_i);
3735 :
3736 3 : syscalTable_0.addRow(syscaltable_i.nrow());
3737 :
3738 : // Prepare row reference object
3739 6 : RefRows refRow(rowIndex,rowIndex+syscaltable_i.nrow()-1);
3740 :
3741 : // Re-index SPW col
3742 3 : Vector<Int> spectralWindowId_output(syscaltable_i.nrow(),mapSubmsSpwid[subms_index]);
3743 3 : spectralWindowId_output += syscalcols_i.spectralWindowId().getColumn();
3744 3 : syscalCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3745 :
3746 : // Columns that can be just copied
3747 3 : syscalCols_0.antennaId().putColumnCells(refRow,syscalcols_i.antennaId().getColumn());
3748 3 : syscalCols_0.feedId().putColumnCells(refRow,syscalcols_i.feedId().getColumn());
3749 3 : syscalCols_0.interval().putColumnCells(refRow,syscalcols_i.interval().getColumn());
3750 3 : syscalCols_0.time().putColumnCells(refRow,syscalcols_i.time().getColumn());
3751 :
3752 : // Optional columns
3753 3 : if (columnOk(syscalcols_i.phaseDiff()))
3754 : {
3755 0 : syscalCols_0.phaseDiff().putColumnCells(refRow,syscalcols_i.phaseDiff().getColumn());
3756 : }
3757 :
3758 3 : if (columnOk(syscalcols_i.phaseDiffFlag()))
3759 : {
3760 0 : syscalCols_0.phaseDiffFlag().putColumnCells(refRow,syscalcols_i.phaseDiffFlag().getColumn());
3761 : }
3762 :
3763 3 : if (columnOk(syscalcols_i.tant()))
3764 : {
3765 0 : syscalCols_0.tant().putColumnCells(refRow,syscalcols_i.tant().getColumn());
3766 : }
3767 :
3768 3 : if (columnOk(syscalcols_i.tantFlag()))
3769 : {
3770 3 : syscalCols_0.tantFlag().putColumnCells(refRow,syscalcols_i.tantFlag().getColumn());
3771 : }
3772 :
3773 3 : if (columnOk(syscalcols_i.tantSpectrum()))
3774 : {
3775 0 : syscalCols_0.tantSpectrum().putColumnCells(refRow,syscalcols_i.tantSpectrum().getColumn());
3776 : }
3777 :
3778 3 : if (columnOk(syscalcols_i.tantTsys()))
3779 : {
3780 0 : syscalCols_0.tantTsys().putColumnCells(refRow,syscalcols_i.tantTsys().getColumn());
3781 : }
3782 :
3783 3 : if (columnOk(syscalcols_i.tantTsysFlag()))
3784 : {
3785 3 : syscalCols_0.tantTsysFlag().putColumnCells(refRow,syscalcols_i.tantTsysFlag().getColumn());
3786 : }
3787 :
3788 3 : if (columnOk(syscalcols_i.tantTsysSpectrum()))
3789 : {
3790 0 : syscalCols_0.tantTsysSpectrum().putColumnCells(refRow,syscalcols_i.tantTsysSpectrum().getColumn());
3791 : }
3792 :
3793 3 : if (columnOk(syscalcols_i.tcal()))
3794 : {
3795 0 : syscalCols_0.tcal().putColumnCells(refRow,syscalcols_i.tcal().getColumn());
3796 : }
3797 :
3798 3 : if (columnOk(syscalcols_i.tcalFlag()))
3799 : {
3800 3 : syscalCols_0.tcalFlag().putColumnCells(refRow,syscalcols_i.tcalFlag().getColumn());
3801 : }
3802 :
3803 3 : if (columnOk(syscalcols_i.tcalSpectrum()))
3804 : {
3805 3 : syscalCols_0.tcalSpectrum().putColumnCells(refRow,syscalcols_i.tcalSpectrum().getColumn());
3806 : }
3807 :
3808 3 : if (columnOk(syscalcols_i.trx()))
3809 : {
3810 0 : syscalCols_0.trx().putColumnCells(refRow,syscalcols_i.trx().getColumn());
3811 : }
3812 :
3813 3 : if (columnOk(syscalcols_i.trxFlag()))
3814 : {
3815 3 : syscalCols_0.trxFlag().putColumnCells(refRow,syscalcols_i.trxFlag().getColumn());
3816 : }
3817 :
3818 3 : if (columnOk(syscalcols_i.trxSpectrum()))
3819 : {
3820 3 : syscalCols_0.trxSpectrum().putColumnCells(refRow,syscalcols_i.trxSpectrum().getColumn());
3821 : }
3822 :
3823 3 : if (columnOk(syscalcols_i.tsky()))
3824 : {
3825 0 : syscalCols_0.tsky().putColumnCells(refRow,syscalcols_i.tsky().getColumn());
3826 : }
3827 :
3828 3 : if (columnOk(syscalcols_i.tskyFlag()))
3829 : {
3830 3 : syscalCols_0.tskyFlag().putColumnCells(refRow,syscalcols_i.tskyFlag().getColumn());
3831 : }
3832 :
3833 3 : if (columnOk(syscalcols_i.tskySpectrum()))
3834 : {
3835 3 : syscalCols_0.tskySpectrum().putColumnCells(refRow,syscalcols_i.tskySpectrum().getColumn());
3836 : }
3837 :
3838 3 : if (columnOk(syscalcols_i.tsys()))
3839 : {
3840 0 : syscalCols_0.tsys().putColumnCells(refRow,syscalcols_i.tsys().getColumn());
3841 : }
3842 :
3843 3 : if (columnOk(syscalcols_i.tsysFlag()))
3844 : {
3845 3 : syscalCols_0.tsysFlag().putColumnCells(refRow,syscalcols_i.tsysFlag().getColumn());
3846 : }
3847 :
3848 3 : if (columnOk(syscalcols_i.tsysSpectrum()))
3849 : {
3850 3 : syscalCols_0.tsysSpectrum().putColumnCells(refRow,syscalcols_i.tsysSpectrum().getColumn());
3851 : }
3852 :
3853 : // Increment row offset
3854 3 : rowIndex += syscalcols_i.nrow();
3855 : }
3856 : }
3857 :
3858 : /*
3859 : }
3860 : else
3861 : {
3862 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3863 : << "SYSCAL sub-table found but has no valid content" << LogIO::POST;
3864 : return false;
3865 : }
3866 : */
3867 : }
3868 : else
3869 : {
3870 6 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3871 6 : << "SYSCAL sub-table not found " << LogIO::POST;
3872 3 : return false;
3873 : }
3874 :
3875 25 : return true;
3876 : }
3877 :
3878 :
3879 : // -----------------------------------------------------------------------
3880 : // Method to merge FreqOffset sub-tables from SubMSs to create the MMS-level FreqOffset sub-table
3881 : // -----------------------------------------------------------------------
3882 28 : bool MSTransformDataHandler::mergeFreqOffsetTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3883 : {
3884 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3885 :
3886 28 : if (filenames.size() != mapSubmsSpwid.size())
3887 : {
3888 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3889 0 : return false;
3890 : }
3891 :
3892 56 : String filename_0 = filenames(0);
3893 56 : MeasurementSet ms_0(filename_0,Table::Update);
3894 :
3895 28 : if (Table::isReadable(ms_0.freqOffsetTableName()) and !ms_0.freqOffset().isNull())
3896 : {
3897 0 : MSFreqOffset freqoffsetTable_0 = ms_0.freqOffset();
3898 :
3899 0 : if (freqoffsetTable_0.nrow() > 0)
3900 : {
3901 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3902 0 : << "Merging FREQ_OFFSET sub-tables from all sub-MSs to form MMS-level FREQ_OFFSET sub-table" << LogIO::POST;
3903 :
3904 0 : MSFreqOffsetColumns freqoffsetCols_0(freqoffsetTable_0);
3905 :
3906 0 : uInt rowIndex = freqoffsetTable_0.nrow();
3907 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3908 : {
3909 0 : String filename_i = filenames(subms_index);
3910 0 : MeasurementSet ms_i(filename_i);
3911 0 : MSFreqOffset freqoffsettable_i = ms_i.freqOffset();
3912 :
3913 0 : if (freqoffsettable_i.nrow() > 0)
3914 : {
3915 0 : MSFreqOffsetColumns freqoffsetcols_i(freqoffsettable_i);
3916 :
3917 0 : freqoffsetTable_0.addRow(freqoffsettable_i.nrow());
3918 :
3919 : // Prepare row reference object
3920 0 : RefRows refRow(rowIndex,rowIndex+freqoffsettable_i.nrow()-1);
3921 :
3922 : // Re-index SPW col
3923 0 : Vector<Int> spectralWindowId_output(freqoffsettable_i.nrow(),mapSubmsSpwid[subms_index]);
3924 0 : spectralWindowId_output += freqoffsetcols_i.spectralWindowId().getColumn();
3925 0 : freqoffsetCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3926 :
3927 : // Columns that can be just copied
3928 0 : freqoffsetCols_0.antenna1().putColumnCells(refRow,freqoffsetcols_i.antenna1().getColumn());
3929 0 : freqoffsetCols_0.antenna2().putColumnCells(refRow,freqoffsetcols_i.antenna2().getColumn());
3930 0 : freqoffsetCols_0.feedId().putColumnCells(refRow,freqoffsetcols_i.feedId().getColumn());
3931 0 : freqoffsetCols_0.interval().putColumnCells(refRow,freqoffsetcols_i.interval().getColumn());
3932 0 : freqoffsetCols_0.offset().putColumnCells(refRow,freqoffsetcols_i.offset().getColumn());
3933 0 : freqoffsetCols_0.time().putColumnCells(refRow,freqoffsetcols_i.time().getColumn());
3934 :
3935 : // NOTE (jagonzal): FreqOffset does not have optional columns
3936 :
3937 : // Increment row offset
3938 0 : rowIndex += freqoffsettable_i.nrow();
3939 : }
3940 : }
3941 :
3942 : }
3943 : else
3944 : {
3945 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3946 0 : << "FREQ_OFFSET sub-table found but has no valid content" << LogIO::POST;
3947 0 : return false;
3948 : }
3949 :
3950 : }
3951 : else
3952 : {
3953 56 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3954 56 : << "FREQ_OFFSET sub-table not found " << LogIO::POST;
3955 28 : return false;
3956 : }
3957 :
3958 0 : return true;
3959 : }
3960 :
3961 : // -----------------------------------------------------------------------
3962 : // Method to merge CalDevice sub-tables from SubMSs to create the MMS-level CalDevice sub-table
3963 : // -----------------------------------------------------------------------
3964 28 : bool MSTransformDataHandler::mergeCalDeviceSubtables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3965 : {
3966 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3967 :
3968 28 : if (filenames.size() != mapSubmsSpwid.size())
3969 : {
3970 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3971 0 : return false;
3972 : }
3973 :
3974 56 : String filename_0 = filenames(0);
3975 56 : MeasurementSet ms_0(filename_0,Table::Update);
3976 :
3977 28 : if (Table::isReadable(ms_0.tableName() + "/CALDEVICE"))
3978 : {
3979 8 : Table subtable_0(ms_0.tableName() + "/CALDEVICE", Table::Update);
3980 :
3981 8 : if (subtable_0.nrow() > 0)
3982 : {
3983 16 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3984 16 : << "Merging CALDEVICE sub-tables from all sub-MSs to form MMS-level CALDEVICE sub-table" << LogIO::POST;
3985 :
3986 : // Get RW access to columns
3987 16 : ScalarColumn<Int> antennaIdCol_0(subtable_0, "ANTENNA_ID");
3988 16 : ScalarColumn<Int> feedIdCol_0(subtable_0, "FEED_ID");
3989 16 : ScalarColumn<Int> spectralWindowIdCol_0(subtable_0, "SPECTRAL_WINDOW_ID");
3990 16 : ScalarColumn<Double> timeCol_0(subtable_0, "TIME");
3991 16 : ScalarColumn<Double> intervalCol_0(subtable_0, "INTERVAL");
3992 :
3993 16 : ScalarColumn<Int> numCalLoadCol_0(subtable_0, "NUM_CAL_LOAD");
3994 16 : ArrayColumn<String> calLoadNamesCol_0(subtable_0, "CAL_LOAD_NAMES");
3995 16 : ScalarColumn<Int> numReceptorCol_0(subtable_0, "NUM_RECEPTOR");
3996 16 : ArrayColumn<Float> noiseCalCol_0(subtable_0, "NOISE_CAL");
3997 16 : ArrayColumn<Float> calEffCol_0(subtable_0, "CAL_EFF");
3998 16 : ArrayColumn<Double> temperatureLoadCol_0(subtable_0, "TEMPERATURE_LOAD");
3999 :
4000 : // Get original content of columns
4001 16 : Vector<Int> antennaId_0;
4002 8 : if (columnOk(antennaIdCol_0)) antennaId_0 = antennaIdCol_0.getColumn();
4003 16 : Vector<Int> feedId_0;
4004 8 : if (columnOk(feedIdCol_0)) feedId_0 = feedIdCol_0.getColumn();
4005 16 : Vector<Int> spectralWindowId_0;
4006 8 : if (columnOk(spectralWindowIdCol_0)) spectralWindowId_0 = spectralWindowIdCol_0.getColumn();
4007 16 : Vector<Double> time_0;
4008 8 : if (columnOk(timeCol_0)) time_0 = timeCol_0.getColumn();
4009 16 : Vector<Double> interval_0;
4010 8 : if (columnOk(intervalCol_0)) interval_0 = intervalCol_0.getColumn();
4011 :
4012 16 : Vector<Int> numCalLoad_0;
4013 8 : if (columnOk(numCalLoadCol_0)) numCalLoad_0 = numCalLoadCol_0.getColumn();
4014 16 : Array<String> calLoadNames_0;
4015 8 : if (columnOk(calLoadNamesCol_0)) calLoadNames_0 = calLoadNamesCol_0.getColumn();
4016 16 : Vector<Int> numReceptor_0;
4017 8 : if (columnOk(numReceptorCol_0)) numReceptor_0 = numReceptorCol_0.getColumn();
4018 16 : Array<Float> noiseCal_0;
4019 8 : if (columnOk(noiseCalCol_0)) noiseCal_0 = noiseCalCol_0.getColumn();
4020 16 : Array<Float> calEff_0;
4021 8 : if (columnOk(calEffCol_0)) calEff_0 = calEffCol_0.getColumn();
4022 16 : Array<Double> temperatureLoad_0;
4023 8 : if (columnOk(temperatureLoadCol_0)) temperatureLoad_0 = temperatureLoadCol_0.getColumn();
4024 :
4025 8 : uInt rowIndex = subtable_0.nrow();
4026 26 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
4027 : {
4028 36 : String filename_i = filenames(subms_index);
4029 36 : MeasurementSet ms_i(filename_i);
4030 36 : Table subtable_i(ms_i.tableName() + "/CALDEVICE", Table::Update);
4031 :
4032 18 : if (subtable_i.nrow() > 0)
4033 : {
4034 : // Get RW access to columns
4035 36 : ScalarColumn<Int> antennaIdCol_i(subtable_i, "ANTENNA_ID");
4036 36 : ScalarColumn<Int> feedIdCol_i(subtable_i, "FEED_ID");
4037 36 : ScalarColumn<Int> spectralWindowIdCol_i(subtable_i, "SPECTRAL_WINDOW_ID");
4038 36 : ScalarColumn<Double> timeCol_i(subtable_i, "TIME");
4039 36 : ScalarColumn<Double> intervalCol_i(subtable_i, "INTERVAL");
4040 :
4041 36 : ScalarColumn<Int> numCalLoadCol_i(subtable_i, "NUM_CAL_LOAD");
4042 36 : ArrayColumn<String> calLoadNamesCol_i(subtable_i, "CAL_LOAD_NAMES");
4043 36 : ScalarColumn<Int> numReceptorCol_i(subtable_i, "NUM_RECEPTOR");
4044 36 : ArrayColumn<Float> noiseCalCol_i(subtable_i, "NOISE_CAL");
4045 36 : ArrayColumn<Float> calEffCol_i(subtable_i, "CAL_EFF");
4046 36 : ArrayColumn<Double> temperatureLoadCol_i(subtable_i, "TEMPERATURE_LOAD");
4047 :
4048 : // Get original content of columns
4049 36 : Vector<Int> antennaId_i;
4050 18 : if (columnOk(antennaIdCol_i)) antennaId_i = antennaIdCol_i.getColumn();
4051 36 : Vector<Int> feedId_i;
4052 18 : if (columnOk(feedIdCol_i)) feedId_i = feedIdCol_i.getColumn();
4053 36 : Vector<Int> spectralWindowId_i;
4054 18 : if (columnOk(spectralWindowIdCol_i)) spectralWindowId_i = spectralWindowIdCol_i.getColumn();
4055 36 : Vector<Double> time_i;
4056 18 : if (columnOk(timeCol_i)) time_i = timeCol_i.getColumn();
4057 36 : Vector<Double> interval_i;
4058 18 : if (columnOk(intervalCol_i)) interval_i = intervalCol_i.getColumn();
4059 :
4060 36 : Vector<Int> numCalLoad_i;
4061 18 : if (columnOk(numCalLoadCol_i)) numCalLoad_i = numCalLoadCol_i.getColumn();
4062 36 : Array<String> calLoadNames_i;
4063 18 : if (columnOk(calLoadNamesCol_i)) calLoadNames_i = calLoadNamesCol_i.getColumn();
4064 36 : Vector<Int> numReceptor_i;
4065 18 : if (columnOk(numReceptorCol_i)) numReceptor_i = numReceptorCol_i.getColumn();
4066 36 : Array<Float> noiseCal_i;
4067 18 : if (columnOk(noiseCalCol_i)) noiseCal_i = noiseCalCol_i.getColumn();
4068 36 : Array<Float> calEff_i;
4069 18 : if (columnOk(calEffCol_i)) calEff_i = calEffCol_i.getColumn();
4070 36 : Array<Double> temperatureLoad_i;
4071 18 : if (columnOk(temperatureLoadCol_i)) temperatureLoad_i = temperatureLoadCol_i.getColumn();
4072 :
4073 : // Add n# rows to subtable_i equivalent to n# rows from subtable_0
4074 18 : subtable_0.addRow(subtable_i.nrow());
4075 :
4076 : // Prepare row reference object
4077 36 : RefRows refRow(rowIndex,rowIndex+subtable_i.nrow()-1);
4078 :
4079 : // Re-index SPW col
4080 18 : Vector<Int> spectralWindowId_output(spectralWindowId_i.size(),mapSubmsSpwid[subms_index]);
4081 18 : spectralWindowId_output += spectralWindowId_i;
4082 18 : spectralWindowIdCol_0.putColumnCells(refRow,spectralWindowId_output);
4083 :
4084 : // Columns that can be just copied
4085 18 : if (columnOk(antennaIdCol_i))
4086 : {
4087 18 : antennaIdCol_0.putColumnCells(refRow,antennaId_i);
4088 : }
4089 :
4090 18 : if (columnOk(feedIdCol_i))
4091 : {
4092 18 : feedIdCol_0.putColumnCells(refRow,feedId_i);
4093 : }
4094 :
4095 18 : if (columnOk(timeCol_i))
4096 : {
4097 18 : timeCol_0.putColumnCells(refRow,time_i);
4098 : }
4099 :
4100 18 : if (columnOk(intervalCol_i))
4101 : {
4102 18 : intervalCol_0.putColumnCells(refRow,interval_i);
4103 : }
4104 :
4105 :
4106 18 : if (columnOk(numCalLoadCol_i))
4107 : {
4108 18 : numCalLoadCol_0.putColumnCells(refRow,numCalLoad_i);
4109 : }
4110 :
4111 18 : if (columnOk(calLoadNamesCol_i))
4112 : {
4113 18 : calLoadNamesCol_0.putColumnCells(refRow,calLoadNames_i);
4114 : }
4115 :
4116 18 : if (columnOk(numReceptorCol_i))
4117 : {
4118 18 : numReceptorCol_0.putColumnCells(refRow,numReceptor_i);
4119 : }
4120 :
4121 18 : if (columnOk(noiseCalCol_i))
4122 : {
4123 8 : noiseCalCol_0.putColumnCells(refRow,noiseCal_i);
4124 : }
4125 :
4126 18 : if (columnOk(calEffCol_i))
4127 : {
4128 0 : calEffCol_0.putColumnCells(refRow,calEff_i);
4129 : }
4130 :
4131 18 : if (columnOk(temperatureLoadCol_i))
4132 : {
4133 10 : temperatureLoadCol_0.putColumnCells(refRow,temperatureLoad_i);
4134 : }
4135 :
4136 : // Increment row offset
4137 18 : rowIndex += subtable_i.nrow();
4138 : }
4139 : }
4140 :
4141 : }
4142 : else
4143 : {
4144 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4145 0 : << "CALDEVICE sub-table found but has no valid content" << LogIO::POST;
4146 0 : return false;
4147 : }
4148 : }
4149 : else
4150 : {
4151 40 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4152 40 : << "CALDEVICE sub-table not found " << LogIO::POST;
4153 20 : return false;
4154 : }
4155 :
4156 8 : return true;
4157 : }
4158 :
4159 :
4160 : // -----------------------------------------------------------------------
4161 : // Method to merge SysPower sub-tables from SubMSs to create the MMS-level SysPower sub-table
4162 : // -----------------------------------------------------------------------
4163 28 : bool MSTransformDataHandler::mergeSysPowerSubtables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
4164 : {
4165 84 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
4166 :
4167 28 : if (filenames.size() != mapSubmsSpwid.size())
4168 : {
4169 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
4170 0 : return false;
4171 : }
4172 :
4173 56 : String filename_0 = filenames(0);
4174 56 : MeasurementSet ms_0(filename_0,Table::Update);
4175 :
4176 28 : if (Table::isReadable(ms_0.tableName() + "/SYSPOWER"))
4177 : {
4178 8 : Table subtable_0(ms_0.tableName() + "/SYSPOWER", Table::Update);
4179 :
4180 8 : if (subtable_0.nrow() > 0)
4181 : {
4182 12 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4183 12 : << "Merging SYSPOWER sub-tables from all sub-MSs to form MMS-level SYSPOWER sub-table" << LogIO::POST;
4184 :
4185 : // Get RW access to columns
4186 12 : ScalarColumn<Int> antennaIdCol_0(subtable_0, "ANTENNA_ID");
4187 12 : ScalarColumn<Int> feedIdCol_0(subtable_0, "FEED_ID");
4188 12 : ScalarColumn<Int> spectralWindowIdCol_0(subtable_0, "SPECTRAL_WINDOW_ID");
4189 12 : ScalarColumn<Double> timeCol_0(subtable_0, "TIME");
4190 12 : ScalarColumn<Double> intervalCol_0(subtable_0, "INTERVAL");
4191 :
4192 12 : ArrayColumn<Float> switchedDiffCol_0(subtable_0, "SWITCHED_DIFF");
4193 12 : ArrayColumn<Float> switchedSumCol_0(subtable_0, "SWITCHED_SUM");
4194 12 : ArrayColumn<Float> requantizerGainCol_0(subtable_0, "REQUANTIZER_GAIN");
4195 :
4196 : // Get original content of columns
4197 12 : Vector<Int> antennaId_0;
4198 6 : if (columnOk(antennaIdCol_0)) antennaId_0 = antennaIdCol_0.getColumn();
4199 12 : Vector<Int> feedId_0;
4200 6 : if (columnOk(feedIdCol_0)) feedId_0 = feedIdCol_0.getColumn();
4201 12 : Vector<Int> spectralWindowId_0;
4202 6 : if (columnOk(spectralWindowIdCol_0)) spectralWindowId_0 = spectralWindowIdCol_0.getColumn();
4203 12 : Vector<Double> time_0;
4204 6 : if (columnOk(timeCol_0)) time_0 = timeCol_0.getColumn();
4205 12 : Vector<Double> interval_0;
4206 6 : if (columnOk(intervalCol_0)) interval_0 = intervalCol_0.getColumn();
4207 :
4208 12 : Array<Float> switchedDiff_0;
4209 6 : if (columnOk(switchedDiffCol_0)) switchedDiff_0 = switchedDiffCol_0.getColumn();
4210 12 : Array<Float> switchedSum_0;
4211 6 : if (columnOk(switchedSumCol_0)) switchedSum_0 = switchedSumCol_0.getColumn();
4212 12 : Array<Float> requantizerGain_0;
4213 6 : if (columnOk(requantizerGainCol_0)) requantizerGain_0 = requantizerGainCol_0.getColumn();
4214 :
4215 6 : uInt rowIndex = subtable_0.nrow();
4216 14 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
4217 : {
4218 16 : String filename_i = filenames(subms_index);
4219 16 : MeasurementSet ms_i(filename_i);
4220 16 : Table subtable_i(ms_i.tableName() + "/SYSPOWER", Table::Update);
4221 :
4222 8 : if (subtable_i.nrow() > 0)
4223 : {
4224 : // Get RW access to columns
4225 16 : ScalarColumn<Int> antennaIdCol_i(subtable_i, "ANTENNA_ID");
4226 16 : ScalarColumn<Int> feedIdCol_i(subtable_i, "FEED_ID");
4227 16 : ScalarColumn<Int> spectralWindowIdCol_i(subtable_i, "SPECTRAL_WINDOW_ID");
4228 16 : ScalarColumn<Double> timeCol_i(subtable_i, "TIME");
4229 16 : ScalarColumn<Double> intervalCol_i(subtable_i, "INTERVAL");
4230 :
4231 16 : ArrayColumn<Float> switchedDiffCol_i(subtable_i, "SWITCHED_DIFF");
4232 16 : ArrayColumn<Float> switchedSumCol_i(subtable_i, "SWITCHED_SUM");
4233 16 : ArrayColumn<Float> requantizerGainCol_i(subtable_i, "REQUANTIZER_GAIN");
4234 :
4235 : // Get original content of columns
4236 16 : Vector<Int> antennaId_i;
4237 8 : if (columnOk(antennaIdCol_i)) antennaId_i = antennaIdCol_i.getColumn();
4238 16 : Vector<Int> feedId_i;
4239 8 : if (columnOk(feedIdCol_i)) feedId_i = feedIdCol_i.getColumn();
4240 16 : Vector<Int> spectralWindowId_i;
4241 8 : if (columnOk(spectralWindowIdCol_i)) spectralWindowId_i = spectralWindowIdCol_i.getColumn();
4242 16 : Vector<Double> time_i;
4243 8 : if (columnOk(timeCol_i)) time_i = timeCol_i.getColumn();
4244 16 : Vector<Double> interval_i;
4245 8 : if (columnOk(intervalCol_i)) interval_i = intervalCol_i.getColumn();
4246 :
4247 16 : Array<Float> switchedDiff_i;
4248 8 : if (columnOk(switchedDiffCol_i)) switchedDiff_i = switchedDiffCol_i.getColumn();
4249 16 : Array<Float> switchedSum_i;
4250 8 : if (columnOk(switchedSumCol_i)) switchedSum_i = switchedSumCol_i.getColumn();
4251 16 : Array<Float> requantizerGain_i;
4252 8 : if (columnOk(requantizerGainCol_i)) requantizerGain_i = requantizerGainCol_i.getColumn();
4253 :
4254 : // Add n# rows to subtable_i equivalent to n# rows from subtable_0
4255 8 : subtable_0.addRow(subtable_i.nrow());
4256 :
4257 : // Prepare row reference object
4258 16 : RefRows refRow(rowIndex,rowIndex+subtable_i.nrow()-1);
4259 :
4260 : // Re-index SPW col
4261 8 : Vector<Int> spectralWindowId_output(spectralWindowId_i.size(),mapSubmsSpwid[subms_index]);
4262 8 : spectralWindowId_output += spectralWindowId_i;
4263 8 : spectralWindowIdCol_0.putColumnCells(refRow,spectralWindowId_output);
4264 :
4265 : // Columns that can be just copied
4266 8 : if (columnOk(antennaIdCol_i))
4267 : {
4268 8 : antennaIdCol_0.putColumnCells(refRow,antennaId_i);
4269 : }
4270 :
4271 8 : if (columnOk(feedIdCol_i))
4272 : {
4273 8 : feedIdCol_0.putColumnCells(refRow,feedId_i);
4274 : }
4275 :
4276 8 : if (columnOk(timeCol_i))
4277 : {
4278 8 : timeCol_0.putColumnCells(refRow,time_i);
4279 : }
4280 :
4281 8 : if (columnOk(intervalCol_i))
4282 : {
4283 8 : intervalCol_0.putColumnCells(refRow,interval_i);
4284 : }
4285 :
4286 :
4287 8 : if (columnOk(switchedDiffCol_i))
4288 : {
4289 8 : switchedDiffCol_0.putColumnCells(refRow,switchedDiff_i);
4290 : }
4291 :
4292 8 : if (columnOk(switchedSumCol_i))
4293 : {
4294 8 : switchedSumCol_0.putColumnCells(refRow,switchedSum_i);
4295 : }
4296 :
4297 8 : if (columnOk(requantizerGainCol_i))
4298 : {
4299 8 : requantizerGainCol_0.putColumnCells(refRow,requantizerGain_i);
4300 : }
4301 :
4302 : // Increment row offset
4303 8 : rowIndex += subtable_i.nrow();
4304 : }
4305 : }
4306 :
4307 : }
4308 : else
4309 : {
4310 4 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4311 4 : << "SYSPOWER sub-table found but has no valid content" << LogIO::POST;
4312 2 : return false;
4313 : }
4314 : }
4315 : else
4316 : {
4317 40 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4318 40 : << "SYSPOWER sub-table not found " << LogIO::POST;
4319 20 : return false;
4320 : }
4321 :
4322 6 : return true;
4323 : }
4324 :
4325 : // -----------------------------------------------------------------------
4326 : //
4327 : // -----------------------------------------------------------------------
4328 : // template <class T> bool MSTransformDataHandler::columnOk (ArrayColumn<T> column)
4329 : // {
4330 : // bool ret;
4331 : // if (column.isNull()==false and column.hasContent()==true and column.ndimColumn() > 0)
4332 : // {
4333 : // ret = true;
4334 : // }
4335 : // else
4336 : // {
4337 : // ret = false;
4338 : // }
4339 :
4340 : // return ret;
4341 : // }
4342 :
4343 : // // -----------------------------------------------------------------------
4344 : // //
4345 : // // -----------------------------------------------------------------------
4346 : // template <class T> bool MSTransformDataHandler::columnOk (ScalarColumn<T> column)
4347 : // {
4348 : // bool ret;
4349 : // if (column.isNull()==false and column.hasContent()==true)
4350 : // {
4351 : // ret = true;
4352 : // }
4353 : // else
4354 : // {
4355 : // ret = false;
4356 : // }
4357 :
4358 : // return ret;
4359 : // }
4360 :
4361 :
4362 : // -----------------------------------------------------------------------
4363 : // Work-around to copy the keywords of the FLOAT_DATA column to the output MS
4364 : // -----------------------------------------------------------------------
4365 2209 : void MSTransformDataHandler::copyMainTableKeywords (TableRecord& outKeys,
4366 : const TableRecord& inKeys)
4367 : {
4368 2592 : for (uInt i=0; i<inKeys.nfields(); i++) {
4369 383 : if (inKeys.type(i) == TpString) {
4370 : // Add keywords for MAIN table columns such as FLOAT_DATA
4371 766 : String ikey = inKeys.name(i);
4372 383 : if (!outKeys.isDefined (ikey)) {
4373 381 : String keyval;
4374 381 : inKeys.get(ikey, keyval);
4375 381 : outKeys.define(ikey,keyval);
4376 : }
4377 :
4378 : }
4379 :
4380 : }
4381 2209 : }
4382 :
4383 :
4384 : } //# NAMESPACE CASA - END
|
