Line data Source code
1 : //# MSTransformDataHandler.cc: This file contains the implementation of the MSTransformDataHandler class.
2 : //#
3 : //# CASA - Common Astronomy Software Applications (http://casa.nrao.edu/)
4 : //# Copyright (C) Associated Universities, Inc. Washington DC, USA 2011, All rights reserved.
5 : //# Copyright (C) European Southern Observatory, 2011, All rights reserved.
6 : //#
7 : //# This library is free software; you can redistribute it and/or
8 : //# modify it under the terms of the GNU Lesser General Public
9 : //# License as published by the Free software Foundation; either
10 : //# version 2.1 of the License, or (at your option) any later version.
11 : //#
12 : //# This library is distributed in the hope that it will be useful,
13 : //# but WITHOUT ANY WARRANTY, without even the implied warranty of
14 : //# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : //# Lesser General Public License for more details.
16 : //#
17 : //# You should have received a copy of the GNU Lesser General Public
18 : //# License along with this library; if not, write to the Free Software
19 : //# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20 : //# MA 02111-1307 USA
21 : //# $Id: $
22 :
23 : #include <mstransform/MSTransform/MSTransformDataHandler.h>
24 : #include <casacore/tables/Tables/TableProxy.h>
25 : #include <casacore/tables/TaQL/TableParse.h>
26 : #include <casacore/ms/MSOper/MSMetaData.h>
27 : #include <asdmstman/AsdmStMan.h>
28 :
29 :
30 : using namespace casacore;
31 : namespace casa { //# NAMESPACE CASA - BEGIN
32 :
33 : /////////////////////////////////////////////
34 : /// MSTransformDataHandler implementation ///
35 : /////////////////////////////////////////////
36 :
37 : // -----------------------------------------------------------------------
38 : //
39 : // -----------------------------------------------------------------------
40 0 : MSTransformDataHandler::MSTransformDataHandler(const String& theMS, Table::TableOption option,
41 : bool virtualModelCol,bool virtualCorrectedCol,
42 0 : bool reindex) :
43 : ms_p(MeasurementSet(theMS, option)),
44 0 : mssel_p(ms_p),
45 : msc_p(NULL),
46 : mscIn_p(NULL),
47 : keepShape_p(true),
48 : antennaSel_p(false),
49 : timeBin_p(-1.0),
50 : scanString_p(""),
51 : intentString_p(""),
52 : obsString_p(""),
53 : uvrangeString_p(""),
54 : taqlString_p(""),
55 : timeRange_p(""),
56 : arrayExpr_p(""),
57 : combine_p(""),
58 : fitorder_p(-1),
59 : fitspw_p("*"),
60 : fitoutspw_p("*"),
61 : virtualModelCol_p(virtualModelCol),
62 : virtualCorrectedCol_p(virtualCorrectedCol),
63 0 : reindex_p(reindex)
64 : {
65 0 : return;
66 : }
67 :
68 : // -----------------------------------------------------------------------
69 : //
70 : // -----------------------------------------------------------------------
71 0 : MSTransformDataHandler::MSTransformDataHandler(const MeasurementSet& ms,
72 : bool virtualModelCol,bool virtualCorrectedCol,
73 0 : bool reindex) :
74 : ms_p(ms),
75 0 : mssel_p(ms_p),
76 : msc_p(NULL),
77 : mscIn_p(NULL),
78 : keepShape_p(true),
79 : antennaSel_p(false),
80 : timeBin_p(-1.0),
81 : scanString_p(""),
82 : intentString_p(""),
83 : obsString_p(""),
84 : uvrangeString_p(""),
85 : taqlString_p(""),
86 : timeRange_p(""),
87 : arrayExpr_p(""),
88 : combine_p(""),
89 : fitorder_p(-1),
90 : fitspw_p("*"),
91 : fitoutspw_p("*"),
92 : virtualModelCol_p(virtualModelCol),
93 : virtualCorrectedCol_p(virtualCorrectedCol),
94 0 : reindex_p(reindex)
95 : {
96 0 : return;
97 : }
98 :
99 : // -----------------------------------------------------------------------
100 : //
101 : // -----------------------------------------------------------------------
102 0 : MSTransformDataHandler::~MSTransformDataHandler()
103 : {
104 0 : if (msc_p) delete msc_p;
105 0 : msc_p = nullptr;
106 :
107 0 : if (mscIn_p) delete mscIn_p;
108 0 : mscIn_p = nullptr;
109 :
110 0 : msOut_p=MeasurementSet();
111 :
112 : // parseColumnNames unavoidably has a static String and Vector<MS::PredefinedColumns>.
113 : // Collapse them down to free most of that memory.
114 0 : parseColumnNames("None");
115 :
116 0 : return;
117 0 : }
118 :
119 : // -----------------------------------------------------------------------
120 : //
121 : // -----------------------------------------------------------------------
122 0 : const Vector<MS::PredefinedColumns>& MSTransformDataHandler::parseColumnNames(String col)
123 : {
124 : // Memorize both for efficiency and so that the info
125 : // message at the bottom isn't unnecessarily repeated.
126 0 : static String my_colNameStr = "";
127 0 : static Vector<MS::PredefinedColumns> my_colNameVect;
128 :
129 0 : col.upcase();
130 0 : if(col == my_colNameStr && col != "")
131 : {
132 0 : return my_colNameVect;
133 : }
134 0 : else if(col == "NONE")
135 : {
136 0 : my_colNameStr = "";
137 0 : my_colNameVect.resize(0);
138 0 : return my_colNameVect;
139 : }
140 :
141 : uInt nNames;
142 :
143 0 : if(col.contains("ALL"))
144 : {
145 0 : nNames = 3;
146 0 : my_colNameVect.resize(nNames);
147 0 : my_colNameVect[0] = MS::DATA;
148 0 : my_colNameVect[1] = MS::MODEL_DATA;
149 0 : my_colNameVect[2] = MS::CORRECTED_DATA;
150 : }
151 : else
152 : {
153 0 : nNames = dataColStrToEnums(col, my_colNameVect);
154 : }
155 :
156 : // Whether or not the MS has the columns is checked by verifyColumns().
157 : // Unfortunately it cannot be done here because this is a static method.
158 :
159 :
160 : /*
161 : * jagonzal: Redundant logging message (this info is already provided by MSTransformManager)
162 : *
163 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
164 : // "NONE" is used by the destructor
165 : if(col != "NONE")
166 : {
167 : os << LogIO::NORMAL << "Using ";
168 : for(uInt i = 0; i < nNames; ++i)
169 : {
170 : os << MS::columnName(my_colNameVect[i]) << " ";
171 : }
172 :
173 : os << " column" << (my_colNameVect.nelements() > 1 ? "s." : ".") << LogIO::POST;
174 : }
175 : */
176 :
177 0 : my_colNameStr = col;
178 0 : return my_colNameVect;
179 : }
180 :
181 : // -----------------------------------------------------------------------
182 : //
183 : // -----------------------------------------------------------------------
184 0 : const Vector<MS::PredefinedColumns>& MSTransformDataHandler::parseColumnNames( String col,
185 : bool produceModel,
186 : const MeasurementSet& msref,
187 : bool virtualModelCol,
188 : bool virtualCorrectedCol)
189 : {
190 : // Memorize both for efficiency and so that the info
191 : // message at the bottom isn't unnecessarily repeated.
192 0 : static String my_colNameStr = "";
193 0 : static Vector<MS::PredefinedColumns> my_colNameVect;
194 :
195 : // Data columns to pick up if present.
196 0 : Vector<MS::PredefinedColumns> wanted;
197 :
198 0 : col.upcase();
199 :
200 : // This version of parseColumnNames does not reuse results of previous calls
201 : // but always checks the given columns because it cannot be certain that msref
202 : // refers to the same MS with every call.
203 :
204 0 : if (col == "NONE")
205 : {
206 0 : my_colNameStr = "";
207 0 : my_colNameVect.resize(0);
208 0 : return my_colNameVect;
209 : }
210 :
211 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
212 :
213 : // Are we choosy?
214 0 : const bool doAny = col.contains("ALL") || col.contains("ANY");
215 :
216 : uInt nPoss;
217 0 : if (doAny)
218 : {
219 0 : nPoss = 5;
220 0 : wanted.resize(nPoss);
221 0 : wanted[0] = MS::DATA;
222 0 : wanted[1] = MS::MODEL_DATA;
223 0 : wanted[2] = MS::CORRECTED_DATA;
224 0 : wanted[3] = MS::FLOAT_DATA;
225 0 : wanted[4] = MS::LAG_DATA;
226 : }
227 : // split name string into individual names
228 : else
229 : {
230 0 : nPoss = dataColStrToEnums(col, wanted);
231 : }
232 :
233 : // Add MODEL_DATA separately from 'datacolumn' selection
234 0 : if (produceModel) {
235 0 : const auto nelem = wanted.nelements();
236 0 : wanted.resize(nelem + 1, true);
237 0 : wanted[nelem] = MS::MODEL_DATA;
238 0 : ++nPoss;
239 : // When producing output MS with DATA,MODEL from CORRECTED, setupMS will have
240 : // to add MS::DATA to the output MS, not MS::CORRECTED_DATA.
241 : // An alternative would be to add a special case in setupMS, similar
242 : // to mustWriteOnlyToData, when wanted == CORRECTED_DATA, MODEL_DATA.
243 0 : if (MS::CORRECTED_DATA == wanted[0]) {
244 0 : wanted[0] = MS::DATA;
245 : }
246 : }
247 :
248 0 : uInt nFound = 0;
249 0 : my_colNameVect.resize(0);
250 0 : for (uInt i = 0; i < nPoss; ++i)
251 : {
252 0 : if (msref.tableDesc().isColumn(MS::columnName(wanted[i])))
253 : {
254 0 : ++nFound;
255 0 : my_colNameVect.resize(nFound, true);
256 0 : my_colNameVect[nFound - 1] = wanted[i];
257 : }
258 : // CAS-5348 (jagonzal): Model parameters check is done at construction time
259 : // (produceModel comes from uvcontsub, doesn't require MODEL in input MS)
260 0 : else if ((wanted[i] == MS::MODEL_DATA and virtualModelCol) or produceModel)
261 : {
262 0 : ++nFound;
263 0 : my_colNameVect.resize(nFound, true);
264 0 : my_colNameVect[nFound - 1] = wanted[i];
265 : }
266 0 : else if (wanted[i] == MS::CORRECTED_DATA and virtualCorrectedCol)
267 : {
268 0 : ++nFound;
269 0 : my_colNameVect.resize(nFound, true);
270 0 : my_colNameVect[nFound - 1] = wanted[i];
271 : }
272 0 : else if (!doAny)
273 : {
274 0 : ostringstream ostr;
275 0 : ostr << "Desired column (" << MS::columnName(wanted[i])
276 0 : << ") not found in the input MS (" << msref.tableName()
277 0 : << ").";
278 0 : throw(AipsError(ostr.str()));
279 : }
280 : }
281 0 : if (nFound == 0) throw(AipsError("Did not find and select any data columns."));
282 :
283 0 : my_colNameStr = col;
284 0 : return my_colNameVect;
285 : }
286 :
287 : // -----------------------------------------------------------------------
288 : //
289 : // -----------------------------------------------------------------------
290 0 : uInt MSTransformDataHandler::dataColStrToEnums(const String& col, Vector<MS::PredefinedColumns>& colvec)
291 : {
292 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
293 0 : String tmpNames(col);
294 0 : Vector<String> tokens;
295 0 : tmpNames.upcase();
296 :
297 : // split name string into individual names
298 : char * pch;
299 0 : Int it = 0;
300 0 : pch = strtok((char*) tmpNames.c_str(), " ,");
301 0 : while (pch != NULL)
302 : {
303 0 : tokens.resize(it + 1, true);
304 0 : tokens[it] = String(pch);
305 0 : ++it;
306 0 : pch = strtok(NULL, " ,");
307 : }
308 :
309 0 : uInt nNames = tokens.nelements();
310 :
311 0 : uInt nFound = 0;
312 0 : for (uInt i = 0; i < nNames; ++i)
313 : {
314 0 : colvec.resize(nFound + 1, true);
315 0 : colvec[nFound] = MS::UNDEFINED_COLUMN;
316 :
317 0 : if ( tokens[i] == "OBSERVED"
318 0 : || tokens[i] == "DATA"
319 0 : || tokens[i] == MS::columnName(MS::DATA))
320 : {
321 0 : colvec[nFound++] = MS::DATA;
322 : }
323 0 : else if ( tokens[i] == "FLOAT"
324 0 : || tokens[i] == "FLOAT_DATA"
325 0 : || tokens[i] == MS::columnName(MS::FLOAT_DATA))
326 : {
327 0 : colvec[nFound++] = MS::FLOAT_DATA;
328 : }
329 0 : else if ( tokens[i] == "LAG"
330 0 : || tokens[i] == "LAG_DATA"
331 0 : || tokens[i] == MS::columnName(MS::LAG_DATA))
332 : {
333 0 : colvec[nFound++] = MS::LAG_DATA;
334 : }
335 0 : else if ( tokens[i] == "MODEL"
336 0 : || tokens[i] == "MODEL_DATA"
337 0 : || tokens[i] == MS::columnName(MS::MODEL_DATA))
338 : {
339 0 : colvec[nFound++] = MS::MODEL_DATA;
340 : }
341 0 : else if ( tokens[i] == "CORRECTED"
342 0 : || tokens[i] == "CORRECTED_DATA"
343 0 : || tokens[i] == MS::columnName(MS::CORRECTED_DATA))
344 : {
345 0 : colvec[nFound++] = MS::CORRECTED_DATA;
346 : }
347 : // "NONE" is used by the destructor
348 0 : else if (tmpNames != "NONE")
349 : {
350 0 : os << LogIO::SEVERE;
351 :
352 0 : if (nFound == 0)
353 : {
354 0 : colvec[0] = MS::DATA;
355 0 : os << "Unrecognized data column " << tokens[i] << "...trying DATA.";
356 : }
357 : else
358 : {
359 0 : os << "Skipping unrecognized data column " << tokens[i];
360 : }
361 :
362 0 : os << LogIO::POST;
363 : }
364 : }
365 0 : return nFound;
366 : }
367 :
368 : // -----------------------------------------------------------------------
369 : //
370 : // -----------------------------------------------------------------------
371 0 : bool MSTransformDataHandler::setmsselect( const String& spw, const String& field,
372 : const String& baseline, const String& scan,
373 : const String& uvrange, const String& taql,
374 : const Vector<Int>& step, const String& subarray,
375 : const String& correlation, const String& intent,
376 : const String& obs, const String& feed)
377 : {
378 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
379 : bool ok;
380 :
381 0 : String myspwstr(spw == "" ? "*" : spw);
382 0 : Record selrec = ms_p.msseltoindex(myspwstr, field);
383 :
384 0 : ok = selectSource(selrec.asArrayInt("field"));
385 :
386 : // All of the requested selection functions will be tried, even if an
387 : // earlier one has indicated its failure. This allows all of the selection
388 : // strings to be tested, yielding more complete feedback for the user
389 : // (fewer retries). This is a matter of taste, though. If the selections
390 : // turn out to be slow, this function should return on the first false.
391 :
392 0 : if (!selectSpw(myspwstr, step))
393 : {
394 0 : os << LogIO::SEVERE << "No channels selected." << LogIO::POST;
395 0 : ok = false;
396 : }
397 :
398 0 : if (baseline != "")
399 : {
400 0 : Vector<Int> antid(0);
401 0 : Vector<String> antstr(1, baseline);
402 0 : selectAntenna(antid, antstr);
403 : }
404 :
405 0 : scanString_p = scan;
406 0 : intentString_p = intent;
407 0 : obsString_p = obs;
408 0 : uvrangeString_p = uvrange;
409 0 : taqlString_p = taql;
410 0 : feedString_p = feed;
411 :
412 0 : if (subarray != "") selectArray(subarray);
413 :
414 0 : if (!selectCorrelations(correlation))
415 : {
416 0 : os << LogIO::SEVERE << "No correlations selected." << LogIO::POST;
417 0 : ok = false;
418 : }
419 :
420 0 : return ok;
421 : }
422 :
423 : // -----------------------------------------------------------------------
424 : //
425 : // -----------------------------------------------------------------------
426 0 : bool MSTransformDataHandler::selectSource(const Vector<Int>& fieldid)
427 : {
428 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
429 :
430 0 : bool cando = true;
431 :
432 0 : if (fieldid.nelements() < 1)
433 : {
434 0 : fieldid_p = Vector<Int> (1, -1);
435 : }
436 0 : else if (fieldid.nelements() > ms_p.field().nrow())
437 : {
438 0 : os << LogIO::SEVERE << "More fields were requested than are in the input MS." << LogIO::POST;
439 0 : cando = false;
440 : }
441 0 : else if (max(fieldid) >= static_cast<Int> (ms_p.field().nrow()))
442 : {
443 : // Arriving here is very unlikely since if fieldid came from MSSelection
444 : // bad fields were presumably already quietly dropped.
445 0 : os << LogIO::SEVERE << "At least 1 field was requested that is not in the input MS." << LogIO::POST;
446 0 : cando = false;
447 : }
448 : else
449 : {
450 0 : fieldid_p = fieldid;
451 : }
452 :
453 0 : if (fieldid_p.nelements() == 1 && fieldid_p[0] < 0)
454 : {
455 0 : fieldid_p.resize(ms_p.field().nrow());
456 0 : indgen(fieldid_p);
457 : }
458 :
459 0 : return cando;
460 : }
461 :
462 : // -----------------------------------------------------------------------
463 : //
464 : // -----------------------------------------------------------------------
465 0 : bool MSTransformDataHandler::selectSpw(const String& spwstr,const Vector<Int>& steps)
466 : {
467 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
468 :
469 0 : MSSelection mssel;
470 0 : String myspwstr(spwstr == "" ? "*" : spwstr);
471 0 : spwString_p = myspwstr;
472 :
473 0 : mssel.setSpwExpr(myspwstr);
474 :
475 0 : widths_p = steps.copy();
476 0 : if (widths_p.nelements() < 1)
477 : {
478 0 : widths_p.resize(1);
479 0 : widths_p[0] = 1;
480 : }
481 : else
482 : {
483 0 : for (uInt k = 0; k < widths_p.nelements(); ++k)
484 : {
485 0 : if (widths_p[k] == 0)
486 : {
487 0 : os << LogIO::WARN << "0 cannot be used for channel width...using 1 instead." << LogIO::POST;
488 0 : widths_p[k] = 1;
489 : }
490 : }
491 : }
492 :
493 : // Each row should have spw, start, stop, step
494 : // A single width is a default, but multiple widths should be used literally.
495 0 : Matrix<Int> chansel = mssel.getChanList(&ms_p, 1);
496 :
497 0 : if (chansel.nrow() > 0)
498 : {
499 : // Use myspwstr if it selected anything...
500 0 : spw_p = chansel.column(0);
501 0 : chanStart_p = chansel.column(1);
502 0 : chanEnd_p = chansel.column(2);
503 0 : chanStep_p = chansel.column(3);
504 :
505 0 : uInt nspw = chanEnd_p.nelements();
506 0 : nchan_p.resize(nspw);
507 :
508 : // A single width is a default, but multiple widths should be used literally.
509 0 : if (widths_p.nelements() > 1 && widths_p.nelements() != spw_p.nelements())
510 : {
511 : os << LogIO::SEVERE
512 : << "Mismatch between the # of widths specified by width and the # of spws."
513 0 : << LogIO::POST;
514 0 : return false;
515 : }
516 :
517 : // Copy the default width to all spws.
518 0 : if (widths_p.nelements() < nspw)
519 : {
520 0 : widths_p.resize(nspw, true);
521 0 : for (uInt k = 1; k < nspw; ++k)
522 : {
523 0 : widths_p[k] = widths_p[0];
524 : }
525 : }
526 :
527 0 : for (uInt k = 0; k < nspw; ++k)
528 : {
529 : // CAS-2224, triggered by spw='0:2' (as opposed to '0:2~2').
530 0 : if (chanStep_p[k] == 0) chanStep_p[k] = 1;
531 :
532 0 : nchan_p[k] = 1 + (chanEnd_p[k] - chanStart_p[k]) / (chanStep_p[k]* widths_p[k]);
533 0 : if (nchan_p[k] < 1) nchan_p[k] = 1;
534 : }
535 : }
536 : else
537 : {
538 : // Select everything and rely on widths.
539 0 : MSSpWindowColumns mySpwTab(ms_p.spectralWindow());
540 0 : uInt nspw = mySpwTab.nrow();
541 :
542 0 : nchan_p = mySpwTab.numChan().getColumn();
543 :
544 0 : spw_p.resize(nspw);
545 0 : indgen(spw_p);
546 :
547 0 : chanStart_p.resize(nspw);
548 0 : chanStep_p.resize(nspw);
549 0 : for (uInt k = 0; k < nspw; ++k)
550 : {
551 0 : chanStart_p[k] = 0;
552 0 : chanEnd_p[k] = nchan_p[k] - 1;
553 0 : chanStep_p[k] = 1;
554 : }
555 :
556 0 : if (widths_p.nelements() != spw_p.nelements())
557 : {
558 0 : if (widths_p.nelements() == 1)
559 : {
560 0 : widths_p.resize(spw_p.nelements(), true);
561 0 : for (uInt k = 1; k < spw_p.nelements(); ++k)
562 : {
563 0 : widths_p[k] = widths_p[0];
564 : }
565 :
566 : }
567 : else
568 : {
569 : os << LogIO::SEVERE
570 : << "Mismatch between the # of widths specified by width and the # of spws."
571 0 : << LogIO::POST;
572 0 : return false;
573 : }
574 : }
575 :
576 0 : for (uInt k = 0; k < nspw; ++k)
577 : {
578 0 : nchan_p[k] = 1 + (nchan_p[k] - 1) / widths_p[k];
579 : }
580 : }
581 :
582 : // Check for and filter out selected spws that aren't included in DATA_DESCRIPTION.
583 : // (See CAS-1673 for an example.)
584 0 : std::set<Int> selectedSpwNotInDD(MSTransformDataHandler::findSpwsNotInDD(ms_p, spw_p));
585 0 : uInt nSelectedSpwNotInDD = selectedSpwNotInDD.size();
586 0 : if (nSelectedSpwNotInDD > 0)
587 : {
588 0 : os << LogIO::NORMAL << "The following a priori selected input spw(s)\n";
589 0 : for (std::set<Int>::iterator spwit = selectedSpwNotInDD.begin();
590 0 : spwit != selectedSpwNotInDD.end(); ++spwit)
591 : {
592 0 : os << spw_p[*spwit] << " ";
593 : }
594 : os << "\nwere not found in DATA_DESCRIPTION (i. e. no rows in the main "
595 : "table reference them) and therefore "
596 0 : "are not included to the output." << LogIO::POST;
597 :
598 0 : uInt nSelSpw = spw_p.nelements();
599 0 : uInt ngoodSelSpwSlots = nSelSpw - nSelectedSpwNotInDD;
600 0 : Vector<Int> spwc(ngoodSelSpwSlots);
601 0 : Vector<Int> chanStartc(ngoodSelSpwSlots);
602 0 : Vector<Int> chanEndc(ngoodSelSpwSlots);
603 0 : Vector<Int> nchanc(ngoodSelSpwSlots);
604 0 : Vector<Int> chanStepc(ngoodSelSpwSlots);
605 0 : std::set<Int>::iterator spwNotDDEnd = selectedSpwNotInDD.end();
606 :
607 0 : uInt j = 0;
608 0 : for (uInt k = 0; k < nSelSpw; ++k)
609 : {
610 0 : if (selectedSpwNotInDD.find(k) == spwNotDDEnd)
611 : {
612 0 : spwc[j] = spw_p[k];
613 0 : chanStartc[j] = chanStart_p[k];
614 0 : chanEndc[j] = chanEnd_p[k];
615 0 : nchanc[j] = nchan_p[k];
616 0 : chanStepc[j] = chanStep_p[k];
617 0 : ++j;
618 : }
619 : }
620 0 : spw_p.resize(ngoodSelSpwSlots);
621 0 : spw_p = spwc;
622 0 : chanStart_p.resize(ngoodSelSpwSlots);
623 0 : chanStart_p = chanStartc;
624 0 : chanEnd_p.resize(ngoodSelSpwSlots);
625 0 : chanEnd_p = chanEndc;
626 0 : nchan_p.resize(ngoodSelSpwSlots);
627 0 : nchan_p = nchanc;
628 0 : chanStep_p.resize(ngoodSelSpwSlots);
629 0 : chanStep_p = chanStepc;
630 : }
631 :
632 0 : mssel.getChanSlices(chanSlices_p, &ms_p, 1);
633 0 : return true;
634 : }
635 :
636 : // -----------------------------------------------------------------------
637 : //
638 : // -----------------------------------------------------------------------
639 0 : std::set<Int> MSTransformDataHandler::findSpwsNotInDD(MeasurementSet& ms,Vector<Int> spwv)
640 : {
641 0 : ScalarColumn<Int> spws_in_dd( ms.dataDescription(),
642 0 : MSDataDescription::columnName(MSDataDescription::SPECTRAL_WINDOW_ID));
643 0 : std::set<Int> uniqSpwsInDD;
644 0 : uInt nspwsInDD = spws_in_dd.nrow();
645 :
646 0 : for (uInt ddrow = 0; ddrow < nspwsInDD; ++ddrow)
647 : {
648 0 : uniqSpwsInDD.insert(spws_in_dd(ddrow));
649 : }
650 :
651 0 : std::set<Int> badSelSpwSlots;
652 0 : std::set<Int>::iterator ddend = uniqSpwsInDD.end();
653 0 : for (uInt k = 0; k < spwv.nelements(); ++k)
654 : {
655 0 : if (uniqSpwsInDD.find(spwv[k]) == ddend)
656 : {
657 0 : badSelSpwSlots.insert(k);
658 : }
659 : }
660 :
661 0 : return badSelSpwSlots;
662 : }
663 :
664 : // -----------------------------------------------------------------------
665 : //
666 : // -----------------------------------------------------------------------
667 0 : void MSTransformDataHandler::selectAntenna(const Vector<Int>& antennaids,const Vector<String>& antennaSel)
668 : {
669 0 : antennaSel_p = MSTransformDataHandler::pickAntennas(antennaId_p, antennaSelStr_p, antennaids,antennaSel);
670 0 : return;
671 : }
672 :
673 : // -----------------------------------------------------------------------
674 : //
675 : // -----------------------------------------------------------------------
676 0 : bool MSTransformDataHandler::pickAntennas( Vector<Int>& selected_antennaids,
677 : Vector<String>& selected_antenna_strs,
678 : const Vector<Int>& antennaids,
679 : const Vector<String>& antennaSel)
680 : {
681 0 : bool didSelect = true;
682 :
683 0 : if ((antennaids.nelements() == 1) && (antennaids[0] == -1))
684 : {
685 0 : if (antennaSel[0] == "")
686 : {
687 0 : didSelect = false;
688 : }
689 : else
690 : {
691 0 : selected_antennaids.resize();
692 : }
693 : }
694 : else
695 : {
696 0 : selected_antennaids = antennaids;
697 : }
698 :
699 0 : selected_antenna_strs = antennaSel;
700 :
701 0 : return didSelect;
702 : }
703 :
704 : // -----------------------------------------------------------------------
705 : //
706 : // -----------------------------------------------------------------------
707 0 : void MSTransformDataHandler::selectArray(const String& subarray)
708 : {
709 0 : arrayExpr_p = subarray;
710 0 : return;
711 : }
712 :
713 : // -----------------------------------------------------------------------
714 : //
715 : // -----------------------------------------------------------------------
716 0 : bool MSTransformDataHandler::selectCorrelations(const String& corrstr)
717 : {
718 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
719 :
720 0 : corrString_p = corrstr;
721 0 : const bool areSelecting = corrstr != "" && corrstr != "*";
722 :
723 : // Get correlation slices
724 0 : MSSelection mssel1;
725 0 : if (areSelecting) mssel1.setPolnExpr(corrstr.c_str());
726 0 : mssel1.getCorrSlices(corrSlices_p, &ms_p);
727 :
728 : // Get correlation map
729 : // jagonzal (CAS-6951): We have to use another MSSelection because the first one corrupts the correlation
730 : // expression for instance "XX;YY" is turned into "XX" after calling getCorrSlices
731 0 : MSSelection mssel2;
732 0 : if (areSelecting) mssel2.setPolnExpr(corrstr.c_str());
733 0 : return MSTransformDataHandler::getCorrMaps(mssel2, ms_p, inPolOutCorrToInCorrMap_p, areSelecting);
734 : }
735 :
736 : // -----------------------------------------------------------------------
737 : //
738 : // -----------------------------------------------------------------------
739 0 : bool MSTransformDataHandler::getCorrMaps(MSSelection& mssel,
740 : const MeasurementSet& ms,
741 : Vector<Vector<Int> >& outToIn,
742 : const bool areSelecting)
743 : {
744 :
745 : // ?? This always returns true!!!?!!
746 0 : bool cando = true;
747 :
748 : // The total number of polids
749 0 : uInt npol = ms.polarization().nrow();
750 :
751 : // Nominally empty selection for all polids
752 0 : outToIn.resize(npol);
753 0 : outToIn.set(Vector<Int> ());
754 0 : if (areSelecting)
755 : {
756 : // Get the corr indices as an ordered map
757 0 : std::map<Int, Vector<Vector<Int> > > corrmap(mssel.getCorrMap(&ms));
758 :
759 : // Iterate over the ordered map to fill the vector maps
760 0 : for ( auto mi = corrmap.begin( ); mi != corrmap.end( ); ++mi)
761 : {
762 0 : Int pol = mi->first;
763 0 : std::vector<int> correlations_idx = mi->second[0].tovector();
764 0 : std::sort(correlations_idx.begin(), correlations_idx.end());
765 0 : outToIn[pol] = Vector<Int>(correlations_idx);
766 : }
767 : }
768 : else
769 : { // Make outToIn an identity map.
770 0 : ScalarColumn<Int> numCorr(ms.polarization(),MSPolarization::columnName(MSPolarization::NUM_CORR));
771 :
772 0 : for (uInt polid = 0; polid < npol; ++polid)
773 : {
774 0 : uInt ncorr = numCorr(polid);
775 0 : outToIn[polid].resize(ncorr);
776 0 : for (uInt cid = 0; cid < ncorr; ++cid)
777 : {
778 0 : outToIn[polid][cid] = cid;
779 : }
780 : }
781 : }
782 :
783 0 : return cando;
784 : }
785 :
786 : // -----------------------------------------------------------------------
787 : //
788 : // -----------------------------------------------------------------------
789 0 : void MSTransformDataHandler::selectTime(Double timeBin, String timerng)
790 : {
791 0 : timeBin_p = timeBin;
792 0 : timeRange_p = timerng;
793 0 : }
794 :
795 : // -----------------------------------------------------------------------
796 : //
797 : // -----------------------------------------------------------------------
798 0 : bool MSTransformDataHandler::makeMSBasicStructure(String& msname,
799 : String& colname,
800 : bool produceModel,
801 : bool createWeightSpectrumCols,
802 : const Vector<Int>& tileShape,
803 : const String& combine,
804 : Table::TableOption option)
805 : {
806 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
807 : os << LogIO::DEBUG1 << "Preparing to setup output MS with createWeightSpectrumCols: "
808 0 : << createWeightSpectrumCols << LogIO::POST;;
809 :
810 0 : if ((spw_p.nelements() > 0) && (max(spw_p) >= Int(ms_p.spectralWindow().nrow())))
811 : {
812 0 : os << LogIO::SEVERE << "SpectralWindow selection contains elements that do not exist in this MS" << LogIO::POST;
813 0 : ms_p = MeasurementSet();
814 0 : return false;
815 : }
816 :
817 : // Watch out! This throws an AipsError if ms_p doesn't have the requested columns.
818 : const Vector<MS::PredefinedColumns> colNamesTok =
819 0 : parseColumnNames(colname,produceModel, ms_p,virtualModelCol_p,virtualCorrectedCol_p);
820 :
821 0 : if (!makeSelection())
822 : {
823 0 : ms_p = MeasurementSet();
824 0 : throw(MSSelectionNullSelection("MSSelectionNullSelection : The selected table has zero rows."));
825 : return false;
826 : }
827 :
828 0 : mscIn_p = new MSColumns(mssel_p);
829 :
830 : // Note again the parseColumnNames() a few lines back that stops setupMS()
831 : // from being called if the MS doesn't have the requested columns.
832 0 : MeasurementSet* outpointer = 0;
833 :
834 0 : if (tileShape.nelements() == 3)
835 : {
836 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0], colNamesTok,
837 : createWeightSpectrumCols, tileShape);
838 : }
839 :
840 : // the following calls MSTileLayout... disabled for now because it
841 : // forces tiles to be the full spw bandwidth in width (gmoellen, 2010/11/07)
842 0 : else if ((tileShape.nelements() == 1) && (tileShape[0] == 0 || tileShape[0]== 1))
843 : {
844 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0],
845 0 : mscIn_p->observation().telescopeName()(0),
846 0 : colNamesTok, createWeightSpectrumCols, tileShape[0]);
847 : }
848 : else {
849 : // Sweep all other cases of bad tileshape to a default one.
850 : // (this probably never happens)
851 0 : outpointer = setupMS(msname, nchan_p[0], ncorr_p[0],
852 0 : mscIn_p->observation().telescopeName()(0),
853 : colNamesTok, createWeightSpectrumCols, 0);
854 : }
855 :
856 0 : combine_p = combine;
857 :
858 0 : msOut_p = *outpointer;
859 :
860 : // handle column keywords copy for CORRECTED_DATA -> DATA
861 0 : if (colNamesTok.nelements() == 1 && colNamesTok[0] == MS::CORRECTED_DATA && mssel_p.isColumn(MS::CORRECTED_DATA)) {
862 0 : TableColumn outCol(msOut_p, "DATA");
863 0 : TableColumn inCol(mssel_p, "CORRECTED_DATA");
864 : // Copy the keywords CORRECTED_DATA -> DATA
865 0 : copyMainTableKeywords(outCol.rwKeywordSet(), inCol.keywordSet());
866 : }
867 :
868 0 : bool ret = true;
869 : try
870 : {
871 0 : if (option == Table::Scratch)
872 : {
873 : // Set up pointing (has to be done in the copied MS)
874 0 : SetupNewTable pointingSetup(msOut_p.pointingTableName(),MSPointing::requiredTableDesc(), Table::New);
875 0 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::POINTING),Table(pointingSetup));
876 0 : msOut_p.initRefs();
877 :
878 : // Add additional columns to SPECTRAL_WINDOW sub-table
879 0 : addOptionalColumns(mssel_p.spectralWindow(), msOut_p.spectralWindow(), true);
880 :
881 : // Initialize output MS Columns
882 0 : msc_p = new MSColumns(msOut_p);
883 :
884 : // Write transformed SPECTRAL_WINDOW, DATA_DESCRIPTION_ID and POLARIZATION
885 0 : ret &= fillDDTables();
886 : }
887 : else
888 : {
889 0 : ret = fillSubTables(colNamesTok);
890 : }
891 : }
892 0 : catch (AipsError ex)
893 : {
894 0 : ret = false;
895 : os << LogIO::SEVERE
896 : << "Exception filling the sub-tables: " << ex.getMesg() << endl
897 0 : << "Stack Trace: " << ex.getStackTrace()
898 0 : << LogIO::POST;
899 : }
900 :
901 0 : if (!ret)
902 : {
903 0 : delete outpointer;
904 0 : ms_p = MeasurementSet();
905 0 : msOut_p = MeasurementSet();
906 0 : os << LogIO::SEVERE << msname << " left unfinished." << LogIO::POST;
907 0 : return false;
908 : }
909 :
910 : //Detaching the selected part
911 0 : ms_p = MeasurementSet();
912 :
913 0 : delete outpointer;
914 0 : return true;
915 : }
916 :
917 : // -----------------------------------------------------------------------
918 : //
919 : // -----------------------------------------------------------------------
920 0 : bool MSTransformDataHandler::isAllColumns(const Vector<MS::PredefinedColumns>& colNames)
921 : {
922 0 : bool dCol = false, mCol = false, cCol = false;
923 0 : for (uInt i = 0; i < colNames.nelements(); i++)
924 : {
925 0 : if (colNames[i] == MS::DATA) dCol = true;
926 0 : else if (colNames[i] == MS::MODEL_DATA) mCol = true;
927 0 : else if (colNames[i] == MS::CORRECTED_DATA) cCol = true;
928 : // else turn off all?
929 : }
930 :
931 0 : return (dCol && mCol && cCol);
932 : }
933 :
934 : // -----------------------------------------------------------------------
935 : // Modified version of makeSelection that uses the new getter methods
936 : // MSS::getSPWDDIDList() and MSS::getDDIDList()
937 : // -----------------------------------------------------------------------
938 0 : bool MSTransformDataHandler::makeSelection()
939 : {
940 :
941 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
942 :
943 : // VisSet/MSIter will check if the SORTED exists
944 : // jagonzal (CAS-5327): Commenting this out, since this implies all sorts of memory leaks
945 : // Block<Int> sort;
946 : // ROVisibilityIterator(ms_p, sort);
947 :
948 : const MeasurementSet *elms;
949 0 : elms = &ms_p;
950 0 : MeasurementSet sorted;
951 0 : if (ms_p.keywordSet().isDefined("SORTED_TABLE"))
952 : {
953 0 : sorted = ms_p.keywordSet().asTable("SORTED_TABLE");
954 :
955 : //If ms is not writable and sort is a subselection...use original ms
956 0 : if (ms_p.nrow() == sorted.nrow()) elms = &sorted;
957 : }
958 :
959 0 : MSSelection thisSelection;
960 0 : if (fieldid_p.nelements() > 0)
961 : {
962 0 : thisSelection.setFieldExpr(MSSelection::indexExprStr(fieldid_p));
963 : }
964 :
965 0 : if (spw_p.nelements() > 0)
966 : {
967 0 : thisSelection.setSpwExpr(spwString_p);
968 : }
969 :
970 0 : if (antennaSel_p)
971 : {
972 0 : if (antennaId_p.nelements() > 0)
973 : {
974 0 : thisSelection.setAntennaExpr(MSSelection::indexExprStr(antennaId_p));
975 : }
976 0 : if (antennaSelStr_p[0] != "")
977 : {
978 0 : thisSelection.setAntennaExpr(MSSelection::nameExprStr(antennaSelStr_p));
979 : }
980 :
981 : }
982 :
983 0 : if (timeRange_p != "")
984 : {
985 0 : thisSelection.setTimeExpr(timeRange_p);
986 : }
987 :
988 :
989 0 : thisSelection.setUvDistExpr(uvrangeString_p);
990 0 : thisSelection.setScanExpr(scanString_p);
991 0 : thisSelection.setStateExpr(intentString_p);
992 0 : thisSelection.setObservationExpr(obsString_p);
993 :
994 0 : if (arrayExpr_p != "")
995 : {
996 0 : thisSelection.setArrayExpr(arrayExpr_p);
997 : }
998 :
999 0 : if (corrString_p != "")
1000 : {
1001 0 : thisSelection.setPolnExpr(corrString_p.c_str());
1002 : }
1003 :
1004 0 : thisSelection.setTaQLExpr(taqlString_p);
1005 0 : thisSelection.setFeedExpr(feedString_p);
1006 :
1007 0 : TableExprNode exprNode = thisSelection.toTableExprNode(elms);
1008 0 : selTimeRanges_p = thisSelection.getTimeList();
1009 0 : selObsId_p = thisSelection.getObservationList();
1010 :
1011 : // Get the list of DDI for the selected spws
1012 0 : spw2ddid_p = thisSelection.getSPWDDIDList(elms);
1013 :
1014 0 : const MSDataDescription ddtable = elms->dataDescription();
1015 0 : ScalarColumn<Int> polId(ddtable,MSDataDescription::columnName(MSDataDescription::POLARIZATION_ID));
1016 0 : const MSPolarization poltable = elms->polarization();
1017 0 : ArrayColumn<Int> pols(poltable,MSPolarization::columnName(MSPolarization::CORR_TYPE));
1018 :
1019 : // Get the list of DDI for the selected polarizations
1020 0 : Vector<Int> polDDIList = thisSelection.getDDIDList(elms);
1021 :
1022 : // When polDDIList is empty, do not do an intersection
1023 0 : bool doIntersection = true;
1024 :
1025 0 : if (polDDIList.size() == 0){
1026 0 : doIntersection = false;
1027 : }
1028 :
1029 : // intersection between selected DDI from spw selection and
1030 : // selected DDI from polarization selection
1031 0 : if (doIntersection) {
1032 0 : Vector<Int> intersectedDDI = set_intersection(spw2ddid_p, polDDIList);
1033 0 : uInt nddids = intersectedDDI.size();
1034 0 : if (nddids > 0){
1035 0 : spw2ddid_p.resize(nddids);
1036 0 : for (uInt ii = 0; ii < nddids; ++ii){
1037 0 : spw2ddid_p[ii] = intersectedDDI[ii];
1038 : }
1039 : }
1040 : else {
1041 : os << LogIO::SEVERE << "None of the selected correlations are in spectral window "
1042 0 : << LogIO::POST;
1043 : }
1044 : }
1045 :
1046 :
1047 : // This is actually the number of selected DDI
1048 0 : uInt nDDIs = spw2ddid_p.size();
1049 :
1050 0 : inNumCorr_p.resize(nDDIs);
1051 0 : ncorr_p.resize(nDDIs);
1052 :
1053 : // Map the correlations from input selected DDI to output
1054 0 : for (uInt k = 0; k < nDDIs; ++k)
1055 : {
1056 0 : Int ddid = spw2ddid_p[k];
1057 :
1058 : // Number of input correlations for each DDI
1059 : // It reads the nelements of the CORR_TYPE column cell
1060 0 : inNumCorr_p[k] = pols(polId(ddid)).nelements();
1061 :
1062 : // Corresponding number of output correlations for each DDI
1063 0 : ncorr_p[k] = inPolOutCorrToInCorrMap_p[polId(ddid)].nelements();
1064 0 : if (ncorr_p[k] == 0)
1065 : {
1066 : os << LogIO::SEVERE
1067 : << "None of the selected correlations are in spectral window "
1068 0 : << spw_p[k] << LogIO::POST;
1069 0 : return false;
1070 : }
1071 : }
1072 :
1073 :
1074 : // Now remake the selected ms
1075 0 : if (!(exprNode.isNull()))
1076 : {
1077 0 : mssel_p = MeasurementSet((*elms)(exprNode));
1078 : }
1079 : else
1080 : {
1081 : // Null take all the ms ...setdata() blank means that
1082 0 : mssel_p = MeasurementSet((*elms));
1083 : }
1084 :
1085 0 : if (mssel_p.nrow() == 0) return false;
1086 :
1087 : // Setup antNewIndex_p now that mssel_p is ready.
1088 0 : if (antennaSel_p and reindex_p)
1089 : {
1090 : /*
1091 : // Watch out! getAntenna*List() and getBaselineList() return negative numbers for negated antennas!
1092 : ScalarColumn<Int> ant1c(mssel_p, MS::columnName(MS::ANTENNA1));
1093 : ScalarColumn<Int> ant2c(mssel_p, MS::columnName(MS::ANTENNA2));
1094 : Vector<Int> selAnts(ant1c.getColumn());
1095 : uInt nAnts = selAnts.nelements();
1096 :
1097 : selAnts.resize(2 * nAnts, true);
1098 : selAnts(Slice(nAnts, nAnts)) = ant2c.getColumn();
1099 : nAnts = GenSort<Int>::sort(selAnts, Sort::Ascending,Sort::NoDuplicates);
1100 : selAnts.resize(nAnts, true);
1101 : Int maxAnt = max(selAnts);
1102 : */
1103 :
1104 : // jagonzal: Scanning the main table is extremely inefficient, and depends on TaQL
1105 : // Therefore simply remove the negated antennas from getAntenna1List
1106 0 : vector<Int> antsSel;
1107 :
1108 : // Get antennas selected on position 1
1109 0 : Vector<Int> ant1List = thisSelection.getAntenna1List();
1110 0 : for (uInt idx=0;idx<ant1List.size();idx++)
1111 : {
1112 0 : if (ant1List(idx) >= 0) antsSel.push_back(ant1List(idx));
1113 : }
1114 :
1115 : // Get antennas selected on position 2
1116 0 : Vector<Int> ant2List = thisSelection.getAntenna2List();
1117 0 : for (uInt idx=0;idx<ant2List.size();idx++)
1118 : {
1119 0 : if (ant2List(idx) >= 0) antsSel.push_back(ant2List(idx));
1120 : }
1121 :
1122 : // Sort and remove duplicates
1123 0 : std::sort(antsSel.begin(), antsSel.end());
1124 0 : antsSel.erase(std::unique(antsSel.begin(), antsSel.end()),antsSel.end());
1125 :
1126 0 : Vector<Int> selAnts(antsSel);
1127 0 : uInt nAnts = selAnts.size();
1128 0 : Int maxAnt = max(selAnts);
1129 :
1130 0 : if (maxAnt < 0)
1131 : {
1132 : os << LogIO::SEVERE
1133 : << "The maximum selected antenna number, " << maxAnt
1134 : << ", seems to be < 0."
1135 0 : << LogIO::POST;
1136 0 : return false;
1137 : }
1138 :
1139 0 : antNewIndex_p.resize(maxAnt + 1);
1140 : //So if you see -1 in the main, feed, or pointing tables, fix it
1141 0 : antNewIndex_p.set(-1);
1142 :
1143 0 : for (uInt k = 0; k < nAnts; ++k)
1144 0 : antNewIndex_p[selAnts[k]] = k;
1145 :
1146 : //If the total number of output antennas is the same as the input antennas
1147 : //this means that the selection of baselines includes at the end
1148 : //all the input antennas. Therefore setting antenna selection to false.
1149 : //See CAS-11111
1150 0 : if(nAnts == elms->antenna().nrow())
1151 0 : antennaSel_p = false;
1152 : }
1153 : // This still gets tripped up by VLA:OUT.
1154 : else
1155 : {
1156 : // Make a default antNewIndex_p.
1157 0 : antNewIndex_p.resize(mssel_p.antenna().nrow());
1158 0 : indgen(antNewIndex_p);
1159 : }
1160 :
1161 0 : if (mssel_p.nrow() < ms_p.nrow())
1162 : {
1163 : os << LogIO::NORMAL
1164 : << mssel_p.nrow() << " out of " << ms_p.nrow()
1165 : << " rows are going to be considered due to the selection criteria."
1166 0 : << LogIO::POST;
1167 : }
1168 :
1169 0 : return true;
1170 : }
1171 :
1172 :
1173 : // -----------------------------------------------------------------------
1174 : //
1175 : // -----------------------------------------------------------------------
1176 0 : MeasurementSet* MSTransformDataHandler::setupMS(const String& MSFileName, const Int nchan,
1177 : const Int nCorr, const String& telescop,
1178 : const Vector<MS::PredefinedColumns>& colNames,
1179 : bool createWeightSpectrumCols,
1180 : const Int obstype, const bool compress,
1181 : const asdmStManUseAlternatives asdmStManUse,
1182 : Table::TableOption option)
1183 : {
1184 : //Choose an appropriate tileshape
1185 0 : IPosition dataShape(2, nCorr, nchan);
1186 0 : IPosition tileShape = MSTileLayout::tileShape(dataShape, obstype, telescop);
1187 0 : return setupMS(MSFileName, nchan, nCorr, colNames, createWeightSpectrumCols,
1188 0 : tileShape.asVector(),compress, asdmStManUse,option);
1189 : }
1190 :
1191 : // -----------------------------------------------------------------------
1192 : //
1193 : // -----------------------------------------------------------------------
1194 0 : MeasurementSet* MSTransformDataHandler::setupMS(const String& MSFileName, const Int nchan,
1195 : const Int nCorr,
1196 : const Vector<MS::PredefinedColumns>& colNamesTok,
1197 : bool createWeightSpectrumCols,
1198 : const Vector<Int>& tshape, const bool compress,
1199 : const asdmStManUseAlternatives asdmStManUse,
1200 : Table::TableOption option)
1201 : {
1202 0 : if (tshape.nelements() != 3) throw(AipsError("TileShape has to have 3 elements "));
1203 :
1204 : // This is more to shush a compiler warning than to warn users.
1205 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1206 0 : if (tshape[0] != nCorr)
1207 0 : os << LogIO::DEBUG1 << "Warning: using " << tshape[0]
1208 : << " from the tileshape instead of " << nCorr
1209 0 : << " for the number of correlations." << LogIO::POST;
1210 0 : if (tshape[1] != nchan)
1211 0 : os << LogIO::DEBUG1 << "Warning: using " << tshape[1]
1212 : << " from the tileshape instead of " << nchan
1213 0 : << " for the number of channels." << LogIO::POST;
1214 :
1215 : // Choose an appropriate tileshape //////////////////
1216 :
1217 0 : IPosition tileShape(tshape);
1218 :
1219 : // Make the MS table
1220 0 : TableDesc td = MS::requiredTableDesc();
1221 0 : Vector<String> tiledDataNames;
1222 :
1223 0 : if (option == Table::Scratch)
1224 : {
1225 0 : SetupNewTable newtab(MSFileName, td, option);
1226 0 : TableLock lock(TableLock::AutoLocking);
1227 0 : MeasurementSet *ms = new MeasurementSet(newtab, lock);
1228 :
1229 : // Set up default sub-tables for the MS
1230 0 : SetupNewTable dataDescSetup(ms->dataDescriptionTableName(),MSDataDescription::requiredTableDesc(), option);
1231 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::DATA_DESCRIPTION),Table(dataDescSetup));
1232 0 : SetupNewTable polarizationSetup(ms->polarizationTableName(),MSPolarization::requiredTableDesc(), option);
1233 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::POLARIZATION),Table(polarizationSetup));
1234 0 : SetupNewTable spectralWindowSetup(ms->spectralWindowTableName(),MSSpectralWindow::requiredTableDesc(), option);
1235 0 : ms->rwKeywordSet().defineTable(MS::keywordName(MS::SPECTRAL_WINDOW),Table(spectralWindowSetup));
1236 0 : ms->initRefs();
1237 :
1238 0 : return ms;
1239 : }
1240 :
1241 : // Even though we know the data is going to be the same shape throughout I'll
1242 : // still create a column that has a variable shape as this will permit MS's
1243 : // with other shapes to be appended.
1244 0 : uInt ncols = colNamesTok.nelements();
1245 0 : const bool mustWriteOnlyToData = mustConvertToData(ncols, colNamesTok);
1246 0 : if (mustWriteOnlyToData)
1247 : {
1248 0 : MS::addColumnToDesc(td, MS::DATA, 2);
1249 0 : if (asdmStManUse == DONT)
1250 : {
1251 0 : if (compress) MS::addColumnCompression(td, MS::DATA, true);
1252 0 : String hcolName = String("Tiled") + String("DATA");
1253 0 : td.defineHypercolumn(hcolName, 3, stringToVector("DATA"));
1254 0 : tiledDataNames.resize(1);
1255 0 : tiledDataNames[0] = hcolName;
1256 : }
1257 : }
1258 : else
1259 : {
1260 0 : tiledDataNames.resize(ncols);
1261 0 : for (uInt i = 0; i < ncols; ++i)
1262 : {
1263 : // Unfortunately MS::PredefinedColumns aren't ordered so that
1264 : // I can just check if colNamesTok[i] is in the "data range".
1265 0 : if ( colNamesTok[i] == MS::DATA
1266 0 : || colNamesTok[i] == MS::MODEL_DATA
1267 0 : || colNamesTok[i] == MS::CORRECTED_DATA
1268 0 : || colNamesTok[i] == MS::FLOAT_DATA
1269 0 : || colNamesTok[i] == MS::LAG_DATA)
1270 : {
1271 0 : if (asdmStManUse == DONT || colNamesTok[i] != MS::DATA)
1272 : {
1273 0 : MS::addColumnToDesc(td, colNamesTok[i], 2);
1274 0 : if (compress) MS::addColumnCompression(td, colNamesTok[i], true);
1275 : }
1276 : }
1277 : else
1278 : {
1279 0 : throw(AipsError( MS::columnName(colNamesTok[i]) + " is not a recognized data column "));
1280 : }
1281 0 : if (asdmStManUse == DONT || colNamesTok[i] != MS::DATA)
1282 : {
1283 0 : String hcolName = String("Tiled") + MS::columnName(colNamesTok[i]);
1284 0 : td.defineHypercolumn(hcolName, 3,stringToVector(MS::columnName(colNamesTok[i])));
1285 0 : tiledDataNames[i] = hcolName;
1286 : }
1287 : }
1288 : }
1289 :
1290 : //other cols for compression
1291 0 : if (compress && asdmStManUse != USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1292 : {
1293 0 : MS::addColumnCompression(td, MS::WEIGHT, true);
1294 0 : MS::addColumnCompression(td, MS::SIGMA, true);
1295 : }
1296 :
1297 0 : if (createWeightSpectrumCols) {
1298 0 : MS::addColumnToDesc(td, MS::WEIGHT_SPECTRUM, 2);
1299 0 : MS::addColumnToDesc(td, MS::SIGMA_SPECTRUM, 2);
1300 :
1301 0 : td.defineHypercolumn("TiledWgtSpectrum", 3,stringToVector(MS::columnName(MS::WEIGHT_SPECTRUM)));
1302 0 : td.defineHypercolumn("TiledSigmaSpectrum", 3,stringToVector(MS::columnName(MS::SIGMA_SPECTRUM)));
1303 : }
1304 :
1305 0 : td.defineHypercolumn("TiledFlagCategory", 4,stringToVector(MS::columnName(MS::FLAG_CATEGORY)));
1306 0 : td.defineHypercolumn("TiledUVW", 2, stringToVector(MS::columnName(MS::UVW)));
1307 :
1308 0 : if (asdmStManUse != USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1309 : {
1310 0 : td.defineHypercolumn("TiledFlag", 3,stringToVector(MS::columnName(MS::FLAG)));
1311 0 : td.defineHypercolumn("TiledWgt", 2,stringToVector(MS::columnName(MS::WEIGHT)));
1312 0 : td.defineHypercolumn("TiledSigma", 2,stringToVector(MS::columnName(MS::SIGMA)));
1313 : }
1314 :
1315 0 : SetupNewTable newtab(MSFileName, td, option);
1316 :
1317 0 : uInt cache_val = 32768;
1318 :
1319 : // Set the default Storage Manager to be the Incr one
1320 0 : IncrementalStMan incrStMan("ISMData", cache_val);
1321 0 : newtab.bindAll(incrStMan, true);
1322 :
1323 : //Override the binding for specific columns
1324 0 : IncrementalStMan incrStMan0("Array_ID", cache_val);
1325 0 : newtab.bindColumn(MS::columnName(MS::ARRAY_ID), incrStMan0);
1326 0 : IncrementalStMan incrStMan1("EXPOSURE", cache_val);
1327 0 : newtab.bindColumn(MS::columnName(MS::EXPOSURE), incrStMan1);
1328 0 : IncrementalStMan incrStMan2("FEED1", cache_val);
1329 0 : newtab.bindColumn(MS::columnName(MS::FEED1), incrStMan2);
1330 0 : IncrementalStMan incrStMan3("FEED2", cache_val);
1331 0 : newtab.bindColumn(MS::columnName(MS::FEED2), incrStMan3);
1332 0 : IncrementalStMan incrStMan4("FIELD_ID", cache_val);
1333 0 : newtab.bindColumn(MS::columnName(MS::FIELD_ID), incrStMan4);
1334 :
1335 : // jagonzal (CAS-6746): Don't use IncrementalStMan with RW cols ///////////////////////////////////////////////////
1336 : // IncrementalStMan incrStMan5("FLAG_ROW",cache_val/4);
1337 : // newtab.bindColumn(MS::columnName(MS::FLAG_ROW), incrStMan5);
1338 0 : StandardStMan aipsStManFlagRow("FLAG_ROW", cache_val/4);
1339 0 : newtab.bindColumn(MS::columnName(MS::FLAG_ROW), aipsStManFlagRow);
1340 : ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
1341 :
1342 0 : IncrementalStMan incrStMan6("INTERVAL", cache_val);
1343 0 : newtab.bindColumn(MS::columnName(MS::INTERVAL), incrStMan6);
1344 0 : IncrementalStMan incrStMan7("OBSERVATION_ID", cache_val);
1345 0 : newtab.bindColumn(MS::columnName(MS::OBSERVATION_ID), incrStMan7);
1346 0 : IncrementalStMan incrStMan8("PROCESSOR_ID", cache_val);
1347 0 : newtab.bindColumn(MS::columnName(MS::PROCESSOR_ID), incrStMan8);
1348 0 : IncrementalStMan incrStMan9("SCAN_NUMBER", cache_val);
1349 0 : newtab.bindColumn(MS::columnName(MS::SCAN_NUMBER), incrStMan9);
1350 0 : IncrementalStMan incrStMan10("STATE_ID", cache_val);
1351 0 : newtab.bindColumn(MS::columnName(MS::STATE_ID), incrStMan10);
1352 0 : IncrementalStMan incrStMan11("TIME", cache_val);
1353 0 : newtab.bindColumn(MS::columnName(MS::TIME), incrStMan11);
1354 0 : IncrementalStMan incrStMan12("TIME_CENTROID", cache_val);
1355 0 : newtab.bindColumn(MS::columnName(MS::TIME_CENTROID), incrStMan12);
1356 :
1357 : // Bind ANTENNA1, ANTENNA2 and DATA_DESC_ID to the standardStMan
1358 : // as they may change sufficiently frequently to make the
1359 : // incremental storage manager inefficient for these columns.
1360 0 : StandardStMan aipsStMan0("ANTENNA1", cache_val);
1361 0 : newtab.bindColumn(MS::columnName(MS::ANTENNA1), aipsStMan0);
1362 0 : StandardStMan aipsStMan1("ANTENNA2", cache_val);
1363 0 : newtab.bindColumn(MS::columnName(MS::ANTENNA2), aipsStMan1);
1364 0 : StandardStMan aipsStMan2("DATA_DESC_ID", cache_val);
1365 0 : newtab.bindColumn(MS::columnName(MS::DATA_DESC_ID), aipsStMan2);
1366 :
1367 : // Bind the DATA, FLAG & WEIGHT/SIGMA_SPECTRUM columns to the tiled stman or
1368 : // asdmStMan
1369 0 : AsdmStMan sm;
1370 :
1371 0 : if (mustWriteOnlyToData)
1372 : {
1373 0 : if (asdmStManUse == DONT)
1374 : {
1375 0 : TiledShapeStMan tiledStMan1Data("TiledDATA", tileShape);
1376 0 : newtab.bindColumn(MS::columnName(MS::DATA), tiledStMan1Data);
1377 : }
1378 : else
1379 : {
1380 0 : newtab.bindColumn(MS::columnName(MS::DATA), sm);
1381 : }
1382 : }
1383 : else
1384 : {
1385 0 : for (uInt i = 0; i < ncols; ++i)
1386 : {
1387 0 : TiledShapeStMan tiledStMan1Data(tiledDataNames[i], tileShape);
1388 0 : newtab.bindColumn(MS::columnName(colNamesTok[i]), tiledStMan1Data);
1389 : }
1390 0 : if (asdmStManUse != DONT)
1391 : {
1392 0 : newtab.bindColumn(MS::columnName(MS::DATA), sm);
1393 : }
1394 : }
1395 :
1396 0 : TiledShapeStMan tiledStMan1fc("TiledFlagCategory",IPosition(4, tileShape(0), tileShape(1), 1, tileShape(2)));
1397 0 : newtab.bindColumn(MS::columnName(MS::FLAG_CATEGORY), tiledStMan1fc);
1398 :
1399 0 : if (createWeightSpectrumCols) {
1400 0 : TiledShapeStMan tiledStMan2("TiledWgtSpectrum", tileShape);
1401 0 : TiledShapeStMan tiledStMan6("TiledSigmaSpectrum", tileShape);
1402 0 : newtab.bindColumn(MS::columnName(MS::WEIGHT_SPECTRUM), tiledStMan2);
1403 0 : newtab.bindColumn(MS::columnName(MS::SIGMA_SPECTRUM), tiledStMan6);
1404 : }
1405 :
1406 0 : TiledColumnStMan tiledStMan3("TiledUVW",IPosition(2, 3, (tileShape(0) * tileShape(1) * tileShape(2)) / 3));
1407 0 : newtab.bindColumn(MS::columnName(MS::UVW), tiledStMan3);
1408 0 : if (asdmStManUse == USE_FOR_DATA_WEIGHT_SIGMA_FLAG)
1409 : {
1410 0 : newtab.bindColumn(MS::columnName(MS::FLAG), sm);
1411 0 : newtab.bindColumn(MS::columnName(MS::WEIGHT), sm);
1412 0 : newtab.bindColumn(MS::columnName(MS::SIGMA), sm);
1413 : }
1414 : else
1415 : {
1416 0 : TiledShapeStMan tiledStMan1f("TiledFlag", tileShape);
1417 0 : TiledShapeStMan tiledStMan4("TiledWgt", IPosition(2, tileShape(0),
1418 0 : tileShape(1) * tileShape(2)));
1419 0 : TiledShapeStMan tiledStMan5("TiledSigma",IPosition(2, tileShape(0),
1420 0 : tileShape(1) * tileShape(2)));
1421 :
1422 0 : newtab.bindColumn(MS::columnName(MS::FLAG), tiledStMan1f);
1423 0 : newtab.bindColumn(MS::columnName(MS::WEIGHT), tiledStMan4);
1424 0 : newtab.bindColumn(MS::columnName(MS::SIGMA), tiledStMan5);
1425 : }
1426 :
1427 : // Avoid lock overheads by locking the table permanently
1428 0 : TableLock lock(TableLock::AutoLocking);
1429 0 : MeasurementSet *ms = new MeasurementSet(newtab, lock);
1430 :
1431 : // Set up the sub-tables for the UVFITS MS (we make new tables with 0 rows)
1432 : // Table::TableOption option = Table::New;
1433 0 : createSubtables(*ms, option);
1434 :
1435 : // Set the TableInfo
1436 0 : TableInfo& info(ms->tableInfo());
1437 0 : info.setType(TableInfo::type(TableInfo::MEASUREMENTSET));
1438 0 : info.setSubType(String("UVFITS"));
1439 0 : info.readmeAddLine("This is a measurement set Table holding astronomical observations");
1440 :
1441 0 : return ms;
1442 : }
1443 :
1444 : // -----------------------------------------------------------------------
1445 : //
1446 : // -----------------------------------------------------------------------
1447 0 : void MSTransformDataHandler::createSubtables(MeasurementSet& ms, Table::TableOption option)
1448 : {
1449 0 : SetupNewTable antennaSetup(ms.antennaTableName(),MSAntenna::requiredTableDesc(), option);
1450 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::ANTENNA),Table(antennaSetup));
1451 0 : SetupNewTable dataDescSetup(ms.dataDescriptionTableName(),MSDataDescription::requiredTableDesc(), option);
1452 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::DATA_DESCRIPTION),Table(dataDescSetup));
1453 0 : SetupNewTable feedSetup(ms.feedTableName(), MSFeed::requiredTableDesc(),option);
1454 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FEED), Table(feedSetup));
1455 0 : SetupNewTable flagCmdSetup(ms.flagCmdTableName(),MSFlagCmd::requiredTableDesc(), option);
1456 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FLAG_CMD),Table(flagCmdSetup));
1457 0 : SetupNewTable fieldSetup(ms.fieldTableName(), MSField::requiredTableDesc(),option);
1458 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::FIELD), Table(fieldSetup));
1459 0 : SetupNewTable historySetup(ms.historyTableName(),MSHistory::requiredTableDesc(), option);
1460 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::HISTORY),Table(historySetup));
1461 0 : SetupNewTable observationSetup(ms.observationTableName(),MSObservation::requiredTableDesc(), option);
1462 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::OBSERVATION),Table(observationSetup));
1463 0 : SetupNewTable polarizationSetup(ms.polarizationTableName(),MSPolarization::requiredTableDesc(), option);
1464 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::POLARIZATION),Table(polarizationSetup));
1465 0 : SetupNewTable processorSetup(ms.processorTableName(),MSProcessor::requiredTableDesc(), option);
1466 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::PROCESSOR),Table(processorSetup));
1467 0 : SetupNewTable spectralWindowSetup(ms.spectralWindowTableName(),MSSpectralWindow::requiredTableDesc(), option);
1468 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::SPECTRAL_WINDOW),Table(spectralWindowSetup));
1469 0 : SetupNewTable stateSetup(ms.stateTableName(), MSState::requiredTableDesc(),option);
1470 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::STATE), Table(stateSetup));
1471 :
1472 : // Add the optional Source sub table to allow for specification of the rest frequency
1473 0 : SetupNewTable sourceSetup(ms.sourceTableName(),MSSource::requiredTableDesc(), option);
1474 0 : ms.rwKeywordSet().defineTable(MS::keywordName(MS::SOURCE),Table(sourceSetup, 0));
1475 :
1476 : // Update the references to the sub-table keywords
1477 0 : ms.initRefs();
1478 :
1479 0 : return;
1480 : }
1481 :
1482 : // -----------------------------------------------------------------------
1483 : //
1484 : // -----------------------------------------------------------------------
1485 0 : bool MSTransformDataHandler::fillSubTables(const Vector<MS::PredefinedColumns>&)
1486 : {
1487 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1488 0 : bool success = true;
1489 :
1490 : // Copy the sub-tables before doing anything with the main table.
1491 : // Otherwise MSColumns won't work.
1492 :
1493 : // fill or update
1494 0 : Timer timer;
1495 :
1496 0 : timer.mark();
1497 0 : success &= copyPointing();
1498 0 : os << LogIO::DEBUG1 << "copyPointing took " << timer.real() << "s." << LogIO::POST;
1499 :
1500 : // Optional columns should be set up before msc_p.
1501 0 : addOptionalColumns(mssel_p.spectralWindow(), msOut_p.spectralWindow(), true);
1502 :
1503 : // Force the Measures frames for all the time type columns to
1504 : // have the same reference as the TIME column of the main table.
1505 : // Disable the empty table check (with false) because some of the
1506 : // sub-tables (like POINTING) might already have been written.
1507 : // However, empty tables are still empty after setting up the reference codes here.
1508 0 : msc_p = new MSColumns(msOut_p);
1509 0 : msc_p->setEpochRef(MEpoch::castType(mscIn_p->timeMeas().getMeasRef().getType()), false);
1510 :
1511 : // UVW is the only other Measures column in the main table.
1512 0 : msc_p->uvwMeas().setDescRefCode(Muvw::castType(mscIn_p->uvwMeas().getMeasRef().getType()));
1513 :
1514 0 : if (!mscIn_p->flagCategory().isNull() && mscIn_p->flagCategory().isDefined(0))
1515 : {
1516 0 : msc_p->setFlagCategories(mscIn_p->flagCategories());
1517 : }
1518 :
1519 :
1520 0 : timer.mark();
1521 0 : if (!fillDDTables()) return false;
1522 0 : os << LogIO::DEBUG1 << "fillDDTables took " << timer.real() << "s." << LogIO::POST;
1523 :
1524 : // SourceIDs need to be re-mapped around here
1525 : // (It cannot not be done in selectSource() because mssel_p is not set up yet)
1526 0 : timer.mark();
1527 0 : relabelSources();
1528 0 : os << LogIO::DEBUG1 << "relabelSources took " << timer.real() << "s." << LogIO::POST;
1529 :
1530 0 : success &= fillFieldTable();
1531 0 : success &= copySource();
1532 :
1533 0 : success &= copyAntenna();
1534 : // Feed table writing has to be after antenna
1535 0 : if (!copyFeed()) return false;
1536 :
1537 0 : success &= copyFlag_Cmd();
1538 0 : success &= copyHistory();
1539 0 : success &= copyObservation();
1540 0 : success &= copyProcessor();
1541 0 : success &= copyState();
1542 :
1543 0 : timer.mark();
1544 0 : success &= copySyscal();
1545 0 : os << LogIO::DEBUG1 << "copySyscal took " << timer.real() << "s." << LogIO::POST;
1546 :
1547 0 : timer.mark();
1548 0 : success &= copyWeather();
1549 0 : os << LogIO::DEBUG1 << "copyWeather took " << timer.real() << "s." << LogIO::POST;
1550 :
1551 0 : timer.mark();
1552 0 : success &= filterOptSubtable("CALDEVICE");
1553 0 : os << LogIO::DEBUG1 << "CALDEVICE took " << timer.real() << "s." << LogIO::POST;
1554 :
1555 0 : timer.mark();
1556 0 : success &= filterOptSubtable("SYSPOWER");
1557 0 : os << LogIO::DEBUG1 << "SYSPOWER took " << timer.real() << "s." << LogIO::POST;
1558 :
1559 : // Run this after running the other copy*()s.
1560 : // Maybe there should be an option to *not* run it.
1561 0 : success &= copyGenericSubtables();
1562 0 : return success;
1563 : }
1564 :
1565 : // -----------------------------------------------------------------------
1566 : //
1567 : // -----------------------------------------------------------------------
1568 0 : bool MSTransformDataHandler::fillFieldTable()
1569 : {
1570 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1571 :
1572 0 : uInt nAddedCols = addOptionalColumns(mssel_p.field(), msOut_p.field(), true);
1573 :
1574 0 : MSFieldColumns msField(msOut_p.field());
1575 :
1576 0 : const MSFieldColumns& fieldIn = mscIn_p->field();
1577 0 : ScalarColumn<String> code(fieldIn.code());
1578 0 : ArrayColumn<Double> delayDir(fieldIn.delayDir());
1579 0 : ScalarColumn<bool> flagRow(fieldIn.flagRow());
1580 0 : ScalarColumn<String> name(fieldIn.name());
1581 0 : ScalarColumn<Int> numPoly(fieldIn.numPoly());
1582 0 : ArrayColumn<Double> phaseDir(fieldIn.phaseDir());
1583 0 : ArrayColumn<Double> refDir(fieldIn.referenceDir());
1584 0 : ScalarColumn<Int> sourceId(fieldIn.sourceId());
1585 0 : ScalarColumn<Double> time(fieldIn.time());
1586 :
1587 0 : String refstr;
1588 0 : String nameVarRefColDelayDir, nameVarRefColPhaseDir, nameVarRefColRefDir;
1589 :
1590 : // Need to correctly define the direction measures.
1591 :
1592 : // DelayDir
1593 0 : if (delayDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1594 : {
1595 0 : delayDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1596 0 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("Ref",refstr);
1597 : }
1598 :
1599 : // it's a variable ref. column
1600 0 : if (delayDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1601 : {
1602 0 : delayDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1603 0 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1604 0 : nameVarRefColDelayDir = refstr;
1605 :
1606 0 : Vector<String> refTypeV;
1607 0 : delayDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1608 0 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1609 :
1610 0 : Vector<uInt> refCodeV;
1611 0 : delayDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1612 0 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1613 0 : Int refid = msField.delayDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1614 :
1615 : // Erase the redundant Ref keyword
1616 0 : if (refid >= 0)
1617 : {
1618 0 : msField.delayDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1619 : }
1620 : }
1621 :
1622 : // PhaseDir
1623 0 : if (phaseDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1624 : {
1625 0 : phaseDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1626 0 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("Ref",refstr);
1627 : }
1628 :
1629 : // It's a variable ref. column
1630 0 : if (phaseDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1631 : {
1632 0 : phaseDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1633 0 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1634 0 : nameVarRefColPhaseDir = refstr;
1635 :
1636 0 : Vector<String> refTypeV;
1637 0 : phaseDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1638 0 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1639 :
1640 0 : Vector<uInt> refCodeV;
1641 0 : phaseDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1642 0 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1643 :
1644 0 : Int refid = msField.phaseDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1645 0 : if (refid >= 0)
1646 : {
1647 0 : msField.phaseDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1648 : }
1649 : }
1650 :
1651 : // ReferenceDir
1652 0 : if (refDir.keywordSet().asRecord("MEASINFO").isDefined("Ref"))
1653 : {
1654 0 : refDir.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1655 0 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define(
1656 : "Ref", refstr);
1657 : }
1658 :
1659 : // It's a variable ref. column
1660 0 : if (refDir.keywordSet().asRecord("MEASINFO").isDefined("VarRefCol"))
1661 : {
1662 0 : refDir.keywordSet().asRecord("MEASINFO").get("VarRefCol", refstr);
1663 0 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("VarRefCol", refstr);
1664 0 : nameVarRefColRefDir = refstr;
1665 :
1666 0 : Vector<String> refTypeV;
1667 0 : refDir.keywordSet().asRecord("MEASINFO").get("TabRefTypes", refTypeV);
1668 0 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefTypes", refTypeV);
1669 :
1670 0 : Vector<uInt> refCodeV;
1671 0 : refDir.keywordSet().asRecord("MEASINFO").get("TabRefCodes", refCodeV);
1672 0 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").define("TabRefCodes", refCodeV);
1673 :
1674 0 : Int refid = msField.referenceDir().keywordSet().asRecord("MEASINFO").fieldNumber("Ref");
1675 0 : if (refid >= 0)
1676 : {
1677 0 : msField.referenceDir().rwKeywordSet().asrwRecord("MEASINFO").removeField(RecordFieldId(refid));
1678 : }
1679 : }
1680 :
1681 : // ...and the time measure...
1682 0 : time.keywordSet().asRecord("MEASINFO").get("Ref", refstr);
1683 0 : msField.time().rwKeywordSet().asrwRecord("MEASINFO").define("Ref", refstr);
1684 :
1685 0 : if (!reindex_p)
1686 : {
1687 0 : const MSField &inputField = mssel_p.field();
1688 0 : MSField &outputField = msOut_p.field();
1689 0 : TableCopy::copyRows(outputField, inputField);
1690 0 : copyEphemerisTable(msField);
1691 0 : return true;
1692 : }
1693 :
1694 : // fieldRelabel_p size: nrow of a input MS, -1 for unselected field ids
1695 0 : fieldRelabel_p.resize(mscIn_p->field().nrow());
1696 0 : fieldRelabel_p.set(-1);
1697 :
1698 : os << LogIO::DEBUG1 << fieldid_p.nelements()
1699 0 : << " fields selected out of " << mscIn_p->field().nrow()
1700 0 : << LogIO::POST;
1701 :
1702 : try {
1703 :
1704 0 : msOut_p.field().addRow(fieldid_p.nelements());
1705 :
1706 0 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1707 : {
1708 0 : fieldRelabel_p[fieldid_p[k]] = k;
1709 :
1710 0 : msField.code().put(k, code(fieldid_p[k]));
1711 0 : msField.delayDir().put(k, delayDir(fieldid_p[k]));
1712 0 : msField.flagRow().put(k, flagRow(fieldid_p[k]));
1713 0 : msField.name().put(k, name(fieldid_p[k]));
1714 0 : msField.numPoly().put(k, numPoly(fieldid_p[k]));
1715 0 : msField.phaseDir().put(k, phaseDir(fieldid_p[k]));
1716 0 : msField.referenceDir().put(k, refDir(fieldid_p[k]));
1717 0 : msField.time().put(k, time(fieldid_p[k]));
1718 :
1719 0 : Int inSrcID = sourceId(fieldid_p[k]);
1720 0 : if (inSrcID < 0)
1721 : {
1722 0 : msField.sourceId().put(k, -1);
1723 : }
1724 : else
1725 : {
1726 0 : msField.sourceId().put(k, sourceRelabel_p[inSrcID]);
1727 : }
1728 : }
1729 :
1730 0 : if (nAddedCols > 0)
1731 : {
1732 0 : copyEphemerisTable(msField);
1733 :
1734 : // need to copy the reference column
1735 0 : if (!nameVarRefColDelayDir.empty())
1736 : {
1737 0 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColDelayDir);
1738 0 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColDelayDir);
1739 0 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1740 : {
1741 0 : cdMDirRef.put(k, dM(fieldid_p[k]));
1742 : }
1743 : }
1744 :
1745 : // need to copy the reference column
1746 0 : if (!nameVarRefColPhaseDir.empty())
1747 : {
1748 0 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColPhaseDir);
1749 0 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColPhaseDir);
1750 0 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1751 : {
1752 0 : cdMDirRef.put(k, dM(fieldid_p[k]));
1753 : }
1754 : }
1755 :
1756 : // need to copy the reference column
1757 0 : if (!nameVarRefColRefDir.empty())
1758 : {
1759 0 : ScalarColumn<Int> dM(mssel_p.field(), nameVarRefColRefDir);
1760 0 : ScalarColumn<Int> cdMDirRef(msOut_p.field(),nameVarRefColRefDir);
1761 0 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
1762 : {
1763 0 : cdMDirRef.put(k, dM(fieldid_p[k]));
1764 : }
1765 : }
1766 : }
1767 :
1768 : }
1769 0 : catch (AipsError x)
1770 : {
1771 0 : os << LogIO::EXCEPTION << "Error " << x.getMesg() << " setting up the output FIELD table." << LogIO::POST;
1772 : }
1773 0 : catch (...)
1774 : {
1775 0 : throw(AipsError("Unknown exception caught and released in fillFieldTable()"));
1776 : }
1777 :
1778 0 : return true;
1779 : }
1780 :
1781 0 : bool MSTransformDataHandler::copyEphemerisTable(MSFieldColumns & msField)
1782 : {
1783 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1784 0 : const MSFieldColumns& fieldIn = mscIn_p->field();
1785 0 : ScalarColumn<Int> eID(fieldIn.ephemerisId());
1786 :
1787 0 : if (eID.hasContent())
1788 : {
1789 : uInt nField;
1790 0 : if (reindex_p)
1791 : {
1792 0 : nField = fieldid_p.nelements();
1793 : }
1794 : else
1795 : {
1796 0 : nField = eID.nrow();
1797 : }
1798 :
1799 0 : String destPathName = Path(msOut_p.field().tableName()).absoluteName();
1800 0 : ScalarColumn<String> name(fieldIn.name());
1801 :
1802 0 : for (uInt k = 0; k < nField; ++k)
1803 : {
1804 : uInt fieldId;
1805 0 : if (reindex_p)
1806 : {
1807 0 : fieldId = fieldid_p[k];
1808 : }
1809 : else
1810 : {
1811 0 : fieldId = k;
1812 : }
1813 :
1814 0 : Int theEphId = eID(fieldId);
1815 :
1816 : // There is an ephemeris attached to this field
1817 0 : if (theEphId > -1)
1818 : {
1819 0 : Path ephPath = Path(fieldIn.ephemPath(fieldId));
1820 :
1821 : // Copy the ephemeris table over to the output FIELD table
1822 0 : if (ephPath.length() > 0)
1823 : {
1824 0 : Directory origEphemDir(ephPath);
1825 0 : origEphemDir.copy(destPathName + "/" + ephPath.baseName());
1826 :
1827 : os << LogIO::NORMAL
1828 0 : << "Transferring ephemeris " << ephPath.baseName()
1829 0 : << " for output field " << name(fieldId)
1830 0 : << LogIO::POST;
1831 : }
1832 : }
1833 :
1834 0 : if (reindex_p)
1835 : {
1836 0 : msField.ephemerisId().put(k, theEphId);
1837 : }
1838 : }
1839 : }
1840 :
1841 0 : return true;
1842 : }
1843 :
1844 : // -----------------------------------------------------------------------
1845 : // Modified version of fillDDTables
1846 : // -----------------------------------------------------------------------
1847 0 : bool MSTransformDataHandler::fillDDTables()
1848 : {
1849 0 : fillPolTable();
1850 0 : fillDDITable();
1851 0 : fillSPWTable();
1852 :
1853 0 : return true;
1854 : }
1855 :
1856 : // -----------------------------------------------------------------------
1857 : //
1858 : // -----------------------------------------------------------------------
1859 0 : bool MSTransformDataHandler::fillPolTable()
1860 : {
1861 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1862 :
1863 : // Input polarization table
1864 0 : const MSPolarization &poltable = mssel_p.polarization();
1865 0 : ScalarColumn<Int> numCorr(poltable,MSPolarization::columnName(MSPolarization::NUM_CORR));
1866 0 : ArrayColumn<Int> corrType(poltable,MSPolarization::columnName(MSPolarization::CORR_TYPE));
1867 0 : ArrayColumn<Int> corrProd(poltable,MSPolarization::columnName(MSPolarization::CORR_PRODUCT));
1868 0 : ScalarColumn<bool> flagRow(poltable,MSPolarization::columnName(MSPolarization::FLAG_ROW));
1869 :
1870 : // Output polarization table
1871 0 : MSPolarizationColumns& msPol(msc_p->polarization());
1872 :
1873 : // Fill output polarization table
1874 0 : uInt nPol = poltable.nrow(); // nOutputPol = nInputPol (no PolId re-index)
1875 0 : corrSlice_p.resize(nPol);
1876 0 : for (uInt polId = 0; polId < nPol; polId++)
1877 : {
1878 0 : uInt ncorr = inPolOutCorrToInCorrMap_p[polId].nelements();
1879 0 : const Vector<Int> inCT(corrType(polId));
1880 :
1881 : // Add row
1882 0 : msOut_p.polarization().addRow();
1883 0 : msPol.numCorr().put(polId, ncorr);
1884 0 : msPol.flagRow().put(polId, flagRow(polId));
1885 :
1886 : // Setup correlation slices
1887 0 : if (ncorr > 0 && ncorr < inCT.nelements())
1888 : {
1889 0 : keepShape_p = false;
1890 :
1891 : // Check whether the requested correlations can be accessed by slicing.
1892 : // That means there must be a constant stride. The most likely (only?)
1893 : // way to violate that is to ask for 3 out of 4 correlations.
1894 0 : if (ncorr > 2)
1895 : {
1896 : os << LogIO::SEVERE
1897 : << "Sorry, the requested correlation selection is not unsupported.\n"
1898 : << "Try selecting fewer or all of the correlations."
1899 0 : << LogIO::POST;
1900 0 : return false;
1901 : }
1902 :
1903 0 : size_t increment = 2;
1904 0 : if (ncorr > 1)
1905 : {
1906 0 : increment = inPolOutCorrToInCorrMap_p[polId][1] - inPolOutCorrToInCorrMap_p[polId][0];
1907 : }
1908 0 : corrSlice_p[polId] = Slice(inPolOutCorrToInCorrMap_p[polId][0],ncorr,increment);
1909 : }
1910 : else
1911 : {
1912 0 : corrSlice_p[polId] = Slice(0, ncorr);
1913 : }
1914 :
1915 : // Apply slices to correlation type and product
1916 0 : Vector<Int> outCT;
1917 0 : const Matrix<Int> inCP(corrProd(polId));
1918 0 : Matrix<Int> outCP;
1919 0 : outCT.resize(ncorr);
1920 0 : outCP.resize(2, ncorr);
1921 0 : for (uInt k = 0; k < ncorr; ++k)
1922 : {
1923 0 : Int inCorrInd = inPolOutCorrToInCorrMap_p[polId][k];
1924 :
1925 0 : outCT[k] = inCT[inCorrInd];
1926 0 : for (uInt feedind = 0; feedind < 2; ++feedind)
1927 : {
1928 0 : outCP(feedind, k) = inCP(feedind, inCorrInd);
1929 : }
1930 :
1931 : }
1932 :
1933 : // Fill correlation type and product
1934 0 : msPol.corrType().put(polId, outCT);
1935 0 : msPol.corrProduct().put(polId, outCP);
1936 : }
1937 :
1938 0 : return true;
1939 : }
1940 :
1941 : // -----------------------------------------------------------------------
1942 : //
1943 : // -----------------------------------------------------------------------
1944 0 : bool MSTransformDataHandler::fillDDITable()
1945 : {
1946 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
1947 :
1948 : // Input selected DDIs based on joint SPW-Pol selection
1949 0 : uInt nddid = spw2ddid_p.size();
1950 :
1951 : // Input ddi table
1952 0 : const MSDataDescription &inputDDI = mssel_p.dataDescription();
1953 :
1954 0 : ScalarColumn<Int> polIdCol(inputDDI, MSDataDescription::columnName(MSDataDescription::POLARIZATION_ID));
1955 0 : ScalarColumn<Int> spwIdCol(inputDDI,MSDataDescription::columnName(MSDataDescription::SPECTRAL_WINDOW_ID));
1956 :
1957 : // Get selected SPWs
1958 0 : Vector<Int> selectedSpwIds(nddid);
1959 0 : for (uInt row=0; row<spw2ddid_p.size(); ++row)
1960 : {
1961 0 : selectedSpwIds[row] = spwIdCol(spw2ddid_p[row]);
1962 : }
1963 :
1964 : // Get list of unique selected SPWs
1965 0 : bool option(false);
1966 0 : Sort sortSpws(spw_p.getStorage(option), sizeof(Int));
1967 0 : sortSpws.sortKey((uInt) 0, TpInt);
1968 0 : Vector<uInt> spwsortindex, spwuniqinds;
1969 0 : sortSpws.sort(spwsortindex, spw_p.nelements());
1970 0 : uInt nuniqSpws = sortSpws.unique(spwuniqinds, spwsortindex);
1971 0 : spw_uniq_p.resize(nuniqSpws);
1972 :
1973 : // Make map from input to output SPWs
1974 0 : spwRelabel_p.resize(mscIn_p->spectralWindow().nrow());
1975 0 : spwRelabel_p.set(-1);
1976 0 : for (uInt k = 0; k < nuniqSpws; ++k)
1977 : {
1978 0 : spw_uniq_p[k] = spw_p[spwuniqinds[k]];
1979 0 : spwRelabel_p[spw_uniq_p[k]] = k;
1980 : }
1981 :
1982 0 : if (!reindex_p)
1983 : {
1984 0 : MSDataDescription &outputDDI = msOut_p.dataDescription();
1985 0 : TableCopy::copyRows(outputDDI, inputDDI);
1986 0 : return true;
1987 : }
1988 :
1989 : // Output SPECTRAL_WINDOW_ID column
1990 0 : Vector<Int> newSPWId(nddid);
1991 0 : for (uInt ddi=0; ddi<nddid; ddi++)
1992 : {
1993 0 : newSPWId[ddi] = spwRelabel_p[spwIdCol(spw2ddid_p[ddi])];
1994 : }
1995 :
1996 : // Output POLARIZATION_ID column
1997 0 : Vector<Int> newPolId(nddid);
1998 0 : for (uInt ddi=0; ddi<nddid; ddi++)
1999 : {
2000 0 : newPolId[ddi] = polIdCol(spw2ddid_p[ddi]);
2001 : }
2002 :
2003 : // Fill output DDI table
2004 0 : MSDataDescColumns& outputDDI(msc_p->dataDescription());
2005 0 : for (uInt ddi=0; ddi<nddid; ddi++)
2006 : {
2007 0 : msOut_p.dataDescription().addRow();
2008 0 : outputDDI.flagRow().put(ddi, false);
2009 0 : outputDDI.polarizationId().put(ddi, newPolId[ddi]);
2010 0 : outputDDI.spectralWindowId().put(ddi, newSPWId[ddi]);
2011 : }
2012 :
2013 :
2014 0 : return true;
2015 : }
2016 :
2017 : // -----------------------------------------------------------------------
2018 : //
2019 : // -----------------------------------------------------------------------
2020 0 : bool MSTransformDataHandler::fillSPWTable()
2021 : {
2022 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2023 :
2024 : // Selected input MS SPW Table Columns (read-only)
2025 0 : MSSpWindowColumns inSpWCols(mssel_p.spectralWindow());
2026 :
2027 : // SPW Table Columns of output MS (read-write)
2028 0 : MSSpWindowColumns& msSpW(msc_p->spectralWindow());
2029 :
2030 : // Detect which optional columns of SPECTRAL_WINDOW are present.
2031 : // inSpWCols and msSpW should agree because addOptionalColumns() was done for
2032 : // SPECTRAL_WINDOW in fillAllTables() before making msc_p or calling fillDDTables
2033 0 : bool haveSpwAN = columnOk(inSpWCols.assocNature());
2034 0 : bool haveSpwASI = columnOk(inSpWCols.assocSpwId());
2035 0 : bool haveSpwBN = columnOk(inSpWCols.bbcNo());
2036 0 : bool haveSpwBS = columnOk(inSpWCols.bbcSideband());
2037 0 : bool haveSpwDI = columnOk(inSpWCols.dopplerId());
2038 0 : bool haveSpwSWF = mssel_p.spectralWindow().tableDesc().isColumn("SDM_WINDOW_FUNCTION") &&
2039 0 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_WINDOW_FUNCTION");
2040 0 : bool haveSpwSNB = mssel_p.spectralWindow().tableDesc().isColumn("SDM_NUM_BIN") &&
2041 0 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_NUM_BIN");
2042 0 : bool haveSpwCorrBit = mssel_p.spectralWindow().tableDesc().isColumn("SDM_CORR_BIT") &&
2043 0 : mssel_p.spectralWindow().tableDesc().columnDescSet().isDefined("SDM_CORR_BIT");
2044 :
2045 0 : uInt nuniqSpws = spw_uniq_p.size();
2046 :
2047 : // This sets the number of input channels for each spw. But
2048 : // it considers that a SPW ID contains only one set of channels.
2049 : // I hope this is true!!
2050 : // Write to SPECTRAL_WINDOW table
2051 0 : inNumChan_p.resize(spw_p.nelements());
2052 0 : for (uInt k = 0; k < spw_p.nelements(); ++k)
2053 : {
2054 0 : inNumChan_p[k] = inSpWCols.numChan()(spw_p[k]);
2055 : }
2056 :
2057 0 : if (reindex_p)
2058 : {
2059 0 : msOut_p.spectralWindow().addRow(nuniqSpws);
2060 : }
2061 : else
2062 : {
2063 0 : const MSSpectralWindow &inputSPW = mssel_p.spectralWindow();
2064 0 : MSSpectralWindow &outputSPW = msOut_p.spectralWindow();
2065 0 : TableCopy::copyRows(outputSPW, inputSPW);
2066 : }
2067 :
2068 :
2069 0 : Vector<Vector<Int> > spwinds_of_uniq_spws(nuniqSpws);
2070 0 : totnchan_p.resize(nuniqSpws);
2071 0 : for (uInt k = 0; k < nuniqSpws; ++k)
2072 : {
2073 0 : Int maxchan = 0;
2074 0 : uInt j = 0;
2075 :
2076 0 : totnchan_p[k] = 0;
2077 0 : spwinds_of_uniq_spws[k].resize();
2078 0 : for (uInt spwind = 0; spwind < spw_p.nelements(); ++spwind)
2079 : {
2080 0 : if (spw_p[spwind] == spw_uniq_p[k])
2081 : {
2082 0 : Int highchan = nchan_p[spwind] * chanStep_p[spwind] + chanStart_p[spwind];
2083 :
2084 0 : if (highchan > maxchan) maxchan = highchan;
2085 :
2086 0 : totnchan_p[k] += nchan_p[spwind];
2087 :
2088 : // The true is necessary to avoid scrambling previously assigned values.
2089 0 : spwinds_of_uniq_spws[k].resize(j + 1, true);
2090 :
2091 : // Warning! spwinds_of_uniq_spws[k][j] will compile without warning, but dump core at runtime.
2092 0 : (spwinds_of_uniq_spws[k])[j] = spwind;
2093 :
2094 0 : ++j;
2095 : }
2096 : }
2097 0 : if (maxchan > inSpWCols.numChan()(spw_uniq_p[k]))
2098 : {
2099 : os << LogIO::SEVERE
2100 : << " Channel settings wrong; exceeding number of channels in spw "
2101 0 : << spw_uniq_p[k] << LogIO::POST;
2102 0 : return false;
2103 : }
2104 : }
2105 :
2106 : // min_k is an index for getting an spw index via spw_uniq_p[min_k].
2107 : // k is an index for getting an spw index via spw_p[k].
2108 0 : for (uInt min_k = 0; min_k < nuniqSpws; ++min_k)
2109 : {
2110 0 : uInt k = spwinds_of_uniq_spws[min_k][0];
2111 0 : uInt outSPWId = reindex_p? min_k : spw_p[k];
2112 :
2113 0 : if (spwinds_of_uniq_spws[min_k].nelements() > 1 || nchan_p[k] != inSpWCols.numChan()(spw_p[k]))
2114 : {
2115 0 : Vector<Double> effBWIn = inSpWCols.effectiveBW()(spw_uniq_p[min_k]);
2116 0 : Int nOutChan = totnchan_p[min_k];
2117 0 : Vector<Double> chanFreqOut(nOutChan);
2118 0 : Vector<Double> chanFreqIn = inSpWCols.chanFreq()(spw_uniq_p[min_k]);
2119 0 : Vector<Double> chanWidthOut(nOutChan);
2120 0 : Vector<Double> chanWidthIn = inSpWCols.chanWidth()(spw_uniq_p[min_k]);
2121 0 : Vector<Double> spwResolOut(nOutChan);
2122 0 : Vector<Double> spwResolIn = inSpWCols.resolution()(spw_uniq_p[min_k]);
2123 0 : Vector<Double> effBWOut(nOutChan);
2124 0 : Int outChan = 0;
2125 0 : Int outChanNotDropped = 0;
2126 :
2127 0 : keepShape_p = false;
2128 :
2129 : // The sign of CHAN_WIDTH defaults to +. Its determination assumes that
2130 : // chanFreqIn is monotonic, but not that the sign of the chanWidthIn is correct.
2131 0 : bool neginc = chanFreqIn[chanFreqIn.nelements() - 1] < chanFreqIn[0];
2132 :
2133 0 : effBWOut.set(0.0);
2134 0 : Double totalBW = 0.0;
2135 0 : for (uInt rangeNum = 0; rangeNum < spwinds_of_uniq_spws[min_k].nelements(); ++rangeNum)
2136 : {
2137 0 : k = spwinds_of_uniq_spws[min_k][rangeNum];
2138 :
2139 0 : Int span = chanStep_p[k] * widths_p[k];
2140 :
2141 0 : for (Int j = 0; j < nchan_p[k]; ++j)
2142 : {
2143 0 : Int inpChan = chanStart_p[k] + j * span;
2144 :
2145 0 : if (span >= 1)
2146 : {
2147 0 : Int lastChan = inpChan + span - 1;
2148 :
2149 0 : if (lastChan > chanEnd_p[k])
2150 : {
2151 : // The averaging width is not a factor of the number of
2152 : // selected input channels, so the last output bin receives
2153 : // fewer input channels than the other bins.
2154 0 : lastChan = chanEnd_p[k];
2155 :
2156 0 : Int nchan = lastChan - inpChan + 1;
2157 : os << LogIO::NORMAL
2158 : << "The last output channel of spw "
2159 0 : << spw_p[k] << " has only " << nchan
2160 0 : << " input channel";
2161 0 : if (nchan > 1) os << "s.";
2162 0 : os << LogIO::POST;
2163 :
2164 : // jagonzal (CAS-8618): We may have more than one channel dropped per SPW
2165 : // due to multiple channel selections
2166 0 : spwDropChannelMap_p[spw_p[k]].push_back(outChan);
2167 : }
2168 : else
2169 : {
2170 0 : for (Int inputChan = inpChan;inputChan<=lastChan;inputChan++)
2171 : {
2172 0 : spwSelectedChannelMap_p[spw_p[k]][outChanNotDropped].push_back(inputChan);
2173 : }
2174 0 : outChanNotDropped++;
2175 : }
2176 :
2177 0 : chanFreqOut[outChan] = (chanFreqIn[inpChan]
2178 0 : + chanFreqIn[lastChan]) / 2;
2179 :
2180 0 : Double sep = chanFreqIn[lastChan] - chanFreqIn[inpChan];
2181 :
2182 0 : if (neginc) sep = -sep;
2183 :
2184 : // The internal abs is necessary because the sign of chanWidthIn may be wrong.
2185 0 : chanWidthOut[outChan] = sep + 0.5 * abs(chanWidthIn[inpChan] + chanWidthIn[lastChan]);
2186 0 : if (neginc) chanWidthOut[outChan] = -chanWidthOut[outChan];
2187 :
2188 0 : spwResolOut[outChan] = 0.5 * (spwResolIn[inpChan] + spwResolIn[lastChan]) + sep;
2189 :
2190 0 : for (Int avgChan = inpChan; avgChan <= lastChan; avgChan += chanStep_p[k])
2191 : {
2192 0 : effBWOut[outChan] += effBWIn[avgChan];
2193 : }
2194 :
2195 : }
2196 : else
2197 : {
2198 0 : chanFreqOut[outChan] = chanFreqIn[inpChan];
2199 0 : spwResolOut[outChan] = spwResolIn[inpChan];
2200 0 : chanWidthOut[outChan] = chanWidthIn[inpChan];
2201 0 : effBWOut[outChan] = effBWIn[inpChan];
2202 : }
2203 : // Use CHAN_WIDTH instead of EFFECTIVE_BW
2204 0 : totalBW += abs(chanWidthOut[outChan]);
2205 0 : ++outChan;
2206 : }
2207 : }
2208 0 : --outChan;
2209 :
2210 0 : msSpW.chanFreq().put(outSPWId, chanFreqOut);
2211 0 : msSpW.refFrequency().put(outSPWId,min(chanFreqOut[0], chanFreqOut[chanFreqOut.size() - 1]));
2212 0 : msSpW.resolution().put(outSPWId, spwResolOut);
2213 0 : msSpW.numChan().put(outSPWId, nOutChan);
2214 0 : msSpW.chanWidth().put(outSPWId, chanWidthOut);
2215 0 : msSpW.effectiveBW().put(outSPWId, spwResolOut);
2216 0 : msSpW.totalBandwidth().put(outSPWId, totalBW);
2217 : }
2218 : else
2219 : {
2220 0 : msSpW.chanFreq().put(outSPWId, inSpWCols.chanFreq()(spw_p[k]));
2221 0 : msSpW.refFrequency().put(outSPWId, inSpWCols.refFrequency()(spw_p[k]));
2222 0 : msSpW.resolution().put(outSPWId, inSpWCols.resolution()(spw_p[k]));
2223 0 : msSpW.numChan().put(outSPWId, inSpWCols.numChan()(spw_p[k]));
2224 0 : msSpW.chanWidth().put(outSPWId, inSpWCols.chanWidth()(spw_p[k]));
2225 0 : msSpW.effectiveBW().put(outSPWId, inSpWCols.effectiveBW()(spw_p[k]));
2226 0 : msSpW.totalBandwidth().put(outSPWId,inSpWCols.totalBandwidth()(spw_p[k]));
2227 : }
2228 :
2229 0 : msSpW.flagRow().put(outSPWId, inSpWCols.flagRow()(spw_p[k]));
2230 0 : msSpW.freqGroup().put(outSPWId, inSpWCols.freqGroup()(spw_p[k]));
2231 0 : msSpW.freqGroupName().put(outSPWId, inSpWCols.freqGroupName()(spw_p[k]));
2232 0 : msSpW.ifConvChain().put(outSPWId, inSpWCols.ifConvChain()(spw_p[k]));
2233 0 : msSpW.measFreqRef().put(outSPWId, inSpWCols.measFreqRef()(spw_p[k]));
2234 0 : msSpW.name().put(outSPWId, inSpWCols.name()(spw_p[k]));
2235 0 : msSpW.netSideband().put(outSPWId, inSpWCols.netSideband()(spw_p[k]));
2236 :
2237 :
2238 0 : if (haveSpwBN) msSpW.bbcNo().put(outSPWId, inSpWCols.bbcNo()(spw_p[k]));
2239 0 : if (haveSpwBS) msSpW.bbcSideband().put(outSPWId, inSpWCols.bbcSideband()(spw_p[k]));
2240 0 : if (haveSpwDI) msSpW.dopplerId().put(outSPWId, inSpWCols.dopplerId()(spw_p[k]));
2241 0 : if (haveSpwSWF)
2242 : {
2243 0 : ScalarColumn<String> inSwfCol(mssel_p.spectralWindow(), "SDM_WINDOW_FUNCTION");
2244 0 : ScalarColumn<String> outSwfCol(msOut_p.spectralWindow(), "SDM_WINDOW_FUNCTION");
2245 0 : outSwfCol.put(outSPWId, inSwfCol(spw_p[k]));
2246 : }
2247 0 : if (haveSpwSNB)
2248 : {
2249 0 : ScalarColumn<Int> inSnbCol(mssel_p.spectralWindow(), "SDM_NUM_BIN");
2250 0 : ScalarColumn<Int> outSnbCol(msOut_p.spectralWindow(), "SDM_NUM_BIN");
2251 0 : outSnbCol.put(outSPWId, inSnbCol(spw_p[k]));
2252 : }
2253 0 : if (haveSpwCorrBit)
2254 : {
2255 0 : ScalarColumn<String> inCorrBitCol(mssel_p.spectralWindow(), "SDM_CORR_BIT");
2256 0 : ScalarColumn<String> outCorrBitCol(msOut_p.spectralWindow(), "SDM_CORR_BIT");
2257 0 : outCorrBitCol.put(outSPWId, inCorrBitCol(spw_p[k]));
2258 : }
2259 :
2260 0 : if (haveSpwASI)
2261 : {
2262 0 : if (reindex_p)
2263 : {
2264 : // Get list of SPWs associated to his one
2265 0 : std::vector<Int> selectedSPWs = spw_p.tovector();
2266 :
2267 : // Get the list of selected SPWs and association nature
2268 0 : Array<Int> assocSpwId = inSpWCols.assocSpwId()(spw_p[k]);
2269 0 : Array<String> assocNature = inSpWCols.assocNature()(spw_p[k]);
2270 :
2271 : // Find which associated SPWs are selected, and store the transformed Id
2272 0 : std::vector<Int>::iterator findIt;
2273 0 : std::vector<Int> selectedAssocSpwId;
2274 0 : std::vector<String> selectedAssocNature;
2275 0 : for (uInt idx=0;idx<assocSpwId.size();idx++)
2276 : {
2277 0 : IPosition pos(1,idx);
2278 0 : Int spw = assocSpwId(pos);
2279 0 : findIt = find (selectedSPWs.begin(), selectedSPWs.end(), spw);
2280 0 : if (findIt != selectedSPWs.end())
2281 : {
2282 0 : selectedAssocSpwId.push_back(spwRelabel_p[spw]);
2283 0 : if (haveSpwAN) selectedAssocNature.push_back(assocNature(pos));
2284 : }
2285 : }
2286 :
2287 : // Store selected associated SPW Ids
2288 0 : Vector<Int> selectedAssocSpwIdVector(selectedAssocSpwId);
2289 0 : msSpW.assocSpwId().put(min_k, selectedAssocSpwIdVector);
2290 :
2291 : // Store selected association nature
2292 0 : if (haveSpwAN)
2293 : {
2294 0 : Vector<String> selectedAssocNatureVector(selectedAssocNature);
2295 0 : msSpW.assocNature().put(min_k, selectedAssocNatureVector);
2296 : }
2297 : }
2298 : else
2299 : {
2300 0 : msSpW.assocNature().put(outSPWId, inSpWCols.assocNature()(outSPWId));
2301 : }
2302 : }
2303 : }
2304 :
2305 :
2306 0 : return true;
2307 : }
2308 :
2309 : // -----------------------------------------------------------------------
2310 : //
2311 : // -----------------------------------------------------------------------
2312 0 : uInt MSTransformDataHandler::addOptionalColumns(const Table& inTab, Table& outTab,const bool beLazy)
2313 : {
2314 0 : uInt nAdded = 0;
2315 0 : const TableDesc& inTD = inTab.actualTableDesc();
2316 :
2317 : // Only rely on the # of columns if you are sure that inTab and outTab
2318 : // can't have the same # of columns without having _different_ columns,
2319 : // i.e. use beLazy if outTab.actualTableDesc() is in its default state.
2320 0 : uInt nInCol = inTD.ncolumn();
2321 0 : if (!beLazy || nInCol > outTab.actualTableDesc().ncolumn())
2322 : {
2323 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2324 :
2325 0 : Vector<String> oldColNames = inTD.columnNames();
2326 :
2327 0 : for (uInt k = 0; k < nInCol; ++k)
2328 : {
2329 0 : if (!outTab.actualTableDesc().isColumn(oldColNames[k]))
2330 : {
2331 : try
2332 : {
2333 0 : outTab.addColumn(inTD.columnDesc(k), false);
2334 0 : ++nAdded;
2335 : }
2336 : // NOT AipsError x
2337 0 : catch (...)
2338 : {
2339 : os << LogIO::WARN << "Could not add column "
2340 0 : << oldColNames[k] << " to " << outTab.tableName()
2341 0 : << LogIO::POST;
2342 : }
2343 : }
2344 : }
2345 : }
2346 :
2347 0 : return nAdded;
2348 : }
2349 :
2350 : // -----------------------------------------------------------------------
2351 : //
2352 : // -----------------------------------------------------------------------
2353 0 : void MSTransformDataHandler::relabelSources()
2354 : {
2355 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2356 :
2357 : //Source is an optional table, so it may not exist
2358 0 : if (Table::isReadable(mssel_p.sourceTableName()))
2359 : {
2360 : // Note that mscIn_p->field().sourceId() has ALL of the
2361 : // sourceIDs in the input MS, not just the selected ones.
2362 0 : const Vector<Int>& inSrcIDs = mscIn_p->field().sourceId().getColumn();
2363 :
2364 0 : Int highestInpSrc = max(inSrcIDs);
2365 :
2366 : // Ensure space for -1.
2367 0 : if (highestInpSrc < 0) highestInpSrc = 0;
2368 :
2369 0 : sourceRelabel_p.resize(highestInpSrc + 1);
2370 : // Default to "any".
2371 0 : sourceRelabel_p.set(-1);
2372 :
2373 : // Enable sourceIDs that are actually referred
2374 : // by selected fields, and remap them using j.
2375 0 : uInt j = 0;
2376 0 : for (uInt k = 0; k < fieldid_p.nelements(); ++k)
2377 : {
2378 0 : Int fldInSrcID = inSrcIDs[fieldid_p[k]];
2379 :
2380 0 : if (fldInSrcID > -1)
2381 : {
2382 : // Multiple fields can use the same
2383 0 : if (sourceRelabel_p[fldInSrcID] == -1)
2384 : {
2385 : // source in a mosaic.
2386 0 : sourceRelabel_p[fldInSrcID] = j;
2387 0 : ++j;
2388 : }
2389 : }
2390 : }
2391 : }
2392 : else
2393 : {
2394 : // Default to "any".
2395 : os << LogIO::NORMAL
2396 : << "The input MS does not have the optional SOURCE table.\n"
2397 0 : << "-1 will be used as a generic source ID." << LogIO::POST;
2398 0 : sourceRelabel_p.resize(1);
2399 0 : sourceRelabel_p.set(-1);
2400 : }
2401 :
2402 0 : return;
2403 : }
2404 :
2405 : // -----------------------------------------------------------------------
2406 : //
2407 : // -----------------------------------------------------------------------
2408 0 : void MSTransformDataHandler::copySubtable(const String& tabName, const Table& inTab,const bool doFilter)
2409 : {
2410 0 : String outName(msOut_p.tableName() + '/' + tabName);
2411 :
2412 0 : if (PlainTable::tableCache()(outName)) PlainTable::tableCache().remove(outName);
2413 0 : inTab.deepCopy(outName, Table::New, false, Table::AipsrcEndian, doFilter);
2414 0 : Table outTab(outName, Table::Update);
2415 0 : msOut_p.rwKeywordSet().defineTable(tabName, outTab);
2416 0 : msOut_p.initRefs();
2417 :
2418 0 : return;
2419 : }
2420 :
2421 : // -----------------------------------------------------------------------
2422 : //
2423 : // -----------------------------------------------------------------------
2424 0 : void MSTransformDataHandler::make_map(std::map<Int, Int>& mapper, const Vector<Int>& inv)
2425 : {
2426 0 : std::set<Int> valSet;
2427 :
2428 : // Strange, but slightly more efficient than going forward.
2429 0 : for (Int i = inv.nelements(); i--;)
2430 : {
2431 0 : valSet.insert(inv[i]);
2432 : }
2433 :
2434 0 : uInt remaval = 0;
2435 0 : for (std::set<Int>::const_iterator vs_iter = valSet.begin(); vs_iter != valSet.end(); ++vs_iter)
2436 : {
2437 0 : mapper[*vs_iter] = remaval;
2438 0 : ++remaval;
2439 : }
2440 :
2441 0 : return;
2442 : }
2443 :
2444 : // -----------------------------------------------------------------------
2445 : //
2446 : // -----------------------------------------------------------------------
2447 0 : bool MSTransformDataHandler::copyPointing()
2448 : {
2449 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2450 :
2451 : //Pointing is allowed to not exist
2452 0 : if (Table::isReadable(mssel_p.pointingTableName()))
2453 : {
2454 0 : const MSPointing& oldPoint = mssel_p.pointing();
2455 :
2456 0 : if ((!antennaSel_p && timeRange_p == "") or !reindex_p)
2457 : {
2458 : // jagonzal: copySubtables works with POINTING because it
2459 : // is not created by default in the method createSubtables
2460 0 : copySubtable(MS::keywordName(MS::POINTING), oldPoint);
2461 : }
2462 : else
2463 : {
2464 0 : setupNewPointing();
2465 :
2466 0 : if (oldPoint.nrow() > 0)
2467 : {
2468 : // Could be declared as Table&
2469 0 : MSPointing& newPoint = msOut_p.pointing();
2470 :
2471 : // Add optional columns if present in oldPoint.
2472 0 : uInt nAddedCols = addOptionalColumns(oldPoint, newPoint, true);
2473 0 : os << LogIO::DEBUG1 << "POINTING has " << nAddedCols << " optional columns." << LogIO::POST;
2474 :
2475 0 : const MSPointingColumns oldPCs(oldPoint);
2476 0 : MSPointingColumns newPCs(newPoint);
2477 0 : newPCs.setEpochRef(MEpoch::castType(oldPCs.timeMeas().getMeasRef().getType()));
2478 0 : newPCs.setDirectionRef(MDirection::castType(oldPCs.directionMeasCol().getMeasRef().getType()));
2479 0 : newPCs.setEncoderDirectionRef(MDirection::castType(oldPCs.encoderMeas().getMeasRef().getType()));
2480 :
2481 0 : const ScalarColumn<Int>& antIds = oldPCs.antennaId();
2482 0 : const ScalarColumn<Double>& time = oldPCs.time();
2483 0 : ScalarColumn<Int>& outants = newPCs.antennaId();
2484 :
2485 0 : uInt nTRanges = selTimeRanges_p.ncolumn();
2486 :
2487 0 : uInt outRow = 0;
2488 :
2489 : // Int for comparison
2490 0 : Int maxSelAntp1 = antNewIndex_p.nelements();
2491 : // with newAntInd.
2492 0 : for (uInt inRow = 0; inRow < antIds.nrow(); ++inRow)
2493 : {
2494 0 : Int newAntInd = antIds(inRow);
2495 0 : if (antennaSel_p)
2496 : {
2497 0 : newAntInd = newAntInd < maxSelAntp1 ? antNewIndex_p[newAntInd] : -1;
2498 : }
2499 :
2500 0 : Double t = time(inRow);
2501 :
2502 0 : if (newAntInd > -1)
2503 : {
2504 0 : bool matchT = false;
2505 0 : if (nTRanges == 0)
2506 : {
2507 0 : matchT = true;
2508 : }
2509 : else
2510 : {
2511 0 : for (uInt tr = 0; tr < nTRanges; ++tr)
2512 : {
2513 0 : if (t >= selTimeRanges_p(0, tr) && t <= selTimeRanges_p(1, tr))
2514 : {
2515 0 : matchT = true;
2516 0 : break;
2517 : }
2518 : }
2519 : }
2520 :
2521 0 : if (matchT)
2522 : {
2523 0 : TableCopy::copyRows(newPoint, oldPoint, outRow,inRow, 1, false);
2524 0 : outants.put(outRow, newAntInd);
2525 0 : ++outRow;
2526 : }
2527 : }
2528 : }
2529 : }
2530 : }
2531 : }
2532 : else
2533 : {
2534 : // Make an empty stub for MSColumns.
2535 0 : setupNewPointing();
2536 : }
2537 :
2538 :
2539 0 : return true;
2540 : }
2541 :
2542 : // -----------------------------------------------------------------------
2543 : //
2544 : // -----------------------------------------------------------------------
2545 0 : void MSTransformDataHandler::setupNewPointing()
2546 : {
2547 0 : SetupNewTable pointingSetup(msOut_p.pointingTableName(),
2548 0 : MSPointing::requiredTableDesc(), Table::New);
2549 :
2550 : // POINTING can be large, set some sensible defaults for storageMgrs
2551 0 : IncrementalStMan ismPointing("ISMPointing");
2552 0 : StandardStMan ssmPointing("SSMPointing", 32768);
2553 0 : pointingSetup.bindAll(ismPointing, true);
2554 0 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::DIRECTION),ssmPointing);
2555 0 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::TARGET),ssmPointing);
2556 0 : pointingSetup.bindColumn(MSPointing::columnName(MSPointing::TIME),ssmPointing);
2557 0 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::POINTING),Table(pointingSetup));
2558 0 : msOut_p.initRefs();
2559 :
2560 0 : return;
2561 : }
2562 :
2563 : // -----------------------------------------------------------------------
2564 : //
2565 : // -----------------------------------------------------------------------
2566 0 : bool MSTransformDataHandler::copySource()
2567 : {
2568 : //Source is an optional table, so it may not exist
2569 0 : if (Table::isReadable(mssel_p.sourceTableName()))
2570 : {
2571 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2572 :
2573 0 : const MSSource& oldSource = mssel_p.source();
2574 0 : MSSource& newSource = msOut_p.source();
2575 :
2576 : // Add optional columns if present in oldSource.
2577 0 : uInt nAddedCols = addOptionalColumns(oldSource, newSource, true);
2578 0 : os << LogIO::DEBUG1 << "SOURCE has " << nAddedCols << " optional columns." << LogIO::POST;
2579 :
2580 0 : const MSSourceColumns incols(oldSource);
2581 0 : MSSourceColumns outcols(newSource);
2582 :
2583 : // Copy the Measures frame info. This has to be done before filling the rows.
2584 0 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2585 0 : outcols.setDirectionRef(MDirection::castType(incols.directionMeas().getMeasRef().getType()));
2586 0 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2587 0 : outcols.setFrequencyRef(MFrequency::castType(incols.restFrequencyMeas().getMeasRef().getType()));
2588 0 : outcols.setRadialVelocityRef(MRadialVelocity::castType(incols.sysvelMeas().getMeasRef().getType()));
2589 :
2590 0 : if (!reindex_p)
2591 : {
2592 0 : TableCopy::copyRows(newSource, oldSource);
2593 0 : return true;
2594 : }
2595 :
2596 0 : const ScalarColumn<Int>& inSId = incols.sourceId();
2597 0 : ScalarColumn<Int>& outSId = outcols.sourceId();
2598 0 : const ScalarColumn<Int>& inSPW = incols.spectralWindowId();
2599 0 : ScalarColumn<Int>& outSPW = outcols.spectralWindowId();
2600 :
2601 : // row number in output.
2602 0 : uInt outrn = 0;
2603 0 : uInt nInputRows = inSId.nrow();
2604 : // inSidVal is Int.
2605 0 : Int maxSId = sourceRelabel_p.nelements();
2606 0 : Int maxSPWId = spwRelabel_p.nelements();
2607 0 : for (uInt inrn = 0; inrn < nInputRows; ++inrn)
2608 : {
2609 0 : Int inSidVal = inSId(inrn);
2610 : // -1 means the source is valid for any SPW.
2611 0 : Int inSPWVal = inSPW(inrn);
2612 0 : if (inSidVal >= maxSId)
2613 : {
2614 0 : os << LogIO::WARN << "Invalid SOURCE ID in SOURCE table row " << inrn << LogIO::POST;
2615 : }
2616 0 : if (inSPWVal >= maxSPWId)
2617 : {
2618 0 : os << LogIO::WARN << "Invalid SPW ID in SOURCE table row " << inrn << LogIO::POST;
2619 : }
2620 :
2621 0 : if ( (inSidVal > -1) &&
2622 0 : (inSidVal < maxSId) &&
2623 0 : (sourceRelabel_p[inSidVal] > -1) &&
2624 0 : ((inSPWVal == -1) || (inSPWVal < maxSPWId && spwRelabel_p[inSPWVal] > -1)))
2625 : {
2626 : // Copy inrn to outrn.
2627 0 : TableCopy::copyRows(newSource, oldSource, outrn, inrn, 1);
2628 0 : outSId.put(outrn, sourceRelabel_p[inSidVal]);
2629 0 : outSPW.put(outrn, inSPWVal > -1 ? spwRelabel_p[inSPWVal] : -1);
2630 0 : ++outrn;
2631 : }
2632 : }
2633 :
2634 : }
2635 :
2636 0 : return true;
2637 : }
2638 :
2639 : // -----------------------------------------------------------------------
2640 : //
2641 : // -----------------------------------------------------------------------
2642 0 : bool MSTransformDataHandler::copyAntenna()
2643 : {
2644 0 : const MSAntenna& oldAnt = mssel_p.antenna();
2645 0 : MSAntenna& newAnt = msOut_p.antenna();
2646 0 : const MSAntennaColumns incols(oldAnt);
2647 0 : MSAntennaColumns outcols(newAnt);
2648 0 : bool retval = false;
2649 :
2650 0 : outcols.setOffsetRef(MPosition::castType(incols.offsetMeas().getMeasRef().getType()));
2651 0 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2652 :
2653 : //TableCopy::copyRows(newAnt, oldAnt);
2654 : //retval = true;
2655 :
2656 0 : if (!antennaSel_p or !reindex_p)
2657 : {
2658 0 : TableCopy::copyRows(newAnt, oldAnt);
2659 0 : retval = true;
2660 : }
2661 : else
2662 : {
2663 0 : uInt nAnt = antNewIndex_p.nelements();
2664 : // Don't use min() here, it's too overloaded.
2665 0 : if (nAnt > oldAnt.nrow()) nAnt = oldAnt.nrow();
2666 :
2667 0 : for (uInt k = 0; k < nAnt; ++k)
2668 : {
2669 0 : if (antNewIndex_p[k] > -1) TableCopy::copyRows(newAnt, oldAnt, antNewIndex_p[k], k, 1, false);
2670 : }
2671 0 : retval = true;
2672 : }
2673 :
2674 0 : return retval;
2675 : }
2676 :
2677 : // -----------------------------------------------------------------------
2678 : //
2679 : // -----------------------------------------------------------------------
2680 0 : bool MSTransformDataHandler::copyFeed()
2681 : {
2682 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2683 :
2684 0 : const MSFeed& oldFeed = mssel_p.feed();
2685 0 : MSFeed& newFeed = msOut_p.feed();
2686 0 : const MSFeedColumns incols(oldFeed);
2687 0 : MSFeedColumns outcols(newFeed);
2688 :
2689 0 : outcols.setDirectionRef(MDirection::castType(incols.beamOffsetMeas().getMeasRef().getType()));
2690 0 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2691 0 : outcols.setPositionRef(MPosition::castType(incols.positionMeas().getMeasRef().getType()));
2692 :
2693 0 : if ((!antennaSel_p && allEQ(spwRelabel_p, spw_p)) or !reindex_p)
2694 : {
2695 0 : TableCopy::copyRows(newFeed, oldFeed);
2696 : }
2697 : else
2698 : {
2699 0 : const Vector<Int>& antIds = incols.antennaId().getColumn();
2700 0 : const Vector<Int>& spwIds = incols.spectralWindowId().getColumn();
2701 :
2702 : // Copy selected rows.
2703 0 : uInt totNFeeds = antIds.nelements();
2704 0 : uInt totalSelFeeds = 0;
2705 0 : Int maxSelAntp1 = antNewIndex_p.nelements();
2706 0 : for (uInt k = 0; k < totNFeeds; ++k)
2707 : {
2708 : // antenna must be selected, and spwId must be -1 (any) or selected.
2709 0 : if ( antIds[k] < maxSelAntp1 &&
2710 0 : antNewIndex_p[antIds[k]] > -1 &&
2711 0 : (spwIds[k] < 0 || spwRelabel_p[spwIds[k]] > -1)
2712 : )
2713 : {
2714 0 : TableCopy::copyRows(newFeed, oldFeed, totalSelFeeds, k, 1,false);
2715 0 : ++totalSelFeeds;
2716 : }
2717 : }
2718 :
2719 : // Remap antenna and spw #s.
2720 0 : ScalarColumn<Int>& antCol = outcols.antennaId();
2721 0 : ScalarColumn<Int>& spwCol = outcols.spectralWindowId();
2722 :
2723 0 : for (uInt k = 0; k < totalSelFeeds; ++k)
2724 : {
2725 :
2726 0 : antCol.put(k, antNewIndex_p[antCol(k)]);
2727 0 : if (spwCol(k) > -1) spwCol.put(k, spwRelabel_p[spwCol(k)]);
2728 : }
2729 : }
2730 :
2731 : // Check if selected spw is WVR data. WVR data does not have FEED data
2732 : // so it is not a failure if newFeed.nrow == 0
2733 0 : if (newFeed.nrow() < 1 and spw_p.size() == 1){
2734 : // Using the MSMetaData class
2735 0 : MSMetaData msmeta(&mssel_p, 0);
2736 0 : std::set<uInt> wvrspw = msmeta.getWVRSpw();
2737 0 : for (std::set<uInt>::iterator bbit = wvrspw.begin(); bbit != wvrspw.end(); ++bbit){
2738 0 : if ((uInt)spw_p[0] == *bbit){
2739 0 : os << LogIO::DEBUG1 << "Skipping spw="<<*bbit<<" because it is WVR and has no FEED content" << LogIO::POST;
2740 0 : return true;
2741 : }
2742 : }
2743 : }
2744 :
2745 0 : if (newFeed.nrow() < 1)
2746 : {
2747 : // LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2748 0 : os << LogIO::SEVERE << "No feeds were selected." << LogIO::POST;
2749 0 : return false;
2750 : }
2751 :
2752 0 : return true;
2753 : }
2754 :
2755 : // -----------------------------------------------------------------------
2756 : // Get the processorId corresponding to a given DDI
2757 : // -----------------------------------------------------------------------
2758 0 : Int MSTransformDataHandler::getProcessorId(Int dataDescriptionId, String msname)
2759 : {
2760 0 : ostringstream taql;
2761 0 : taql << "SELECT PROCESSOR_ID from " << msname;
2762 0 : taql << " WHERE DATA_DESC_ID ==" << dataDescriptionId;
2763 0 : taql << " LIMIT 1";
2764 :
2765 0 : casacore::TableProxy *firstSelectedRow = new TableProxy(tableCommand(taql.str()).table());
2766 0 : Record colWrapper = firstSelectedRow->getVarColumn(String("PROCESSOR_ID"),0,1,1);
2767 0 : casacore::Vector<Int> processorId = colWrapper.asArrayInt("r1");
2768 :
2769 0 : delete firstSelectedRow;
2770 0 : return processorId[0];
2771 : }
2772 :
2773 :
2774 : // -----------------------------------------------------------------------
2775 : //
2776 : // -----------------------------------------------------------------------
2777 0 : bool MSTransformDataHandler::copyFlag_Cmd()
2778 : {
2779 :
2780 : // Like POINTING, FLAG_CMD is supposed to exist but is allowed not to.
2781 0 : if (Table::isReadable(mssel_p.flagCmdTableName()))
2782 : {
2783 0 : const MSFlagCmd& oldFlag_Cmd = mssel_p.flagCmd();
2784 :
2785 0 : if (oldFlag_Cmd.nrow() > 0)
2786 : {
2787 :
2788 : // Could be declared as Table&
2789 0 : MSFlagCmd& newFlag_Cmd = msOut_p.flagCmd();
2790 :
2791 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2792 :
2793 : // Add optional columns if present in oldFlag_Cmd.
2794 0 : uInt nAddedCols = addOptionalColumns(oldFlag_Cmd, newFlag_Cmd, true);
2795 0 : os << LogIO::DEBUG1 << "FLAG_CMD has " << nAddedCols << " optional columns." << LogIO::POST;
2796 :
2797 0 : const MSFlagCmdColumns oldFCs(oldFlag_Cmd);
2798 0 : MSFlagCmdColumns newFCs(newFlag_Cmd);
2799 0 : newFCs.setEpochRef(MEpoch::castType(oldFCs.timeMeas().getMeasRef().getType()));
2800 :
2801 0 : TableCopy::copyRows(newFlag_Cmd, oldFlag_Cmd);
2802 :
2803 : }
2804 : }
2805 :
2806 0 : return true;
2807 : }
2808 :
2809 : // -----------------------------------------------------------------------
2810 : //
2811 : // -----------------------------------------------------------------------
2812 0 : bool MSTransformDataHandler::copyHistory()
2813 : {
2814 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2815 :
2816 0 : const MSHistory& oldHistory = mssel_p.history();
2817 :
2818 : // Could be declared as Table&
2819 0 : MSHistory& newHistory = msOut_p.history();
2820 :
2821 : // Add optional columns if present in oldHistory.
2822 0 : uInt nAddedCols = addOptionalColumns(oldHistory, newHistory, true);
2823 0 : os << LogIO::DEBUG1 << "HISTORY has " << nAddedCols << " optional columns." << LogIO::POST;
2824 :
2825 0 : const MSHistoryColumns oldHCs(oldHistory);
2826 0 : MSHistoryColumns newHCs(newHistory);
2827 0 : newHCs.setEpochRef(MEpoch::castType(oldHCs.timeMeas().getMeasRef().getType()));
2828 :
2829 0 : TableCopy::copyRows(newHistory, oldHistory);
2830 :
2831 0 : return true;
2832 : }
2833 :
2834 : // -----------------------------------------------------------------------
2835 : //
2836 : // -----------------------------------------------------------------------
2837 0 : bool MSTransformDataHandler::copyObservation()
2838 : {
2839 0 : const MSObservation& oldObs = mssel_p.observation();
2840 0 : MSObservation& newObs = msOut_p.observation();
2841 0 : const MSObservationColumns oldObsCols(oldObs);
2842 0 : MSObservationColumns newObsCols(newObs);
2843 0 : newObsCols.setEpochRef(MEpoch::castType(oldObsCols.releaseDateMeas().getMeasRef().getType()));
2844 :
2845 0 : uInt nObs = selObsId_p.nelements();
2846 0 : if (nObs > 0 and reindex_p)
2847 : {
2848 0 : for (uInt outrn = 0; outrn < nObs; ++outrn)
2849 : {
2850 0 : TableCopy::copyRows(newObs, oldObs, outrn, selObsId_p[outrn], 1);
2851 0 : }
2852 :
2853 : }
2854 : else
2855 : {
2856 0 : TableCopy::copyRows(newObs, oldObs);
2857 : }
2858 :
2859 0 : return true;
2860 : }
2861 :
2862 : // -----------------------------------------------------------------------
2863 : //
2864 : // -----------------------------------------------------------------------
2865 0 : bool MSTransformDataHandler::copyProcessor()
2866 : {
2867 0 : const MSProcessor& oldProc = mssel_p.processor();
2868 0 : MSProcessor& newProc = msOut_p.processor();
2869 0 : TableCopy::copyRows(newProc, oldProc);
2870 :
2871 0 : return true;
2872 : }
2873 :
2874 : // -----------------------------------------------------------------------
2875 : //
2876 : // -----------------------------------------------------------------------
2877 0 : bool MSTransformDataHandler::copyState()
2878 : {
2879 : // STATE is allowed to not exist, even though it is not optional in
2880 : // the MS def'n. For one thing, split dropped it for quite a while.
2881 0 : if (Table::isReadable(mssel_p.stateTableName()))
2882 : {
2883 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2884 0 : const MSState& oldState = mssel_p.state();
2885 :
2886 0 : if (oldState.nrow() > 0)
2887 : {
2888 0 : if (!intentString_p.empty() and reindex_p)
2889 : {
2890 0 : MSState& newState = msOut_p.state();
2891 0 : const MSStateColumns oldStateCols(oldState);
2892 0 : MSStateColumns newStateCols(newState);
2893 :
2894 : // Initialize stateRemapper_p if necessary.
2895 : // if (stateRemapper_p.size() < 1) make_map(stateRemapper_p, mscIn_p->stateId().getColumn());
2896 :
2897 : // jagonzal (CAS-6351): Do not apply implicit re-indexing //////////////////////////////////////////////////
2898 : //
2899 : // Get list of selected scan intent indexes
2900 0 : MSSelection mssel;
2901 0 : mssel.setStateExpr(intentString_p);
2902 0 : Vector<Int> scanIntentList = mssel.getStateObsModeList(getInputMS());
2903 : //
2904 : // Populate state re-mapper using all selected indexes (not only the implicit ones)
2905 0 : stateRemapper_p.clear();
2906 0 : for (uInt index=0; index < scanIntentList.size(); index++)
2907 : {
2908 0 : stateRemapper_p[scanIntentList(index)] = index;
2909 : }
2910 : ///////////////////////////////////////////////////////////////////////////////////////////////////////////
2911 :
2912 0 : uInt nStates = stateRemapper_p.size();
2913 :
2914 : // stateRemapper_p goes from input to output, as is wanted in most
2915 : // places. Here we need a map going the other way, so make one.
2916 0 : Vector<Int> outStateToInState(nStates);
2917 0 : std::map<Int, Int>::iterator mit;
2918 :
2919 0 : for (mit = stateRemapper_p.begin(); mit != stateRemapper_p.end(); ++mit)
2920 : {
2921 0 : outStateToInState[(*mit).second] = (*mit).first;
2922 : }
2923 :
2924 :
2925 0 : for (uInt outrn = 0; outrn < nStates; ++outrn)
2926 : {
2927 0 : TableCopy::copyRows(newState, oldState, outrn,outStateToInState[outrn], 1);
2928 : }
2929 : }
2930 : // jagonzal (CAS-6351): Do not apply implicit re-indexing
2931 : // Therefore just copy the input state sub-table to the output state sub-table
2932 : else
2933 : {
2934 0 : MSState& newState = msOut_p.state();
2935 0 : TableCopy::copyRows(newState, oldState);
2936 : }
2937 :
2938 : }
2939 : }
2940 0 : return true;
2941 : }
2942 :
2943 : // -----------------------------------------------------------------------
2944 : //
2945 : // -----------------------------------------------------------------------
2946 0 : bool MSTransformDataHandler::copySyscal()
2947 : {
2948 : // SYSCAL is allowed to not exist.
2949 0 : if (Table::isReadable(mssel_p.sysCalTableName()))
2950 : {
2951 0 : const MSSysCal& oldSysc = mssel_p.sysCal();
2952 :
2953 0 : if (oldSysc.nrow() > 0)
2954 : {
2955 : // Add a SYSCAL subtable to msOut_p with 0 rows for now.
2956 0 : Table::TableOption option = Table::New;
2957 0 : TableDesc sysCalTD = MSSysCal::requiredTableDesc();
2958 0 : SetupNewTable sysCalSetup(msOut_p.sysCalTableName(), sysCalTD,option);
2959 0 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::SYSCAL),Table(sysCalSetup, 0));
2960 :
2961 : // update the references to the subtable keywords
2962 0 : msOut_p.initRefs();
2963 :
2964 : // Could be declared as Table&.
2965 0 : MSSysCal& newSysc = msOut_p.sysCal();
2966 :
2967 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
2968 :
2969 0 : uInt nAddedCols = addOptionalColumns(oldSysc, newSysc, true);
2970 0 : os << LogIO::DEBUG1 << "SYSCAL has " << nAddedCols << " optional columns." << LogIO::POST;
2971 :
2972 0 : const MSSysCalColumns incols(oldSysc);
2973 0 : MSSysCalColumns outcols(newSysc);
2974 0 : outcols.setEpochRef(MEpoch::castType(incols.timeMeas().getMeasRef().getType()));
2975 :
2976 0 : if ((!antennaSel_p && allEQ(spwRelabel_p, spw_p)) or !reindex_p)
2977 : {
2978 0 : TableCopy::copyRows(newSysc, oldSysc);
2979 : }
2980 : else
2981 : {
2982 0 : const Vector<Int>& antIds = incols.antennaId().getColumn();
2983 0 : const Vector<Int>& spwIds = incols.spectralWindowId().getColumn();
2984 :
2985 : // Copy selected rows.
2986 0 : uInt totNSyscals = antIds.nelements();
2987 0 : uInt totalSelSyscals = 0;
2988 0 : Int maxSelAntp1 = antNewIndex_p.nelements(); // Int for comparison with antIds.
2989 0 : for (uInt k = 0; k < totNSyscals; ++k)
2990 : {
2991 : // antenna must be selected, and spwId must be -1 (any) or selected.
2992 0 : if ( antIds[k] < maxSelAntp1 &&
2993 0 : antNewIndex_p[antIds[k]] > -1 &&
2994 0 : (spwIds[k] < 0 || spwRelabel_p[spwIds[k]] > -1)
2995 : )
2996 : {
2997 0 : TableCopy::copyRows(newSysc, oldSysc, totalSelSyscals,k, 1, false);
2998 0 : ++totalSelSyscals;
2999 : }
3000 : }
3001 :
3002 : // Remap antenna and spw #s.
3003 0 : ScalarColumn<Int>& antCol = outcols.antennaId();
3004 0 : ScalarColumn<Int>& spwCol = outcols.spectralWindowId();
3005 :
3006 0 : for (uInt k = 0; k < totalSelSyscals; ++k)
3007 : {
3008 0 : antCol.put(k, antNewIndex_p[antCol(k)]);
3009 0 : if (spwCol(k) > -1) spwCol.put(k, spwRelabel_p[spwCol(k)]);
3010 : }
3011 : }
3012 : }
3013 : }
3014 :
3015 0 : return true;
3016 : }
3017 :
3018 0 : bool MSTransformDataHandler::copyWeather()
3019 : {
3020 : // Weather is allowed to not exist.
3021 0 : if (Table::isReadable(mssel_p.weatherTableName()))
3022 : {
3023 0 : const MSWeather& oldWeath = mssel_p.weather();
3024 :
3025 0 : if (oldWeath.nrow() > 0)
3026 : {
3027 : // Add a WEATHER subtable to msOut_p with 0 rows for now.
3028 0 : Table::TableOption option = Table::New;
3029 0 : TableDesc weatherTD = MSWeather::requiredTableDesc();
3030 0 : SetupNewTable weatherSetup(msOut_p.weatherTableName(), weatherTD,option);
3031 0 : msOut_p.rwKeywordSet().defineTable(MS::keywordName(MS::WEATHER),Table(weatherSetup, 0));
3032 :
3033 : // update the references to the subtable keywords
3034 0 : msOut_p.initRefs();
3035 :
3036 0 : MSWeather& newWeath = msOut_p.weather(); // Could be declared as Table&
3037 :
3038 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3039 :
3040 0 : uInt nAddedCols = addOptionalColumns(oldWeath, newWeath, true);
3041 0 : os << LogIO::DEBUG1 << "WEATHER has " << nAddedCols << " optional columns." << LogIO::POST;
3042 :
3043 0 : const MSWeatherColumns oldWCs(oldWeath);
3044 0 : MSWeatherColumns newWCs(newWeath);
3045 0 : newWCs.setEpochRef(MEpoch::castType(oldWCs.timeMeas().getMeasRef().getType()));
3046 :
3047 0 : if (!antennaSel_p or !reindex_p)
3048 : {
3049 0 : TableCopy::copyRows(newWeath, oldWeath);
3050 : }
3051 : else
3052 : {
3053 0 : const Vector<Int>& antIds(oldWCs.antennaId().getColumn());
3054 0 : ScalarColumn<Int>& outants = newWCs.antennaId();
3055 :
3056 0 : uInt selRow = 0;
3057 0 : Int maxSelAntp1 = antNewIndex_p.nelements();
3058 :
3059 0 : for (uInt k = 0; k < antIds.nelements(); ++k)
3060 : {
3061 : // Weather station is on antenna?
3062 0 : if (antIds[k] > -1)
3063 : {
3064 : // remap ant num
3065 0 : if (antIds[k] < maxSelAntp1)
3066 : {
3067 0 : Int newAntInd = antNewIndex_p[antIds[k]];
3068 :
3069 : // Ant selected?
3070 0 : if (newAntInd > -1)
3071 : {
3072 0 : TableCopy::copyRows(newWeath, oldWeath, selRow,k, 1);
3073 0 : outants.put(selRow, newAntInd);
3074 0 : ++selRow;
3075 : }
3076 : }
3077 : }
3078 : else
3079 : {
3080 : // means valid for all antennas.
3081 0 : TableCopy::copyRows(newWeath, oldWeath, selRow, k, 1);
3082 0 : outants.put(selRow, -1);
3083 0 : ++selRow;
3084 : }
3085 : }
3086 : }
3087 : }
3088 : }
3089 :
3090 0 : return true;
3091 : }
3092 :
3093 : // -----------------------------------------------------------------------
3094 : //
3095 : // -----------------------------------------------------------------------
3096 0 : bool MSTransformDataHandler::filterOptSubtable(const String& subtabname)
3097 : {
3098 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3099 :
3100 : // subtabname is allowed to not exist. Use ms_p instead of mssel_p, because
3101 : // the latter has a random name which does NOT necessarily lead to
3102 : // subtabname. (And if selection took care of subtables, we probably
3103 : // wouldn't have to do it here.)
3104 0 : if (Table::isReadable(ms_p.tableName() + '/' + subtabname))
3105 : {
3106 0 : const Table intab(ms_p.tableName() + '/' + subtabname);
3107 :
3108 0 : if (intab.nrow() > 0) {
3109 :
3110 : // Add feed if selecting by it is ever added.
3111 0 : bool doFilter = (antennaSel_p || !allEQ(spwRelabel_p, spw_p)) && reindex_p;
3112 :
3113 0 : copySubtable(subtabname, intab, doFilter);
3114 :
3115 0 : if (doFilter) {
3116 :
3117 : // At this point msOut_p has subtab with 0 rows.
3118 0 : Table outtab(msOut_p.tableName() + '/' + subtabname,Table::Update);
3119 0 : ScalarColumn<Int> inAntIdCol(intab, "ANTENNA_ID");
3120 0 : ScalarColumn<Int> inSpwIdCol(intab, "SPECTRAL_WINDOW_ID");
3121 0 : const Vector<Int>& antIds = inAntIdCol.getColumn();
3122 0 : const Vector<Int>& spwIds = inSpwIdCol.getColumn();
3123 :
3124 : // Copy selected rows.
3125 0 : uInt totNOuttabs = antIds.nelements();
3126 0 : uInt totalSelOuttabs = 0;
3127 :
3128 : // Int for comparison with antIds.
3129 0 : Int maxSelAntp1 = antNewIndex_p.nelements();
3130 :
3131 0 : bool haveRemappingProblem = false;
3132 0 : for (uInt inrow = 0; inrow < totNOuttabs; ++inrow)
3133 : {
3134 : // antenna must be selected, and spwId must be -1 (any) or selected.
3135 : // Extra care must be taken because for a while MSes had CALDEVICE
3136 : // and SYSPOWER, but the ANTENNA_ID and SPECTRAL_WINDOW_ID of those
3137 : // sub-tables were not being re-mapped by split.
3138 0 : if (antIds[inrow] < maxSelAntp1 && antNewIndex_p[antIds[inrow]] > -1)
3139 : {
3140 :
3141 0 : if (spwIds[inrow] < 0 ||
3142 0 : (spwIds[inrow] < static_cast<Int> (spwRelabel_p.nelements()) &&
3143 0 : spwRelabel_p[spwIds[inrow]] > -1))
3144 : {
3145 0 : TableCopy::copyRows(outtab, intab, totalSelOuttabs,inrow, 1, false);
3146 0 : ++totalSelOuttabs;
3147 : }
3148 :
3149 : // Ideally we'd like to catch antenna errors too. They are
3150 : // avoided, but because of the way antNewIndex_p is made,
3151 : // antIds[inrow] >= maxSelAntp1
3152 : // is not necessarily an error. It's not even possible to catch
3153 : // all the spw errors, so we settle for catching the ones we can
3154 : // and reliably avoiding segfaults.
3155 0 : else if (spwIds[inrow] >= static_cast<Int> (spwRelabel_p.nelements()))
3156 : {
3157 0 : haveRemappingProblem = true;
3158 : }
3159 :
3160 : }
3161 : }
3162 :
3163 0 : if (haveRemappingProblem)
3164 : {
3165 : os << LogIO::WARN << "At least one row of "
3166 : << intab.tableName()
3167 : << " has an antenna or spw mismatch.\n"
3168 : << "(Presumably from an older split, sorry.)\n"
3169 : << "If " << subtabname
3170 : << " is important, it should be fixed with tb or browsetable,\n"
3171 : << "or by redoing the split that made "
3172 : << ms_p.tableName() << " (check its history)."
3173 0 : << LogIO::POST;
3174 : }
3175 :
3176 :
3177 : // Remap antenna and spw #s.
3178 0 : ScalarColumn<Int> outAntCol(outtab, "ANTENNA_ID");
3179 0 : ScalarColumn<Int> outSpwCol(outtab, "SPECTRAL_WINDOW_ID");
3180 :
3181 0 : for (uInt k = 0; k < totalSelOuttabs; ++k)
3182 : {
3183 0 : outAntCol.put(k, antNewIndex_p[outAntCol(k)]);
3184 0 : if (outSpwCol(k) > -1) outSpwCol.put(k, spwRelabel_p[outSpwCol(k)]);
3185 : }
3186 : }
3187 : }
3188 : }
3189 :
3190 0 : return true;
3191 : }
3192 :
3193 : // -----------------------------------------------------------------------
3194 : //
3195 : // -----------------------------------------------------------------------
3196 0 : bool MSTransformDataHandler::copyGenericSubtables()
3197 : {
3198 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3199 :
3200 : // Already handled subtables will be removed from this,
3201 : // so a modifiable copy is needed.
3202 0 : TableRecord inkws(mssel_p.keywordSet());
3203 :
3204 : // Some of the standard subtables need special handling,
3205 : // e.g. DATA_DESCRIPTION, SPECTRAL_WINDOW, and ANTENNA, so they are already
3206 : // defined in msOut_p. Several more (e.g. FLAG_CMD) were also already
3207 : // created by MS::createDefaultSubtables(). Don't try to write over them - a
3208 : // locking error will result.
3209 0 : const TableRecord& outkws = msOut_p.keywordSet();
3210 0 : for (uInt i = 0; i < outkws.nfields(); ++i)
3211 : {
3212 0 : if (outkws.type(i) == TpTable && inkws.isDefined(outkws.name(i)))
3213 : {
3214 0 : inkws.removeField(outkws.name(i));
3215 : }
3216 : }
3217 :
3218 : // Find ephemerides files
3219 0 : std::vector<String> ephemerides;
3220 0 : for (uInt i = 0; i < inkws.nfields(); ++i)
3221 : {
3222 0 : if (inkws.type(i) == TpTable && inkws.name(i).contains("EPHEM"))
3223 : {
3224 0 : ephemerides.push_back(inkws.name(i));
3225 : }
3226 : }
3227 :
3228 : // Remove ephemerides files
3229 0 : for (uInt i = 0; i < ephemerides.size(); ++i)
3230 : {
3231 0 : inkws.removeField(ephemerides.at(i));
3232 : }
3233 :
3234 :
3235 : // msOut_p.rwKeywordSet() (pass a reference, not a copy) will put a lock on msOut_p.
3236 0 : TableCopy::copySubTables(msOut_p.rwKeywordSet(), inkws, msOut_p.tableName(), msOut_p.tableType(), mssel_p);
3237 :
3238 : // TableCopy::copySubTables(Table, Table, bool) includes this other code,
3239 : // which seems to be copying subtables at one level deeper, but not recursively?
3240 0 : const TableDesc& inDesc = mssel_p.tableDesc();
3241 0 : const TableDesc& outDesc = msOut_p.tableDesc();
3242 0 : for (uInt i = 0; i < outDesc.ncolumn(); ++i)
3243 : {
3244 : // Only writable cols can have keywords (and thus subtables) defined.
3245 0 : if (msOut_p.isColumnWritable(i))
3246 : {
3247 0 : const String& name = outDesc[i].name();
3248 :
3249 0 : if (inDesc.isColumn(name))
3250 : {
3251 0 : TableColumn outCol(msOut_p, name);
3252 0 : TableColumn inCol(mssel_p, name);
3253 :
3254 0 : TableCopy::copySubTables(outCol.rwKeywordSet(),
3255 : inCol.keywordSet(),
3256 : msOut_p.tableName(),
3257 : msOut_p.tableType(),
3258 : mssel_p);
3259 : // Copy the keywords if column is [FLOAT_|CORRECTED_]DATA
3260 : // KS NOTE CORRECTED_DATA -> DATA case should be handled separately.
3261 0 : if (name == "FLOAT_DATA" || name == "DATA" || name == "CORRECTED_DATA")
3262 0 : copyMainTableKeywords(outCol.rwKeywordSet(), inCol.keywordSet());
3263 :
3264 : }
3265 : }
3266 : }
3267 :
3268 0 : return true;
3269 : }
3270 :
3271 : // -----------------------------------------------------------------------
3272 : // Method to merge SPW sub-tables from SubMSs to create the MMS level SPW sub-table
3273 : // -----------------------------------------------------------------------
3274 0 : bool MSTransformDataHandler::mergeSpwSubTables(Vector<String> filenames)
3275 : {
3276 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3277 :
3278 0 : String filename_0 = filenames(0);
3279 0 : MeasurementSet ms_0(filename_0,Table::Update);
3280 :
3281 0 : if (Table::isReadable(ms_0.spectralWindowTableName()) and !ms_0.spectralWindow().isNull())
3282 : {
3283 0 : MSSpectralWindow spwTable_0 = ms_0.spectralWindow();
3284 :
3285 0 : if (spwTable_0.nrow() > 0)
3286 : {
3287 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3288 0 : << "Merging SPECTRAL_WINDOW sub-tables from all sub-MSs to form MMS-level SPECTRAL_WINDOW sub-table" << LogIO::POST;
3289 :
3290 0 : MSSpWindowColumns spwCols_0(spwTable_0);
3291 :
3292 : // Map subMS with spw_id to merge the FEED table later
3293 0 : Vector<uInt> mapSubmsSpwid;
3294 :
3295 : // subMS_0000 starts with spw 0
3296 0 : uInt spwStart = 0;
3297 0 : mapSubmsSpwid.resize(filenames.size());
3298 0 : mapSubmsSpwid[0] = spwStart;
3299 :
3300 : // for next subMS
3301 0 : uInt rowIndex = spwTable_0.nrow();
3302 0 : spwStart = spwStart + rowIndex;
3303 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3304 : {
3305 0 : String filename_i = filenames(subms_index);
3306 0 : MeasurementSet ms_i(filename_i);
3307 0 : MSSpectralWindow spwTable_i = ms_i.spectralWindow();
3308 :
3309 0 : if (spwTable_i.nrow() > 0)
3310 : {
3311 0 : MSSpWindowColumns spwCols_i(spwTable_i);
3312 :
3313 0 : uInt nrow = spwTable_i.nrow();
3314 0 : spwTable_0.addRow(nrow);
3315 :
3316 : // Map of this subMS to spw ID
3317 0 : mapSubmsSpwid[subms_index] = spwStart;
3318 :
3319 : // for next subMS
3320 0 : spwStart = spwStart + nrow;
3321 :
3322 0 : for (uInt subms_row_index=0;subms_row_index<spwTable_i.nrow();subms_row_index++)
3323 : {
3324 0 : spwCols_0.measFreqRef().put(rowIndex,spwCols_i.measFreqRef()(subms_row_index));
3325 0 : spwCols_0.chanFreq().put(rowIndex,spwCols_i.chanFreq()(subms_row_index));
3326 0 : spwCols_0.refFrequency().put(rowIndex,spwCols_i.refFrequency()(subms_row_index));
3327 0 : spwCols_0.chanWidth().put(rowIndex,spwCols_i.chanWidth()(subms_row_index));
3328 0 : spwCols_0.effectiveBW().put(rowIndex,spwCols_i.effectiveBW()(subms_row_index));
3329 0 : spwCols_0.resolution().put(rowIndex,spwCols_i.resolution()(subms_row_index));
3330 0 : spwCols_0.flagRow().put(rowIndex,spwCols_i.flagRow()(subms_row_index));
3331 0 : spwCols_0.freqGroup().put(rowIndex,spwCols_i.freqGroup()(subms_row_index));
3332 0 : spwCols_0.freqGroupName().put(rowIndex,spwCols_i.freqGroupName()(subms_row_index));
3333 0 : spwCols_0.ifConvChain().put(rowIndex,spwCols_i.ifConvChain()(subms_row_index));
3334 0 : spwCols_0.name().put(rowIndex,spwCols_i.name()(subms_row_index));
3335 0 : spwCols_0.netSideband().put(rowIndex,spwCols_i.netSideband()(subms_row_index));
3336 0 : spwCols_0.numChan().put(rowIndex,spwCols_i.numChan()(subms_row_index));
3337 0 : spwCols_0.totalBandwidth().put(rowIndex,spwCols_i.totalBandwidth()(subms_row_index));
3338 :
3339 : // Optional columns
3340 0 : if (columnOk(spwCols_i.bbcNo()))
3341 0 : spwCols_0.bbcNo().put(rowIndex,spwCols_i.bbcNo()(subms_row_index));
3342 :
3343 0 : if (columnOk(spwCols_i.assocSpwId()))
3344 0 : spwCols_0.assocSpwId().put(rowIndex,spwCols_i.assocSpwId()(subms_row_index));
3345 :
3346 0 : if(columnOk(spwCols_i.assocNature()))
3347 0 : spwCols_0.assocNature().put(rowIndex,spwCols_i.assocNature()(subms_row_index));
3348 :
3349 0 : if (columnOk(spwCols_i.bbcSideband()))
3350 0 : spwCols_0.bbcSideband().put(rowIndex,spwCols_i.bbcSideband()(subms_row_index));
3351 :
3352 0 : if (columnOk(spwCols_i.dopplerId()))
3353 0 : spwCols_0.dopplerId().put(rowIndex,spwCols_i.dopplerId()(subms_row_index));
3354 :
3355 0 : if (columnOk(spwCols_i.receiverId()))
3356 0 : spwCols_0.receiverId().put(rowIndex,spwCols_i.receiverId()(subms_row_index));
3357 :
3358 0 : if (spwTable_i.tableDesc().isColumn("SDM_WINDOW_FUNCTION") &&
3359 0 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_WINDOW_FUNCTION"))
3360 : {
3361 0 : ScalarColumn<String> swfCol_i(spwTable_i, "SDM_WINDOW_FUNCTION");
3362 0 : ScalarColumn<String> swfCol_0(spwTable_0, "SDM_WINDOW_FUNCTION");
3363 0 : swfCol_0.put(rowIndex, swfCol_i(subms_row_index));
3364 : }
3365 :
3366 0 : if (spwTable_i.tableDesc().isColumn("SDM_NUM_BIN") &&
3367 0 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_NUM_BIN"))
3368 : {
3369 0 : ScalarColumn<Int> snbCol_i(spwTable_i, "SDM_NUM_BIN");
3370 0 : ScalarColumn<Int> snbCol_0(spwTable_0, "SDM_NUM_BIN");
3371 0 : snbCol_0.put(rowIndex, snbCol_i(subms_row_index));
3372 : }
3373 0 : if (spwTable_i.tableDesc().isColumn("SDM_CORR_BIT") &&
3374 0 : spwTable_i.tableDesc().columnDescSet().isDefined("SDM_CORR_BIT"))
3375 : {
3376 0 : ScalarColumn<String> corrBitCol_i(spwTable_i, "SDM_CORR_BIT");
3377 0 : ScalarColumn<String> corrBitCol_0(spwTable_0, "SDM_CORR_BIT");
3378 0 : corrBitCol_0.put(rowIndex, corrBitCol_i(subms_row_index));
3379 : }
3380 :
3381 0 : rowIndex += 1;
3382 : }
3383 : }
3384 : }
3385 :
3386 : // Merge the other sub-tables using SPW map generated here
3387 0 : mergeDDISubTables(filenames);
3388 0 : mergeFeedSubTables(filenames, mapSubmsSpwid);
3389 0 : mergeSourceSubTables(filenames, mapSubmsSpwid);
3390 0 : mergeFreqOffsetTables(filenames, mapSubmsSpwid);
3391 0 : mergeCalDeviceSubtables(filenames, mapSubmsSpwid);
3392 0 : mergeSysPowerSubtables(filenames, mapSubmsSpwid);
3393 0 : mergeSyscalSubTables(filenames, mapSubmsSpwid);
3394 : }
3395 : else
3396 : {
3397 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3398 0 : << "SPECTRAL_WINDOW sub-table found but has no valid content" << LogIO::POST;
3399 0 : return false;
3400 : }
3401 : }
3402 : else
3403 : {
3404 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3405 0 : << "SPECTRAL_WINDOW sub-table not found " << LogIO::POST;
3406 0 : return false;
3407 : }
3408 :
3409 0 : return true;
3410 : }
3411 :
3412 : // -----------------------------------------------------------------------
3413 : // Method to merge DDI sub-tables from SubMSs to create the MMS-level DDI sub-table
3414 : // -----------------------------------------------------------------------
3415 0 : bool MSTransformDataHandler::mergeDDISubTables(Vector<String> filenames)
3416 : {
3417 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3418 :
3419 0 : String filename_0 = filenames(0);
3420 0 : MeasurementSet ms_0(filename_0,Table::Update);
3421 :
3422 0 : if (Table::isReadable(ms_0.dataDescriptionTableName()) and !ms_0.dataDescription().isNull())
3423 : {
3424 0 : MSDataDescription ddiTable_0 = ms_0.dataDescription();
3425 :
3426 0 : if (ddiTable_0.nrow() > 0)
3427 : {
3428 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3429 0 : << "Merging DDI sub-tables from all sub-MSs to form MMS-level DDI sub-table" << LogIO::POST;
3430 :
3431 0 : MSDataDescColumns ddiCols_0(ddiTable_0);
3432 :
3433 0 : uInt rowIndex = ddiTable_0.nrow();
3434 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3435 : {
3436 0 : String filename_i = filenames(subms_index);
3437 0 : MeasurementSet ms_i(filename_i);
3438 0 : MSDataDescription dditable_i = ms_i.dataDescription();
3439 :
3440 0 : if (dditable_i.nrow() > 0)
3441 : {
3442 0 : MSDataDescColumns ddicols_i(dditable_i);
3443 :
3444 0 : ddiTable_0.addRow(dditable_i.nrow());
3445 :
3446 0 : for (uInt subms_row_index=0;subms_row_index<dditable_i.nrow();subms_row_index++)
3447 : {
3448 : // get the last spw id entered in the 0th DD table
3449 0 : uInt spwid = ddiCols_0.spectralWindowId().get(rowIndex-1);
3450 :
3451 0 : ddiCols_0.flagRow().put(rowIndex,ddicols_i.flagRow()(subms_row_index));
3452 0 : ddiCols_0.polarizationId().put(rowIndex,ddicols_i.polarizationId()(subms_row_index));
3453 :
3454 : // Take into account that some SPW may be pointed by several DDIs
3455 0 : uInt deltaDDI = 1;
3456 0 : if (subms_row_index>0)
3457 : {
3458 0 : deltaDDI = ddicols_i.spectralWindowId()(subms_row_index) - ddicols_i.spectralWindowId()(subms_row_index-1);
3459 : }
3460 :
3461 0 : ddiCols_0.spectralWindowId().put(rowIndex,spwid+deltaDDI);
3462 0 : rowIndex += 1;
3463 : }
3464 : }
3465 : }
3466 : }
3467 : else
3468 : {
3469 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3470 0 : << "DDI sub-table found but has no valid content" << LogIO::POST;
3471 0 : return false;
3472 : }
3473 : }
3474 : else
3475 : {
3476 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3477 0 : << "DDI sub-table not found " << LogIO::POST;
3478 0 : return false;
3479 : }
3480 :
3481 0 : return true;
3482 : }
3483 :
3484 :
3485 : // -----------------------------------------------------------------------
3486 : // Method to merge FEED sub-tables from SubMSs to create the MMS-level FEED sub-table
3487 : // -----------------------------------------------------------------------
3488 0 : bool MSTransformDataHandler::mergeFeedSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3489 : {
3490 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3491 :
3492 0 : if (filenames.size() != mapSubmsSpwid.size())
3493 : {
3494 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3495 0 : return false;
3496 : }
3497 :
3498 0 : String filename_0 = filenames(0);
3499 0 : MeasurementSet ms_0(filename_0,Table::Update);
3500 :
3501 0 : if (Table::isReadable(ms_0.feedTableName()) and !ms_0.feed().isNull())
3502 : {
3503 0 : MSFeed feedTable_0 = ms_0.feed();
3504 :
3505 : // CAS-7167. The WVR spw has no FEED content.
3506 : // if (feedTable_0.nrow() >= 0)
3507 : // {
3508 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3509 0 : << "Merging FEED sub-tables from all sub-MSs to form MMS-level FEED sub-table" << LogIO::POST;
3510 :
3511 0 : MSFeedColumns feedCols_0(feedTable_0);
3512 :
3513 0 : uInt rowIndex = feedTable_0.nrow();
3514 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3515 : {
3516 0 : String filename_i = filenames(subms_index);
3517 0 : MeasurementSet ms_i(filename_i);
3518 0 : MSFeed feedtable_i = ms_i.feed();
3519 :
3520 0 : if (feedtable_i.nrow() > 0)
3521 : {
3522 0 : MSFeedColumns feedcols_i(feedtable_i);
3523 :
3524 0 : feedTable_0.addRow(feedtable_i.nrow());
3525 :
3526 : // Prepare row reference object
3527 0 : RefRows refRow(rowIndex,rowIndex+feedtable_i.nrow()-1);
3528 :
3529 : // Re-index SPW col
3530 0 : Vector<Int> spectralWindowId_output(feedtable_i.nrow(),mapSubmsSpwid[subms_index]);
3531 0 : spectralWindowId_output += feedcols_i.spectralWindowId().getColumn();
3532 0 : feedCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3533 :
3534 : // Columns that can be just copied
3535 0 : feedCols_0.position().putColumnCells(refRow,feedcols_i.position().getColumn());
3536 0 : feedCols_0.beamOffset().putColumnCells(refRow,feedcols_i.beamOffset().getColumn());
3537 0 : feedCols_0.polarizationType().putColumnCells(refRow,feedcols_i.polarizationType().getColumn());
3538 0 : feedCols_0.polResponse().putColumnCells(refRow,feedcols_i.polResponse().getColumn());
3539 0 : feedCols_0.receptorAngle().putColumnCells(refRow,feedcols_i.receptorAngle().getColumn());
3540 0 : feedCols_0.antennaId().putColumnCells(refRow,feedcols_i.antennaId().getColumn());
3541 0 : feedCols_0.beamId().putColumnCells(refRow,feedcols_i.beamId().getColumn());
3542 0 : feedCols_0.feedId().putColumnCells(refRow,feedcols_i.feedId().getColumn());
3543 0 : feedCols_0.interval().putColumnCells(refRow,feedcols_i.interval().getColumn());
3544 0 : feedCols_0.numReceptors().putColumnCells(refRow,feedcols_i.numReceptors().getColumn());
3545 0 : feedCols_0.time().putColumnCells(refRow,feedcols_i.time().getColumn());
3546 :
3547 : // optional columns
3548 0 : if (columnOk(feedcols_i.focusLength()))
3549 : {
3550 0 : feedCols_0.focusLength().putColumnCells(refRow,feedcols_i.focusLength().getColumn());
3551 : }
3552 :
3553 0 : if (columnOk(feedcols_i.phasedFeedId()))
3554 : {
3555 0 : feedCols_0.phasedFeedId().putColumnCells(refRow,feedcols_i.phasedFeedId().getColumn());
3556 : }
3557 :
3558 : // Increment row offset
3559 0 : rowIndex += feedtable_i.nrow();
3560 : }
3561 : }
3562 :
3563 : // }
3564 : /*
3565 : else
3566 : {
3567 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3568 : << "FEED sub-table found but has no valid content" << LogIO::POST;
3569 : return false;
3570 : }
3571 : */
3572 : }
3573 : else
3574 : {
3575 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__) <<
3576 0 : "FEED sub-table not found " << LogIO::POST;
3577 0 : return false;
3578 : }
3579 :
3580 0 : return true;
3581 : }
3582 :
3583 : // -----------------------------------------------------------------------
3584 : // Method to merge Source sub-tables from SubMSs to create the MMS-level FEED sub-table
3585 : // -----------------------------------------------------------------------
3586 0 : bool MSTransformDataHandler::mergeSourceSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3587 : {
3588 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3589 :
3590 0 : if (filenames.size() != mapSubmsSpwid.size())
3591 : {
3592 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3593 0 : return false;
3594 : }
3595 :
3596 0 : String filename_0 = filenames(0);
3597 0 : MeasurementSet ms_0(filename_0,Table::Update);
3598 :
3599 0 : if (Table::isReadable(ms_0.sourceTableName()) and !ms_0.source().isNull())
3600 : {
3601 0 : MSSource sourceTable_0 = ms_0.source();
3602 :
3603 0 : if (sourceTable_0.nrow() > 0)
3604 : {
3605 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3606 0 : << "Merging SOURCE sub-tables from all sub-MSs to form MMS-level SOURCE sub-table" << LogIO::POST;
3607 :
3608 0 : MSSourceColumns sourceCols_0(sourceTable_0);
3609 :
3610 0 : uInt rowIndex = sourceTable_0.nrow();
3611 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3612 : {
3613 0 : String filename_i = filenames(subms_index);
3614 0 : MeasurementSet ms_i(filename_i);
3615 0 : MSSource sourcetable_i = ms_i.source();
3616 :
3617 0 : if (sourcetable_i.nrow() > 0)
3618 : {
3619 0 : MSSourceColumns sourcecols_i(sourcetable_i);
3620 :
3621 0 : sourceTable_0.addRow(sourcetable_i.nrow());
3622 :
3623 : // Prepare row reference object
3624 0 : RefRows refRow(rowIndex,rowIndex+sourcetable_i.nrow()-1);
3625 :
3626 : // Re-index SPW col
3627 0 : Vector<Int> spectralWindowId_output(sourcetable_i.nrow(),mapSubmsSpwid[subms_index]);
3628 0 : spectralWindowId_output += sourcecols_i.spectralWindowId().getColumn();
3629 0 : sourceCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3630 :
3631 : // Columns that can be just copied
3632 0 : sourceCols_0.direction().putColumnCells(refRow,sourcecols_i.direction().getColumn());
3633 0 : sourceCols_0.properMotion().putColumnCells(refRow,sourcecols_i.properMotion().getColumn());
3634 0 : sourceCols_0.calibrationGroup().putColumnCells(refRow,sourcecols_i.calibrationGroup().getColumn());
3635 0 : sourceCols_0.code().putColumnCells(refRow,sourcecols_i.code().getColumn());
3636 0 : sourceCols_0.interval().putColumnCells(refRow,sourcecols_i.interval().getColumn());
3637 0 : sourceCols_0.name().putColumnCells(refRow,sourcecols_i.name().getColumn());
3638 0 : sourceCols_0.numLines().putColumnCells(refRow,sourcecols_i.numLines().getColumn());
3639 0 : sourceCols_0.sourceId().putColumnCells(refRow,sourcecols_i.sourceId().getColumn());
3640 0 : sourceCols_0.time().putColumnCells(refRow,sourcecols_i.time().getColumn());
3641 :
3642 : // Optional columns
3643 0 : if (columnOk(sourcecols_i.position()))
3644 : {
3645 0 : sourceCols_0.position().putColumnCells(refRow,sourcecols_i.position().getColumn());
3646 : }
3647 :
3648 0 : if (columnOk(sourcecols_i.transition()))
3649 : {
3650 0 : sourceCols_0.transition().putColumnCells(refRow,sourcecols_i.transition().getColumn());
3651 : }
3652 :
3653 0 : if (columnOk(sourcecols_i.restFrequency()))
3654 : {
3655 0 : sourceCols_0.restFrequency().putColumnCells(refRow,sourcecols_i.restFrequency().getColumn());
3656 : }
3657 :
3658 0 : if (columnOk(sourcecols_i.sysvel()))
3659 : {
3660 0 : sourceCols_0.sysvel().putColumnCells(refRow,sourcecols_i.sysvel().getColumn());
3661 : }
3662 :
3663 0 : if (columnOk(sourcecols_i.pulsarId()))
3664 : {
3665 0 : sourceCols_0.pulsarId().putColumnCells(refRow,sourcecols_i.pulsarId().getColumn());
3666 : }
3667 :
3668 0 : if (columnOk(sourcecols_i.sourceModel()))
3669 : {
3670 0 : sourceCols_0.sourceModel().putColumnCells(refRow,sourcecols_i.sourceModel().getColumn());
3671 : }
3672 :
3673 : // Increment row offset
3674 0 : rowIndex += sourcetable_i.nrow();
3675 : }
3676 : }
3677 :
3678 : }
3679 : else
3680 : {
3681 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3682 0 : << "SOURCE sub-table found but has no valid content" << LogIO::POST;
3683 0 : return false;
3684 : }
3685 : }
3686 : else
3687 : {
3688 0 : os << LogIO::SEVERE << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3689 0 : << "SOURCE sub-table not found " << LogIO::POST;
3690 0 : return false;
3691 : }
3692 :
3693 0 : return true;
3694 : }
3695 :
3696 :
3697 : // -----------------------------------------------------------------------
3698 : // Method to merge Syscal sub-tables from SubMSs to create the MMS-level Syscal sub-table
3699 : // -----------------------------------------------------------------------
3700 0 : bool MSTransformDataHandler::mergeSyscalSubTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3701 : {
3702 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3703 :
3704 0 : if (filenames.size() != mapSubmsSpwid.size())
3705 : {
3706 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3707 0 : return false;
3708 : }
3709 :
3710 0 : String filename_0 = filenames(0);
3711 0 : MeasurementSet ms_0(filename_0,Table::Update);
3712 :
3713 0 : if(Table::isReadable(ms_0.sysCalTableName()) and !ms_0.sysCal().isNull())
3714 : {
3715 0 : MSSysCal syscalTable_0 = ms_0.sysCal();
3716 :
3717 : // CAS-7167. The WVR spw has no FEED content.
3718 : // if (syscalTable_0.nrow() >= 0)
3719 : // {
3720 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3721 0 : << "Merging SYSCAL sub-tables from all sub-MSs to form MMS-level SYSCAL sub-table" << LogIO::POST;
3722 :
3723 0 : MSSysCalColumns syscalCols_0(syscalTable_0);
3724 :
3725 0 : uInt rowIndex = syscalTable_0.nrow();
3726 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3727 : {
3728 0 : String filename_i = filenames(subms_index);
3729 0 : MeasurementSet ms_i(filename_i);
3730 0 : MSSysCal syscaltable_i = ms_i.sysCal();
3731 :
3732 0 : if (syscaltable_i.nrow() > 0)
3733 : {
3734 0 : MSSysCalColumns syscalcols_i(syscaltable_i);
3735 :
3736 0 : syscalTable_0.addRow(syscaltable_i.nrow());
3737 :
3738 : // Prepare row reference object
3739 0 : RefRows refRow(rowIndex,rowIndex+syscaltable_i.nrow()-1);
3740 :
3741 : // Re-index SPW col
3742 0 : Vector<Int> spectralWindowId_output(syscaltable_i.nrow(),mapSubmsSpwid[subms_index]);
3743 0 : spectralWindowId_output += syscalcols_i.spectralWindowId().getColumn();
3744 0 : syscalCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3745 :
3746 : // Columns that can be just copied
3747 0 : syscalCols_0.antennaId().putColumnCells(refRow,syscalcols_i.antennaId().getColumn());
3748 0 : syscalCols_0.feedId().putColumnCells(refRow,syscalcols_i.feedId().getColumn());
3749 0 : syscalCols_0.interval().putColumnCells(refRow,syscalcols_i.interval().getColumn());
3750 0 : syscalCols_0.time().putColumnCells(refRow,syscalcols_i.time().getColumn());
3751 :
3752 : // Optional columns
3753 0 : if (columnOk(syscalcols_i.phaseDiff()))
3754 : {
3755 0 : syscalCols_0.phaseDiff().putColumnCells(refRow,syscalcols_i.phaseDiff().getColumn());
3756 : }
3757 :
3758 0 : if (columnOk(syscalcols_i.phaseDiffFlag()))
3759 : {
3760 0 : syscalCols_0.phaseDiffFlag().putColumnCells(refRow,syscalcols_i.phaseDiffFlag().getColumn());
3761 : }
3762 :
3763 0 : if (columnOk(syscalcols_i.tant()))
3764 : {
3765 0 : syscalCols_0.tant().putColumnCells(refRow,syscalcols_i.tant().getColumn());
3766 : }
3767 :
3768 0 : if (columnOk(syscalcols_i.tantFlag()))
3769 : {
3770 0 : syscalCols_0.tantFlag().putColumnCells(refRow,syscalcols_i.tantFlag().getColumn());
3771 : }
3772 :
3773 0 : if (columnOk(syscalcols_i.tantSpectrum()))
3774 : {
3775 0 : syscalCols_0.tantSpectrum().putColumnCells(refRow,syscalcols_i.tantSpectrum().getColumn());
3776 : }
3777 :
3778 0 : if (columnOk(syscalcols_i.tantTsys()))
3779 : {
3780 0 : syscalCols_0.tantTsys().putColumnCells(refRow,syscalcols_i.tantTsys().getColumn());
3781 : }
3782 :
3783 0 : if (columnOk(syscalcols_i.tantTsysFlag()))
3784 : {
3785 0 : syscalCols_0.tantTsysFlag().putColumnCells(refRow,syscalcols_i.tantTsysFlag().getColumn());
3786 : }
3787 :
3788 0 : if (columnOk(syscalcols_i.tantTsysSpectrum()))
3789 : {
3790 0 : syscalCols_0.tantTsysSpectrum().putColumnCells(refRow,syscalcols_i.tantTsysSpectrum().getColumn());
3791 : }
3792 :
3793 0 : if (columnOk(syscalcols_i.tcal()))
3794 : {
3795 0 : syscalCols_0.tcal().putColumnCells(refRow,syscalcols_i.tcal().getColumn());
3796 : }
3797 :
3798 0 : if (columnOk(syscalcols_i.tcalFlag()))
3799 : {
3800 0 : syscalCols_0.tcalFlag().putColumnCells(refRow,syscalcols_i.tcalFlag().getColumn());
3801 : }
3802 :
3803 0 : if (columnOk(syscalcols_i.tcalSpectrum()))
3804 : {
3805 0 : syscalCols_0.tcalSpectrum().putColumnCells(refRow,syscalcols_i.tcalSpectrum().getColumn());
3806 : }
3807 :
3808 0 : if (columnOk(syscalcols_i.trx()))
3809 : {
3810 0 : syscalCols_0.trx().putColumnCells(refRow,syscalcols_i.trx().getColumn());
3811 : }
3812 :
3813 0 : if (columnOk(syscalcols_i.trxFlag()))
3814 : {
3815 0 : syscalCols_0.trxFlag().putColumnCells(refRow,syscalcols_i.trxFlag().getColumn());
3816 : }
3817 :
3818 0 : if (columnOk(syscalcols_i.trxSpectrum()))
3819 : {
3820 0 : syscalCols_0.trxSpectrum().putColumnCells(refRow,syscalcols_i.trxSpectrum().getColumn());
3821 : }
3822 :
3823 0 : if (columnOk(syscalcols_i.tsky()))
3824 : {
3825 0 : syscalCols_0.tsky().putColumnCells(refRow,syscalcols_i.tsky().getColumn());
3826 : }
3827 :
3828 0 : if (columnOk(syscalcols_i.tskyFlag()))
3829 : {
3830 0 : syscalCols_0.tskyFlag().putColumnCells(refRow,syscalcols_i.tskyFlag().getColumn());
3831 : }
3832 :
3833 0 : if (columnOk(syscalcols_i.tskySpectrum()))
3834 : {
3835 0 : syscalCols_0.tskySpectrum().putColumnCells(refRow,syscalcols_i.tskySpectrum().getColumn());
3836 : }
3837 :
3838 0 : if (columnOk(syscalcols_i.tsys()))
3839 : {
3840 0 : syscalCols_0.tsys().putColumnCells(refRow,syscalcols_i.tsys().getColumn());
3841 : }
3842 :
3843 0 : if (columnOk(syscalcols_i.tsysFlag()))
3844 : {
3845 0 : syscalCols_0.tsysFlag().putColumnCells(refRow,syscalcols_i.tsysFlag().getColumn());
3846 : }
3847 :
3848 0 : if (columnOk(syscalcols_i.tsysSpectrum()))
3849 : {
3850 0 : syscalCols_0.tsysSpectrum().putColumnCells(refRow,syscalcols_i.tsysSpectrum().getColumn());
3851 : }
3852 :
3853 : // Increment row offset
3854 0 : rowIndex += syscalcols_i.nrow();
3855 : }
3856 : }
3857 :
3858 : /*
3859 : }
3860 : else
3861 : {
3862 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3863 : << "SYSCAL sub-table found but has no valid content" << LogIO::POST;
3864 : return false;
3865 : }
3866 : */
3867 : }
3868 : else
3869 : {
3870 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3871 0 : << "SYSCAL sub-table not found " << LogIO::POST;
3872 0 : return false;
3873 : }
3874 :
3875 0 : return true;
3876 : }
3877 :
3878 :
3879 : // -----------------------------------------------------------------------
3880 : // Method to merge FreqOffset sub-tables from SubMSs to create the MMS-level FreqOffset sub-table
3881 : // -----------------------------------------------------------------------
3882 0 : bool MSTransformDataHandler::mergeFreqOffsetTables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3883 : {
3884 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3885 :
3886 0 : if (filenames.size() != mapSubmsSpwid.size())
3887 : {
3888 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3889 0 : return false;
3890 : }
3891 :
3892 0 : String filename_0 = filenames(0);
3893 0 : MeasurementSet ms_0(filename_0,Table::Update);
3894 :
3895 0 : if (Table::isReadable(ms_0.freqOffsetTableName()) and !ms_0.freqOffset().isNull())
3896 : {
3897 0 : MSFreqOffset freqoffsetTable_0 = ms_0.freqOffset();
3898 :
3899 0 : if (freqoffsetTable_0.nrow() > 0)
3900 : {
3901 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3902 0 : << "Merging FREQ_OFFSET sub-tables from all sub-MSs to form MMS-level FREQ_OFFSET sub-table" << LogIO::POST;
3903 :
3904 0 : MSFreqOffsetColumns freqoffsetCols_0(freqoffsetTable_0);
3905 :
3906 0 : uInt rowIndex = freqoffsetTable_0.nrow();
3907 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
3908 : {
3909 0 : String filename_i = filenames(subms_index);
3910 0 : MeasurementSet ms_i(filename_i);
3911 0 : MSFreqOffset freqoffsettable_i = ms_i.freqOffset();
3912 :
3913 0 : if (freqoffsettable_i.nrow() > 0)
3914 : {
3915 0 : MSFreqOffsetColumns freqoffsetcols_i(freqoffsettable_i);
3916 :
3917 0 : freqoffsetTable_0.addRow(freqoffsettable_i.nrow());
3918 :
3919 : // Prepare row reference object
3920 0 : RefRows refRow(rowIndex,rowIndex+freqoffsettable_i.nrow()-1);
3921 :
3922 : // Re-index SPW col
3923 0 : Vector<Int> spectralWindowId_output(freqoffsettable_i.nrow(),mapSubmsSpwid[subms_index]);
3924 0 : spectralWindowId_output += freqoffsetcols_i.spectralWindowId().getColumn();
3925 0 : freqoffsetCols_0.spectralWindowId().putColumnCells(refRow,spectralWindowId_output);
3926 :
3927 : // Columns that can be just copied
3928 0 : freqoffsetCols_0.antenna1().putColumnCells(refRow,freqoffsetcols_i.antenna1().getColumn());
3929 0 : freqoffsetCols_0.antenna2().putColumnCells(refRow,freqoffsetcols_i.antenna2().getColumn());
3930 0 : freqoffsetCols_0.feedId().putColumnCells(refRow,freqoffsetcols_i.feedId().getColumn());
3931 0 : freqoffsetCols_0.interval().putColumnCells(refRow,freqoffsetcols_i.interval().getColumn());
3932 0 : freqoffsetCols_0.offset().putColumnCells(refRow,freqoffsetcols_i.offset().getColumn());
3933 0 : freqoffsetCols_0.time().putColumnCells(refRow,freqoffsetcols_i.time().getColumn());
3934 :
3935 : // NOTE (jagonzal): FreqOffset does not have optional columns
3936 :
3937 : // Increment row offset
3938 0 : rowIndex += freqoffsettable_i.nrow();
3939 : }
3940 : }
3941 :
3942 : }
3943 : else
3944 : {
3945 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3946 0 : << "FREQ_OFFSET sub-table found but has no valid content" << LogIO::POST;
3947 0 : return false;
3948 : }
3949 :
3950 : }
3951 : else
3952 : {
3953 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3954 0 : << "FREQ_OFFSET sub-table not found " << LogIO::POST;
3955 0 : return false;
3956 : }
3957 :
3958 0 : return true;
3959 : }
3960 :
3961 : // -----------------------------------------------------------------------
3962 : // Method to merge CalDevice sub-tables from SubMSs to create the MMS-level CalDevice sub-table
3963 : // -----------------------------------------------------------------------
3964 0 : bool MSTransformDataHandler::mergeCalDeviceSubtables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
3965 : {
3966 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
3967 :
3968 0 : if (filenames.size() != mapSubmsSpwid.size())
3969 : {
3970 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
3971 0 : return false;
3972 : }
3973 :
3974 0 : String filename_0 = filenames(0);
3975 0 : MeasurementSet ms_0(filename_0,Table::Update);
3976 :
3977 0 : if (Table::isReadable(ms_0.tableName() + "/CALDEVICE"))
3978 : {
3979 0 : Table subtable_0(ms_0.tableName() + "/CALDEVICE", Table::Update);
3980 :
3981 0 : if (subtable_0.nrow() > 0)
3982 : {
3983 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
3984 0 : << "Merging CALDEVICE sub-tables from all sub-MSs to form MMS-level CALDEVICE sub-table" << LogIO::POST;
3985 :
3986 : // Get RW access to columns
3987 0 : ScalarColumn<Int> antennaIdCol_0(subtable_0, "ANTENNA_ID");
3988 0 : ScalarColumn<Int> feedIdCol_0(subtable_0, "FEED_ID");
3989 0 : ScalarColumn<Int> spectralWindowIdCol_0(subtable_0, "SPECTRAL_WINDOW_ID");
3990 0 : ScalarColumn<Double> timeCol_0(subtable_0, "TIME");
3991 0 : ScalarColumn<Double> intervalCol_0(subtable_0, "INTERVAL");
3992 :
3993 0 : ScalarColumn<Int> numCalLoadCol_0(subtable_0, "NUM_CAL_LOAD");
3994 0 : ArrayColumn<String> calLoadNamesCol_0(subtable_0, "CAL_LOAD_NAMES");
3995 0 : ScalarColumn<Int> numReceptorCol_0(subtable_0, "NUM_RECEPTOR");
3996 0 : ArrayColumn<Float> noiseCalCol_0(subtable_0, "NOISE_CAL");
3997 0 : ArrayColumn<Float> calEffCol_0(subtable_0, "CAL_EFF");
3998 0 : ArrayColumn<Double> temperatureLoadCol_0(subtable_0, "TEMPERATURE_LOAD");
3999 :
4000 : // Get original content of columns
4001 0 : Vector<Int> antennaId_0;
4002 0 : if (columnOk(antennaIdCol_0)) antennaId_0 = antennaIdCol_0.getColumn();
4003 0 : Vector<Int> feedId_0;
4004 0 : if (columnOk(feedIdCol_0)) feedId_0 = feedIdCol_0.getColumn();
4005 0 : Vector<Int> spectralWindowId_0;
4006 0 : if (columnOk(spectralWindowIdCol_0)) spectralWindowId_0 = spectralWindowIdCol_0.getColumn();
4007 0 : Vector<Double> time_0;
4008 0 : if (columnOk(timeCol_0)) time_0 = timeCol_0.getColumn();
4009 0 : Vector<Double> interval_0;
4010 0 : if (columnOk(intervalCol_0)) interval_0 = intervalCol_0.getColumn();
4011 :
4012 0 : Vector<Int> numCalLoad_0;
4013 0 : if (columnOk(numCalLoadCol_0)) numCalLoad_0 = numCalLoadCol_0.getColumn();
4014 0 : Array<String> calLoadNames_0;
4015 0 : if (columnOk(calLoadNamesCol_0)) calLoadNames_0 = calLoadNamesCol_0.getColumn();
4016 0 : Vector<Int> numReceptor_0;
4017 0 : if (columnOk(numReceptorCol_0)) numReceptor_0 = numReceptorCol_0.getColumn();
4018 0 : Array<Float> noiseCal_0;
4019 0 : if (columnOk(noiseCalCol_0)) noiseCal_0 = noiseCalCol_0.getColumn();
4020 0 : Array<Float> calEff_0;
4021 0 : if (columnOk(calEffCol_0)) calEff_0 = calEffCol_0.getColumn();
4022 0 : Array<Double> temperatureLoad_0;
4023 0 : if (columnOk(temperatureLoadCol_0)) temperatureLoad_0 = temperatureLoadCol_0.getColumn();
4024 :
4025 0 : uInt rowIndex = subtable_0.nrow();
4026 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
4027 : {
4028 0 : String filename_i = filenames(subms_index);
4029 0 : MeasurementSet ms_i(filename_i);
4030 0 : Table subtable_i(ms_i.tableName() + "/CALDEVICE", Table::Update);
4031 :
4032 0 : if (subtable_i.nrow() > 0)
4033 : {
4034 : // Get RW access to columns
4035 0 : ScalarColumn<Int> antennaIdCol_i(subtable_i, "ANTENNA_ID");
4036 0 : ScalarColumn<Int> feedIdCol_i(subtable_i, "FEED_ID");
4037 0 : ScalarColumn<Int> spectralWindowIdCol_i(subtable_i, "SPECTRAL_WINDOW_ID");
4038 0 : ScalarColumn<Double> timeCol_i(subtable_i, "TIME");
4039 0 : ScalarColumn<Double> intervalCol_i(subtable_i, "INTERVAL");
4040 :
4041 0 : ScalarColumn<Int> numCalLoadCol_i(subtable_i, "NUM_CAL_LOAD");
4042 0 : ArrayColumn<String> calLoadNamesCol_i(subtable_i, "CAL_LOAD_NAMES");
4043 0 : ScalarColumn<Int> numReceptorCol_i(subtable_i, "NUM_RECEPTOR");
4044 0 : ArrayColumn<Float> noiseCalCol_i(subtable_i, "NOISE_CAL");
4045 0 : ArrayColumn<Float> calEffCol_i(subtable_i, "CAL_EFF");
4046 0 : ArrayColumn<Double> temperatureLoadCol_i(subtable_i, "TEMPERATURE_LOAD");
4047 :
4048 : // Get original content of columns
4049 0 : Vector<Int> antennaId_i;
4050 0 : if (columnOk(antennaIdCol_i)) antennaId_i = antennaIdCol_i.getColumn();
4051 0 : Vector<Int> feedId_i;
4052 0 : if (columnOk(feedIdCol_i)) feedId_i = feedIdCol_i.getColumn();
4053 0 : Vector<Int> spectralWindowId_i;
4054 0 : if (columnOk(spectralWindowIdCol_i)) spectralWindowId_i = spectralWindowIdCol_i.getColumn();
4055 0 : Vector<Double> time_i;
4056 0 : if (columnOk(timeCol_i)) time_i = timeCol_i.getColumn();
4057 0 : Vector<Double> interval_i;
4058 0 : if (columnOk(intervalCol_i)) interval_i = intervalCol_i.getColumn();
4059 :
4060 0 : Vector<Int> numCalLoad_i;
4061 0 : if (columnOk(numCalLoadCol_i)) numCalLoad_i = numCalLoadCol_i.getColumn();
4062 0 : Array<String> calLoadNames_i;
4063 0 : if (columnOk(calLoadNamesCol_i)) calLoadNames_i = calLoadNamesCol_i.getColumn();
4064 0 : Vector<Int> numReceptor_i;
4065 0 : if (columnOk(numReceptorCol_i)) numReceptor_i = numReceptorCol_i.getColumn();
4066 0 : Array<Float> noiseCal_i;
4067 0 : if (columnOk(noiseCalCol_i)) noiseCal_i = noiseCalCol_i.getColumn();
4068 0 : Array<Float> calEff_i;
4069 0 : if (columnOk(calEffCol_i)) calEff_i = calEffCol_i.getColumn();
4070 0 : Array<Double> temperatureLoad_i;
4071 0 : if (columnOk(temperatureLoadCol_i)) temperatureLoad_i = temperatureLoadCol_i.getColumn();
4072 :
4073 : // Add n# rows to subtable_i equivalent to n# rows from subtable_0
4074 0 : subtable_0.addRow(subtable_i.nrow());
4075 :
4076 : // Prepare row reference object
4077 0 : RefRows refRow(rowIndex,rowIndex+subtable_i.nrow()-1);
4078 :
4079 : // Re-index SPW col
4080 0 : Vector<Int> spectralWindowId_output(spectralWindowId_i.size(),mapSubmsSpwid[subms_index]);
4081 0 : spectralWindowId_output += spectralWindowId_i;
4082 0 : spectralWindowIdCol_0.putColumnCells(refRow,spectralWindowId_output);
4083 :
4084 : // Columns that can be just copied
4085 0 : if (columnOk(antennaIdCol_i))
4086 : {
4087 0 : antennaIdCol_0.putColumnCells(refRow,antennaId_i);
4088 : }
4089 :
4090 0 : if (columnOk(feedIdCol_i))
4091 : {
4092 0 : feedIdCol_0.putColumnCells(refRow,feedId_i);
4093 : }
4094 :
4095 0 : if (columnOk(timeCol_i))
4096 : {
4097 0 : timeCol_0.putColumnCells(refRow,time_i);
4098 : }
4099 :
4100 0 : if (columnOk(intervalCol_i))
4101 : {
4102 0 : intervalCol_0.putColumnCells(refRow,interval_i);
4103 : }
4104 :
4105 :
4106 0 : if (columnOk(numCalLoadCol_i))
4107 : {
4108 0 : numCalLoadCol_0.putColumnCells(refRow,numCalLoad_i);
4109 : }
4110 :
4111 0 : if (columnOk(calLoadNamesCol_i))
4112 : {
4113 0 : calLoadNamesCol_0.putColumnCells(refRow,calLoadNames_i);
4114 : }
4115 :
4116 0 : if (columnOk(numReceptorCol_i))
4117 : {
4118 0 : numReceptorCol_0.putColumnCells(refRow,numReceptor_i);
4119 : }
4120 :
4121 0 : if (columnOk(noiseCalCol_i))
4122 : {
4123 0 : noiseCalCol_0.putColumnCells(refRow,noiseCal_i);
4124 : }
4125 :
4126 0 : if (columnOk(calEffCol_i))
4127 : {
4128 0 : calEffCol_0.putColumnCells(refRow,calEff_i);
4129 : }
4130 :
4131 0 : if (columnOk(temperatureLoadCol_i))
4132 : {
4133 0 : temperatureLoadCol_0.putColumnCells(refRow,temperatureLoad_i);
4134 : }
4135 :
4136 : // Increment row offset
4137 0 : rowIndex += subtable_i.nrow();
4138 : }
4139 : }
4140 :
4141 : }
4142 : else
4143 : {
4144 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4145 0 : << "CALDEVICE sub-table found but has no valid content" << LogIO::POST;
4146 0 : return false;
4147 : }
4148 : }
4149 : else
4150 : {
4151 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4152 0 : << "CALDEVICE sub-table not found " << LogIO::POST;
4153 0 : return false;
4154 : }
4155 :
4156 0 : return true;
4157 : }
4158 :
4159 :
4160 : // -----------------------------------------------------------------------
4161 : // Method to merge SysPower sub-tables from SubMSs to create the MMS-level SysPower sub-table
4162 : // -----------------------------------------------------------------------
4163 0 : bool MSTransformDataHandler::mergeSysPowerSubtables(Vector<String> filenames, Vector<uInt> mapSubmsSpwid)
4164 : {
4165 0 : LogIO os(LogOrigin("MSTransformDataHandler", __FUNCTION__));
4166 :
4167 0 : if (filenames.size() != mapSubmsSpwid.size())
4168 : {
4169 0 : os << LogIO::SEVERE << "List of Sub-MSs does not match size of SPW re-indexing map" << LogIO::POST;
4170 0 : return false;
4171 : }
4172 :
4173 0 : String filename_0 = filenames(0);
4174 0 : MeasurementSet ms_0(filename_0,Table::Update);
4175 :
4176 0 : if (Table::isReadable(ms_0.tableName() + "/SYSPOWER"))
4177 : {
4178 0 : Table subtable_0(ms_0.tableName() + "/SYSPOWER", Table::Update);
4179 :
4180 0 : if (subtable_0.nrow() > 0)
4181 : {
4182 0 : os << LogIO::NORMAL << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4183 0 : << "Merging SYSPOWER sub-tables from all sub-MSs to form MMS-level SYSPOWER sub-table" << LogIO::POST;
4184 :
4185 : // Get RW access to columns
4186 0 : ScalarColumn<Int> antennaIdCol_0(subtable_0, "ANTENNA_ID");
4187 0 : ScalarColumn<Int> feedIdCol_0(subtable_0, "FEED_ID");
4188 0 : ScalarColumn<Int> spectralWindowIdCol_0(subtable_0, "SPECTRAL_WINDOW_ID");
4189 0 : ScalarColumn<Double> timeCol_0(subtable_0, "TIME");
4190 0 : ScalarColumn<Double> intervalCol_0(subtable_0, "INTERVAL");
4191 :
4192 0 : ArrayColumn<Float> switchedDiffCol_0(subtable_0, "SWITCHED_DIFF");
4193 0 : ArrayColumn<Float> switchedSumCol_0(subtable_0, "SWITCHED_SUM");
4194 0 : ArrayColumn<Float> requantizerGainCol_0(subtable_0, "REQUANTIZER_GAIN");
4195 :
4196 : // Get original content of columns
4197 0 : Vector<Int> antennaId_0;
4198 0 : if (columnOk(antennaIdCol_0)) antennaId_0 = antennaIdCol_0.getColumn();
4199 0 : Vector<Int> feedId_0;
4200 0 : if (columnOk(feedIdCol_0)) feedId_0 = feedIdCol_0.getColumn();
4201 0 : Vector<Int> spectralWindowId_0;
4202 0 : if (columnOk(spectralWindowIdCol_0)) spectralWindowId_0 = spectralWindowIdCol_0.getColumn();
4203 0 : Vector<Double> time_0;
4204 0 : if (columnOk(timeCol_0)) time_0 = timeCol_0.getColumn();
4205 0 : Vector<Double> interval_0;
4206 0 : if (columnOk(intervalCol_0)) interval_0 = intervalCol_0.getColumn();
4207 :
4208 0 : Array<Float> switchedDiff_0;
4209 0 : if (columnOk(switchedDiffCol_0)) switchedDiff_0 = switchedDiffCol_0.getColumn();
4210 0 : Array<Float> switchedSum_0;
4211 0 : if (columnOk(switchedSumCol_0)) switchedSum_0 = switchedSumCol_0.getColumn();
4212 0 : Array<Float> requantizerGain_0;
4213 0 : if (columnOk(requantizerGainCol_0)) requantizerGain_0 = requantizerGainCol_0.getColumn();
4214 :
4215 0 : uInt rowIndex = subtable_0.nrow();
4216 0 : for (uInt subms_index=1;subms_index < filenames.size();subms_index++)
4217 : {
4218 0 : String filename_i = filenames(subms_index);
4219 0 : MeasurementSet ms_i(filename_i);
4220 0 : Table subtable_i(ms_i.tableName() + "/SYSPOWER", Table::Update);
4221 :
4222 0 : if (subtable_i.nrow() > 0)
4223 : {
4224 : // Get RW access to columns
4225 0 : ScalarColumn<Int> antennaIdCol_i(subtable_i, "ANTENNA_ID");
4226 0 : ScalarColumn<Int> feedIdCol_i(subtable_i, "FEED_ID");
4227 0 : ScalarColumn<Int> spectralWindowIdCol_i(subtable_i, "SPECTRAL_WINDOW_ID");
4228 0 : ScalarColumn<Double> timeCol_i(subtable_i, "TIME");
4229 0 : ScalarColumn<Double> intervalCol_i(subtable_i, "INTERVAL");
4230 :
4231 0 : ArrayColumn<Float> switchedDiffCol_i(subtable_i, "SWITCHED_DIFF");
4232 0 : ArrayColumn<Float> switchedSumCol_i(subtable_i, "SWITCHED_SUM");
4233 0 : ArrayColumn<Float> requantizerGainCol_i(subtable_i, "REQUANTIZER_GAIN");
4234 :
4235 : // Get original content of columns
4236 0 : Vector<Int> antennaId_i;
4237 0 : if (columnOk(antennaIdCol_i)) antennaId_i = antennaIdCol_i.getColumn();
4238 0 : Vector<Int> feedId_i;
4239 0 : if (columnOk(feedIdCol_i)) feedId_i = feedIdCol_i.getColumn();
4240 0 : Vector<Int> spectralWindowId_i;
4241 0 : if (columnOk(spectralWindowIdCol_i)) spectralWindowId_i = spectralWindowIdCol_i.getColumn();
4242 0 : Vector<Double> time_i;
4243 0 : if (columnOk(timeCol_i)) time_i = timeCol_i.getColumn();
4244 0 : Vector<Double> interval_i;
4245 0 : if (columnOk(intervalCol_i)) interval_i = intervalCol_i.getColumn();
4246 :
4247 0 : Array<Float> switchedDiff_i;
4248 0 : if (columnOk(switchedDiffCol_i)) switchedDiff_i = switchedDiffCol_i.getColumn();
4249 0 : Array<Float> switchedSum_i;
4250 0 : if (columnOk(switchedSumCol_i)) switchedSum_i = switchedSumCol_i.getColumn();
4251 0 : Array<Float> requantizerGain_i;
4252 0 : if (columnOk(requantizerGainCol_i)) requantizerGain_i = requantizerGainCol_i.getColumn();
4253 :
4254 : // Add n# rows to subtable_i equivalent to n# rows from subtable_0
4255 0 : subtable_0.addRow(subtable_i.nrow());
4256 :
4257 : // Prepare row reference object
4258 0 : RefRows refRow(rowIndex,rowIndex+subtable_i.nrow()-1);
4259 :
4260 : // Re-index SPW col
4261 0 : Vector<Int> spectralWindowId_output(spectralWindowId_i.size(),mapSubmsSpwid[subms_index]);
4262 0 : spectralWindowId_output += spectralWindowId_i;
4263 0 : spectralWindowIdCol_0.putColumnCells(refRow,spectralWindowId_output);
4264 :
4265 : // Columns that can be just copied
4266 0 : if (columnOk(antennaIdCol_i))
4267 : {
4268 0 : antennaIdCol_0.putColumnCells(refRow,antennaId_i);
4269 : }
4270 :
4271 0 : if (columnOk(feedIdCol_i))
4272 : {
4273 0 : feedIdCol_0.putColumnCells(refRow,feedId_i);
4274 : }
4275 :
4276 0 : if (columnOk(timeCol_i))
4277 : {
4278 0 : timeCol_0.putColumnCells(refRow,time_i);
4279 : }
4280 :
4281 0 : if (columnOk(intervalCol_i))
4282 : {
4283 0 : intervalCol_0.putColumnCells(refRow,interval_i);
4284 : }
4285 :
4286 :
4287 0 : if (columnOk(switchedDiffCol_i))
4288 : {
4289 0 : switchedDiffCol_0.putColumnCells(refRow,switchedDiff_i);
4290 : }
4291 :
4292 0 : if (columnOk(switchedSumCol_i))
4293 : {
4294 0 : switchedSumCol_0.putColumnCells(refRow,switchedSum_i);
4295 : }
4296 :
4297 0 : if (columnOk(requantizerGainCol_i))
4298 : {
4299 0 : requantizerGainCol_0.putColumnCells(refRow,requantizerGain_i);
4300 : }
4301 :
4302 : // Increment row offset
4303 0 : rowIndex += subtable_i.nrow();
4304 : }
4305 : }
4306 :
4307 : }
4308 : else
4309 : {
4310 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4311 0 : << "SYSPOWER sub-table found but has no valid content" << LogIO::POST;
4312 0 : return false;
4313 : }
4314 : }
4315 : else
4316 : {
4317 0 : os << LogIO::DEBUG1 << LogOrigin("MSTransformDataHandler", __FUNCTION__)
4318 0 : << "SYSPOWER sub-table not found " << LogIO::POST;
4319 0 : return false;
4320 : }
4321 :
4322 0 : return true;
4323 : }
4324 :
4325 : // -----------------------------------------------------------------------
4326 : //
4327 : // -----------------------------------------------------------------------
4328 : // template <class T> bool MSTransformDataHandler::columnOk (ArrayColumn<T> column)
4329 : // {
4330 : // bool ret;
4331 : // if (column.isNull()==false and column.hasContent()==true and column.ndimColumn() > 0)
4332 : // {
4333 : // ret = true;
4334 : // }
4335 : // else
4336 : // {
4337 : // ret = false;
4338 : // }
4339 :
4340 : // return ret;
4341 : // }
4342 :
4343 : // // -----------------------------------------------------------------------
4344 : // //
4345 : // // -----------------------------------------------------------------------
4346 : // template <class T> bool MSTransformDataHandler::columnOk (ScalarColumn<T> column)
4347 : // {
4348 : // bool ret;
4349 : // if (column.isNull()==false and column.hasContent()==true)
4350 : // {
4351 : // ret = true;
4352 : // }
4353 : // else
4354 : // {
4355 : // ret = false;
4356 : // }
4357 :
4358 : // return ret;
4359 : // }
4360 :
4361 :
4362 : // -----------------------------------------------------------------------
4363 : // Work-around to copy the keywords of the FLOAT_DATA column to the output MS
4364 : // -----------------------------------------------------------------------
4365 0 : void MSTransformDataHandler::copyMainTableKeywords (TableRecord& outKeys,
4366 : const TableRecord& inKeys)
4367 : {
4368 0 : for (uInt i=0; i<inKeys.nfields(); i++) {
4369 0 : if (inKeys.type(i) == TpString) {
4370 : // Add keywords for MAIN table columns such as FLOAT_DATA
4371 0 : String ikey = inKeys.name(i);
4372 0 : if (!outKeys.isDefined (ikey)) {
4373 0 : String keyval;
4374 0 : inKeys.get(ikey, keyval);
4375 0 : outKeys.define(ikey,keyval);
4376 : }
4377 :
4378 : }
4379 :
4380 : }
4381 0 : }
4382 :
4383 :
4384 : } //# NAMESPACE CASA - END
|
