Line data Source code
1 : ///# AgentFlagger.cc: this defines AgentFlagger
2 : //# Copyright (C) 2000,2001,2002
3 : //# Associated Universities, Inc. Washington DC, USA.
4 : //#
5 : //# This library is free software; you can redistribute it and/or modify it
6 : //# under the terms of the GNU Library General Public License as published by
7 : //# the Free Software Foundation; either version 2 of the License, or (at your
8 : //# option) any later version.
9 : //#
10 : //# This library is distributed in the hope that it will be useful, but WITHOUT
11 : //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 : //# License for more details.
14 : //#
15 : //# You should have received a copy of the GNU Library General Public License
16 : //# along with this library; if not, write to the Free Software Foundation,
17 : //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 : //#
19 : //# Correspondence concerning AIPS++ should be addressed as follows:
20 : //# Internet email: aips2-request@nrao.edu.
21 : //# Postal address: AIPS++ Project Office
22 : //# National Radio Astronomy Observatory
23 : //# 520 Edgemont Road
24 : //# Charlottesville, VA 22903-2475 USA
25 : //#
26 : //# $Id$
27 : #include <casacore/casa/Arrays/Vector.h>
28 : #include <casacore/casa/Arrays/ArrayMath.h>
29 : #include <casacore/casa/Arrays/ArrayLogical.h>
30 : #include <casacore/casa/BasicSL/Complex.h>
31 : #include <casacore/casa/Utilities/Regex.h>
32 : #include <casacore/casa/OS/HostInfo.h>
33 : #include <casacore/casa/Exceptions/Error.h>
34 : #include <flagging/Flagging/AgentFlagger.h>
35 : #include <flagging/Flagging/FlagVersion.h>
36 : #include <stdarg.h>
37 : #include <sstream>
38 : #include <iostream>
39 : #include <vector>
40 :
41 : #include <flagging/Flagging/FlagCalTableHandler.h>
42 : #include <flagging/Flagging/FlagMSHandler.h>
43 : #ifdef USE_GRPC
44 : #include <flagging/Flagging/grpcFlagAgentDisplay.h>
45 : #endif
46 :
47 : using namespace casacore;
48 : namespace casa {
49 :
50 : const bool AgentFlagger::dbg = false;
51 :
52 :
53 : // -----------------------------------------------------------------------
54 : // Default Constructor
55 : // -----------------------------------------------------------------------
56 1729 : AgentFlagger::AgentFlagger ()
57 : {
58 1729 : fdh_p = NULL;
59 1729 : summaryAgent_p = NULL;
60 1729 : displayAgent_p = NULL;
61 :
62 1729 : done();
63 1729 : }
64 :
65 :
66 : // -----------------------------------------------------------------------
67 : // Default Destructor
68 : // -----------------------------------------------------------------------
69 1729 : AgentFlagger::~AgentFlagger ()
70 : {
71 1729 : done();
72 1729 : }
73 :
74 : void
75 3458 : AgentFlagger::done()
76 : {
77 3458 : if(fdh_p){
78 1707 : delete fdh_p;
79 1707 : fdh_p = NULL;
80 : }
81 :
82 : // Default values of parameters
83 3458 : msname_p = "";
84 3458 : iterationApproach_p = FlagDataHandler::SUB_INTEGRATION;
85 3458 : timeInterval_p = 0.0;
86 3458 : isMS_p = true;
87 3458 : combinescans_p = false;
88 3458 : extendflags_p = true;
89 3458 : spw_p = "";
90 3458 : scan_p = "";
91 3458 : field_p = "";
92 3458 : antenna_p = "";
93 3458 : timerange_p = "";
94 3458 : correlation_p = "";
95 3458 : intent_p = "";
96 3458 : feed_p = "";
97 3458 : array_p = "";
98 3458 : uvrange_p = "";
99 3458 : observation_p = "";
100 :
101 3458 : max_p = 0.0;
102 :
103 3458 : dataselection_p = Record();
104 :
105 : /*
106 : if (! dataselection_p.empty()) {
107 : Record temp;
108 : dataselection_p = temp;
109 : }
110 : */
111 :
112 3458 : if (! agentParams_p.empty()) {
113 2554 : Record temp;
114 1277 : agentParams_p = temp;
115 : }
116 :
117 3458 : if(summaryAgent_p){
118 0 : summaryAgent_p = NULL;
119 : }
120 :
121 3458 : if(displayAgent_p){
122 0 : displayAgent_p = NULL;
123 : }
124 :
125 3458 : mode_p = "";
126 3458 : agents_config_list_p.clear();
127 3458 : agents_config_list_copy_p.clear();
128 3458 : agents_list_p.clear();
129 :
130 3458 : return;
131 : }
132 :
133 : // ---------------------------------------------------------------------
134 : // AgentFlagger::open
135 : // Create a FlagDataHandler object based on the input type:
136 : // MS or a calibration file. Open the MS or cal table and
137 : // attach it to the tool.
138 : // ---------------------------------------------------------------------
139 : bool
140 1707 : AgentFlagger::open(String msname, Double ntime)
141 : {
142 :
143 5121 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
144 :
145 1707 : if (msname.empty()) {
146 : os << LogIO::SEVERE << "No Measurement Set has been parsed"
147 0 : << LogIO::POST;
148 0 : return false;
149 : }
150 :
151 1707 : msname_p = msname;
152 :
153 1707 : if (ntime)
154 3 : timeInterval_p = ntime;
155 :
156 1707 : max_p = timeInterval_p;
157 :
158 : if(dbg){
159 : os << LogIO::NORMAL << "msname = " << msname_p << " ntime = " << timeInterval_p << LogIO::POST;
160 : }
161 :
162 1707 : if(fdh_p) delete fdh_p;
163 :
164 : // create a FlagDataHandler object
165 3414 : Table table(msname_p,TableLock(TableLock::AutoNoReadLocking));
166 1707 : TableInfo& info = table.tableInfo();
167 1707 : String type = info.type();
168 1707 : table.flush();
169 1707 : table.relinquishAutoLocks(true);
170 1707 : table.unlock();
171 1707 : os << LogIO::NORMAL << "Table type is " << type << LogIO::POST;
172 :
173 : // For a measurement set
174 1707 : if (type == "Measurement Set")
175 : {
176 1471 : isMS_p = true;
177 1471 : fdh_p = new FlagMSHandler(msname_p, iterationApproach_p, timeInterval_p);
178 : }
179 : // For a calibration file
180 : else
181 : {
182 236 : isMS_p = false;
183 236 : fdh_p = new FlagCalTableHandler(msname_p, iterationApproach_p, timeInterval_p);
184 : }
185 :
186 : // Open the MS
187 1707 : fdh_p->open();
188 :
189 1707 : return true;
190 : }
191 :
192 : // ---------------------------------------------------------------------
193 : // AgentFlagger::selectData
194 : // Get a record with data selection parameters and
195 : // Parse it to the FlagDataHandler to select the data
196 : // ---------------------------------------------------------------------
197 : bool
198 1291 : AgentFlagger::selectData(Record selrec)
199 : {
200 :
201 3873 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
202 : if (dbg)
203 : os << LogIO::NORMAL << "Called from selectData(Record)" << LogIO::POST;
204 :
205 1291 : if (! fdh_p){
206 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
207 0 : return false;
208 : }
209 :
210 :
211 1291 : if (! selrec.empty()) {
212 :
213 1291 : dataselection_p = selrec;
214 :
215 : // Check if all the parameters are in the record. If not,
216 : // use the default values
217 1291 : if (dataselection_p.isDefined("spw"))
218 1243 : dataselection_p.get("spw", spw_p);
219 1291 : if (dataselection_p.isDefined("scan"))
220 1259 : dataselection_p.get("scan", scan_p);
221 1291 : if (dataselection_p.isDefined("field"))
222 1226 : dataselection_p.get("field", field_p);
223 1291 : if (dataselection_p.isDefined("antenna"))
224 1226 : dataselection_p.get("antenna", antenna_p);
225 1291 : if (dataselection_p.isDefined("timerange"))
226 1227 : dataselection_p.get("timerange", timerange_p);
227 1291 : if (dataselection_p.isDefined("correlation"))
228 1223 : dataselection_p.get("correlation", correlation_p);
229 1291 : if (dataselection_p.isDefined("intent"))
230 1224 : dataselection_p.get("intent", intent_p);
231 1291 : if (dataselection_p.isDefined("feed"))
232 1223 : dataselection_p.get("feed", feed_p);
233 1291 : if (dataselection_p.isDefined("array"))
234 1223 : dataselection_p.get("array", array_p);
235 1291 : if (dataselection_p.isDefined("uvrange"))
236 1223 : dataselection_p.get("uvrange", uvrange_p);
237 1291 : if (dataselection_p.isDefined("observation"))
238 1227 : dataselection_p.get("observation", observation_p);
239 :
240 : }
241 :
242 1291 : bool ret_status = true;
243 :
244 : // Set the data selection
245 1291 : ret_status = fdh_p->setDataSelection(dataselection_p);
246 1291 : if (!ret_status) {
247 : os << LogIO::SEVERE << "Failed to set the data selection."
248 0 : << LogIO::POST;
249 0 : return false;
250 : }
251 :
252 : // Select the data
253 1291 : ret_status = fdh_p->selectData();
254 1277 : if (!ret_status) {
255 : os << LogIO::SEVERE << "Failed to select the data."
256 0 : << LogIO::POST;
257 0 : return false;
258 : }
259 :
260 :
261 1277 : return true;
262 : }
263 :
264 :
265 : // ---------------------------------------------------------------------
266 : // AgentFlagger::selectData
267 : // Create a record with data selection parameters.
268 : // ---------------------------------------------------------------------
269 : bool
270 1223 : AgentFlagger::selectData(String field, String spw, String array,
271 : String feed, String scan, String antenna,
272 : String uvrange, String timerange, String correlation,
273 : String intent, String observation)
274 : {
275 :
276 3669 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
277 :
278 : if (dbg)
279 : os << LogIO::NORMAL << "Called from selectData(String....)" << LogIO::POST;
280 :
281 : // Create a record with the parameters
282 1237 : Record selection = Record();
283 :
284 1223 : selection.define("spw", spw);
285 1223 : selection.define("scan", scan);
286 1223 : selection.define("field", field);
287 1223 : selection.define("antenna", antenna);
288 1223 : selection.define("timerange", timerange);
289 1223 : selection.define("correlation", correlation);
290 1223 : selection.define("intent", intent);
291 1223 : selection.define("feed", feed);
292 1223 : selection.define("array", array);
293 1223 : selection.define("uvrange", uvrange);
294 1223 : selection.define("observation", observation);
295 :
296 : // Call the main selectData() method
297 1237 : selectData(selection);
298 :
299 2418 : return true;
300 :
301 : }
302 :
303 :
304 : // ---------------------------------------------------------------------
305 : // AgentFlagger::parseAgentParameters
306 : // Create a vector of agents and parameters
307 : // Each input record contains data selection parameters
308 : // and agent's specific parameters
309 : // ---------------------------------------------------------------------
310 : bool
311 2231 : AgentFlagger::parseAgentParameters(Record agent_params)
312 : {
313 6693 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
314 :
315 2231 : if (! fdh_p){
316 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
317 0 : return false;
318 : }
319 :
320 : // Default values for some parameters
321 4462 : String mode = "";
322 4462 : String agent_name = "";
323 2231 : Bool apply = true;
324 4462 : String datacolumn = "DATA";
325 :
326 : // create a temporary vector of agents
327 4462 : std::vector<Record> listOfAgents;
328 :
329 :
330 2231 : if(agent_params.empty()){
331 : os << LogIO::SEVERE << "No agent's record has been provided."
332 0 : << LogIO::POST;
333 0 : return false;
334 : }
335 :
336 : // Use the given Record of parameters
337 2231 : agentParams_p = agent_params;
338 :
339 2231 : if (! agentParams_p.isDefined("mode")) {
340 : os << LogIO::WARN << "No mode has been provided"
341 0 : << LogIO::POST;
342 0 : return false;
343 : }
344 :
345 : // Get mode
346 2231 : agentParams_p.get("mode", mode);
347 :
348 : // Validate mode against known modes
349 2231 : if (! isModeValid(mode)){
350 : os << LogIO::WARN << "Mode "<< mode << " is not valid or doesn't exist"
351 7 : << LogIO::POST;
352 7 : return false;
353 : }
354 :
355 : // Name for the logging output
356 2224 : if (! agentParams_p.isDefined("agentname")){
357 248 : agent_name = mode;
358 248 : agent_name.capitalize();
359 248 : agentParams_p.define("agentname", agent_name);
360 : }
361 :
362 2224 : agentParams_p.get("agentname", agent_name);
363 :
364 2224 : os << LogIO::DEBUG1 << "Parsing parameters for agent: "<<mode<< LogIO::POST;
365 :
366 : // Validate datacolumn
367 4421 : if (mode.compare("tfcrop") == 0 or mode.compare("clip") == 0 or mode.compare("rflag") == 0
368 4421 : or mode.compare("antint") == 0 or mode.compare("display") == 0){
369 :
370 213 : if (agentParams_p.isDefined("datacolumn"))
371 158 : agentParams_p.get("datacolumn", datacolumn);
372 :
373 213 : os << LogIO::NORMAL3 << "Validating data column "<<datacolumn<<" based on input type"<< LogIO::POST;
374 :
375 213 : if(!validateDataColumn(datacolumn)){
376 6 : os << LogIO::WARN<< "Cannot parse agent parameters "<< LogIO::POST;
377 6 : return false;
378 : }
379 :
380 207 : os << LogIO::DEBUG1 << "Will use data column "<< datacolumn << LogIO::POST;
381 207 : agentParams_p.define("datacolumn", datacolumn);
382 : }
383 :
384 : // Enforce a default value for the apply parameter
385 2218 : if (! agentParams_p.isDefined("apply")){
386 0 : agentParams_p.define("apply", apply);
387 : }
388 :
389 : // If there is a tfcrop, extend or rflag agent in the list,
390 : // get the maximum value of ntime and the combinescans parameter
391 2218 : if (mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or mode.compare("rflag") == 0) {
392 : Double ntime;
393 109 : if (agentParams_p.isDefined("ntime")){
394 68 : agentParams_p.get("ntime", ntime);
395 68 : getMax(ntime);
396 : }
397 :
398 : // Get the combinescans parameter. If any of them is true,
399 : // it will be true for the whole list
400 109 : Bool combine = false;
401 109 : if (agentParams_p.isDefined("combinescans"))
402 93 : agentParams_p.get("combinescans", combine);
403 :
404 109 : combinescans_p = combinescans_p || combine;
405 :
406 : os << LogIO::DEBUG2 << "max ntime="<<max_p<<" and combinescans="<<
407 109 : combinescans_p << LogIO::POST;
408 :
409 : }
410 :
411 : // Activate async i/o if tfcrop/clip/rflag is present
412 2218 : if (mode.compare("tfcrop") == 0 or mode.compare("clip") == 0 or mode.compare("rflag") == 0)
413 : {
414 195 : fdh_p->enableAsyncIO(true);
415 : }
416 :
417 : // Default for all modes
418 2218 : String correlation = "";
419 :
420 : // Make correlation always uppercase
421 2218 : if (agentParams_p.isDefined("correlation")) {
422 658 : agentParams_p.get("correlation", correlation);
423 658 : correlation.upcase();
424 658 : agentParams_p.define("correlation", correlation);
425 : }
426 :
427 :
428 : // Create one agent for each polarization
429 2218 : if (mode.compare("tfcrop") == 0) {
430 :
431 27 : if (not agentParams_p.isDefined("correlation")) {
432 : // Default for tfcrop
433 20 : correlation = "ABS_ALL";
434 20 : if (datacolumn.compare("FLOAT_DATA") == 0)
435 1 : correlation = "REAL_ALL";
436 20 : agentParams_p.define("correlation", correlation);
437 : }
438 : if (dbg){
439 : cout << "mode="<<mode<<" correlation="<<correlation<<endl;
440 : }
441 :
442 : // Is the expression polarization an ALL?
443 27 : if (isExpressionPolarizationAll(correlation)) {
444 :
445 : // Get the complex unitary function (ABS, NORM, REAL, IMAG, ARG)
446 42 : String function = getExpressionFunction(correlation);
447 :
448 : // Get all the polarizations in the MS
449 21 : std::vector<String> *allpol = fdh_p->corrProducts_p;
450 :
451 101 : for (size_t i=0; i < allpol->size(); i++){
452 : // compose the full expression
453 160 : String func = function;
454 160 : String pol = allpol->at(i);
455 160 : String exp = func.append("_");
456 80 : exp = func.append(pol);
457 :
458 : // Save the record to a list of agents
459 80 : agentParams_p.define("correlation", exp);
460 160 : String name = agent_name;
461 80 : name = name.append("_");
462 80 : name = name.append(pol);
463 80 : agentParams_p.define("agentname", name);
464 80 : listOfAgents.push_back(agentParams_p);
465 : }
466 : }
467 : }
468 :
469 : // Check if extend agent should be added to the list later
470 2218 : if (mode.compare("tfcrop") == 0 or mode.compare("rflag") == 0){
471 80 : if (agentParams_p.isDefined("extendflags"))
472 76 : agentParams_p.get("extendflags", extendflags_p);
473 : }
474 :
475 2218 : if (listOfAgents.size() > 0) {
476 : // add the agent(s) to the list
477 101 : for (size_t i=0; i < listOfAgents.size(); i++) {
478 80 : agents_config_list_p.push_back(listOfAgents.at(i));
479 : }
480 : }
481 : else {
482 2197 : agents_config_list_p.push_back(agentParams_p);
483 : }
484 :
485 : if (dbg){
486 : for (size_t i=0; i < agents_config_list_p.size(); i++) {
487 : ostringstream os;
488 : agents_config_list_p.at(i).print(os);
489 : String str(os.str());
490 : cout << str << endl;
491 : }
492 : }
493 :
494 : // Keep a copy of the vector of Agents parameters
495 2218 : agents_config_list_copy_p = agents_config_list_p;
496 :
497 : // Parse the parameters of the extend agent
498 2298 : if ((mode.compare("tfcrop") == 0 or mode.compare("rflag") == 0) and
499 80 : extendflags_p==true){
500 :
501 : os << LogIO::NORMAL << "Will extend the flags after " << mode <<
502 21 : "; extendflags=true" << LogIO::POST;
503 :
504 21 : extendflags_p = false;
505 21 : parseExtendParameters(
506 : "", // field
507 : "", // spw
508 : "", // array
509 : "", // feed
510 : "", // scan
511 : "", // antenna
512 : "", // uvrange
513 : "", // timerange
514 : "", // correlation
515 : "", // intent
516 : "", // observation
517 : max_p, // ntime
518 21 : combinescans_p, // combinescans
519 : true, // extendpols
520 : 50.0, // growtime
521 : 80.0 // growfreq
522 : // false, // growaround
523 : // false, // flagneartime
524 : // false, // flagnearfreq
525 : // true // apply
526 : );
527 : }
528 :
529 2218 : return true;
530 : }
531 :
532 : /*
533 : * Build a string with a list of agents, for logging purposes. Uses the names they have in
534 : * the mode parameter, for example display, rflag, summary, etc. rather than the more
535 : * internal names of the agent classes. When the list of agents is very long, a short
536 : * cut of the list is produced.
537 : *
538 : * @param agents list of agents (items as casacore records, with a "mode" field)
539 : * @return a comma separated list of agents
540 : */
541 2512 : std::string buildListAgentNames(const std::vector<Record> &agents) {
542 2512 : const size_t MAX_LIST_PRINT = 10;
543 :
544 2512 : std::string all;
545 2512 : if (!agents.empty()) {
546 2511 : auto elem = agents.cbegin();
547 5022 : String mode;
548 2511 : elem->get("mode", mode);
549 2511 : all += mode;
550 2511 : size_t count = 1;
551 :
552 3084 : while (++elem != agents.cend() && ++count <= MAX_LIST_PRINT) {
553 573 : elem->get("mode", mode);
554 573 : all += ", " + mode;
555 : }
556 2511 : if (agents.size() > MAX_LIST_PRINT) {
557 44 : all += ", ... (" + std::to_string(agents.size()-MAX_LIST_PRINT) +
558 22 : " more, not shown)";
559 : }
560 : }
561 2512 : return all;
562 : }
563 :
564 : /*
565 : * For a mode (agent) configuration check:
566 : * if using time or channel averaging in auto-flagging modes (clip, tfcrop, rflag),
567 : * the agents can only be used in combination with a subset of other agents. These
568 : * 'whitelisted' agents are: + display + extend + antint.
569 : * See CAS-12294 for discussions.
570 : *
571 : * Additionally, check that timeavg and channelavg are never set for
572 : * any of the non-autof-lagging methods. This is in principle never
573 : * possible via the task interface but the list mode parser is weaks
574 : * and accepts such misconfigurations.
575 :
576 : *
577 : * @param mode flagging mode, using flagdata naming convention
578 : * @param agent_rec record with the agent configuration
579 : * @param anyNotAvg non-empty name if any avg-disallowed agent is in the list
580 : *
581 : * @throws AipsError if there is any configuration error
582 : */
583 2274 : void AgentFlagger::checkAveragingConfig(const std::string &mode, const Record &agent_rec,
584 : const std::string &anyNotAvg) {
585 2274 : Bool tavg = false;
586 2274 : int exists = agent_rec.fieldNumber ("timeavg");
587 2274 : if (exists >= 0) {
588 213 : agent_rec.get("timeavg", tavg);
589 : }
590 :
591 2274 : Bool cavg = false;
592 2274 : exists = agent_rec.fieldNumber ("channelavg");
593 2274 : if (exists >= 0) {
594 214 : agent_rec.get("channelavg", cavg);
595 : }
596 :
597 2274 : if ("clip" == mode or "rflag" == mode or "tfcrop" == mode) {
598 253 : if ((tavg or cavg) and not anyNotAvg.empty()) {
599 16 : ostringstream msg;
600 : msg << "Cannot use " << mode << " mode with timeavg=True or channelavg=True "
601 8 : "and additional modes other than extend, display, and antint. timeavg="
602 8 : << tavg << ", channelavg=" << cavg << ", and the following mode has been "
603 8 : "set up: " << anyNotAvg + ". Refusing to accept this configuration.";
604 8 : throw AipsError(msg);
605 : }
606 : } else {
607 2021 : if ((tavg or cavg)) {
608 28 : ostringstream msg;
609 14 : msg << "Cannot use timeavg or channelavg in mode " << mode << ". timeavg="
610 14 : << tavg << ", channelavg=" << cavg << ". These averaging "
611 : "options are only allowed for auto-flagging methods. Refusing this "
612 14 : "configuration.";
613 14 : throw AipsError(msg);
614 : }
615 : }
616 2252 : }
617 :
618 : /*
619 : * Check: the auto-flagging agents (clip, tfcrop, rflag) can only be combined with some
620 : * other selected agents: display + extend + antint.
621 : * See CAS-12294 for discussions.
622 : *
623 : * @param configs list of agent configurations
624 : *
625 : * @throws AipsError if there is any configuration error
626 : */
627 1267 : std::string AgentFlagger::searchAnyAgentsNotAvg(const std::vector<Record> &configs)
628 : {
629 12670 : const std::vector<std::string> whitelist = { "clip", "rflag", "tfcrop", "extend", "display", "antint" };
630 :
631 1546 : for (const auto &agent : configs) {
632 1363 : String recname;
633 1363 : agent.get("mode", recname);
634 1363 : std::string name = recname;
635 1363 : const auto &found = std::find(whitelist.cbegin(), whitelist.cend(), name);
636 1363 : if (found == whitelist.cend()) {
637 1084 : return name;
638 : }
639 : }
640 :
641 183 : return "";
642 : }
643 :
644 : // ---------------------------------------------------------------------
645 : // AgentFlagger::initAgents
646 : // Initialize the Agents
647 : // Call parseAgentParameters and selectData first
648 : // ---------------------------------------------------------------------
649 : bool
650 1267 : AgentFlagger::initAgents()
651 : {
652 :
653 3801 : LogIO os(LogOrigin("AgentFlagger",__FUNCTION__));
654 :
655 1267 : if (! fdh_p){
656 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
657 0 : return false;
658 : }
659 :
660 1267 : if (dataselection_p.empty()){
661 0 : os << LogIO::SEVERE << "There is no MS Selection available." << LogIO::POST;
662 0 : return false;
663 : }
664 :
665 1267 : if (agents_config_list_p.empty()){
666 0 : return false;
667 : }
668 :
669 :
670 : os << LogIO::NORMAL << "There are initially "<< agents_config_list_p.size()<<
671 : " agents in the list. Agents: "
672 1267 : << buildListAgentNames(agents_config_list_p) << LogIO::POST;
673 :
674 :
675 : // Check once here, to re-use then with every agent in the list
676 2534 : const auto &anyAgentNotAvg = searchAnyAgentsNotAvg(agents_config_list_p);
677 :
678 1267 : size_t list_size = agents_config_list_p.size();
679 :
680 : // Send the logging of the re-applying agents to the debug
681 : // as we are only interested in seeing the unapply action (flagcmd)
682 1267 : Bool retstate = true;
683 : // Just to log this info
684 1289 : std::vector<Record> agents_config_list_filtered;
685 : // To set some parameters only once
686 1267 : bool itersetDone = false;
687 1267 : bool timesetDone = false;
688 : // Loop through the vector of agents
689 3519 : for (size_t i=0; i < list_size; i++) {
690 :
691 : // Get agent record
692 2296 : Record agent_rec = agents_config_list_p[i];
693 : if (dbg){
694 : os<< LogIO::NORMAL<< "Record["<<i<<"].nfields()="<<agent_rec.nfields()<<LogIO::POST;
695 : ostringstream os;
696 : agent_rec.print(os);
697 : String str(os.str());
698 : cout << str << endl;
699 :
700 : }
701 :
702 : // Get the mode
703 2296 : String mode;
704 2274 : agent_rec.get("mode", mode);
705 :
706 : /*
707 : * Special constraints for some agents
708 : */
709 :
710 : // constraints that will produce exceptions if not met
711 2274 : checkAveragingConfig(mode, agent_rec, anyAgentNotAvg);
712 :
713 : // If quack mode with quackincrement = true, skip it
714 2252 : if (mode.compare("quack") == 0 and i > 0 and list_size > 1){
715 2 : Bool quackincrement = false;
716 2 : int exists = agent_rec.fieldNumber ("quackincrement");
717 2 : if (exists >= 0) agent_rec.get("quackincrement", quackincrement);
718 :
719 2 : if (quackincrement){
720 1 : os << LogIO::WARN << "Cannot have quackincrement=True in list mode. Agent quack will be ignored!" << LogIO::POST;
721 1 : continue;
722 : }
723 : }
724 :
725 : // Set the new time interval only once
726 4384 : if (!timesetDone and (mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or
727 2133 : mode.compare("rflag") == 0)) {
728 87 : fdh_p->setTimeInterval(max_p);
729 87 : timesetDone = true;
730 : }
731 :
732 : // Change the new iteration approach only once
733 4384 : if (!itersetDone and (mode.compare("tfcrop") == 0 or mode.compare("extend") == 0
734 2133 : or mode.compare("rflag") == 0 or mode.compare("display") == 0)) {
735 87 : if (combinescans_p)
736 0 : fdh_p->setIterationApproach(FlagDataHandler::COMBINE_SCANS_MAP_ANTENNA_PAIRS_ONLY);
737 : else
738 87 : fdh_p->setIterationApproach(FlagDataHandler::COMPLETE_SCAN_MAP_ANTENNA_PAIRS_ONLY);
739 :
740 87 : itersetDone = true;
741 : }
742 :
743 : // Agent's name
744 2251 : String agent_name;
745 2251 : agent_rec.get("agentname",agent_name);
746 :
747 2251 : FlagAgentBase *fa = 0;
748 :
749 : try
750 : {
751 : // CAS-3943 (flagdata seg-faults when non-existent data column is to be read)
752 2251 : Bool createAgent = true;
753 2165 : if (((mode.compare("tfcrop") == 0 or mode.compare("rflag") == 0
754 4118 : or mode.compare("clip") == 0 or mode.compare("display") == 0 or
755 2006 : mode.compare("antint") == 0))
756 4416 : and (agent_rec.fieldNumber ("datacolumn") >= 0))
757 : {
758 510 : String datacolumn;
759 255 : agent_rec.get("datacolumn", datacolumn);
760 255 : datacolumn.upcase();
761 :
762 255 : if ((datacolumn.compare("CORRECTED") == 0) or (datacolumn.compare("RESIDUAL") == 0))
763 : {
764 8 : createAgent = fdh_p->checkIfColumnExists(MS::columnName(MS::CORRECTED_DATA));
765 8 : if (!createAgent)
766 : {
767 0 : os << LogIO::WARN << "Agent " << agent_name << " cannot be created, necessary CORRECTED_DATA column is not available" << LogIO::POST;
768 0 : os << LogIO::WARN << "If wishing to flag residuals, use datacolumn=\'RESIDUAL_DATA\', which will calculate DATA-MODEL" << LogIO::POST;
769 : }
770 : }
771 : }
772 :
773 : // Create this agent if the necessary columns are available
774 2251 : if (createAgent)
775 : {
776 2255 : FlagAgentBase *tfa = FlagAgentBase::create(fdh_p, agent_rec);
777 2247 : fa = tfa;
778 : }
779 : }
780 8 : catch(AipsError x)
781 : {
782 4 : fa = 0;
783 : // Send out a useful message, and stop adding agents to the list.
784 : // All valid agents before the problematic one, will remain in agents_list_p
785 : // A subsequent call to initAgents() will add to the list.
786 8 : ostringstream oss;
787 4 : agent_rec.print(oss);
788 12 : String recstr(oss.str());
789 27 : while(recstr.contains('\n'))
790 : {
791 23 : recstr = recstr.replace(recstr.index('\n'),1,", ");
792 : }
793 :
794 : os << LogIO::WARN << "Cannot create agent "<< agent_name << ": "
795 4 : << x.getMesg() << LogIO::POST;
796 4 : os << LogIO::DEBUG1<< "Input parameters : " << recstr << LogIO::POST;
797 4 : continue;
798 :
799 : }
800 :
801 2247 : if (fa == 0){
802 0 : os << LogIO::WARN << "Agent "<< agent_name<< " is NULL. Skipping it."<<LogIO::POST;
803 0 : continue;
804 : }
805 :
806 : /*
807 : // Get the last summary agent to list the results back to the task
808 : if (mode.compare("summary") == 0) {
809 : summaryAgent_p = (FlagAgentSummary *) fa;
810 : }
811 : */
812 :
813 : // Get the display agent.
814 2247 : if (mode.compare("display") == 0){
815 0 : displayAgent_p = (FlagAgentDisplay *) fa;
816 : }
817 :
818 : // Add the agent to the FlagAgentList
819 2247 : agents_list_p.push_back(fa);
820 2247 : agents_config_list_filtered.push_back(agent_rec);
821 : }
822 :
823 : os << LogIO::NORMAL << "There are " << agents_list_p.size()
824 : << " valid agents in the list. Agents: "
825 1245 : << buildListAgentNames(agents_config_list_filtered) << LogIO::POST;
826 :
827 : // Clear the list so that this method cannot be called twice
828 1245 : agents_config_list_p.clear();
829 :
830 1245 : if (agents_list_p.size() < 1)
831 1 : retstate = false;
832 :
833 1245 : return retstate;
834 : }
835 :
836 :
837 : // ---------------------------------------------------------------------
838 : // AgentFlagger::run
839 : // Run the agents
840 : // It assumes that initAgents has been called first
841 : // ---------------------------------------------------------------------
842 : Record
843 1245 : AgentFlagger::run(Bool writeflags, Bool sequential)
844 : {
845 :
846 3735 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
847 :
848 1245 : if (! fdh_p){
849 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
850 0 : return Record();
851 : }
852 :
853 1245 : if (agents_list_p.empty()) {
854 1 : os << LogIO::SEVERE << "There is no agent to run in list"<< LogIO::POST;
855 1 : return Record();
856 : }
857 :
858 :
859 : // Use the maximum ntime of the list
860 1244 : os << LogIO::DEBUG2 << "ntime for all agents will be "<< max_p << LogIO::POST;
861 1244 : os << LogIO::DEBUG2 << "combinescans for all agents will be "<< combinescans_p << LogIO::POST;
862 :
863 : // Report to return
864 3732 : FlagReport combinedReport;
865 :
866 :
867 : // Generate the iterators
868 : // It will iterate through the data to evaluate the necessary memory
869 : // and get the START and STOP values of the scans for the quack agent
870 1244 : fdh_p->generateIterator();
871 :
872 1244 : agents_list_p.start();
873 : os << LogIO::DEBUGGING << "Generated iterators. Start loop over chunks / agents. "
874 1244 : "Size of agents list is " << agents_list_p.size() << LogIO::POST;
875 :
876 : // iterate over chunks
877 22859 : while (fdh_p->nextChunk())
878 : {
879 : // iterate over visBuffers
880 622328 : while (fdh_p->nextBuffer())
881 : {
882 :
883 : // Apply or unapply the flags, in sequential or in parallel.
884 : // By default it is set to true, which will preserve the order
885 : // in which the agents are added to the list.
886 600713 : agents_list_p.apply(sequential);
887 :
888 : // Flush flags to MS
889 600713 : if (writeflags) {
890 494412 : fdh_p->flushFlags();
891 : }
892 : }
893 :
894 : // Print the chunk summary stats
895 21615 : if (fdh_p->summarySignal())
896 : {
897 15778 : agents_list_p.chunkSummary();
898 15778 : fdh_p->progressCounts_p = 0;
899 : }
900 : }
901 :
902 : // Print the MS summary stats
903 1244 : agents_list_p.tableSummary();
904 1244 : if (writeflags)
905 830 : os << LogIO::NORMAL << "=> " << "Writing flags to the MS" << LogIO::POST;
906 :
907 1244 : agents_list_p.terminate();
908 1244 : agents_list_p.join();
909 :
910 : // Gather the display reports from all agents
911 1244 : combinedReport = agents_list_p.gatherReports();
912 :
913 : // Send reports to display agent
914 1244 : if (displayAgent_p)
915 0 : displayAgent_p->displayReports(combinedReport);
916 :
917 1244 : agents_list_p.clear();
918 :
919 1244 : return combinedReport;
920 : }
921 :
922 : // ---------------------------------------------------------------------
923 : // AgentFlagger::isExpressionPolarizationAll
924 : // Returns true if expression contains a polarization ALL
925 : //
926 : // ---------------------------------------------------------------------
927 : bool
928 27 : AgentFlagger::isExpressionPolarizationAll(String correlation)
929 : {
930 :
931 27 : if (correlation.find("ALL") == string::npos){
932 6 : return false;
933 : }
934 :
935 21 : return true;
936 : }
937 :
938 : // ---------------------------------------------------------------------
939 : // AgentFlagger::getExpressionFunction
940 : // Get the unitary function of a polarization expression
941 : // returns a String with the function name
942 : //
943 : // ---------------------------------------------------------------------
944 : String
945 21 : AgentFlagger::getExpressionFunction(String expression)
946 : {
947 :
948 42 : String func;
949 :
950 : // Parse complex unitary function
951 21 : if (expression.find("REAL") != string::npos)
952 : {
953 1 : func = "REAL";
954 : }
955 20 : else if (expression.find("IMAG") != string::npos)
956 : {
957 0 : func = "IMAG";
958 : }
959 20 : else if (expression.find("ARG") != string::npos)
960 : {
961 0 : func = "ARG";
962 : }
963 20 : else if (expression.find("ABS") != string::npos)
964 : {
965 20 : func = "ABS";
966 : }
967 0 : else if (expression.find("NORM") != string::npos)
968 : {
969 0 : func = "NORM";
970 : }
971 : else
972 : {
973 0 : return "";
974 : }
975 :
976 21 : return func;
977 : }
978 :
979 : // ---------------------------------------------------------------------
980 : // AgentFlagger::getMax
981 : // Get the maximum between two values and
982 : // assign it to the max_p class member.
983 : // ---------------------------------------------------------------------
984 : void
985 68 : AgentFlagger::getMax(Double value)
986 : {
987 68 : if (value > max_p)
988 0 : max_p = value;
989 :
990 68 : return;
991 :
992 : }
993 :
994 :
995 : // ---------------------------------------------------------------------
996 : // AgentFlagger::getFlagVersionList
997 : // Get the flag versions list from the file FLAG_VERSION_LIST in the
998 : // MS directory
999 : //
1000 : // ---------------------------------------------------------------------
1001 : bool
1002 315 : AgentFlagger::getFlagVersionList(Vector<String> &verlist)
1003 : {
1004 :
1005 945 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE));
1006 :
1007 : try
1008 : {
1009 :
1010 315 : if (! fdh_p){
1011 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
1012 0 : return false;
1013 : }
1014 :
1015 315 : verlist.resize(0);
1016 : Int num;
1017 :
1018 945 : FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW");
1019 630 : Vector<String> vlist = fv.getVersionList();
1020 :
1021 315 : num = verlist.nelements();
1022 315 : verlist.resize( num + vlist.nelements() + 1, true );
1023 315 : verlist[num] = String("\nMS : ") + fdh_p->getTableName() + String("\n");
1024 :
1025 808 : for(Int j=0; j<(Int)vlist.nelements(); j++)
1026 493 : verlist[num+j+1] = vlist[j];
1027 : }
1028 0 : catch (AipsError x)
1029 : {
1030 0 : os << LogIO::SEVERE << "Could not get Flag Version List : " << x.getMesg() << LogIO::POST;
1031 0 : return false;
1032 : }
1033 :
1034 315 : return true;
1035 : }
1036 :
1037 :
1038 : // ---------------------------------------------------------------------
1039 : // AgentFlagger::printFlagSelection
1040 : // Print the flag versions list
1041 : //
1042 : // ---------------------------------------------------------------------
1043 : bool
1044 0 : AgentFlagger::printFlagSelections()
1045 : {
1046 :
1047 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE));
1048 :
1049 : // Use the copy of the agent records list because the original
1050 : // was deallocated in the init() method.
1051 0 : if (! agents_config_list_copy_p.empty())
1052 : {
1053 0 : os << "Current list of agents : " << LogIO::POST;
1054 :
1055 : // TODO: loop through list
1056 : // Duplicate the vector... ???
1057 0 : for (size_t i=0; i < agents_config_list_copy_p.size(); i++) {
1058 0 : ostringstream out;
1059 0 : Record agent_rec;
1060 0 : agent_rec = agents_config_list_copy_p.at(i);
1061 0 : agent_rec.print(out);
1062 0 : os << out.str() << LogIO::POST;
1063 : }
1064 : if (dbg)
1065 : cout << "Size of original list " << agents_config_list_copy_p.size() << endl;
1066 :
1067 : }
1068 0 : else os << " No current agents " << LogIO::POST;
1069 :
1070 0 : return true;
1071 : }
1072 :
1073 :
1074 : // ---------------------------------------------------------------------
1075 : // AgentFlagger::saveFlagVersion
1076 : // Save the flag version
1077 : //
1078 : // ---------------------------------------------------------------------
1079 : bool
1080 346 : AgentFlagger::saveFlagVersion(String versionname, String comment, String merge)
1081 : {
1082 1038 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE));
1083 :
1084 346 : if (! fdh_p){
1085 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
1086 0 : return false;
1087 : }
1088 :
1089 692 : FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW");
1090 346 : fv.saveFlagVersion(versionname, comment, merge);
1091 :
1092 346 : return true;
1093 : }
1094 :
1095 :
1096 : // ---------------------------------------------------------------------
1097 : // AgentFlagger::restoreFlagVersion
1098 : // Restore the flag version
1099 : //
1100 : // ---------------------------------------------------------------------
1101 : bool
1102 7 : AgentFlagger::restoreFlagVersion(Vector<String> versionname, String merge)
1103 : {
1104 21 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE));
1105 :
1106 7 : if (! fdh_p){
1107 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
1108 0 : return false;
1109 : }
1110 :
1111 : try
1112 : {
1113 21 : FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW");
1114 14 : for(Int j=0;j<(Int)versionname.nelements();j++)
1115 7 : fv.restoreFlagVersion(versionname[j], merge);
1116 : }
1117 0 : catch (AipsError x)
1118 : {
1119 0 : os << LogIO::SEVERE << "Could not restore Flag Version : " << x.getMesg() << LogIO::POST;
1120 0 : throw AipsError(x);
1121 : // return false;
1122 : }
1123 7 : return true;
1124 : }
1125 :
1126 : // ---------------------------------------------------------------------
1127 : // AgentFlagger::deleteFlagVersion
1128 : // Delete the flag version
1129 : //
1130 : // ---------------------------------------------------------------------
1131 : bool
1132 0 : AgentFlagger::deleteFlagVersion(Vector<String> versionname)
1133 : {
1134 :
1135 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE));
1136 :
1137 0 : if (! fdh_p){
1138 0 : os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST;
1139 0 : return false;
1140 : }
1141 :
1142 : try
1143 : {
1144 0 : FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW");
1145 0 : for(Int j=0;j<(Int)versionname.nelements();j++)
1146 0 : fv.deleteFlagVersion(versionname[j]);
1147 : }
1148 0 : catch (AipsError x)
1149 : {
1150 0 : os << LogIO::SEVERE << "Could not delete Flag Version : " << x.getMesg() << LogIO::POST;
1151 0 : return false;
1152 : }
1153 0 : return true;
1154 : }
1155 :
1156 : // ---------------------------------------------------------------------
1157 : // AgentFlagger::isModeValid
1158 : // Check if mode is valid.
1159 : // Return false if not in the list
1160 : // ---------------------------------------------------------------------
1161 : bool
1162 2231 : AgentFlagger::isModeValid(String mode)
1163 : {
1164 : bool ret;
1165 :
1166 : // The validation depends on the type of input
1167 2231 : if (isMS_p) {
1168 3790 : if (mode.compare("manual") == 0 or mode.compare("clip") == 0 or
1169 2468 : mode.compare("quack") == 0 or mode.compare("shadow") == 0 or
1170 1596 : mode.compare("elevation") == 0 or
1171 1561 : mode.compare("tfcrop") == 0 or
1172 2200 : mode.compare("extend") == 0 or mode.compare("rflag") == 0 or
1173 1368 : mode.compare("antint") == 0 or
1174 1018 : mode.compare("unflag") == 0 or mode.compare("summary") == 0
1175 2946 : or mode.compare("display") == 0) {
1176 :
1177 2015 : ret = true;
1178 : }
1179 : else
1180 0 : ret = false;
1181 : }
1182 : else {
1183 544 : if (mode.compare("manual") == 0 or mode.compare("clip") == 0 or
1184 439 : mode.compare("quack") == 0 or mode.compare("tfcrop") == 0 or
1185 428 : mode.compare("extend") == 0 or mode.compare("rflag") == 0 or
1186 282 : mode.compare("antint") == 0 or
1187 211 : mode.compare("unflag") == 0 or mode.compare("summary") == 0
1188 397 : or mode.compare("display") == 0) {
1189 :
1190 209 : ret = true;
1191 : }
1192 : else
1193 7 : ret = false;
1194 :
1195 : }
1196 :
1197 2231 : return ret;
1198 : }
1199 :
1200 :
1201 : Bool
1202 213 : AgentFlagger::validateDataColumn(String datacol)
1203 : {
1204 :
1205 426 : String datacolumn = "";
1206 213 : Bool ret = false;
1207 213 : datacol.upcase();
1208 :
1209 426 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1210 :
1211 : // The validation depends on the type of input
1212 213 : if (isMS_p) {
1213 176 : if (datacol.compare("DATA") == 0)
1214 142 : datacolumn = "DATA";
1215 34 : else if(datacol.compare("CORRECTED") == 0)
1216 1 : datacolumn = "CORRECTED_DATA";
1217 33 : else if(datacol.compare("MODEL") == 0)
1218 0 : datacolumn = "MODEL_DATA";
1219 33 : else if(datacol.compare("FLOAT_DATA") == 0)
1220 4 : datacolumn = "FLOAT_DATA";
1221 29 : else if(datacol.compare("WEIGHT_SPECTRUM") == 0)
1222 6 : datacolumn = "WEIGHT_SPECTRUM";
1223 23 : else if(datacol.compare("WEIGHT") == 0)
1224 5 : datacolumn = "WEIGHT";
1225 18 : else if(datacol.compare("RESIDUAL") == 0)
1226 8 : datacolumn = "CORRECTED_DATA";
1227 10 : else if(datacol.compare("RESIDUAL_DATA") == 0)
1228 10 : datacolumn = "DATA";
1229 : else
1230 0 : datacolumn = "";
1231 : }
1232 : else {
1233 : // cal tables
1234 43 : if (datacol.compare("FPARAM") == 0 or (datacol.compare("CPARAM") == 0) or
1235 6 : (datacol.compare("SNR") == 0))
1236 35 : datacolumn = datacol;
1237 : }
1238 :
1239 : // Check if main tables exist
1240 213 : if (fdh_p->checkIfColumnExists(datacolumn)) {
1241 210 : ret = true;
1242 : }
1243 : else {
1244 3 : os << LogIO::WARN << "Data column "<< datacolumn << " does not exist in input data" << LogIO::POST;
1245 : }
1246 :
1247 : // Check if other columns were requested also
1248 : // RESIDUAL is calculated later from CORRECTED-MODEL
1249 : // RESIDUAL_DATA is calculated later from DATA-MODEL
1250 :
1251 213 : if (datacol.compare("RESIDUAL") == 0 or datacol.compare("RESIDUAL_DATA")==0){
1252 18 : ret = false;
1253 : // check if MODEL _DATA or virtual MODEL_DATA exist
1254 18 : if (fdh_p->checkIfColumnExists("MODEL_DATA") or fdh_p->checkIfSourceModelColumnExists()){
1255 14 : ret = true;
1256 : }
1257 : else {
1258 4 : os << LogIO::WARN << "Data column MODEL_DATA or virtual MODEL_DATA does not exist in input data" << LogIO::POST;
1259 : }
1260 :
1261 : }
1262 :
1263 426 : return ret;
1264 : }
1265 :
1266 :
1267 : // ---------------------------------------------------------------------
1268 : // AgentFlagger::parseManualParameters
1269 : // Parse data selection parameters and specific manual parameters
1270 : //
1271 : // ---------------------------------------------------------------------
1272 : bool
1273 0 : AgentFlagger::parseManualParameters(String field, String spw, String array,
1274 : String feed, String scan, String antenna,
1275 : String uvrange, String timerange, String correlation,
1276 : String intent, String observation, Bool autocorr,
1277 : Bool apply)
1278 : {
1279 :
1280 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1281 :
1282 : // Default values for some parameters
1283 0 : String mode = "manual";
1284 0 : String agent_name = "Manual";
1285 :
1286 : // Create a record with the parameters
1287 0 : Record agent_record = Record();
1288 :
1289 0 : agent_record.define("mode", mode);
1290 0 : agent_record.define("spw", spw);
1291 0 : agent_record.define("scan", scan);
1292 0 : agent_record.define("field", field);
1293 0 : agent_record.define("antenna", antenna);
1294 0 : agent_record.define("timerange", timerange);
1295 0 : agent_record.define("correlation", correlation);
1296 0 : agent_record.define("intent", intent);
1297 0 : agent_record.define("feed", feed);
1298 0 : agent_record.define("array", array);
1299 0 : agent_record.define("uvrange", uvrange);
1300 0 : agent_record.define("observation", observation);
1301 0 : agent_record.define("apply", apply);
1302 0 : agent_record.define("agentname", agent_name);
1303 :
1304 0 : agent_record.define("autocorr", autocorr);
1305 :
1306 : // Call the main method
1307 0 : parseAgentParameters(agent_record);
1308 :
1309 0 : return true;
1310 : }
1311 :
1312 :
1313 : // ---------------------------------------------------------------------
1314 : // AgentFlagger::parseClipParameters
1315 : // Parse data selection parameters and specific clip parameters
1316 : //
1317 : // ---------------------------------------------------------------------
1318 : bool
1319 0 : AgentFlagger::parseClipParameters(String field, String spw, String array, String feed, String scan,
1320 : String antenna, String uvrange, String timerange, String correlation,
1321 : String intent, String observation, String datacolumn,
1322 : Vector<Double> clipminmax, Bool clipoutside, Bool channelavg, casac::variant chanbin, Bool timeavg,
1323 : String timebin, Bool clipzeros, Bool apply)
1324 : {
1325 :
1326 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1327 :
1328 : // Default values for some parameters
1329 0 : String mode = "clip";
1330 0 : String agent_name = "Clip";
1331 :
1332 : // Create a record with the parameters
1333 0 : Record agent_record = Record();
1334 :
1335 0 : agent_record.define("mode", mode);
1336 0 : agent_record.define("spw", spw);
1337 0 : agent_record.define("scan", scan);
1338 0 : agent_record.define("field", field);
1339 0 : agent_record.define("antenna", antenna);
1340 0 : agent_record.define("timerange", timerange);
1341 0 : agent_record.define("correlation", correlation);
1342 0 : agent_record.define("intent", intent);
1343 0 : agent_record.define("feed", feed);
1344 0 : agent_record.define("array", array);
1345 0 : agent_record.define("uvrange", uvrange);
1346 0 : agent_record.define("observation", observation);
1347 0 : agent_record.define("apply", apply);
1348 0 : agent_record.define("agentname", agent_name);
1349 :
1350 0 : agent_record.define("datacolumn", datacolumn);
1351 0 : agent_record.define("clipminmax", clipminmax);
1352 0 : agent_record.define("clipoutside", clipoutside);
1353 0 : agent_record.define("channelavg", channelavg);
1354 :
1355 0 : if (chanbin.type() == casac::variant::INT)
1356 : {
1357 : /*** need range check ***/
1358 0 : agent_record.define("chanbin", (int) chanbin.toInt());
1359 : }
1360 0 : else if (chanbin.type() == casac::variant::INTVEC)
1361 : {
1362 : /*** need range check ***/
1363 0 : auto cb = chanbin.toIntVec( );
1364 0 : std::vector<int> vec;
1365 0 : std::for_each( cb.begin( ), cb.end( ), [&](long x){vec.push_back((int)x);} );
1366 0 : agent_record.define("chanbin", Array<Int>(vec));
1367 : }
1368 :
1369 0 : agent_record.define("timeavg", timeavg);
1370 0 : agent_record.define("timebin", timebin);
1371 0 : agent_record.define("clipzeros", clipzeros);
1372 :
1373 :
1374 : // Call the main method
1375 0 : parseAgentParameters(agent_record);
1376 :
1377 0 : return true;
1378 :
1379 : }
1380 :
1381 : // ---------------------------------------------------------------------
1382 : // AgentFlagger::parseQuackParameters
1383 : // Parse data selection parameters and specific quack parameters
1384 : //
1385 : // ---------------------------------------------------------------------
1386 : bool
1387 0 : AgentFlagger::parseQuackParameters(String field, String spw, String array, String feed, String scan,
1388 : String antenna, String uvrange, String timerange, String correlation,
1389 : String intent, String observation, String quackmode, Double quackinterval,
1390 : Bool quackincrement, Bool apply)
1391 : {
1392 :
1393 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1394 :
1395 : // Default values for some parameters
1396 0 : String mode = "quack";
1397 0 : String agent_name = "Quack";
1398 :
1399 : // Create a record with the parameters
1400 0 : Record agent_record = Record();
1401 :
1402 0 : agent_record.define("mode", mode);
1403 0 : agent_record.define("spw", spw);
1404 0 : agent_record.define("scan", scan);
1405 0 : agent_record.define("field", field);
1406 0 : agent_record.define("antenna", antenna);
1407 0 : agent_record.define("timerange", timerange);
1408 0 : agent_record.define("correlation", correlation);
1409 0 : agent_record.define("intent", intent);
1410 0 : agent_record.define("feed", feed);
1411 0 : agent_record.define("array", array);
1412 0 : agent_record.define("uvrange", uvrange);
1413 0 : agent_record.define("observation", observation);
1414 0 : agent_record.define("apply", apply);
1415 0 : agent_record.define("agentname", agent_name);
1416 :
1417 0 : agent_record.define("quackmode", quackmode);
1418 0 : agent_record.define("quackinterval", quackinterval);
1419 0 : agent_record.define("quackincrement", quackincrement);
1420 :
1421 : // Call the main method
1422 0 : parseAgentParameters(agent_record);
1423 :
1424 0 : return true;
1425 :
1426 : }
1427 :
1428 : // ---------------------------------------------------------------------
1429 : // AgentFlagger::parseElevationParameters
1430 : // Parse data selection parameters and specific elevation parameters
1431 : //
1432 : // ---------------------------------------------------------------------
1433 : bool
1434 0 : AgentFlagger::parseElevationParameters(String field, String spw, String array, String feed, String scan,
1435 : String antenna, String uvrange, String timerange,String correlation,
1436 : String intent, String observation, Double lowerlimit, Double upperlimit, Bool apply)
1437 : {
1438 :
1439 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1440 :
1441 : // Default values for some parameters
1442 0 : String mode = "elevation";
1443 0 : String agent_name = "Elevation";
1444 :
1445 : // Create a record with the parameters
1446 0 : Record agent_record = Record();
1447 :
1448 0 : agent_record.define("mode", mode);
1449 0 : agent_record.define("spw", spw);
1450 0 : agent_record.define("scan", scan);
1451 0 : agent_record.define("field", field);
1452 0 : agent_record.define("antenna", antenna);
1453 0 : agent_record.define("timerange", timerange);
1454 0 : agent_record.define("correlation", correlation);
1455 0 : agent_record.define("intent", intent);
1456 0 : agent_record.define("feed", feed);
1457 0 : agent_record.define("array", array);
1458 0 : agent_record.define("uvrange", uvrange);
1459 0 : agent_record.define("observation", observation);
1460 0 : agent_record.define("apply", apply);
1461 0 : agent_record.define("agentname", agent_name);
1462 :
1463 0 : agent_record.define("lowerlimit", lowerlimit);
1464 0 : agent_record.define("upperlimit", upperlimit);
1465 :
1466 : // Call the main method
1467 0 : parseAgentParameters(agent_record);
1468 :
1469 0 : return true;
1470 :
1471 : }
1472 :
1473 : // ---------------------------------------------------------------------
1474 : // AgentFlagger::parseTfcropParameters
1475 : // Parse data selection parameters and specific tfcrop parameters
1476 : //
1477 : // ---------------------------------------------------------------------
1478 : bool
1479 0 : AgentFlagger::parseTfcropParameters(String field, String spw, String array, String feed, String scan,
1480 : String antenna, String uvrange, String timerange, String correlation,
1481 : String intent, String observation, Double ntime, Bool combinescans,
1482 : String datacolumn, Double timecutoff, Double freqcutoff, String timefit,
1483 : String freqfit, Int maxnpieces, String flagdimension, String usewindowstats, Int halfwin,
1484 : Bool extendflags, Bool apply, Bool channelavg, casac::variant chanbin, Bool timeavg,
1485 : String timebin)
1486 : {
1487 :
1488 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1489 :
1490 : // Default values for some parameters
1491 0 : String mode = "tfcrop";
1492 0 : String agent_name = "Tfcrop";
1493 :
1494 : // Create a record with the parameters
1495 0 : Record agent_record = Record();
1496 :
1497 0 : agent_record.define("mode", mode);
1498 0 : agent_record.define("spw", spw);
1499 0 : agent_record.define("scan", scan);
1500 0 : agent_record.define("field", field);
1501 0 : agent_record.define("antenna", antenna);
1502 0 : agent_record.define("timerange", timerange);
1503 0 : agent_record.define("correlation", correlation);
1504 0 : agent_record.define("intent", intent);
1505 0 : agent_record.define("feed", feed);
1506 0 : agent_record.define("array", array);
1507 0 : agent_record.define("uvrange", uvrange);
1508 0 : agent_record.define("observation", observation);
1509 0 : agent_record.define("apply", apply);
1510 0 : agent_record.define("agentname", agent_name);
1511 :
1512 0 : agent_record.define("ntime", ntime);
1513 0 : agent_record.define("combinescans", combinescans);
1514 0 : agent_record.define("datacolumn", datacolumn);
1515 0 : agent_record.define("timecutoff", timecutoff);
1516 0 : agent_record.define("freqcutoff", freqcutoff);
1517 0 : agent_record.define("timefit", timefit);
1518 0 : agent_record.define("freqfit", freqfit);
1519 0 : agent_record.define("maxnpieces", maxnpieces);
1520 0 : agent_record.define("flagdimension", flagdimension);
1521 0 : agent_record.define("usewindowstats", usewindowstats);
1522 0 : agent_record.define("halfwin", halfwin);
1523 0 : agent_record.define("extendflags", extendflags);
1524 0 : agent_record.define("apply", apply);
1525 :
1526 0 : if (chanbin.type() == casac::variant::INT)
1527 : {
1528 : /*** need range check ***/
1529 0 : agent_record.define("chanbin", (int) chanbin.toInt());
1530 : }
1531 0 : else if (chanbin.type() == casac::variant::INTVEC)
1532 : {
1533 : /*** need range check ***/
1534 0 : auto cb = chanbin.toIntVec( );
1535 0 : std::vector<int> vec;
1536 0 : std::for_each( cb.begin( ), cb.end( ), [&](long x){vec.push_back((int)x);} );
1537 0 : agent_record.define("chanbin", Array<Int>(vec));
1538 : }
1539 :
1540 0 : agent_record.define("channelavg", channelavg);
1541 0 : agent_record.define("timeavg", timeavg);
1542 0 : agent_record.define("timebin", timebin);
1543 :
1544 : // Call the main method
1545 0 : parseAgentParameters(agent_record);
1546 :
1547 0 : return true;
1548 :
1549 : }
1550 :
1551 : // ---------------------------------------------------------------------
1552 : // AgentFlagger::parseAntIntParameters
1553 : // Parse data selection parameters and specific antint parameters
1554 : //
1555 : // ---------------------------------------------------------------------
1556 : bool
1557 0 : AgentFlagger::parseAntIntParameters(String field, String spw, String /* array */,
1558 : String /* feed */, String scan, String antenna,
1559 : String uvrange, String timerange,
1560 : String correlation, String intent,
1561 : String observation, String antint_ref_antenna,
1562 : Double minchanfrac,
1563 : Bool verbose, Bool apply)
1564 : {
1565 :
1566 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1567 :
1568 : // Mode specific values for some general parameters
1569 0 : String mode = "antint";
1570 0 : String agent_name = "AntennaIntegrations";
1571 :
1572 : // Create a record with the parameters for this mode
1573 0 : Record agent_record = Record();
1574 :
1575 0 : agent_record.define("mode", mode);
1576 0 : agent_record.define("agentname", agent_name);
1577 0 : agent_record.define("field", field);
1578 0 : agent_record.define("spw", spw);
1579 :
1580 0 : agent_record.define("scan", scan);
1581 0 : agent_record.define("antenna", antenna);
1582 0 : agent_record.define("uvrange", uvrange);
1583 0 : agent_record.define("timerange", timerange);
1584 0 : agent_record.define("correlation", correlation);
1585 0 : agent_record.define("intent", intent);
1586 0 : agent_record.define("observation", observation);
1587 0 : agent_record.define("antint_ref_antenna", antint_ref_antenna);
1588 0 : agent_record.define("minchanfrac", minchanfrac);
1589 0 : agent_record.define("verbose", verbose);
1590 0 : agent_record.define("apply", apply);
1591 :
1592 : // Call the main method
1593 0 : parseAgentParameters(agent_record);
1594 :
1595 0 : return true;
1596 : }
1597 :
1598 : // ---------------------------------------------------------------------
1599 : // AgentFlagger::parseExtendParameters
1600 : // Parse data selection parameters and specific extend parameters
1601 : //
1602 : // This method is called from parseAgentParameters, and it calls parseAgentParameters
1603 : // again, passing a record set up for a "extend" agent. That way, an extend agent
1604 : // is added after the main (RFlag or TFcrop) agent.
1605 : // ---------------------------------------------------------------------
1606 : bool
1607 21 : AgentFlagger::parseExtendParameters(String field, String spw, String array, String feed, String scan,
1608 : String antenna, String uvrange, String timerange, String correlation, String intent,
1609 : String observation, Double ntime, Bool combinescans, Bool extendpols, Double growtime,
1610 : Double growfreq, Bool growaround, Bool flagneartime, Bool flagnearfreq, Bool apply)
1611 : {
1612 :
1613 63 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1614 :
1615 : // Default values for some parameters
1616 42 : String mode = "extend";
1617 42 : String agent_name = "Extend";
1618 :
1619 : // Create a record with the parameters
1620 21 : Record agent_record = Record();
1621 :
1622 21 : agent_record.define("mode", mode);
1623 21 : agent_record.define("spw", spw);
1624 21 : agent_record.define("scan", scan);
1625 21 : agent_record.define("field", field);
1626 21 : agent_record.define("antenna", antenna);
1627 21 : agent_record.define("timerange", timerange);
1628 21 : agent_record.define("correlation", correlation);
1629 21 : agent_record.define("intent", intent);
1630 21 : agent_record.define("feed", feed);
1631 21 : agent_record.define("array", array);
1632 21 : agent_record.define("uvrange", uvrange);
1633 21 : agent_record.define("observation", observation);
1634 21 : agent_record.define("apply", apply);
1635 21 : agent_record.define("agentname", agent_name);
1636 :
1637 21 : agent_record.define("ntime", ntime);
1638 21 : agent_record.define("combinescans", combinescans);
1639 21 : agent_record.define("extendpols", extendpols);
1640 21 : agent_record.define("growtime", growtime);
1641 21 : agent_record.define("growfreq", growfreq);
1642 21 : agent_record.define("growaround", growaround);
1643 21 : agent_record.define("flagneartime", flagneartime);
1644 21 : agent_record.define("flagnearfreq", flagnearfreq);
1645 21 : agent_record.define("apply", apply);
1646 :
1647 : // Call the main method
1648 21 : parseAgentParameters(agent_record);
1649 :
1650 42 : return true;
1651 :
1652 : }
1653 :
1654 :
1655 :
1656 : // ---------------------------------------------------------------------
1657 : // AgentFlagger::parseSummaryParameters
1658 : // Parse data selection parameters and specific summary parameters
1659 : //
1660 : // ---------------------------------------------------------------------
1661 : bool
1662 0 : AgentFlagger::parseSummaryParameters(String field, String spw, String array,
1663 : String feed, String scan, String antenna,
1664 : String uvrange, String timerange, String correlation,
1665 : String intent, String observation,
1666 : Bool spwchan, Bool spwcorr, Bool basecnt, Bool fieldcnt, String name)
1667 : {
1668 :
1669 0 : LogIO os(LogOrigin("AgentFlagger", __FUNCTION__));
1670 :
1671 : // Default values for some parameters
1672 0 : String mode = "summary";
1673 0 : String agent_name = "Summary";
1674 :
1675 : // Create a record with the parameters
1676 0 : Record agent_record = Record();
1677 :
1678 0 : agent_record.define("mode", mode);
1679 0 : agent_record.define("spw", spw);
1680 0 : agent_record.define("scan", scan);
1681 0 : agent_record.define("field", field);
1682 0 : agent_record.define("antenna", antenna);
1683 0 : agent_record.define("timerange", timerange);
1684 0 : agent_record.define("correlation", correlation);
1685 0 : agent_record.define("intent", intent);
1686 0 : agent_record.define("feed", feed);
1687 0 : agent_record.define("array", array);
1688 0 : agent_record.define("uvrange", uvrange);
1689 0 : agent_record.define("observation", observation);
1690 0 : agent_record.define("agentname", agent_name);
1691 :
1692 0 : agent_record.define("spwchan", spwchan);
1693 0 : agent_record.define("spwcorr", spwcorr);
1694 0 : agent_record.define("basecnt", basecnt);
1695 0 : agent_record.define("fieldcnt", fieldcnt);
1696 0 : agent_record.define("name", name);
1697 :
1698 : // Call the main method
1699 0 : parseAgentParameters(agent_record);
1700 :
1701 0 : return true;
1702 : }
1703 :
1704 :
1705 : } //#end casa namespace
|
