Line data Source code
1 : //# FlagVersion.cc: Manage flag versions
2 : //# Copyright (C) 1994,1995,1996,1997,1998,1999,2000,2001,2002,2003
3 : //# Associated Universities, Inc. Washington DC, USA.
4 : //#
5 : //# This library is free software; you can redistribute it and/or modify it
6 : //# under the terms of the GNU Library General Public License as published by
7 : //# the Free Software Foundation; either version 2 of the License, or (at your
8 : //# option) any later version.
9 : //#
10 : //# This library is distributed in the hope that it will be useful, but WITHOUT
11 : //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 : //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 : //# License for more details.
14 : //#
15 : //# You should have received a copy of the GNU Library General Public License
16 : //# along with this library; if not, write to the Free Software Foundation,
17 : //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 : //#
19 : //# Correspondence concerning AIPS++ should be addressed as follows:
20 : //# Internet email: aips2-request@nrao.edu.
21 : //# Postal address: AIPS++ Project Office
22 : //# National Radio Astronomy Observatory
23 : //# 520 Edgemont Road
24 : //# Charlottesville, VA 22903-2475 USA
25 : //#
26 : //# $Id$
27 :
28 :
29 :
30 : //# Includes
31 :
32 : #include <casacore/tables/Tables/ScaColDesc.h>
33 : #include <casacore/tables/Tables/ArrColDesc.h>
34 : #include <casacore/tables/Tables/SetupNewTab.h>
35 : #include <casacore/tables/Tables/TableUtil.h>
36 : #include <casacore/tables/DataMan/TiledShapeStMan.h>
37 : #include <casacore/tables/DataMan/DataManError.h>
38 : #include <casacore/casa/IO/ArrayIO.h>
39 : #include <casacore/casa/System/ProgressMeter.h>
40 :
41 : #include <casacore/casa/OS/RegularFile.h>
42 : #include <casacore/casa/OS/Directory.h>
43 : #include <casacore/casa/OS/SymLink.h>
44 :
45 : #include <flagging/Flagging/SLog.h>
46 :
47 : #include <flagging/Flagging/FlagVersion.h>
48 :
49 : using namespace casacore;
50 : namespace casa { //# NAMESPACE CASA - BEGIN
51 :
52 : #define TMR(a) "[User: " << a.user() << "] [System: " << a.system() << "] [Real: " << a.real() << "]"
53 : #define MIN(a,b) ((a)<(b) ? (a) : (b))
54 : #define MAX(a,b) ((a)>(b) ? (a) : (b))
55 :
56 : #define LOG0 0
57 :
58 : String FlagVersion::clname = "FlagVersion";
59 : /* Constructor */
60 668 : FlagVersion::FlagVersion(String intab, String dataflagcolname,
61 668 : String rowflagcolname)
62 : {
63 :
64 1336 : String fnname= "constructor";
65 668 : log = SLog::slog();
66 668 : tab_p = Table(intab, Table::Update);
67 :
68 : /* Check if intab is the root table */
69 668 : if( ! tab_p.isRootTable() )
70 0 : log->out(String("Table: ") + tab_p.tableName()
71 0 : + " is not the root Table. It is a sub-selection. "
72 0 : + "Flag versions will be created, but will not be "
73 0 : + "interchangeable across selections. ",
74 : fnname, clname, LogMessage::WARN);
75 :
76 : /* Record the names of columns to be used for flags */
77 668 : dataflagcolname_p = dataflagcolname;
78 668 : rowflagcolname_p = rowflagcolname;
79 :
80 668 : fcol_p = tab_p.tableDesc().isColumn(dataflagcolname_p);
81 668 : frcol_p = tab_p.tableDesc().isColumn(rowflagcolname_p);
82 :
83 668 : if(!fcol_p && !frcol_p)
84 : {
85 0 : log->out(String("Table: ") + tab_p.tableName()
86 0 : + " has no flag columns. Cannot save flag versions",
87 : fnname, clname, LogMessage::WARN);
88 0 : FlagVersionError(
89 0 : String("No Flag Columns exist for this table. Cannot save flag versions"));
90 : }
91 :
92 668 : tabname_p = tab_p.tableName();
93 :
94 : // Check if tabname_p is a symbolic link. if so - expand !!
95 : //cout << "before : " << tabname_p <<endl;
96 1336 : File rootfile;
97 1336 : Directory fileroot(tabname_p);
98 668 : if (fileroot.exists()) {
99 668 : if (fileroot.isSymLink()) {
100 0 : rootfile = SymLink(fileroot).followSymLink();
101 0 : tabname_p = rootfile.path().expandedName();
102 : }
103 : }
104 : //cout << "after : " << tabname_p <<endl;
105 :
106 668 : nrows_p = tab_p.nrow();
107 :
108 : /* Make this a directory of flag tables */
109 1336 : String flagdir(".flagversions");
110 668 : verlistfile_p = tabname_p + flagdir + String("/FLAG_VERSION_LIST");
111 668 : flagtablename_p = tabname_p + flagdir + String("/flags.");
112 :
113 : //cout << "Working with " << tabname_p + flagdir << endl;
114 :
115 : /* Make sure the directory exists */
116 1336 : Directory dir( tabname_p + flagdir );
117 668 : if( ! dir.exists() ) dir.create();
118 : else
119 : {
120 415 : if ( !dir.isWritable() || !dir.isReadable() )
121 0 : log->out("Flag Table directory is not accesible",
122 : fnname, clname, LogMessage::WARN);
123 : }
124 668 : if( !dir.exists() )
125 0 : FlagVersionError(String("Flag directory does not exist."));
126 :
127 : /* Make sure the version list file exists */
128 1336 : RegularFile listfile(verlistfile_p);
129 668 : if( !listfile.exists() ) listfile.create();
130 : else
131 : {
132 415 : if ( !listfile.isWritable() || !listfile.isReadable() )
133 0 : log->out("Flag Version list file is not accessible",
134 : fnname, clname, LogMessage::WARN);
135 : }
136 668 : if( !listfile.exists() )
137 0 : FlagVersionError(String("flag version file does not exist "));
138 :
139 :
140 : /* Read in the list of versions, and check that the
141 : corresponding flag tables also exist */
142 668 : readVersionList();
143 :
144 668 : }
145 :
146 : /*********************************************************************************/
147 :
148 : /* Destructor */
149 668 : FlagVersion::~FlagVersion()
150 : {
151 : #if LOG0
152 : log->FnPass("~FlagVersion", clname);
153 : #endif
154 668 : }
155 :
156 : /*********************************************************************************/
157 :
158 : /*********************************************************************************/
159 353 : Bool FlagVersion::doesVersionExist( String versionname )
160 : {
161 : /* check if this versionname exists. */
162 353 : Bool exists = false;
163 869 : for(Int i=0;i<(Int)versionlist_p.nelements();i++)
164 516 : if(versionlist_p[i].matches(versionname)) exists = true;
165 :
166 353 : return exists;
167 : }
168 : /*********************************************************************************/
169 :
170 : /* Return a vector of strings, one for each line in the version-list file */
171 315 : Vector<String> FlagVersion::getVersionList()
172 : {
173 315 : return commentlist_p;
174 : }
175 :
176 : /*********************************************************************************/
177 0 : Bool FlagVersion::attachFlagColumns(String version,
178 : ScalarColumn<Bool> &rowflag,
179 : ArrayColumn<Bool> &flag, Table &subtab)
180 : {
181 : //os.FnEnter( "attachFlagColumns",
182 : // "attachFlagColumns( version, rowflag, flag, subtab )" );
183 :
184 : /* Find this version. Set to currentflagtable */
185 0 : Bool exists = doesVersionExist(version);
186 :
187 0 : Table currentflagtable;//, subflagtable;
188 :
189 0 : if(exists && !version.matches("main"))
190 : {
191 0 : String tabvername = flagtablename_p + version;
192 0 : currentflagtable = Table(tabvername, Table::Update);
193 : }
194 : else /* If not found, or if main is asked for, then set to main... tab_p */
195 : {
196 0 : currentflagtable = tab_p;
197 : }
198 :
199 0 : subflagtable_p = currentflagtable(subtab.rowNumbers());
200 :
201 0 : if(frcol_p)
202 0 : rowflag.attach(subflagtable_p,rowflagcolname_p);
203 0 : if(fcol_p)
204 0 : flag.attach(subflagtable_p,dataflagcolname_p);
205 :
206 0 : return true;
207 : }
208 :
209 : /*********************************************************************************/
210 353 : Bool FlagVersion::saveFlagsInto( Table &fromFTab, Table &toFTab, String merge )
211 : {
212 353 : String fnname= "saveFlagsInto";
213 : #if LOG0
214 : log->FnEnter(fnname + "(fromFTAb, toFTab, merge)", clname);
215 : #endif
216 :
217 : #if LOG0
218 : log->out(String("From table: ")+ fromFTab.tableName() +
219 : "To table: "+ toFTab.tableName() +
220 : "Merge type: "+ merge,
221 : fnname, clname, LogMessage::DEBUGGING);
222 : #endif
223 :
224 353 : if( ! merge.matches("replace") &&
225 353 : ! merge.matches("and") &&
226 0 : ! merge.matches("or") ) merge = String("replace");
227 :
228 353 : if(frcol_p)
229 : {
230 674 : ScalarColumn<Bool> fromRowFlag(fromFTab, rowflagcolname_p);
231 674 : ScalarColumn<Bool> toRowFlag(toFTab, rowflagcolname_p);
232 :
233 337 : if( merge.matches("and") )
234 : {
235 0 : Vector<Bool> rfc = fromRowFlag.getColumn();
236 0 : rfc *= toRowFlag.getColumn();
237 0 : toRowFlag.putColumn(rfc);
238 : }
239 337 : if( merge.matches("or") )
240 : {
241 0 : Vector<Bool> rfc = fromRowFlag.getColumn();
242 0 : rfc += toRowFlag.getColumn();
243 0 : toRowFlag.putColumn(rfc);
244 : }
245 337 : if( merge.matches("replace") ) {
246 337 : toRowFlag.putColumn(fromRowFlag.getColumn());
247 : }
248 : }
249 :
250 353 : if(fcol_p)
251 : {
252 706 : ArrayColumn<Bool> fromFlag(fromFTab, dataflagcolname_p);
253 706 : ArrayColumn<Bool> toFlag(toFTab, dataflagcolname_p);
254 :
255 :
256 : try {
257 :
258 706 : IPosition shape(fromFlag.shape(0));
259 : /* Process large chunks at a time in order to avoid overhead, e.g. 100 MB.
260 : Empirically, processing row by row has just a small overhead (3 times)
261 : so we just need to process many rows at a time.
262 : */
263 353 : uInt chunk_rows = 100*1024*1024 / (shape(0) * shape(1)) + 1;
264 706 : Array<Bool> arr1;
265 706 : Array<Bool> arr2;
266 661 : for(unsigned i=0;i<nrows_p;i += chunk_rows)
267 : {
268 357 : unsigned j = i + chunk_rows - 1;
269 357 : if (!(j < nrows_p)) j = nrows_p - 1;
270 714 : RefRows arraySection(i, j);
271 :
272 357 : fromFlag.getColumnCells(arraySection,arr1,true);
273 :
274 308 : if( merge.matches("and") ) {
275 0 : toFlag.getColumnCells(arraySection,arr2,true);
276 0 : arr2 *= arr1;
277 : }
278 308 : else if (merge.matches("or")) {
279 0 : toFlag.getColumnCells(arraySection,arr2,true);
280 0 : arr2 += arr1;
281 : }
282 308 : else if (merge.matches("replace")) {
283 308 : arr2.assign(arr1);
284 : }
285 308 : toFlag.putColumnCells(arraySection,arr2);
286 : }
287 : }
288 98 : catch (DataManError &e)
289 : {
290 : // happens if the FLAG column is of non-fixed shape.
291 : // There is probably not a better way to check the if
292 : // the shape is fixed than assuming it is, and catch
293 : // the error. In particular, note that ColumnDesc::isFixedShape
294 : // is not always accurate.
295 : // Trying to read in the entire column using getColumn()
296 : // would reveal if it is of fixed shape; but this approach
297 : // is memory intensive.
298 :
299 : // Check if DATA_DESC_ID is a column
300 : // if so, use the TableIterator on this.
301 : // For now, go row by row....
302 98 : Array<Bool> arr1;
303 98 : Array<Bool> arr2;
304 98 : ProgressMeter pm(0, nrows_p, "Saving flags");
305 9395335 : for(unsigned i=0;i<nrows_p;i++)
306 : {
307 9395286 : if (i % 16384 == 0) {
308 601 : pm.update(i);
309 : }
310 9395286 : fromFlag.get(i,arr1,true);
311 9395286 : if( merge.matches("and") ) {
312 0 : toFlag.get(i,arr2,true);
313 0 : arr2 *= arr1;
314 : }
315 9395286 : else if (merge.matches("or")) {
316 0 : toFlag.get(i,arr2,true);
317 0 : arr2 += arr1;
318 : }
319 9395286 : else if (merge.matches("replace")) {
320 9395286 : arr2.resize();
321 9395286 : arr2 = arr1;
322 : }
323 9395286 : toFlag.put(i,arr2);
324 : }
325 : }
326 : } // end if fcol_p
327 :
328 353 : toFTab.flush();
329 :
330 706 : return true;
331 : }
332 : /*********************************************************************************/
333 :
334 : /* If this flag version exists, save current flags into this version */
335 : /* If it doesnt exist, create a new flag version and save current flags there */
336 346 : Bool FlagVersion::saveFlagVersion( String versionname ,
337 : String comment , String merge)
338 : {
339 : // Set Log origin to the log messages
340 346 : log->origin(LogOrigin("FlagVersion", __func__));
341 :
342 692 : String fnname= "saveFlagVersion";
343 346 : Bool exists = doesVersionExist(versionname);
344 346 : String tabvername = flagtablename_p + versionname;
345 :
346 : /* If doesn't exist, say so and make a new version. */
347 : /* Attach all current variables to this table */
348 346 : if(!exists)
349 : {
350 346 : log->out(String("Creating new backup flag file called ") +
351 : versionname, fnname, clname, LogMessage::NORMAL);
352 :
353 692 : ofstream listfile;
354 346 : listfile.open(verlistfile_p.data(),ofstream::app);
355 346 : listfile << versionname << " : " << comment << endl;
356 346 : listfile.close();
357 :
358 : /* Create a new Table with the standard Flag Table descriptor */
359 692 : TableDesc td("", versionname, TableDesc::Scratch);
360 346 : td.comment() = "TablePlot Flag Table : " + versionname;
361 :
362 346 : if(fcol_p) {
363 346 : td.addColumn (ArrayColumnDesc<Bool> (dataflagcolname_p, 2));
364 346 : td.defineHypercolumn("TiledFlag", 3,
365 692 : stringToVector(dataflagcolname_p));
366 : }
367 346 : if(frcol_p) td.addColumn (ScalarColumnDesc<Bool> (rowflagcolname_p));
368 :
369 692 : SetupNewTable aNewTab(tabvername, td, Table::New);
370 :
371 : // FLAG hyperColumn
372 : {
373 692 : ArrayColumn<Bool> fromFlag(tab_p, dataflagcolname_p);
374 :
375 : /* Have to figure out the maximum cell shape,
376 : i.e. max number of channels/polarizations and use that
377 : to define the tile shape. There ought to be an easier way. */
378 346 : unsigned npol_max = fromFlag.shape(0)(0);
379 346 : unsigned nchan_max = fromFlag.shape(0)(1);
380 : {
381 346 : unsigned nrow = tab_p.nrow();
382 29308047 : for (unsigned i = 0; i < nrow; i++) {
383 29307701 : unsigned n0 = fromFlag.shape(i)(0);
384 29307701 : unsigned n1 = fromFlag.shape(i)(1);
385 29307701 : if (n0 > npol_max) npol_max = n0;
386 29307701 : if (n1 > nchan_max) nchan_max = n1;
387 : }
388 : }
389 :
390 346 : uInt tile_size = 1024*1024; // bytes
391 : IPosition tileShape(3, npol_max, nchan_max,
392 692 : tile_size*8 / (npol_max * nchan_max));
393 :
394 692 : TiledShapeStMan flagStMan("TiledFlag", tileShape);
395 346 : aNewTab.bindColumn(dataflagcolname_p, flagStMan);
396 : }
397 :
398 692 : Table ftab(aNewTab, Table::Plain, nrows_p);
399 :
400 346 : saveFlagsInto( tab_p, ftab, String("replace") );
401 :
402 346 : readVersionList();
403 : }
404 :
405 : /* Save current main flags into this version. */
406 346 : if(exists && !versionname.matches("main"))
407 : {
408 0 : Table ftab(tabvername, Table::Update);
409 0 : if( ftab.nrow() != tab_p.nrow() )
410 0 : log->out(String("nrows don't match !! "),
411 : fnname, clname, LogMessage::NORMAL);
412 :
413 0 : saveFlagsInto( tab_p, ftab, merge );
414 :
415 0 : ofstream listfile;
416 0 : listfile.open(verlistfile_p.data());
417 0 : for(Int i=0;i<(Int)versionlist_p.nelements();i++)
418 : {
419 0 : if( ! versionlist_p[i].matches("main") )
420 : {
421 0 : if(versionlist_p[i].matches(versionname) && comment.length()>0)
422 0 : commentlist_p[i] = versionname + String(" : ") + comment;
423 0 : listfile << commentlist_p[i] << endl;
424 : }
425 : }
426 0 : listfile.close();
427 : }
428 :
429 692 : return true;
430 : }
431 :
432 : /*********************************************************************************/
433 7 : Bool FlagVersion::restoreFlagVersion( String versionname, String merge )
434 : {
435 14 : String fnname= "restoreFlagVersion";
436 7 : Bool exists = doesVersionExist(versionname);
437 7 : String tabvername = flagtablename_p + versionname;
438 :
439 : /* If doesn't exist, throw an exception. */
440 7 : if(!exists)
441 : {
442 : // log->out(String("Flag version ") + versionname +
443 : // " does not exist", fnname, clname, LogMessage::WARN);
444 0 : throw AipsError("Flag version "+ versionname + " does not exist");
445 : // return false;
446 : }
447 :
448 : /* Save current flags from this version to the main table. */
449 7 : if(exists && !versionname.matches("main"))
450 : {
451 7 : Table ftab(tabvername, Table::Update);
452 7 : saveFlagsInto( ftab, tab_p, merge );
453 : }
454 :
455 14 : return true;
456 : }
457 :
458 : /*********************************************************************************/
459 0 : Bool FlagVersion::deleteFlagVersion( String versionname )
460 : {
461 0 : String fnname= "deleteFlagVersion";
462 : /* Check if this version exists */
463 : /* check if this versionname exists. */
464 0 : Bool exists = false;
465 0 : for(Int i=0;i<(Int)versionlist_p.nelements();i++)
466 0 : if(versionlist_p[i].matches(versionname)) exists = true;
467 :
468 0 : if(!exists)
469 : {
470 0 : log->out(String("Flag version ") + versionname +
471 : " does not exist", fnname, clname, LogMessage::WARN);
472 0 : return false;
473 : }
474 :
475 0 : if(versionname.matches( String("main") ))
476 : {
477 0 : log->out(String("The main flag table cannot be deleted. "),
478 : fnname, clname, LogMessage::WARN);
479 0 : return false;
480 : }
481 :
482 : /* remove the entry from the list file */
483 : /* delete the Table associated with it, and set version to main */
484 :
485 0 : ofstream listfile;
486 0 : listfile.open(verlistfile_p.data());
487 0 : for(Int i=0;i<(Int)versionlist_p.nelements();i++)
488 : {
489 0 : if( ! versionlist_p[i].matches("main") )
490 : {
491 0 : if(versionlist_p[i].matches(versionname))
492 : {
493 0 : log->out(String("Deleting version: ") + versionname,
494 : fnname, clname, LogMessage::WARN);
495 :
496 0 : String tabvername = flagtablename_p + versionname;
497 0 : if(TableUtil::canDeleteTable(tabvername))
498 0 : TableUtil::deleteTable(tabvername);
499 : }
500 0 : else listfile << commentlist_p[i] << endl;
501 : }
502 : }
503 0 : listfile.close();
504 :
505 0 : readVersionList();
506 :
507 0 : return true;
508 : }
509 : /*********************************************************************************
510 : This function should be changed to not use get(i,...) and put(i, ...) instead
511 : of getColumn(...) and putColumn(...). Otherwise out of memory errors happen
512 : for large MSs.
513 : */
514 0 : Bool FlagVersion::clearAllFlags()
515 : {
516 0 : String fnname= "clearAllFlags";
517 : #if LOG0
518 : log->FnEnter(fnname + "()", clname);
519 : #endif
520 :
521 : #if LOG0
522 : log->out(String("Clearing All main-table Flags for : ")+tabname_p,
523 : fnname, clname, LogMessage::DEBUGGING);
524 : #endif
525 :
526 0 : if(frcol_p)
527 : {
528 0 : ScalarColumn<Bool> rfscol;
529 0 : rfscol.attach(tab_p,rowflagcolname_p);
530 :
531 0 : Vector<Bool> frc = rfscol.getColumn();
532 0 : frc = false;
533 0 : rfscol.putColumn(frc);
534 : }
535 0 : if(fcol_p)
536 : {
537 0 : ArrayColumn<Bool> facol;
538 0 : facol.attach(tab_p,dataflagcolname_p);
539 :
540 0 : Array<Bool> fc = facol.getColumn();
541 0 : fc = false;
542 0 : facol.putColumn(fc);
543 : }
544 :
545 0 : tab_p.flush();
546 :
547 0 : return true;
548 : }
549 :
550 : /*********************************************************************************/
551 : /* Read in the list of versions from the listfile */
552 : /* Check that flag tables of the correct names exist and have the same
553 : number of rows as tab_p */
554 1014 : Bool FlagVersion::readVersionList()
555 : {
556 2028 : String fnname= "readVersionList";
557 1014 : versionlist_p.resize(1);
558 1014 : versionlist_p[0] = String("main");
559 :
560 1014 : commentlist_p.resize(1);
561 1014 : commentlist_p[0] = String("main : working copy in main table");
562 :
563 : /* Read the file and make a list of version names */
564 2028 : ifstream listfile;
565 1014 : listfile.open(verlistfile_p.data());
566 :
567 1014 : if( !listfile.good() )
568 0 : log->out("Bad file!", fnname, clname, LogMessage::WARN);
569 :
570 1014 : Int vcount=1;
571 : char vers[500];
572 2867 : while(!listfile.eof())
573 : {
574 1853 : listfile.getline(vers,500,'\n');
575 1853 : if(!listfile.eof())
576 : {
577 839 : versionlist_p.resize(vcount+1,true);
578 839 : commentlist_p.resize(vcount+1,true);
579 839 : versionlist_p[vcount] = String(vers).before(" : ");
580 839 : commentlist_p[vcount] = String(vers);
581 839 : vcount++;
582 : }
583 : }
584 1014 : listfile.close();
585 :
586 : try
587 : {
588 :
589 : /* Try to open each of these flag tables, to check for table validity */
590 2867 : for(Int i=0;i<(Int)versionlist_p.nelements();i++)
591 : {
592 :
593 3706 : String tabvername = flagtablename_p + versionlist_p[i];
594 3706 : Table tab;
595 1853 : if(!versionlist_p[i].matches("main") && ! tab.isReadable(tabvername))
596 0 : log->out(String("Flag Table ") + tabvername +
597 : " does not exist. Please check your flag version file",
598 : fnname, clname, LogMessage::WARN);
599 : }
600 : }
601 0 : catch (AipsError x)
602 : {
603 0 : String err = x.getMesg();
604 0 : log->out(String("Number of rows in the Flag Tables ") +
605 0 : "do not match the number of rows in the main table. " +
606 0 : "Please check your selection.\n" + err,
607 : fnname, clname, LogMessage::SEVERE);
608 0 : return false;
609 : }
610 1014 : return true;
611 : } /*********************************************************************************/
612 0 : void FlagVersion::FlagVersionError(String msg)
613 : {
614 0 : throw AipsError("FlagVersion: " + msg);
615 : }
616 : /*********************************************************************************/
617 :
618 : } //# NAMESPACE CASA - END
619 :
|
