|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import shutil 00002 import unittest 00003 import os 00004 import filecmp 00005 from tasks import * 00006 from taskinit import * 00007 from __main__ import default 00008 from parallel.parallel_task_helper import ParallelTaskHelper 00009 00010 # 00011 # Test of flagdata modes 00012 # 00013 00014 def test_eq(result, total, flagged): 00015 00016 print "%s of %s data was flagged, expected %s of %s" % \ 00017 (result['flagged'], result['total'], flagged, total) 00018 assert result['total'] == total, \ 00019 "%s data in total; %s expected" % (result['total'], total) 00020 assert result['flagged'] == flagged, \ 00021 "%s flags set; %s expected" % (result['flagged'], flagged) 00022 00023 def create_input(str_text, filename): 00024 '''Save the string in a text file''' 00025 00026 inp = filename 00027 cmd = str_text 00028 00029 # remove file first 00030 if os.path.exists(inp): 00031 os.system('rm -f '+ inp) 00032 00033 # save to a file 00034 with open(inp, 'w') as f: 00035 f.write(cmd) 00036 00037 f.close() 00038 00039 return 00040 00041 # Path for data 00042 datapath = os.environ.get('CASAPATH').split()[0] + "/data/regression/unittest/flagdata/" 00043 00044 # Pick up alternative data directory to run tests on MMSs 00045 testmms = False 00046 if os.environ.has_key('TEST_DATADIR'): 00047 DATADIR = str(os.environ.get('TEST_DATADIR'))+'/flagdata/' 00048 if os.path.isdir(DATADIR): 00049 testmms = True 00050 datapath = DATADIR 00051 00052 print 'flagdata tests will use data from '+datapath 00053 00054 # jagonzal (CAS-4287): Add a cluster-less mode to by-pass parallel processing for MMSs as requested 00055 if os.environ.has_key('BYPASS_SEQUENTIAL_PROCESSING'): 00056 ParallelTaskHelper.bypassParallelProcessing(1) 00057 00058 # Base class which defines setUp functions 00059 # for importing different data sets 00060 class test_base(unittest.TestCase): 00061 def setUp_flagdatatest(self): 00062 self.vis = "flagdatatest.ms" 00063 if testmms: 00064 self.vis = "flagdatatest.mms" 00065 00066 if os.path.exists(self.vis): 00067 print "The MS is already around, just unflag" 00068 else: 00069 print "Moving data..." 00070 os.system('cp -r '+datapath + self.vis +' '+ self.vis) 00071 00072 os.system('rm -rf ' + self.vis + '.flagversions') 00073 00074 default(flagdata) 00075 flagdata(vis=self.vis, mode='unflag', savepars=False) 00076 00077 def setUp_ngc5921(self): 00078 self.vis = "ngc5921.ms" 00079 if testmms: 00080 self.vis = 'ngc5921.mms' 00081 00082 if os.path.exists(self.vis): 00083 print "The MS is already around, just unflag" 00084 else: 00085 os.system('cp -r '+datapath + self.vis +' '+ self.vis) 00086 00087 os.system('rm -rf ' + self.vis + '.flagversions') 00088 default(flagdata) 00089 flagdata(vis=self.vis, mode='unflag', savepars=False) 00090 00091 def setUp_flagdatatest_alma(self): 00092 self.vis = "flagdatatest-alma.ms" 00093 if testmms: 00094 self.vis = 'flagdatatest-alma.mms' 00095 00096 if os.path.exists(self.vis): 00097 print "The MS is already around, just unflag" 00098 else: 00099 print "Moving data..." 00100 os.system('cp -r '+datapath + self.vis +' '+ self.vis) 00101 00102 os.system('rm -rf ' + self.vis + '.flagversions') 00103 default(flagdata) 00104 flagdata(vis=self.vis, mode='unflag', savepars=False) 00105 00106 def setUp_data4tfcrop(self): 00107 self.vis = "Four_ants_3C286.ms" 00108 if testmms: 00109 self.vis = 'Four_ants_3C286.mms' 00110 00111 if os.path.exists(self.vis): 00112 print "The MS is already around, just unflag" 00113 else: 00114 print "Moving data..." 00115 os.system('cp -r '+datapath + self.vis +' '+ self.vis) 00116 00117 os.system('rm -rf ' + self.vis + '.flagversions') 00118 default(flagdata) 00119 flagdata(vis=self.vis, mode='unflag', savepars=False) 00120 00121 def setUp_shadowdata2(self): 00122 self.vis = "shadowtest_part.ms" 00123 if testmms: 00124 self.vis = "shadowtest_part.mms" 00125 00126 if os.path.exists(self.vis): 00127 print "The MS is already around, just unflag" 00128 else: 00129 print "Moving data..." 00130 os.system('cp -r '+datapath + self.vis +' '+ self.vis) 00131 00132 os.system('rm -rf ' + self.vis + '.flagversions') 00133 default(flagdata) 00134 flagdata(vis=self.vis, mode='unflag', savepars=False) 00135 00136 def setUp_multi(self): 00137 self.vis = "multiobs.ms" 00138 if testmms: 00139 self.vis = 'multiobs.mms' 00140 00141 if os.path.exists(self.vis): 00142 print "The MS is already around, just unflag" 00143 else: 00144 print "Moving data..." 00145 os.system('cp -r '+datapath + self.vis +' '+ self.vis) 00146 00147 os.system('rm -rf ' + self.vis + '.flagversions') 00148 default(flagdata) 00149 flagdata(vis=self.vis, mode='unflag', savepars=False) 00150 00151 def setUp_tsys_case(self): 00152 self.vis = "X7ef.tsys" 00153 00154 if os.path.exists(self.vis): 00155 print "The CalTable is already around, just unflag" 00156 00157 else: 00158 print "Moving data..." 00159 os.system('cp -r ' + \ 00160 os.environ.get('CASAPATH').split()[0] + 00161 "/data/regression/unittest/flagdata/" + self.vis + ' ' + self.vis) 00162 00163 os.system('rm -rf ' + self.vis + '.flagversions') 00164 00165 flagdata(vis=self.vis, mode='unflag', savepars=False) 00166 00167 def setUp_bpass_case(self): 00168 self.vis = "cal.fewscans.bpass" 00169 00170 if os.path.exists(self.vis): 00171 print "The CalTable is already around, just unflag" 00172 else: 00173 print "Moving data..." 00174 os.system('cp -r ' + \ 00175 os.environ.get('CASAPATH').split()[0] + 00176 "/data/regression/unittest/flagdata/" + self.vis + ' ' + self.vis) 00177 00178 os.system('rm -rf ' + self.vis + '.flagversions') 00179 00180 flagdata(vis=self.vis, mode='unflag', savepars=False) 00181 00182 00183 class test_tfcrop(test_base): 00184 """flagdata:: Test of mode = 'tfcrop'""" 00185 00186 def setUp(self): 00187 self.setUp_data4tfcrop() 00188 00189 def test_tfcrop1(self): 00190 '''flagdata:: Test1 of mode = tfcrop''' 00191 flagdata(vis=self.vis, mode='tfcrop', correlation='ABS_RR',ntime=51.0,spw='9', savepars=False) 00192 res = flagdata(vis=self.vis, mode='summary') 00193 self.assertEqual(res['flagged'], 4489) 00194 self.assertEqual(res['antenna']['ea19']['flagged'], 2294) 00195 self.assertEqual(res['spw']['7']['flagged'], 0) 00196 00197 def test_tfcrop2(self): 00198 '''flagdata:: Test2 of mode = tfcrop ABS_ALL''' 00199 # Note : With ntime=51.0, 64-bit machines get 18696 flags, and 32-bit gets 18695 flags. 00200 # As far as we can determine, this is a genuine precision-related difference. 00201 # With ntime=53.0, there happens to be no difference. 00202 flagdata(vis=self.vis, mode='tfcrop',ntime=53.0,spw='9', savepars=False) 00203 res = flagdata(vis=self.vis, mode='summary') 00204 self.assertEqual(res['flagged'], 18671) 00205 self.assertEqual(res['correlation']['LL']['flagged'], 4250) 00206 self.assertEqual(res['correlation']['RL']['flagged'], 5007) 00207 self.assertEqual(res['correlation']['LR']['flagged'], 4931) 00208 self.assertEqual(res['correlation']['RR']['flagged'], 4483) 00209 00210 def test_extendpols(self): 00211 '''flagdata:: Extend the flags created by clip''' 00212 flagdata(vis=self.vis, mode='clip', correlation='abs_rr', clipminmax=[0,2]) 00213 res = flagdata(vis=self.vis, mode='summary') 00214 self.assertEqual(res['correlation']['RR']['flagged'], 43) 00215 self.assertEqual(res['correlation']['LL']['flagged'], 0) 00216 flagdata(vis=self.vis, mode='extend', extendpols=True, savepars=False) 00217 test_eq(flagdata(vis=self.vis, mode='summary', correlation='Ll'), 1099776, 43) 00218 00219 def test_extendtime(self): 00220 '''flagdata:: Extend the flags created by tfcrop''' 00221 flagdata(vis=self.vis, mode='tfcrop') 00222 # The total per spw:channel/baseline/correlation/scan is 89. 00223 # Show only the ones that are 50% of the total, 44 flags, These should grow 00224 pre = flagdata(vis=self.vis, mode='summary', spwchan=True, basecnt=True, correlation='RR',spw='5', 00225 antenna='ea11&&ea19') 00226 # these should grow later 00227 self.assertEqual(pre['spw:channel']['5:10']['flagged'], 118) 00228 self.assertEqual(pre['spw:channel']['5:10']['total'], 179) 00229 self.assertEqual(pre['spw:channel']['5:28']['flagged'], 128) 00230 self.assertEqual(pre['spw:channel']['5:29']['flagged'], 151) 00231 00232 # these should not grow later. After the consolidation of MMS summaries 00233 # is fixed, revise this test 00234 # self.assertEqual(pre['spw:channel']['5:11']['flagged'], 32) 00235 # self.assertEqual(pre['spw:channel']['5:12']['flagged'], 29) 00236 # self.assertEqual(pre['spw:channel']['5:21']['flagged'], 34) 00237 00238 # Extend in time only 00239 flagdata(vis=self.vis, mode='extend', extendpols=False, growtime=50.0, growfreq=0.0, 00240 growaround=False,flagneartime=False,flagnearfreq=False,savepars=False) 00241 pos = flagdata(vis=self.vis, mode='summary', spwchan=True, basecnt=True, correlation='RR',spw='5', 00242 antenna='ea11&&ea19') 00243 self.assertEqual(pos['spw:channel']['5:10']['flagged'], 179) 00244 self.assertEqual(pos['spw:channel']['5:10']['total'], 179) 00245 self.assertEqual(pos['spw:channel']['5:28']['flagged'], 179) 00246 self.assertEqual(pos['spw:channel']['5:29']['flagged'], 179) 00247 00248 # These did not grow 00249 # self.assertEqual(pos['spw:channel']['5:11']['flagged'], 32) 00250 # self.assertEqual(pos['spw:channel']['5:12']['flagged'], 29) 00251 # self.assertEqual(pos['spw:channel']['5:21']['flagged'], 34) 00252 00253 def test_extendfreq(self): 00254 '''flagdata:: Extend the flags created manually for one scan only''' 00255 flagdata(vis=self.vis, mode='manual',spw='*:0~35',timerange='2010/10/16/14:45:00~14:45:20') 00256 pre = flagdata(vis=self.vis, mode='summary') 00257 self.assertEqual(pre['scan']['31']['flagged'], 0) 00258 self.assertEqual(pre['scan']['30']['flagged'], 165888) 00259 self.assertEqual(pre['flagged'], 165888) 00260 00261 # Extend in frequency only 00262 flagdata(vis=self.vis, mode='extend', extendpols=False, growtime=0.0, growfreq=50.0, savepars=False) 00263 pos = flagdata(vis=self.vis, mode='summary') 00264 self.assertEqual(pos['scan']['31']['flagged'], 0) 00265 self.assertEqual(pos['flagged'], 294912) 00266 00267 00268 00269 class test_rflag(test_base): 00270 """flagdata:: Test of mode = 'rflag'""" 00271 00272 def setUp(self): 00273 self.setUp_data4tfcrop() 00274 00275 def test_rflag1(self): 00276 '''flagdata:: Test1 of mode = rflag : automatic thresholds''' 00277 flagdata(vis=self.vis, mode='rflag', spw='9,10', timedev=[], freqdev=[]); 00278 res = flagdata(vis=self.vis, mode='summary') 00279 self.assertEqual(res['flagged'], 42728.0) 00280 self.assertEqual(res['antenna']['ea19']['flagged'], 18411.0) 00281 self.assertEqual(res['spw']['7']['flagged'], 0) 00282 00283 def test_rflag2(self): 00284 '''flagdata:: Test2 of mode = rflag : partially-specified thresholds''' 00285 flagdata(vis=self.vis, mode='rflag', spw='9,10', timedev=[[1,10,0.1],[1,11,0.07]], \ 00286 freqdev=0.5); 00287 res = flagdata(vis=self.vis, mode='summary',spw='9,10,11') 00288 self.assertEqual(res['flagged'], 52411) 00289 self.assertEqual(res['antenna']['ea19']['flagged'], 24142) 00290 self.assertEqual(res['spw']['11']['flagged'], 0) 00291 00292 def test_rflag3(self): 00293 '''flagdata:: Test3 of mode = rflag : output/input via two methods''' 00294 # (1) Test input/output files, through the task, mode='rflag' 00295 flagdata(vis=self.vis, mode='rflag', spw='9,10', timedev='tdevfile.txt', \ 00296 freqdev='fdevfile.txt', action='calculate'); 00297 flagdata(vis=self.vis, mode='rflag', spw='9,10', timedev='tdevfile.txt', \ 00298 freqdev='fdevfile.txt', action='apply'); 00299 res1 = flagdata(vis=self.vis, mode='summary') 00300 # (2) Test rflag output written to cmd file via mode='rflag' and 'savepars' 00301 # and then read back in via list mode. 00302 # Also test the 'savepars' when timedev and freqdev are specified differently... 00303 flagdata(vis=self.vis,mode='unflag'); 00304 flagdata(vis=self.vis, mode='rflag', spw='9,10', timedev='', \ 00305 freqdev=[],action='calculate',savepars=True,outfile='outcmd.txt'); 00306 flagdata(vis=self.vis, mode='list', inpfile='outcmd.txt'); 00307 res2 = flagdata(vis=self.vis, mode='summary') 00308 00309 #print res1['flagged'], res2['flagged'] 00310 self.assertTrue(abs(res1['flagged']-res2['flagged'])<10000) 00311 self.assertTrue(abs(res1['flagged']-39504.0)<10000) 00312 00313 def test_rflag4(self): 00314 '''flagdata:: Test4 of mode = rflag : correlation selection''' 00315 flagdata(vis=self.vis, mode='rflag', spw='9,10', correlation='rr,ll'); 00316 res = flagdata(vis=self.vis, mode='summary') 00317 self.assertEqual(res['correlation']['RR']['flagged'], 9781.0) 00318 self.assertEqual(res['correlation']['LL']['flagged'], 10355.0) 00319 self.assertEqual(res['correlation']['LR']['flagged'], 0,) 00320 self.assertEqual(res['correlation']['RL']['flagged'], 0,) 00321 00322 00323 class test_shadow(test_base): 00324 def setUp(self): 00325 self.setUp_shadowdata2() 00326 00327 def test_CAS2399(self): 00328 '''flagdata: shadow by antennas not present in MS''' 00329 00330 if os.path.exists("cas2399.txt"): 00331 os.system('rm -rf cas2399.txt') 00332 00333 input = 'name=VLA01\n'+\ 00334 'diameter=25.0\n'+\ 00335 'position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]\n'+\ 00336 'name=VLA02\n'+\ 00337 'diameter=25.0\n'+\ 00338 'position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178]\n'+\ 00339 'name=VLA09\n'+\ 00340 'diameter=25.0\n'+\ 00341 'position=[-1601197.2182404203, -5041974.3604805721, 3554875.1995636248]\n'+\ 00342 'name=VLA10\n'+\ 00343 'diameter=25.0\n'+\ 00344 'position=[-1601227.3367843349,-5041975.7011900628,3554859.1642644769]\n' 00345 00346 filename = 'cas2399.txt' 00347 create_input(input, filename) 00348 00349 flagdata(vis=self.vis, mode='shadow', tolerance=10.0, addantenna=filename) 00350 res = flagdata(vis=self.vis, mode='summary') 00351 self.assertEqual(res['antenna']['VLA18']['flagged'], 3364) 00352 self.assertEqual(res['antenna']['VLA19']['flagged'], 1124) 00353 self.assertEqual(res['antenna']['VLA20']['flagged'], 440) 00354 00355 def test_addantenna(self): 00356 '''flagdata: use antenna file in list mode''' 00357 if os.path.exists("myants.txt"): 00358 os.system('rm -rf myants.txt') 00359 00360 # Create antennafile in disk 00361 input = 'name=VLA01\n'+\ 00362 'diameter=25.0\n'+\ 00363 'position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]\n'+\ 00364 'name=VLA02\n'+\ 00365 'diameter=25.0\n'+\ 00366 'position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178]\n'+\ 00367 'name=VLA09\n'+\ 00368 'diameter=25.0\n'+\ 00369 'position=[-1601197.2182404203, -5041974.3604805721, 3554875.1995636248]\n'+\ 00370 'name=VLA10\n'+\ 00371 'diameter=25.0\n'+\ 00372 'position=[-1601227.3367843349,-5041975.7011900628,3554859.1642644769]\n' 00373 00374 antfile = 'myants.txt' 00375 create_input(input, antfile) 00376 00377 # Create list file 00378 input = "mode='shadow' tolerance=10.0 addantenna='myants.txt'" 00379 filename = 'listfile.txt' 00380 create_input(input, filename) 00381 00382 # Flag 00383 flagdata(vis=self.vis, mode='list', inpfile=filename, savepars=True, outfile='withdict.txt') 00384 00385 # Check flags 00386 res = flagdata(vis=self.vis, mode='summary') 00387 self.assertEqual(res['antenna']['VLA18']['flagged'], 3364) 00388 self.assertEqual(res['antenna']['VLA19']['flagged'], 1124) 00389 self.assertEqual(res['antenna']['VLA20']['flagged'], 440) 00390 00391 class test_flagmanager(test_base): 00392 00393 def setUp(self): 00394 os.system("rm -rf flagdatatest.ms*") # test1 needs a clean start 00395 self.setUp_flagdatatest() 00396 00397 def test1m(self): 00398 '''flagmanager test1m: mode=list, flagbackup=True/False''' 00399 00400 # Create a local copy of the tool 00401 aflocal = casac.agentflagger() 00402 flagmanager(vis=self.vis, mode='list') 00403 aflocal.open(self.vis) 00404 self.assertEqual(len(aflocal.getflagversionlist()), 3) 00405 aflocal.done() 00406 00407 00408 flagdata(vis=self.vis, mode='unflag', flagbackup=False) 00409 flagmanager(vis=self.vis, mode='list') 00410 aflocal.open(self.vis) 00411 self.assertEqual(len(aflocal.getflagversionlist()), 3) 00412 aflocal.done() 00413 00414 flagdata(vis=self.vis, mode='unflag', flagbackup=True) 00415 flagmanager(vis=self.vis, mode='list') 00416 aflocal.open(self.vis) 00417 self.assertEqual(len(aflocal.getflagversionlist()), 4) 00418 aflocal.done() 00419 00420 newname = 'Ha! The best version ever!' 00421 00422 flagmanager(vis=self.vis, mode='rename', oldname='flagdata_2', versionname=newname, 00423 comment='This is a *much* better name') 00424 flagmanager(vis=self.vis, mode='list') 00425 aflocal.open(self.vis) 00426 self.assertEqual(len(aflocal.getflagversionlist()), 4) 00427 aflocal.done() 00428 00429 self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname), 00430 'Flagversion file does not exist: flags.'+newname) 00431 00432 # Specific for MMS 00433 if testmms: 00434 areg = self.vis+'/SUBMSS/*flagversions*' 00435 import glob 00436 print 'Check for .flagversions in the wrong place.' 00437 self.assertEqual(glob.glob(areg), [], 'There should not be any .flagversions in the' 00438 ' SUBMSS directory') 00439 00440 00441 def test2m(self): 00442 """flagmanager test2m: Create, then restore autoflag""" 00443 00444 flagdata(vis=self.vis, mode='summary') 00445 flagmanager(vis=self.vis) 00446 00447 flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True) 00448 00449 flagmanager(vis=self.vis) 00450 ant2 = flagdata(vis=self.vis, mode='summary')['flagged'] 00451 00452 print "After flagging antenna 2 there were", ant2, "flags" 00453 00454 # Change flags, then restore 00455 flagdata(vis=self.vis, mode='manual', antenna="3", flagbackup=True) 00456 flagmanager(vis = self.vis) 00457 ant3 = flagdata(vis=self.vis, mode='summary')['flagged'] 00458 00459 print "After flagging antenna 2 and 3 there were", ant3, "flags" 00460 00461 flagmanager(vis=self.vis, mode='restore', versionname='flagdata_3') 00462 restore2 = flagdata(vis=self.vis, mode='summary')['flagged'] 00463 00464 print "After restoring pre-antenna 3 flagging, there are", restore2, "flags; should be", ant2 00465 00466 self.assertEqual(restore2, ant2) 00467 00468 def test_CAS2701(self): 00469 """flagmanager: Do not allow flagversion=''""" 00470 00471 # Create a local copy of the tool 00472 aflocal = casac.agentflagger() 00473 00474 aflocal.open(self.vis) 00475 l = len(aflocal.getflagversionlist()) 00476 aflocal.done() 00477 00478 flagmanager(vis = self.vis, 00479 mode = "save", 00480 versionname = "non-empty-string") 00481 00482 aflocal.open(self.vis) 00483 self.assertEqual(len(aflocal.getflagversionlist()), l+1) 00484 aflocal.done() 00485 00486 flagmanager(vis = self.vis, 00487 mode = "save", 00488 versionname = "non-empty-string") 00489 00490 aflocal.open(self.vis) 00491 self.assertEqual(len(aflocal.getflagversionlist()), l+1) 00492 aflocal.done() 00493 00494 00495 class test_msselection(test_base): 00496 00497 def setUp(self): 00498 self.setUp_ngc5921() 00499 00500 def test_simple(self): 00501 '''flagdata: select only cross-correlations''' 00502 baselines = flagdata(vis = self.vis, mode="summary", antenna="VA09", basecnt=True)['baseline'].keys() 00503 assert "VA09&&VA09" not in baselines 00504 assert "VA09&&VA10" in baselines 00505 assert "VA09&&VA11" in baselines 00506 assert "VA10&&VA10" not in baselines 00507 assert "VA10&&VA11" not in baselines 00508 00509 baselines = flagdata(vis = self.vis, mode="summary", antenna="VA09,VA10", basecnt=True)['baseline'].keys() 00510 assert "VA09&&VA09" not in baselines 00511 assert "VA09&&VA10" in baselines 00512 assert "VA09&&VA11" in baselines 00513 assert "VA10&&VA10" not in baselines 00514 assert "VA10&&VA11" in baselines 00515 00516 def test_amp(self): 00517 '''flagdata: select only cross-correlations''' 00518 baselines = flagdata(vis = self.vis, mode="summary", antenna="VA09,VA10&",basecnt=True)['baseline'].keys() 00519 assert "VA09&&VA09" not in baselines 00520 assert "VA09&&VA10" in baselines 00521 assert "VA09&&VA11" not in baselines 00522 assert "VA10&&VA10" not in baselines 00523 assert "VA10&&VA11" not in baselines 00524 00525 baselines = flagdata(vis = self.vis, mode="summary", antenna="VA09&VA10",basecnt=True)['baseline'].keys() 00526 assert "VA09&&VA09" not in baselines 00527 assert "VA09&&VA10" in baselines 00528 assert "VA09&&VA11" not in baselines 00529 assert "VA10&&VA10" not in baselines 00530 assert "VA10&&VA11" not in baselines 00531 00532 def test_autocorr1(self): 00533 '''flagdata: flag only auto-correlations with antenna selection''' 00534 flagdata(vis=self.vis, mode='manual', antenna='VA05&&&') 00535 s = flagdata(vis = self.vis, mode="summary",basecnt=True)['baseline'] 00536 assert s['VA05&&VA05']['flagged'] == 7560 00537 assert s['VA01&&VA05']['flagged'] == 0 00538 assert s['VA02&&VA05']['flagged'] == 0 00539 assert s['VA05&&VA10']['flagged'] == 0 00540 assert s['VA05&&VA11']['flagged'] == 0 00541 00542 s = flagdata(vis = self.vis, mode="summary") 00543 self.assertEqual(s['flagged'], 7560) 00544 00545 def test_autocorr2(self): 00546 '''flagdata: flag auto-corrs with parameter''' 00547 flagdata(vis=self.vis, autocorr=True) 00548 s = flagdata(vis = self.vis, mode="summary") 00549 self.assertEqual(s['flagged'], 203994) 00550 00551 def test_autocorr3(self): 00552 '''flagdata: flag auto-corrs in list mode''' 00553 # creat input list 00554 input = "scan='1' mode='manual' autocorr=true reason='AUTO'\n"\ 00555 "scan='3' autocorr=True reason='AUTO'\n"\ 00556 "scan='4' reason='ALL'" 00557 filename = 'listauto.txt' 00558 create_input(input, filename) 00559 00560 # select only the autocorr reasons to flag 00561 flagdata(vis=self.vis, mode='list', inpfile=filename, reason='AUTO', action='apply') 00562 s = flagdata(vis = self.vis, mode="summary", basecnt=True) 00563 self.assertEqual(s['scan']['4']['flagged'], 0) 00564 self.assertEqual(s['baseline']['VA09&&VA28']['flagged'], 0) 00565 self.assertEqual(s['baseline']['VA09&&VA09']['flagged'], 3528) 00566 00567 # select only the third line scan=4 00568 flagdata(vis=self.vis, mode='list', inpfile=filename, reason='ALL', action='apply') 00569 s = flagdata(vis = self.vis, mode="summary", basecnt=True) 00570 self.assertEqual(s['scan']['4']['flagged'], 95256) 00571 self.assertEqual(s['baseline']['VA09&&VA28']['flagged'], 252) 00572 self.assertEqual(s['baseline']['VA09&&VA09']['flagged'], 3780) 00573 self.assertEqual(s['flagged'], 190386) 00574 00575 00576 class test_statistics_queries(test_base): 00577 00578 def setUp(self): 00579 self.setUp_ngc5921() 00580 00581 def test_CAS2021(self): 00582 '''flagdata: test antenna negation selection''' 00583 00584 flagdata(vis=self.vis, antenna='!VA05', savepars=False) 00585 s = flagdata(vis = self.vis, mode="summary",basecnt=True)['baseline'] 00586 assert s['VA01&&VA05']['flagged'] == 0 00587 assert s['VA02&&VA05']['flagged'] == 0 00588 assert s['VA03&&VA05']['flagged'] == 0 00589 assert s['VA04&&VA05']['flagged'] == 0 00590 assert s['VA05&&VA06']['flagged'] == 0 00591 assert s['VA05&&VA07']['flagged'] == 0 00592 assert s['VA05&&VA08']['flagged'] == 0 00593 assert s['VA05&&VA09']['flagged'] == 0 00594 assert s['VA05&&VA10']['flagged'] == 0 00595 assert s['VA05&&VA11']['flagged'] == 0 00596 assert s['VA05&&VA12']['flagged'] == 0 00597 assert s['VA05&&VA13']['flagged'] == 0 00598 assert s['VA05&&VA14']['flagged'] == 0 00599 assert s['VA05&&VA15']['flagged'] == 0 00600 assert s['VA05&&VA16']['flagged'] == 0 00601 assert s['VA05&&VA17']['flagged'] == 0 00602 assert s['VA05&&VA18']['flagged'] == 0 00603 assert s['VA05&&VA19']['flagged'] == 0 00604 assert s['VA05&&VA20']['flagged'] == 0 00605 assert s['VA05&&VA21']['flagged'] == 0 00606 assert s['VA05&&VA22']['flagged'] == 0 00607 assert s['VA05&&VA24']['flagged'] == 0 00608 assert s['VA05&&VA25']['flagged'] == 0 00609 assert s['VA05&&VA26']['flagged'] == 0 00610 assert s['VA05&&VA27']['flagged'] == 0 00611 assert s['VA05&&VA28']['flagged'] == 0 00612 assert s['VA05&&VA05']['flagged'] == 7560 00613 assert s['VA05&&VA05']['total'] == 7560 00614 00615 00616 def test_CAS2212(self): 00617 '''flagdata: Clipping scan selection, CAS-2212, CAS-3496''' 00618 # By default correlation='ABS_ALL' 00619 flagdata(vis=self.vis, mode='clip', scan="2", clipminmax = [0.2, 0.3], savepars=False) 00620 s = flagdata(vis=self.vis, mode='summary') 00621 self.assertEqual(s['flagged'], 85404) 00622 self.assertEqual(s['total'], 2854278) 00623 00624 s = flagdata(vis=self.vis, mode='summary')['scan'] 00625 00626 # Make sure no other scan is clipped 00627 self.assertEqual(s['1']['flagged'], 0) 00628 self.assertEqual(s['3']['flagged'], 0) 00629 self.assertEqual(s['4']['flagged'], 0) 00630 self.assertEqual(s['5']['flagged'], 0) 00631 self.assertEqual(s['6']['flagged'], 0) 00632 self.assertEqual(s['7']['flagged'], 0) 00633 self.assertEqual(s['2']['flagged'], 85404) 00634 00635 def test021(self): 00636 '''flagdata: Test of flagging statistics and queries''' 00637 00638 flagdata(vis=self.vis, correlation='LL', savepars=False) 00639 flagdata(vis=self.vis, spw='0:17~19', savepars=False) 00640 flagdata(vis=self.vis, antenna='VA05&&VA09', savepars=False) 00641 flagdata(vis=self.vis, antenna='VA14', savepars=False) 00642 flagdata(vis=self.vis, field='1', savepars=False) 00643 s = flagdata(vis=self.vis, mode='summary', minrel=0.9, spwchan=True, basecnt=True) 00644 assert s['antenna'].keys() == ['VA14'] 00645 assert 'VA05&&VA09' in s['baseline'].keys() 00646 assert set(s['spw:channel'].keys()) == set(['0:17', '0:18', '0:19']) 00647 assert s['correlation'].keys() == ['LL'] # LL 00648 assert s['field'].keys() == ['1445+09900002_0'] 00649 assert set(s['scan'].keys()) == set(['2', '4', '5', '7']) # field 1 00650 s = flagdata(vis=self.vis, mode='summary', maxrel=0.8) 00651 assert set(s['field'].keys()) == set(['1331+30500002_0', 'N5921_2']) 00652 s = flagdata(vis=self.vis, mode='summary', minabs=400000) 00653 assert set(s['scan'].keys()) == set(['3', '6']) 00654 s = flagdata(vis=self.vis, mode='summary', minabs=400000, maxabs=450000) 00655 assert s['scan'].keys() == ['3'] 00656 00657 def test_chanavg0(self): 00658 print "Test of channel average" 00659 flagdata(vis=self.vis, mode='clip',channelavg=False, clipminmax=[30., 60.], correlation='ABS_RR', 00660 savepars=False) 00661 res = flagdata(vis=self.vis, mode='summary') 00662 self.assertEqual(res['flagged'], 1414186) 00663 00664 def test_chanavg1(self): 00665 flagdata(vis=self.vis, mode='clip',channelavg=True, clipminmax=[30., 60.], correlation='ABS_RR', 00666 savepars=False) 00667 res = flagdata(vis=self.vis, mode='summary') 00668 self.assertEqual(res['flagged'], 1347822) 00669 00670 def test_chanavg2(self): 00671 flagdata(vis=self.vis, mode='clip',channelavg=False, clipminmax=[30., 60.], spw='0:0~10', 00672 correlation='ABS_RR', savepars=False) 00673 res = flagdata(vis=self.vis, mode='summary') 00674 self.assertEqual(res['flagged'], 242053) 00675 00676 def test_chanavg3(self): 00677 flagdata(vis=self.vis, mode='clip',channelavg=True, clipminmax=[30., 60.], spw='0:0~10', 00678 correlation='ABS_RR', savepars=False) 00679 res = flagdata(vis=self.vis, mode='summary') 00680 self.assertEqual(res['flagged'], 231374) 00681 00682 00683 # def test8(self): 00684 # print "Test of mode = 'quack'" 00685 # print "parallel quack" 00686 # flagdata(vis=self.vis, mode='quack', quackinterval=[1.0, 5.0], antenna=['2', '3'], correlation='RR') 00687 # test_eq(flagdata(vis=self.vis, mode='summary'), 2854278, 22365) 00688 # 00689 def test9(self): 00690 '''flagdata: quack mode''' 00691 flagdata(vis=self.vis, mode='quack', quackmode='beg', quackinterval=1, savepars=False) 00692 res = flagdata(vis=self.vis, mode='summary') 00693 self.assertEqual(res['flagged'], 329994) 00694 00695 def test10(self): 00696 '''flagdata: quack mode''' 00697 flagdata(vis=self.vis, mode='quack', quackmode='endb', quackinterval=1, savepars=False) 00698 res = flagdata(vis=self.vis, mode='summary') 00699 self.assertEqual(res['flagged'], 333396) 00700 00701 def test11(self): 00702 '''flagdata: quack mode''' 00703 flagdata(vis=self.vis, mode='quack', quackmode='end', quackinterval=1, savepars=False) 00704 res = flagdata(vis=self.vis, mode='summary') 00705 self.assertEqual(res['flagged'], 2520882) 00706 00707 def test12(self): 00708 '''flagdata: quack mode''' 00709 flagdata(vis=self.vis, mode='quack', quackmode='tail', quackinterval=1, savepars=False) 00710 res = flagdata(vis=self.vis, mode='summary') 00711 self.assertEqual(res['flagged'], 2524284) 00712 00713 def test13(self): 00714 '''flagdata: quack mode, quackincrement''' 00715 flagdata(vis=self.vis, mode='quack', quackinterval=50, quackmode='endb', quackincrement=True, 00716 savepars=False) 00717 test_eq(flagdata(vis=self.vis, mode='summary'), 2854278, 571536) 00718 00719 flagdata(vis=self.vis, mode='quack', quackinterval=20, quackmode='endb', quackincrement=True, 00720 savepars=False) 00721 test_eq(flagdata(vis=self.vis, mode='summary'), 2854278, 857304) 00722 00723 flagdata(vis=self.vis, mode='quack', quackinterval=150, quackmode='endb', quackincrement=True, 00724 savepars=False) 00725 test_eq(flagdata(vis=self.vis, mode='summary'), 2854278, 1571724) 00726 00727 flagdata(vis=self.vis, mode='quack', quackinterval=50, quackmode='endb', quackincrement=True, 00728 savepars=False) 00729 test_eq(flagdata(vis=self.vis, mode='summary'), 2854278, 1762236) 00730 flagdata(vis=self.vis, mode='unflag', savepars=False) 00731 00732 00733 00734 class test_selections(test_base): 00735 """Test various selections""" 00736 00737 def setUp(self): 00738 self.setUp_ngc5921() 00739 00740 def test_scan(self): 00741 '''flagdata: scan selection and manualflag compatibility''' 00742 flagdata(vis=self.vis, scan='3', mode='manualflag', savepars=False) 00743 res = flagdata(vis=self.vis, mode='summary', antenna='VA02') 00744 self.assertEqual(res['flagged'],52416) 00745 00746 def test_antenna(self): 00747 '''flagdata: antenna selection''' 00748 flagdata(vis=self.vis, antenna='VA02', savepars=False) 00749 test_eq(flagdata(vis=self.vis, mode='summary', antenna='VA02'), 196434, 196434) 00750 00751 def test_spw(self): 00752 '''flagdata: spw selection''' 00753 flagdata(vis=self.vis, spw='0', savepars=False) 00754 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 196434) 00755 00756 def test_spw_list(self): 00757 '''flagdata: spw selection in list mode''' 00758 spwfile = 'spwflags.txt' 00759 if os.path.exists(spwfile): 00760 os.system('rm -rf '+spwfile) 00761 00762 flagdata(vis=self.vis, spw='0:1~10', savepars=True, outfile=spwfile) 00763 res0 = flagdata(vis=self.vis, mode='summary', spwchan=True) 00764 self.assertEqual(res0['flagged'], 453060, 'Only channels 1~10 should be flagged') 00765 00766 # Unflag 00767 flagdata(vis=self.vis, mode='unflag') 00768 ures = flagdata(vis=self.vis, mode='summary') 00769 self.assertEqual(ures['flagged'], 0) 00770 00771 # Flag using the saved list 00772 flagdata(vis=self.vis, mode='list', inpfile=spwfile, action='apply') 00773 res = flagdata(vis=self.vis, mode='summary', spwchan=True) 00774 self.assertEqual(res0['flagged'], res['flagged']) 00775 00776 # Only channels 1~10 should be flagged 00777 self.assertEqual(res['spw:channel']['0:0']['flagged'], 0) 00778 self.assertEqual(res['spw:channel']['0:1']['flagged'], 45306) 00779 self.assertEqual(res['spw:channel']['0:2']['flagged'], 45306) 00780 self.assertEqual(res['spw:channel']['0:3']['flagged'], 45306) 00781 self.assertEqual(res['spw:channel']['0:4']['flagged'], 45306) 00782 self.assertEqual(res['spw:channel']['0:5']['flagged'], 45306) 00783 self.assertEqual(res['spw:channel']['0:6']['flagged'], 45306) 00784 self.assertEqual(res['spw:channel']['0:7']['flagged'], 45306) 00785 self.assertEqual(res['spw:channel']['0:8']['flagged'], 45306) 00786 self.assertEqual(res['spw:channel']['0:9']['flagged'], 45306) 00787 self.assertEqual(res['spw:channel']['0:10']['flagged'], 45306) 00788 self.assertEqual(res['spw:channel']['0:11']['flagged'], 0) 00789 00790 def test_correlation(self): 00791 flagdata(vis=self.vis, correlation='LL', savepars=False) 00792 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 98217) 00793 test_eq(flagdata(vis=self.vis, mode='summary', correlation='RR'), 1427139, 0) 00794 flagdata(vis=self.vis, mode='unflag', savepars=False) 00795 flagdata(vis=self.vis, correlation='LL,RR', savepars=False) 00796 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 196434) 00797 00798 flagdata(vis=self.vis, mode='unflag', savepars=False) 00799 flagdata(vis=self.vis, mode='clip', correlation='NORM_RR,LL', clipminmax=[0.,3.]) 00800 res = flagdata(vis=self.vis, mode='summary') 00801 self.assertEqual(res['flagged'], 204979) 00802 # flagdata(vis=self.vis, correlation='LL RR') 00803 # flagdata(vis=self.vis, correlation='LL ,, , ,RR') 00804 # test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 196434) 00805 00806 def test_field(self): 00807 '''flagdata: field selection''' 00808 flagdata(vis=self.vis, field='0', savepars=False) 00809 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 39186) 00810 00811 def test_uvrange(self): 00812 '''flagdata: uvrange selection''' 00813 flagdata(vis=self.vis, uvrange='200~400m', savepars=False) 00814 test_eq(flagdata(vis=self.vis, mode='summary', antenna='VA02'), 196434, 55944) 00815 00816 def test_timerange(self): 00817 '''flagdata: timerange selection''' 00818 flagdata(vis=self.vis, timerange='09:50:00~10:20:00', savepars=False) 00819 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 6552) 00820 00821 def test_array(self): 00822 '''flagdata: array selection''' 00823 flagdata(vis=self.vis, array='0', savepars=False) 00824 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 196434) 00825 00826 def test_action(self): 00827 '''flagdata: action = calculate''' 00828 flagdata(vis=self.vis, antenna='2,3,4', action='calculate') 00829 res = flagdata(vis=self.vis, mode='summary') 00830 self.assertEqual(res['flagged'], 0, 'Nothing should be flagged when action=calculate') 00831 00832 def test_missing_corr_product(self): 00833 '''CAS-4234: Keep going when one of the corr products is not available but others are present''' 00834 flagdata(vis=self.vis, correlation='LL,LR', savepars=False) 00835 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 98217) 00836 test_eq(flagdata(vis=self.vis, mode='summary', correlation='RR'), 1427139, 0) 00837 flagdata(vis=self.vis, mode='unflag', savepars=False) 00838 flagdata(vis=self.vis, correlation='LL,RR,RL', savepars=False) 00839 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 196434, 196434) 00840 00841 00842 class test_selections_alma(test_base): 00843 # Test various selections for alma data 00844 00845 def setUp(self): 00846 self.setUp_flagdatatest_alma() 00847 00848 def test_scanitent(self): 00849 '''flagdata: scanintent selection''' 00850 # flag POINTING CALIBRATION scans 00851 # (CALIBRATE_POINTING_.. from STATE table's OBS_MODE) 00852 flagdata(vis=self.vis, intent='CAL*POINT*', savepars=False) 00853 test_eq(flagdata(vis=self.vis, mode='summary', antenna='2'), 377280, 26200) 00854 00855 def test_wvr(self): 00856 '''flagdata: flag WVR correlation''' 00857 flagdata(vis=self.vis, correlation='I', savepars=False) 00858 test_eq(flagdata(vis=self.vis, mode='summary'),1154592, 22752) 00859 00860 def test_abs_wvr(self): 00861 '''flagdata: clip ABS_WVR''' 00862 flagdata(vis=self.vis, mode='clip',clipminmax=[0,50], correlation='ABS_WVR', savepars=False) 00863 test_eq(flagdata(vis=self.vis, mode='summary'),1154592, 22752) 00864 00865 def test_abs_i(self): 00866 '''flagdata: clip ABS_I. Do not flag WVR''' 00867 flagdata(vis=self.vis, mode='clip', clipminmax=[0,50], correlation='ABS_I', savepars=False) 00868 test_eq(flagdata(vis=self.vis, mode='summary'),1154592, 0) 00869 00870 def test_abs_all(self): 00871 '''flagdata: clip ABS ALL. Do not flag WVR''' 00872 flagdata(vis=self.vis, mode='clip', clipminmax=[0,1], correlation='ABS ALL', savepars=False) 00873 test_eq(flagdata(vis=self.vis, mode='summary'),1154592, 130736) 00874 test_eq(flagdata(vis=self.vis, mode='summary', correlation='I'),22752, 0) 00875 00876 def test_spw(self): 00877 '''flagdata: flag various spw''' 00878 # Test that a white space in the spw parameter is taken correctly 00879 flagdata(vis=self.vis, mode='manual', spw='1,3, 4', savepars=False) 00880 res = flagdata(vis=self.vis, mode='summary') 00881 self.assertEqual(res['spw']['0']['flagged'], 0, 'spw=0 should not be flagged') 00882 self.assertEqual(res['spw']['1']['flagged'], 552960, 'spw=1 should be fully flagged') 00883 self.assertEqual(res['spw']['4']['flagged'], 22752, 'spw=4 should not be flagged') 00884 self.assertEqual(res['spw']['4']['total'], 22752, 'spw=4 should not be flagged') 00885 00886 class test_selections2(test_base): 00887 '''Test other selections''' 00888 00889 def setUp(self): 00890 self.setUp_multi() 00891 00892 def test_observation1(self): 00893 '''flagdata: observation ID selections''' 00894 # string 00895 flagdata(vis=self.vis, observation='1', savepars=False) 00896 res = flagdata(vis=self.vis, mode='summary') 00897 self.assertEqual(res['flagged'], 28500) 00898 self.assertEqual(res['total'], 2882778) 00899 00900 # integer 00901 flagdata(vis=self.vis, mode='unflag', savepars=False) 00902 flagdata(vis=self.vis, observation=1, savepars=False) 00903 res = flagdata(vis=self.vis, mode='summary') 00904 self.assertEqual(res['flagged'], 28500) 00905 self.assertEqual(res['total'], 2882778) 00906 00907 def test_observation2(self): 00908 '''flagdata: observation ID selections in list mode''' 00909 # creat input list 00910 input = "observation='0' mode='manual'" 00911 filename = 'obs2.txt' 00912 create_input(input, filename) 00913 00914 flagdata(vis=self.vis, mode='list', inpfile=filename, savepars=False) 00915 res = flagdata(vis=self.vis, mode='summary') 00916 self.assertEqual(res['observation']['0']['flagged'], 2854278.0) 00917 self.assertEqual(res['observation']['1']['flagged'], 0, 'Only observation 0 should be flagged') 00918 00919 00920 class test_elevation(test_base): 00921 """Test of mode = 'elevation'""" 00922 def setUp(self): 00923 self.setUp_ngc5921() 00924 self.x55 = 666792 # data below 55 degrees, etc. 00925 self.x60 = 1428840 00926 self.x65 = 2854278 00927 self.all = 2854278 00928 00929 def test_lower(self): 00930 flagdata(vis = self.vis, mode = 'elevation', savepars=False) 00931 00932 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, 0) 00933 00934 flagdata(vis = self.vis, mode = 'elevation', lowerlimit = 50, savepars=False) 00935 00936 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, 0) 00937 00938 flagdata(vis = self.vis, mode = 'elevation', lowerlimit = 55, savepars=False) 00939 00940 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, self.x55) 00941 00942 flagdata(vis = self.vis, mode = 'elevation', lowerlimit = 60, savepars=False) 00943 00944 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, self.x60) 00945 00946 flagdata(vis = self.vis, mode = 'elevation', lowerlimit = 65, savepars=False) 00947 00948 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, self.x65) 00949 00950 def test_upper(self): 00951 flagdata(vis = self.vis, mode = 'elevation', upperlimit = 60, savepars=False) 00952 00953 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, self.all - self.x60) 00954 00955 00956 def test_interval(self): 00957 flagdata(vis = self.vis, 00958 mode = 'elevation', 00959 lowerlimit = 55, 00960 upperlimit = 60, 00961 savepars=False) 00962 00963 test_eq(flagdata(vis=self.vis, mode='summary'), self.all, self.all - (self.x60 - self.x55)) 00964 00965 00966 class test_list_file(test_base): 00967 """Test of mode = 'list' using input file""" 00968 00969 def setUp(self): 00970 self.setUp_ngc5921() 00971 00972 def test_file1(self): 00973 '''flagdata: apply flags from a list and do not save''' 00974 # creat input list 00975 input = "scan=1~3 mode=manual\n"+"scan=5 mode=manualflag\n"\ 00976 "#scan='4'" 00977 filename = 'list1.txt' 00978 create_input(input, filename) 00979 00980 # apply and don't save to MS. Ignore comment line 00981 flagdata(vis=self.vis, mode='list', inpfile=filename, savepars=False, action='apply') 00982 res = flagdata(vis=self.vis, mode='summary') 00983 self.assertEqual(res['scan']['4']['flagged'], 0) 00984 self.assertEqual(res['flagged'], 1711206, 'Total flagged does not match') 00985 00986 def test_file2(self): 00987 '''flagdata: only save parameters without running the tool''' 00988 # creat input list 00989 input = "scan=1~3 mode=manual\n"+"scan=5 mode=manual\n" 00990 filename = 'list2.txt' 00991 create_input(input, filename) 00992 00993 # save to another file 00994 if os.path.exists("myflags.txt"): 00995 os.system('rm -rf myflags.txt') 00996 00997 flagdata(vis=self.vis, mode='list', inpfile=filename, savepars=True, action='', outfile='myflags.txt') 00998 self.assertTrue(filecmp.cmp(filename, 'myflags.txt', 1), 'Files should be equal') 00999 01000 res = flagdata(vis=self.vis, mode='summary') 01001 self.assertEqual(res['flagged'], 0, 'No flags should have been applied') 01002 01003 def test_file3(self): 01004 '''flagdata: flag and save list to FLAG_CMD''' 01005 # creat input list 01006 input = "scan=1~3 mode=manual\n"+"scan=5 mode=manual\n" 01007 filename = 'list3.txt' 01008 create_input(input, filename) 01009 01010 # Delete any rows from FLAG_CMD 01011 flagcmd(vis=self.vis, action='clear', clearall=True) 01012 01013 # Flag from list and save to FLAG_CMD 01014 flagdata(vis=self.vis, mode='list', inpfile=filename, savepars=True) 01015 01016 # Verify 01017 if os.path.exists("myflags.txt"): 01018 os.system('rm -rf myflags.txt') 01019 flagcmd(vis=self.vis, action='list', savepars=True, outfile='myflags.txt', useapplied=True) 01020 self.assertTrue(filecmp.cmp(filename, 'myflags.txt', 1), 'Files should be equal') 01021 01022 01023 def test_file4(self): 01024 '''flagdata: save without running and apply in flagcmd''' 01025 # Delete any rows from FLAG_CMD 01026 flagcmd(vis=self.vis, action='clear', clearall=True) 01027 01028 # Test that action='none' is also accepted 01029 flagdata(vis=self.vis, mode='quack', quackmode='tail', quackinterval=1, action='none', 01030 savepars=True) 01031 01032 flagcmd(vis=self.vis, action='apply') 01033 res = flagdata(vis=self.vis, mode='summary') 01034 self.assertEqual(res['flagged'], 2524284) 01035 01036 def test_file5(self): 01037 '''flagdata: clip zeros in mode=list and save reason to FLAG_CMD''' 01038 # get the correct data, by passing the previous setUp() 01039 self.setUp_data4tfcrop() 01040 01041 # creat input list 01042 input = "mode='clip' clipzeros=true reason='CLIP_ZERO'" 01043 filename = 'list5.txt' 01044 create_input(input, filename) 01045 01046 # Save to FLAG_CMD 01047 flagdata(vis=self.vis, mode='list', inpfile=filename, action='', savepars=True) 01048 01049 # Run in flagcmd and select by reason 01050 flagcmd(vis=self.vis, action='apply', reason='CLIP_ZERO') 01051 01052 res = flagdata(vis=self.vis, mode='summary') 01053 self.assertEqual(res['flagged'], 274944, 'Should clip only spw=8') 01054 01055 def test_file6(self): 01056 '''flagdata: select by reason in list mode from a file''' 01057 # creat input list 01058 input = "mode='manual' scan='1' reason='SCAN_1'\n"\ 01059 "mode='manual' scan='2'\n"\ 01060 "scan='3' reason='SCAN_3'\n"\ 01061 "scan='4' reason=''" 01062 filename = 'list6.txt' 01063 create_input(input, filename) 01064 01065 # Select one reason 01066 flagdata(vis=self.vis, mode='list', inpfile=filename, reason='SCAN_3') 01067 res = flagdata(vis=self.vis, mode='summary') 01068 self.assertEqual(res['scan']['3']['flagged'], 762048, 'Should flag only reason=SCAN_3') 01069 self.assertEqual(res['flagged'], 762048, 'Should flag only reason=SCAN_3') 01070 01071 # Select list of reasons 01072 flagdata(vis=self.vis, mode='list', inpfile=filename, reason=['','SCAN_1']) 01073 res = flagdata(vis=self.vis, mode='summary') 01074 self.assertEqual(res['scan']['4']['flagged'], 95256, 'Should flag reason=\'\'') 01075 self.assertEqual(res['scan']['1']['flagged'], 568134, 'Should flag reason=SCAN_1') 01076 01077 # No reason selection 01078 flagdata(vis=self.vis, mode='unflag') 01079 flagdata(vis=self.vis, mode='list', inpfile=filename) 01080 res = flagdata(vis=self.vis, mode='summary') 01081 self.assertEqual(res['scan']['1']['flagged'], 568134) 01082 self.assertEqual(res['scan']['2']['flagged'], 238140) 01083 self.assertEqual(res['scan']['3']['flagged'], 762048) 01084 self.assertEqual(res['scan']['4']['flagged'], 95256) 01085 self.assertEqual(res['flagged'],568134+238140+762048+95256, 'Total flagged') 01086 01087 def test_reason1(self): 01088 '''flagdata: add_reason to FLAG_CMD''' 01089 flagcmd(vis=self.vis, action='clear', clearall=True) 01090 flagdata(vis=self.vis, mode='manual', scan='1,3', savepars=True, cmdreason='SCAN_1_3', 01091 action='') 01092 01093 # Apply flag cmd 01094 flagcmd(vis=self.vis, action='apply', reason='SCAN_1_3') 01095 res = flagdata(vis=self.vis, mode='summary') 01096 self.assertEqual(res['flagged'], 1330182, 'Only scans 1 and 3 should be flagged') 01097 01098 def test_reason2(self): 01099 '''flagdata: add_reason to text file''' 01100 flagdata(vis=self.vis, mode='clip', scan='4', clipminmax=[0, 5], savepars=True, 01101 cmdreason='CLIPSCAN4', outfile='reason2.txt', action='') 01102 01103 flagdata(vis=self.vis, mode='clip', scan='2~3', clipminmax=[ 0, 5], savepars=True, 01104 cmdreason='CLIPSCAN2_3', outfile='reason2.txt', action='') 01105 01106 # Apply flag cmd 01107 flagdata(vis=self.vis, mode='list', inpfile='reason2.txt',reason='CLIPSCAN2_3') 01108 01109 res = flagdata(vis=self.vis, mode='summary') 01110 self.assertEqual(res['flagged'], 69568) 01111 01112 def test_reason3(self): 01113 '''flagdata: replace input reason from file with cmdreason''' 01114 # creat input list 01115 input = "mode='manual' scan='1' reason='SCAN_1'\n"\ 01116 "mode='manual' scan='2'\n"\ 01117 "scan='3' reason='SCAN_3'\n"\ 01118 "scan='4' reason=''" 01119 filename = 'input3.txt' 01120 create_input(input, filename) 01121 01122 flagdata(vis=self.vis, mode='list', inpfile=filename, savepars=True, outfile='reason3.txt', 01123 cmdreason='MANUALFLAG', action='') 01124 01125 # Apply the flag cmds 01126 flagdata(vis=self.vis, mode='list', inpfile='reason3.txt', reason='MANUALFLAG') 01127 01128 res = flagdata(vis=self.vis, mode='summary') 01129 self.assertEqual(res['flagged'], 1663578) 01130 01131 class test_list_list(test_base): 01132 """Test of mode = 'list' using input list""" 01133 01134 def setUp(self): 01135 self.setUp_ngc5921() 01136 01137 def test_list1(self): 01138 '''flagdata: apply flags from a Python list and do not save''' 01139 # creat input list 01140 input = ["scan='1~3' mode='manual'", 01141 "scan='5' mode='manualflag'", 01142 "#scan='4'"] 01143 01144 # apply and don't save to MS. Ignore comment line 01145 flagdata(vis=self.vis, mode='list', inpfile=input, savepars=False, action='apply') 01146 res = flagdata(vis=self.vis, mode='summary') 01147 self.assertEqual(res['scan']['4']['flagged'], 0) 01148 self.assertEqual(res['flagged'], 1711206, 'Total flagged does not match') 01149 01150 def test_list2(self): 01151 '''flagdata: only save parameters without running the tool''' 01152 # creat input list 01153 input = ["scan='1~3' mode='manual'", 01154 "scan='5' mode='manual'"] 01155 01156 # save to another file 01157 if os.path.exists("myflags.txt"): 01158 os.system('rm -rf myflags.txt') 01159 01160 flagdata(vis=self.vis, mode='list', inpfile=input, savepars=True, action='', outfile='myflags.txt') 01161 01162 res = flagdata(vis=self.vis, mode='summary') 01163 self.assertEqual(res['flagged'], 0, 'No flags should have been applied') 01164 01165 def test_list3(self): 01166 '''flagdata: Compare flags from flagdata and flagcmd''' 01167 # creat input list 01168 input = ["scan='1~3' mode='manual'", 01169 "scan='5' mode='manual'"] 01170 01171 # Delete any rows from FLAG_CMD 01172 flagcmd(vis=self.vis, action='clear', clearall=True) 01173 01174 # Flag from list and save to FLAG_CMD 01175 flagdata(vis=self.vis, mode='list', inpfile=input) 01176 res1 = flagdata(vis=self.vis, mode='summary') 01177 01178 # Unflag and save in flagcmd using the cmd mode 01179 flagdata(vis=self.vis, mode='unflag') 01180 flagcmd(vis=self.vis, inpmode='list', inpfile=input) 01181 res2 = flagdata(vis=self.vis, mode='summary') 01182 01183 # Verify 01184 self.assertEqual(res1['flagged'], res2['flagged']) 01185 self.assertEqual(res1['total'], res2['total']) 01186 01187 def test_list4(self): 01188 '''flagdata: clip zeros in mode=list and save reason to FLAG_CMD''' 01189 # get the correct data, by passing the previous setUp() 01190 self.setUp_data4tfcrop() 01191 01192 # creat input list 01193 input = ["mode='clip' clipzeros=true reason='CLIP_ZERO'"] 01194 01195 # Save to FLAG_CMD 01196 flagdata(vis=self.vis, mode='list', inpfile=input, action='', savepars=True) 01197 01198 # Run in flagcmd and select by reason 01199 flagcmd(vis=self.vis, action='apply', reason='CLIP_ZERO') 01200 01201 res = flagdata(vis=self.vis, mode='summary') 01202 self.assertEqual(res['flagged'], 274944, 'Should clip only spw=8') 01203 01204 def test_list5(self): 01205 '''flagdata: select by reason in list mode from a list''' 01206 # creat input list 01207 input = ["mode='manual' scan='1' reason='SCAN_1'", 01208 "mode='manual' scan='2'", 01209 "scan='3' reason='SCAN_3'", 01210 "scan='4' reason=''"] 01211 01212 # Select one reason 01213 flagdata(vis=self.vis, mode='list', inpfile=input, reason='SCAN_3') 01214 res = flagdata(vis=self.vis, mode='summary') 01215 self.assertEqual(res['scan']['3']['flagged'], 762048, 'Should flag only reason=SCAN_3') 01216 self.assertEqual(res['flagged'], 762048, 'Should flag only reason=SCAN_3') 01217 01218 # Select list of reasons 01219 flagdata(vis=self.vis, mode='list', inpfile=input, reason=['','SCAN_1']) 01220 res = flagdata(vis=self.vis, mode='summary') 01221 self.assertEqual(res['scan']['4']['flagged'], 95256, 'Should flag reason=\'\'') 01222 self.assertEqual(res['scan']['1']['flagged'], 568134, 'Should flag reason=SCAN_1') 01223 01224 # No reason selection 01225 flagdata(vis=self.vis, mode='unflag') 01226 flagdata(vis=self.vis, mode='list', inpfile=input) 01227 res = flagdata(vis=self.vis, mode='summary') 01228 self.assertEqual(res['scan']['1']['flagged'], 568134) 01229 self.assertEqual(res['scan']['2']['flagged'], 238140) 01230 self.assertEqual(res['scan']['3']['flagged'], 762048) 01231 self.assertEqual(res['scan']['4']['flagged'], 95256) 01232 self.assertEqual(res['flagged'],568134+238140+762048+95256, 'Total flagged') 01233 01234 01235 def test_reason_list(self): 01236 '''flagdata: replace input reason from list with cmdreason''' 01237 # creat input list 01238 input = ["mode='manual' scan='1' reason='SCAN_1'", 01239 "mode='manual' scan='2'", 01240 "scan='3' reason='SCAN_3'", 01241 "scan='4' reason=''"] 01242 01243 flagdata(vis=self.vis, mode='list', inpfile=input, savepars=True, outfile='reason3.txt', 01244 cmdreason='MANUALFLAG', action='') 01245 01246 # Apply the flag cmds 01247 flagdata(vis=self.vis, mode='list', inpfile='reason3.txt', reason='MANUALFLAG') 01248 01249 res = flagdata(vis=self.vis, mode='summary') 01250 self.assertEqual(res['flagged'], 1663578) 01251 01252 01253 01254 class test_clip(test_base): 01255 """flagdata:: Test of mode = 'clip'""" 01256 01257 def setUp(self): 01258 self.setUp_data4tfcrop() 01259 01260 def test_clipzeros(self): 01261 '''flagdata: clip only zero-value data''' 01262 flagdata(vis=self.vis, mode='clip', clipzeros=True) 01263 res = flagdata(vis=self.vis, mode='summary') 01264 self.assertEqual(res['flagged'],274944,'Should clip only spw=8') 01265 01266 01267 class test_CASA_4_0_bug_fix(test_base): 01268 """flagdata:: Regression test for the fixes introduced during the CASA 4.0 bug fix season""" 01269 01270 def setUp(self): 01271 self.setUp_data4tfcrop() 01272 01273 def test_CAS_4270(self): 01274 """flagdata: Test uvrange given in lambda units""" 01275 01276 flagdata(vis=self.vis,mode='manual',uvrange='<2klambda') 01277 flagdata(vis=self.vis,mode='clip') 01278 summary_ref = flagdata(vis=self.vis,mode='summary') 01279 01280 flagdata(vis=self.vis,mode='unflag') 01281 flagdata(vis=self.vis,mode='list',inpfile=["uvrange='<2Klambda'","mode='clip'"]) 01282 summary_out = flagdata(vis=self.vis,mode='summary') 01283 01284 self.assertEqual(summary_out['flagged'],summary_ref['flagged'],'uvrange given in lambda is not properly translated into meters') 01285 01286 def test_CAS_4312(self): 01287 """flagdata: Test channel selection with Rflag agent""" 01288 01289 flagdata(vis=self.vis,mode='rflag',spw='9:10~20') 01290 summary = flagdata(vis=self.vis,mode='summary') 01291 self.assertEqual(summary['spw']['8']['flagged'],0,'Error in channel selection with Rflag agent') 01292 self.assertEqual(summary['spw']['9']['flagged'],1861,'Error in channel selection with Rflag agent') 01293 self.assertEqual(summary['spw']['10']['flagged'],0,'Error in channel selection with Rflag agent') 01294 01295 01296 def test_CAS_4200(self): 01297 """flagdata: Test quack mode with quackinterval 0""" 01298 01299 flagdata(vis=self.vis,mode='quack',quackinterval=0) 01300 summary_zero = flagdata(vis=self.vis,mode='summary') 01301 self.assertEqual(summary_zero['flagged'],0,'Error in quack mode with quack interval 0') 01302 01303 flagdata(vis=self.vis,mode='quack',quackinterval=1) 01304 summary_one = flagdata(vis=self.vis,mode='summary') 01305 01306 flagdata(vis=self.vis,mode='unflag') 01307 flagdata(vis=self.vis,mode='quack') 01308 summary_default = flagdata(vis=self.vis,mode='summary') 01309 01310 self.assertEqual(summary_one['flagged'],summary_default['flagged'],'Error in quack mode with quack interval 1') 01311 01312 def test_alias(self): 01313 '''flagdata: Test tflagdata alias''' 01314 res = tflagdata(vis=self.vis, mode='summary')['flagged'] 01315 self.assertEqual(res, 0) 01316 01317 def test_spw_freq1(self): 01318 '''flagdata: CAS-3562, flag all spw channels greater than a frequency''' 01319 flagdata(vis=self.vis, spw='>2000MHz', flagbackup=False) 01320 01321 # Flag only spw=6,7 01322 res = flagdata(vis=self.vis, mode='summary') 01323 self.assertEqual(res['spw']['0']['flagged'], 0) 01324 self.assertEqual(res['spw']['10']['flagged'], 0) 01325 self.assertEqual(res['spw']['7']['flagged'], 274944) 01326 self.assertEqual(res['spw']['7']['total'], 274944) 01327 self.assertEqual(res['spw']['6']['flagged'], 274944) 01328 self.assertEqual(res['spw']['6']['total'], 274944) 01329 self.assertEqual(res['flagged'], 549888) 01330 01331 def test_spw_freq2(self): 01332 '''flagdata: CAS-3562, flag the channel with a frequency''' 01333 flagdata(vis=self.vis, spw='*:1956MHz,*:945MHz', flagbackup=False) 01334 01335 # Flag only spw=5,8, first channel (0) 01336 res = flagdata(vis=self.vis, mode='summary', spwchan=True) 01337 self.assertEqual(res['spw:channel']['1:0']['flagged'], 0) 01338 self.assertEqual(res['spw:channel']['15:0']['flagged'], 0) 01339 self.assertEqual(res['spw:channel']['5:0']['flagged'], 4296) 01340 self.assertEqual(res['spw:channel']['5:0']['total'], 4296) 01341 self.assertEqual(res['spw:channel']['8:0']['flagged'], 4296) 01342 self.assertEqual(res['spw:channel']['8:0']['total'], 4296) 01343 self.assertEqual(res['flagged'], 8592) 01344 01345 def test_spw_freq3(self): 01346 '''flagdata: CAS-3562, flag a range of frequencies''' 01347 flagdata(vis=self.vis, spw='1500 ~ 2000MHz', flagbackup=False) 01348 01349 # Flag only spw=0~5 01350 res = flagdata(vis=self.vis, mode='summary', spwchan=True) 01351 self.assertEqual(res['spw']['0']['flagged'], 274944) 01352 self.assertEqual(res['spw']['1']['flagged'], 274944) 01353 self.assertEqual(res['spw']['2']['flagged'], 274944) 01354 self.assertEqual(res['spw']['3']['flagged'], 274944) 01355 self.assertEqual(res['spw']['4']['flagged'], 274944) 01356 self.assertEqual(res['spw']['5']['flagged'], 274944) 01357 self.assertEqual(res['spw']['6']['flagged'], 0) 01358 self.assertEqual(res['flagged'], 1649664) 01359 01360 class test_tsys(test_base): 01361 """Flagdata:: Flagging of Tsys-based CalTable """ 01362 01363 def setUp(self): 01364 self.setUp_tsys_case() 01365 01366 def test_unsupported_elevation(self): 01367 '''Flagdata: Unsupported elevation mode''' 01368 res = flagdata(vis=self.vis, mode='elevation') 01369 self.assertEqual(res, {}) 01370 01371 def test_unsupported_shadow(self): 01372 '''Flagdata: Unsupported shadow mode''' 01373 res = flagdata(vis=self.vis, mode='shadow') 01374 self.assertEqual(res, {}) 01375 01376 def test_mixed_list(self): 01377 '''Flagdata: mixed supported and unsupported modes in a list''' 01378 cmds = ["spw='1'", 01379 "spw='3' mode='elevation'", 01380 "mode='shadow'", 01381 "spw='5'"] 01382 01383 flagdata(vis=self.vis, mode='list', inpfile=cmds) 01384 res = flagdata(vis=self.vis,mode='summary',spw='1,3,5') 01385 self.assertEqual(res['spw']['1']['flagged'], 32256) 01386 self.assertEqual(res['spw']['3']['flagged'], 0) 01387 self.assertEqual(res['spw']['5']['flagged'], 32256) 01388 self.assertEqual(res['flagged'], 32256*2) 01389 01390 01391 def test_invalid_scan(self): 01392 '''Flagdata: unsupported scan selection''' 01393 res = flagdata(vis=self.vis, scan='1,2') 01394 self.assertEqual(res, {}) 01395 01396 def test_default_fparam(self): 01397 '''Flagdata: default data column FPARAM''' 01398 flagdata(vis=self.vis, mode='clip', clipminmax=[0,500]) 01399 res=flagdata(vis=self.vis, mode='summary') 01400 self.assertEqual(res['flagged'], 5325) 01401 01402 def test_manual_field_selection(self): 01403 """Flagdata:: Manually flag a Tsys-based CalTable using field selection""" 01404 01405 flagdata(vis=self.vis, field='0') 01406 res=flagdata(vis=self.vis, mode='summary') 01407 01408 self.assertEqual(res['field']['3c279']['flagged'], 9216) 01409 self.assertEqual(res['field']['Titan']['flagged'], 0) 01410 self.assertEqual(res['field']['TW Hya']['flagged'], 0) 01411 self.assertEqual(res['field']['J1037-295=QSO']['flagged'], 0) 01412 01413 def test_manual_antenna_selection(self): 01414 """Flagdata:: Manually flag a Tsys-based CalTable using antenna selection""" 01415 01416 flagdata(vis=self.vis, antenna='DV09') 01417 res=flagdata(vis=self.vis, mode='summary') 01418 self.assertEqual(res['antenna']['DV09']['flagged'], 14336) 01419 self.assertEqual(res['antenna']['DV10']['flagged'], 0) 01420 01421 def test_clip_fparm_sol1(self): 01422 """Flagdata:: Test clipping first calibration solution product of FPARAM 01423 column using a minmax range """ 01424 01425 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='Sol1', 01426 clipzeros=True, clipminmax=[0.,600.]) 01427 res=flagdata(vis=self.vis, mode='summary') 01428 01429 self.assertEqual(res['total'], 129024.0) 01430 self.assertEqual(res['flagged'], 750.0) 01431 self.assertEqual(res['correlation']['Sol1']['flagged'], 750.0) 01432 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01433 self.assertEqual(res['correlation']['Sol2']['flagged'], 0.0) 01434 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01435 01436 def test_list_fparm_sol1_extension(self): 01437 """Flagdata:: Test list mode to clip first calibration solution product of FPARAM 01438 column using a minmax range, and then extend to the other solution """ 01439 01440 cmds = ["mode='clip' datacolumn='FPARAM' correlation='Sol1' " 01441 "clipzeros=True clipminmax=[0.,600.]", 01442 "mode='extend' extendpols=True growfreq=0.0 growtime=0.0"] 01443 flagdata(vis=self.vis, mode='list', inpfile=cmds) 01444 res=flagdata(vis=self.vis, mode='summary') 01445 self.assertEqual(res['total'], 129024) 01446 self.assertEqual(res['flagged'], 1500) 01447 self.assertEqual(res['correlation']['Sol1']['flagged'], 750) 01448 self.assertEqual(res['correlation']['Sol2']['flagged'], 750) 01449 01450 # Get the same results when flagging using a file 01451 flagdata(vis=self.vis, mode='unflag') 01452 input = "mode='clip' datacolumn='FPARAM' correlation='Sol1' clipzeros=True clipminmax=[0.,600.]\n"\ 01453 "mode='extend' extendpols=True growfreq=0.0 growtime=0.0" 01454 filename = 'callist.txt' 01455 create_input(input, filename) 01456 flagdata(vis=self.vis, mode='list', inpfile=filename) 01457 res=flagdata(vis=self.vis, mode='summary') 01458 self.assertEqual(res['total'], 129024) 01459 self.assertEqual(res['flagged'], 1500) 01460 self.assertEqual(res['correlation']['Sol1']['flagged'], 750) 01461 self.assertEqual(res['correlation']['Sol2']['flagged'], 750) 01462 01463 def test_clip_fparm_sol2(self): 01464 """Flagdata:: Test cliping second calibration solution product of FPARAM 01465 column using a minmax range """ 01466 01467 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='Sol2', 01468 clipzeros=True, clipminmax=[0.,600.]) 01469 res=flagdata(vis=self.vis, mode='summary') 01470 01471 self.assertEqual(res['total'], 129024.0) 01472 self.assertEqual(res['flagged'], 442.0) 01473 self.assertEqual(res['correlation']['Sol1']['flagged'], 0.0) 01474 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01475 self.assertEqual(res['correlation']['Sol2']['flagged'], 442.0) 01476 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01477 01478 def test_clip_fparm_sol1sol2(self): 01479 """Flagdata:: Test cliping first and second calibration solution products of 01480 FPARAM column using a minmax range """ 01481 01482 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='Sol1,Sol2', 01483 clipzeros=True, clipminmax=[0.,600.]) 01484 res=flagdata(vis=self.vis, mode='summary') 01485 01486 self.assertEqual(res['total'], 129024.0) 01487 self.assertEqual(res['flagged'], 1192.0) 01488 self.assertEqual(res['correlation']['Sol1']['flagged'], 750.0) 01489 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01490 self.assertEqual(res['correlation']['Sol2']['flagged'], 442.0) 01491 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01492 01493 def test_invalid_corr(self): 01494 '''Flagdata: default correlation should be REAL_ALL in this case''' 01495 flagdata(vis=self.vis, mode='clip', correlation='ABS_ALL',clipminmax=[0.,600.]) 01496 res=flagdata(vis=self.vis, mode='summary') 01497 self.assertEqual(res['flagged'], 1192.0) 01498 01499 def test_invalid_datacol(self): 01500 '''Flagdata: invalida data column should fall back to default''' 01501 flagdata(vis=self.vis, mode='clip', clipminmax=[0.,600.],datacolumn='PARAMERR') 01502 res=flagdata(vis=self.vis, mode='summary') 01503 self.assertEqual(res['flagged'], 1192.0) 01504 01505 01506 def test_clip_fparm_all(self): 01507 """Flagdata:: Test cliping all calibration solution products of FPARAM 01508 column using a minmax range """ 01509 01510 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='', 01511 clipzeros=True, clipminmax=[0.,600.]) 01512 res=flagdata(vis=self.vis, mode='summary') 01513 self.assertEqual(res['total'], 129024.0) 01514 self.assertEqual(res['flagged'], 1192.0) 01515 self.assertEqual(res['correlation']['Sol1']['flagged'], 750.0) 01516 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01517 self.assertEqual(res['correlation']['Sol2']['flagged'], 442.0) 01518 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01519 01520 def test_clip_fparm_all(self): 01521 """Flagdata:: Test cliping only zeros in all calibration solution 01522 products of FPARAM column""" 01523 01524 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='', 01525 clipzeros=True) 01526 res=flagdata(vis=self.vis, mode='summary') 01527 01528 self.assertEqual(res['total'], 129024.0) 01529 self.assertEqual(res['flagged'], 0.0) 01530 self.assertEqual(res['correlation']['Sol1']['flagged'], 0.0) 01531 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01532 self.assertEqual(res['correlation']['Sol2']['flagged'], 0.0) 01533 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01534 01535 01536 def test_clip_nans_fparm_all(self): 01537 """Flagdata:: Test cliping only NaNs/Infs in all calibration solution products of FPARAM column""" 01538 01539 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='') 01540 res=flagdata(vis=self.vis, mode='summary') 01541 01542 self.assertEqual(res['total'], 129024.0) 01543 self.assertEqual(res['flagged'], 0.0) 01544 self.assertEqual(res['correlation']['Sol1']['flagged'], 0.0) 01545 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01546 self.assertEqual(res['correlation']['Sol2']['flagged'], 0.0) 01547 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01548 01549 01550 def test_clip_fparm_error_absall(self): 01551 """Flagdata:: Error case test when a complex operator is used with CalTables """ 01552 01553 flagdata(vis=self.vis, mode='clip', datacolumn='FPARAM', correlation='ABS_ALL', 01554 clipzeros=True, clipminmax=[0.,600.]) 01555 res=flagdata(vis=self.vis, mode='summary') 01556 01557 self.assertEqual(res['total'], 129024.0) 01558 self.assertEqual(res['flagged'], 1192.0) 01559 self.assertEqual(res['correlation']['Sol1']['flagged'], 750.0) 01560 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01561 self.assertEqual(res['correlation']['Sol2']['flagged'], 442.0) 01562 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01563 01564 def test_clip_snr_all(self): 01565 """Flagdata:: Test cliping all calibration solution products of SNR c 01566 olumn using a minmax range for Tsys CalTable""" 01567 01568 flagdata(vis=self.vis, mode='clip', datacolumn='SNR', correlation='', 01569 clipzeros=True, clipminmax=[0.,2.]) 01570 res=flagdata(vis=self.vis, mode='summary') 01571 01572 self.assertEqual(res['total'], 129024.0) 01573 self.assertEqual(res['flagged'], 0.0) 01574 self.assertEqual(res['correlation']['Sol1']['flagged'], 0.0) 01575 self.assertEqual(res['correlation']['Sol1']['total'], 64512.0) 01576 self.assertEqual(res['correlation']['Sol2']['flagged'], 0.0) 01577 self.assertEqual(res['correlation']['Sol2']['total'], 64512.0) 01578 01579 def test_spw_selection(self): 01580 '''Flagdata: Select spw in cal tables''' 01581 flagdata(vis=self.vis, mode='manual', spw='1,3') 01582 res=flagdata(vis=self.vis, mode='summary') 01583 self.assertEqual(res['spw']['1']['flagged'],32256) 01584 self.assertEqual(res['spw']['3']['flagged'],32256) 01585 self.assertEqual(res['flagged'],32256+32256) 01586 01587 def test_channel_selection(self): 01588 '''Flagdata: Select spw:channel in cal tables''' 01589 flagdata(vis=self.vis, mode='manual', spw='*:0~8,*:120~127') 01590 res=flagdata(vis=self.vis, mode='summary', spwchan=True) 01591 self.assertEqual(res['spw']['1']['flagged'],4284) 01592 self.assertEqual(res['spw']['3']['flagged'],4284) 01593 self.assertEqual(res['spw:channel']['5:0']['flagged'],252) 01594 self.assertEqual(res['spw:channel']['5:0']['total'],252) 01595 self.assertEqual(res['spw:channel']['5:9']['flagged'],0) 01596 self.assertEqual(res['spw:channel']['5:119']['flagged'],0) 01597 self.assertEqual(res['spw:channel']['5:120']['flagged'],252) 01598 self.assertEqual(res['spw:channel']['5:127']['flagged'],252) 01599 self.assertEqual(res['flagged'],4284*4) 01600 01601 class test_bandpass(test_base): 01602 """Flagdata:: Test flagging task with Bpass-based CalTable """ 01603 01604 def setUp(self): 01605 self.setUp_bpass_case() 01606 01607 def test_unsupported_modes(self): 01608 '''Flagdata: elevation and shadow are not supported in cal tables''' 01609 res = flagdata(vis=self.vis, mode='elevation') 01610 self.assertEqual(res, {}) 01611 res = flagdata(vis=self.vis, mode='shadow') 01612 self.assertEqual(res, {}) 01613 01614 def test_nullselections(self): 01615 '''Flagdata: unsupported selections in cal tables''' 01616 res = flagdata(vis=self.vis, scan='1') 01617 self.assertEqual(res, {}) 01618 01619 def test_default_cparam(self): 01620 '''Flagdata: flag CPARAM as the default column''' 01621 flagdata(vis=self.vis, mode='clip', clipzeros=True) 01622 res = flagdata(vis=self.vis, mode='summary') 01623 self.assertEqual(res['flagged'], 9950, 'Should use CPARAM as the default column') 01624 01625 def test_invalid_datacol(self): 01626 '''Flagdata: invalida data column should fall back to default''' 01627 flagdata(vis=self.vis, mode='clip', clipzeros=True, datacolumn='PARAMERR') 01628 res=flagdata(vis=self.vis, mode='summary') 01629 self.assertEqual(res['flagged'], 9950) 01630 01631 01632 def test_manual_field_selection_for_bpass(self): 01633 """Flagdata:: Manually flag a bpass-based CalTable using field selection""" 01634 01635 flagdata(vis=self.vis, field='3C286_A') 01636 summary=flagdata(vis=self.vis, mode='summary') 01637 01638 self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0) 01639 self.assertEqual(summary['field']['3C286_B']['flagged'], 0) 01640 self.assertEqual(summary['field']['3C286_C']['flagged'], 0) 01641 self.assertEqual(summary['field']['3C286_D']['flagged'], 0) 01642 01643 def test_list_field_Selection_for_bpass(self): 01644 """Flagdata:: Manually flag a bpass-based CalTable using list mode """ 01645 01646 flagdata(vis=self.vis, mode='list', inpfile=["field='3C286_A'"]) 01647 summary=flagdata(vis=self.vis, mode='summary') 01648 self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0) 01649 self.assertEqual(summary['field']['3C286_B']['flagged'], 0) 01650 self.assertEqual(summary['field']['3C286_C']['flagged'], 0) 01651 self.assertEqual(summary['field']['3C286_D']['flagged'], 0) 01652 01653 def test_manual_antenna_selection_for_bpass(self): 01654 """Flagdata:: Manually flag a bpass-based CalTable using antenna selection""" 01655 flagdata(vis=self.vis, antenna='ea09') 01656 summary=flagdata(vis=self.vis, mode='summary') 01657 self.assertEqual(summary['antenna']['ea09']['flagged'], 48000.0) 01658 self.assertEqual(summary['antenna']['ea10']['flagged'], 0.0) 01659 01660 def test_list_antenna_Selection_for_bpass(self): 01661 """Flagdata:: Manually flag a bpass-based CalTable using list mode""" 01662 01663 flagdata(vis=self.vis, mode='list', inpfile=["antenna='ea09'"]) 01664 summary=flagdata(vis=self.vis, mode='summary') 01665 self.assertEqual(summary['antenna']['ea09']['flagged'], 48000.0) 01666 self.assertEqual(summary['antenna']['ea10']['flagged'], 0.0) 01667 01668 def test_clip_nan_and_inf_cparam_all_for_bpass(self): 01669 """Flagdata:: Clip only NaNs and Infs in all calibration solutions of CPARAM column""" 01670 01671 flagdata(vis=self.vis, mode='clip',datacolumn='CPARAM', correlation='') 01672 summary=flagdata(vis=self.vis, mode='summary') 01673 self.assertEqual(summary['total'], 1248000.0) 01674 self.assertEqual(summary['flagged'], 0.0) 01675 self.assertEqual(summary['correlation']['Sol1']['flagged'], 0.0) 01676 self.assertEqual(summary['correlation']['Sol1']['total'], 624000.0) 01677 self.assertEqual(summary['correlation']['Sol2']['flagged'], 0.0) 01678 self.assertEqual(summary['correlation']['Sol2']['total'], 624000.0) 01679 01680 def test_clip_minmax_cparam_all_for_bpass(self): 01681 """Flagdata:: Clip all calibration solutions of CPARAM column using a minmax range""" 01682 01683 flagdata(vis=self.vis, mode='clip',clipzeros=True, clipminmax=[0,0.3], datacolumn='CPARAM') 01684 summary=flagdata(vis=self.vis, mode='summary') 01685 self.assertEqual(summary['flagged'], 10047) 01686 self.assertEqual(summary['total'], 1248000) 01687 self.assertEqual(summary['correlation']['Sol1']['flagged'], 10008) 01688 self.assertEqual(summary['correlation']['Sol2']['flagged'], 39) 01689 01690 def test_clip_minmax_snr_all_for_bpass(self): 01691 """Flagdata:: Test cliping all calibration solution products of SNR column using a 01692 minmax range for bpass CalTable""" 01693 01694 flagdata(vis=self.vis, mode='clip', clipzeros=True,clipminmax=[0.,550.],datacolumn='snr', 01695 correlation='') 01696 summary=flagdata(vis=self.vis, mode='summary') 01697 self.assertEqual(summary['total'], 1248000.0) 01698 self.assertEqual(summary['flagged'], 73174.0) 01699 self.assertEqual(summary['correlation']['Sol1']['flagged'], 35130.0) 01700 self.assertEqual(summary['correlation']['Sol1']['total'], 624000.0) 01701 self.assertEqual(summary['correlation']['Sol2']['flagged'], 38044.0) 01702 self.assertEqual(summary['correlation']['Sol2']['total'], 624000.0) 01703 01704 def test_clip_one_list(self): 01705 '''Flagdata: Flag one solution using one command in a list''' 01706 flagdata(vis=self.vis, mode='list', inpfile=["mode='clip' clipminmax=[0,3] correlation='REAL_Sol1'"]) 01707 res = flagdata(vis=self.vis, mode='summary') 01708 self.assertEqual(res['flagged'], 309388) 01709 self.assertEqual(res['correlation']['Sol2']['flagged'], 0) 01710 01711 def test_rflag_cparam_sol2_for_bpass(self): 01712 """Flagdata:: Test rflag solution 2 of CPARAM column for bpass""" 01713 01714 flagdata(vis=self.vis, mode='rflag', correlation='Sol2') 01715 summary=flagdata(vis=self.vis, mode='summary') 01716 self.assertEqual(summary['flagged'], 13197) 01717 self.assertEqual(summary['correlation']['Sol1']['flagged'], 0) 01718 self.assertEqual(summary['correlation']['Sol2']['flagged'], 13197) 01719 01720 def test_tfcrop_cparam_all_for_bpass(self): 01721 """Flagdata:: Test tfcrop in REAL_ALL calibration solutions of CPARAM column""" 01722 01723 flagdata(vis=self.vis, mode='clip', datacolumn='CPARAM',correlation='REAL_ALL',clipzeros=True) 01724 flagdata(vis=self.vis, mode='tfcrop', datacolumn='CPARAM',correlation='REAL_ALL') 01725 summary=flagdata(vis=self.vis, mode='summary') 01726 # self.assertTrue(abs(summary['flagged'] - 63861.0) <= 5) 01727 # self.assertEqual(abs(summary['flagged'] - 69369) <= 5) 01728 assert abs(summary['flagged'] - 79318) <= 10 01729 assert abs(summary['correlation']['Sol1']['flagged'] - 46261) <= 5 01730 assert abs(summary['correlation']['Sol2']['flagged'] - 33057) <= 5 01731 01732 def test_tfcrop_cparam_sol1_extension_for_bpass(self): 01733 """Flagdata:: Test tfcrop first calibration solution product of CPARAM column, 01734 and then extend to the other solution for bpass CalTable""" 01735 01736 flagdata(vis=self.vis, mode='clip', datacolumn='CPARAM',correlation='Sol1',clipzeros=True) 01737 flagdata(vis=self.vis, mode='tfcrop', datacolumn='CPARAM',correlation='Sol1') 01738 pre=flagdata(vis=self.vis, mode='summary') 01739 assert abs(pre['flagged'] - 29299) <= 5 01740 assert abs(pre['correlation']['Sol1']['flagged'] - 29299) <= 5 01741 01742 # Extend to other solution 01743 flagdata(vis=self.vis, mode='extend', extendpols=True, growfreq=0.0, growtime=0.0) 01744 pos=flagdata(vis=self.vis, mode='summary') 01745 assert abs(pos['flagged'] - 2*29299) <= 10 01746 assert abs(pos['correlation']['Sol2']['flagged'] - 29299) <= 5 01747 01748 def test_caltable_flagbackup(self): 01749 '''Flagmanager:: cal table mode=list, flagbackup=True/False''' 01750 01751 # Need a fresh start 01752 os.system('rm -rf cal.fewscans.bpass*') 01753 self.setUp_bpass_case() 01754 01755 # Create a local copy of the tool 01756 aflocal = casac.agentflagger() 01757 flagmanager(vis=self.vis, mode='list') 01758 aflocal.open(self.vis) 01759 self.assertEqual(len(aflocal.getflagversionlist()), 3) 01760 aflocal.done() 01761 01762 flagdata(vis=self.vis, mode='unflag', flagbackup=False) 01763 flagmanager(vis=self.vis, mode='list') 01764 aflocal.open(self.vis) 01765 self.assertEqual(len(aflocal.getflagversionlist()), 3) 01766 aflocal.done() 01767 01768 flagdata(vis=self.vis, mode='unflag', flagbackup=True) 01769 flagmanager(vis=self.vis, mode='list') 01770 aflocal.open(self.vis) 01771 self.assertEqual(len(aflocal.getflagversionlist()), 4) 01772 aflocal.done() 01773 01774 newname = 'Ha! The best version ever!' 01775 01776 flagmanager(vis=self.vis, mode='rename', oldname='flagdata_2', versionname=newname, 01777 comment='This is a *much* better name') 01778 flagmanager(vis=self.vis, mode='list') 01779 aflocal.open(self.vis) 01780 self.assertEqual(len(aflocal.getflagversionlist()), 4) 01781 aflocal.done() 01782 01783 self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname), 01784 'Flagversion file does not exist: flags.'+newname) 01785 01786 01787 01788 # Cleanup class 01789 class cleanup(test_base): 01790 01791 def tearDown(self): 01792 os.system('rm -rf ngc5921.*ms*') 01793 os.system('rm -rf flagdatatest.*ms*') 01794 os.system('rm -rf missing-baseline.*ms*') 01795 os.system('rm -rf multiobs.*ms*') 01796 os.system('rm -rf flagdatatest-alma.*ms*') 01797 os.system('rm -rf Four_ants_3C286.*ms*') 01798 os.system('rm -rf shadowtest_part.*ms*') 01799 os.system('rm -rf cal.fewscans.bpass*') 01800 os.system('rm -rf X7ef.tsys*') 01801 os.system('rm -rf list*txt') 01802 01803 def test_runTest(self): 01804 '''flagdata: Cleanup''' 01805 pass 01806 01807 01808 def suite(): 01809 return [test_tfcrop, 01810 test_rflag, 01811 test_shadow, 01812 test_flagmanager, 01813 test_selections, 01814 test_selections2, 01815 test_selections_alma, 01816 test_statistics_queries, 01817 test_msselection, 01818 test_elevation, 01819 test_list_list, 01820 test_list_file, 01821 test_clip, 01822 test_CASA_4_0_bug_fix, 01823 test_tsys, 01824 test_bandpass, 01825 cleanup]
1.8.0
