|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 ############################################################################# 00002 # $Id:$ 00003 # Test Name: # 00004 # Regression Test Script for ASDM version 1.0 import to MS # 00005 # and the "inverse filler" task importevla 00006 # # 00007 # Rationale for Inclusion: # 00008 # The conversion of ASDM to MS and back needs to be verified. # 00009 # # 00010 # Features tested: # 00011 # 1) Is the import performed without raising exceptions # 00012 # 2) Do all expected tables exist # 00013 # 3) Can the MS be opened # 00014 # 4) Do the tables contain expected values # 00015 # 5) Is exportasdm performed without raising exceptions # 00016 # 6) Is the created ASDM well-formed (XML) and complete # 00017 # 7) Can the resulting ASDM be reimported without raising exceptions # 00018 # 8) Does it have the same number of integrations as the original # 00019 # # 00020 # Input data: # 00021 # one dataset for the filler of ASDM 1.0 # 00022 # one simulated MS dataset # 00023 # # 00024 ############################################################################# 00025 import os 00026 import sys 00027 import shutil 00028 import commands 00029 from __main__ import default 00030 from tasks import * 00031 from taskinit import * 00032 import unittest 00033 00034 00035 def checktable(msname, thename, theexpectation): 00036 global myname 00037 tb.open(msname+"/"+thename) 00038 if thename == "": 00039 thename = "MAIN" 00040 for mycell in theexpectation: 00041 print myname, ": comparing ", mycell 00042 value = tb.getcell(mycell[0], mycell[1]) 00043 # see if value is array 00044 try: 00045 isarray = value.__len__ 00046 except: 00047 # it's not an array 00048 # zero tolerance? 00049 if mycell[3] == 0: 00050 in_agreement = (value == mycell[2]) 00051 else: 00052 in_agreement = ( abs(value - mycell[2]) < mycell[3]) 00053 else: 00054 # it's an array 00055 # zero tolerance? 00056 if mycell[3] == 0: 00057 in_agreement = (value == mycell[2]).all() 00058 else: 00059 in_agreement = (abs(value - mycell[2]) < mycell[3]).all() 00060 if not in_agreement: 00061 print myname, ": Error in MS subtable", thename, ":" 00062 print " column ", mycell[0], " row ", mycell[1], " contains ", value 00063 print " expected value is ", mycell[2] 00064 tb.close() 00065 return False 00066 tb.close() 00067 print myname, ": table ", thename, " as expected." 00068 return True 00069 00070 00071 myname = 'importevla_ut' 00072 00073 00074 class importevla_test(unittest.TestCase): 00075 00076 def setUp(self): 00077 # res = None 00078 00079 origname = 'X_osro_013.55979.93803716435' 00080 asdmname = 'xosro' 00081 if (not os.path.exists(asdmname)): 00082 datapath=os.environ.get('CASAPATH').split()[0]+'/data/regression/unittest/importevla/' 00083 os.system('ln -s '+datapath+origname+' '+asdmname) 00084 00085 self.asdm = asdmname 00086 00087 default(importevla) 00088 00089 00090 def test1(self): 00091 '''Importevla test1: Good input asdm''' 00092 retValue = {'success': True, 'msgs': "", 'error_msgs': '' } 00093 00094 msname = 'xosro1.ms' 00095 # self.res = importevla(asdm=self.asdm, scans='3') 00096 self.res = importevla(asdm=self.asdm, vis=msname, scans='2') 00097 print myname, ": Success! Now checking output ..." 00098 mscomponents = set(["table.dat", 00099 "table.f0", 00100 "table.f1", 00101 "table.f2", 00102 "table.f3", 00103 "table.f4", 00104 "table.f5", 00105 "table.f6", 00106 "table.f7", 00107 "table.f8", 00108 "Antenna.xml", 00109 "Flag.xml", 00110 "SpectralWindow.xml", 00111 "ANTENNA/table.dat", 00112 "DATA_DESCRIPTION/table.dat", 00113 "FEED/table.dat", 00114 "FIELD/table.dat", 00115 "FLAG_CMD/table.dat", 00116 "HISTORY/table.dat", 00117 "OBSERVATION/table.dat", 00118 "POINTING/table.dat", 00119 "POLARIZATION/table.dat", 00120 "PROCESSOR/table.dat", 00121 "SOURCE/table.dat", 00122 "SPECTRAL_WINDOW/table.dat", 00123 "STATE/table.dat", 00124 "SYSCAL/table.dat", 00125 "WEATHER/table.dat", 00126 "ANTENNA/table.f0", 00127 "DATA_DESCRIPTION/table.f0", 00128 "FEED/table.f0", 00129 "FIELD/table.f0", 00130 "FLAG_CMD/table.f0", 00131 "HISTORY/table.f0", 00132 "OBSERVATION/table.f0", 00133 "POINTING/table.f0", 00134 "POLARIZATION/table.f0", 00135 "PROCESSOR/table.f0", 00136 "SOURCE/table.f0", 00137 "SPECTRAL_WINDOW/table.f0", 00138 "STATE/table.f0", 00139 "SYSCAL/table.f0", 00140 "WEATHER/table.f0" 00141 ]) 00142 for name in mscomponents: 00143 if not os.access(msname+"/"+name, os.F_OK): 00144 print myname, ": Error ", msname+"/"+name, "doesn't exist ..." 00145 retValue['success']=False 00146 retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist' 00147 else: 00148 print myname, ": ", name, "present." 00149 print myname, ": MS exists. All tables present. Try opening as MS ..." 00150 try: 00151 ms.open(msname) 00152 except: 00153 print myname, ": Error Cannot open MS table", tablename 00154 retValue['success']=False 00155 retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+tablename 00156 else: 00157 ms.close() 00158 print myname, ": OK. Checking tables in detail ..." 00159 00160 # check main table first 00161 name = "" 00162 # col name, row number, expected value, tolerance 00163 expected = [ 00164 ['UVW', 42, [ 1607.50778695, -1241.40287976 , 584.50368163], 1E-8], 00165 ['EXPOSURE', 42, 1.0, 0] 00166 # ['DATA', 42, [ [10.5526886+0.0j] ], 1E-7] 00167 ] 00168 results = checktable(msname, name, expected) 00169 if not results: 00170 retValue['success']=False 00171 retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed' 00172 else: 00173 retValue['success']=True 00174 00175 expected = [ 00176 # old values using TAI ['UVW', 638, [-65.07623467, 1.05534109, -33.65801386], 1E-8], 00177 ['UVW', 638, [14.20193237, 722.59606805 , 57.57988905], 1E-8], 00178 ['EXPOSURE', 638, 1.0, 0] 00179 # ['DATA', 638, [ [0.00362284+0.00340279j] ], 1E-8] 00180 ] 00181 results = checktable(msname, name, expected) 00182 if not results: 00183 retValue['success']=False 00184 retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed' 00185 else: 00186 retValue['success']=True 00187 00188 name = "ANTENNA" 00189 expected = [ ['OFFSET', 1, [ -4.80000000e-12, 0., 0.], 0], 00190 ['POSITION', 1, [-1599644.8611, -5042953.6623, 3554197.0332], 0.0001], 00191 ['DISH_DIAMETER',1, 25.0, 0] 00192 ] 00193 results = checktable(msname, name, expected) 00194 if not results: 00195 retValue['success']=False 00196 retValue['error_msgs']=retValue['error_msgs']+'Check of table ANTENNA failed' 00197 else: 00198 retValue['success']=True 00199 00200 # name = "POINTING" 00201 # expected = [ ['DIRECTION', 10, [[ 0.],[0.]], 1E-8], 00202 # ['INTERVAL', 10, 4.7672238080000007, 0], 00203 # ['TARGET', 10, [[ 0.], [ 0.]], 1E-8], 00204 # ['TIME', 10, 4777473953.6163883, 0], 00205 # ['TIME_ORIGIN', 10, 0., 0], 00206 # ['POINTING_OFFSET', 10, [[ 0.],[ 0.]], 0], 00207 # ['ENCODER', 10, [ 0., 0.], 1E-8 ] 00208 # ] 00209 # results = checktable(msname, name, expected) 00210 # if not results: 00211 # retValue['success']=False 00212 # retValue['error_msgs']=retValue['error_msgs']+'Check of table POINTING failed' 00213 # else: 00214 # retValue['success']=True 00215 00216 self.assertTrue(results) 00217 00218 00219 def test_apply1(self): 00220 '''importevla: apply all flags and save to file''' 00221 msname = 'online.ms' 00222 cmdfile = 'online_cmd.txt' 00223 if os.path.exists(msname): 00224 os.system('rm -rf '+msname) 00225 if os.path.exists(cmdfile): 00226 os.system('rm -rf '+cmdfile) 00227 00228 importevla(asdm=self.asdm, vis=msname, scans='2',online=True, applyflags=True, 00229 shadow=True,flagzero=True,flagbackup=False, savecmds=True) 00230 00231 # Check flags 00232 res = flagdata(vis=msname, mode='summary') 00233 self.assertEqual(res['flagged'],2446080) 00234 00235 # Check output file existence 00236 self.assertTrue(os.path.exists(cmdfile)) 00237 00238 # Check file content 00239 ff = open(cmdfile,'r') 00240 cmdlist = ff.readlines() 00241 ncmds = cmdlist.__len__() 00242 self.assertEqual(ncmds, 216) 00243 00244 00245 def test_apply2(self): 00246 '''importevla: apply only online flags''' 00247 msname = 'applied.ms' 00248 if os.path.exists(msname): 00249 os.system('rm -rf '+msname) 00250 00251 # Save to different file 00252 importevla(asdm=self.asdm, vis=msname, scans='2',online=True, shadow=False, flagzero=False, 00253 applyflags=True, savecmds=False, flagbackup=False) 00254 00255 # Check flags only in RR and LL 00256 res = flagdata(vis=msname, mode='summary') 00257 self.assertEqual(res['flagged'],2446080) 00258 self.assertEqual(res['scan']['2']['flagged'],2446080) 00259 00260 00261 def test_apply3(self): 00262 '''importevla: apply clip zeros on RR and LL and save to file''' 00263 msname = 'zeros.ms' 00264 cmdfile = 'zeros_cmd.txt' 00265 if os.path.exists(msname): 00266 os.system('rm -rf '+msname) 00267 if os.path.exists(cmdfile): 00268 os.system('rm -rf '+cmdfile) 00269 00270 00271 importevla(asdm=self.asdm, vis=msname, scans='2,13',online=False, applyflags=True, 00272 shadow=False,flagzero=True,flagpol=False, flagbackup=False, savecmds=True) 00273 00274 # Check flags 00275 res = flagdata(vis=msname, mode='summary') 00276 self.assertEqual(res['flagged'],0,'There are no zeros in this data set') 00277 self.assertEqual(res['scan']['2']['flagged'],0,'No flags should have been applied') 00278 self.assertEqual(res['scan']['13']['flagged'],0,'No flags should have been applied') 00279 00280 # Check output file existence 00281 self.assertTrue(os.path.exists(cmdfile)) 00282 00283 # Check output file existence 00284 self.assertTrue(os.path.exists(cmdfile)) 00285 00286 # Check file content 00287 ff = open(cmdfile,'r') 00288 cmdlist = ff.readlines() 00289 ncmds = cmdlist.__len__() 00290 self.assertEqual(ncmds, 2, 'Only clip zeros should be saved to file') 00291 00292 def test_apply4(self): 00293 '''importevla: Save online flags to FLAG_CMD and file; do not apply''' 00294 00295 # Use default msname and outfile 00296 msname = 'xosro4.ms' 00297 cmdfile = msname.replace('.ms','_cmd.txt') 00298 if os.path.exists(msname): 00299 os.system('rm -rf '+msname) 00300 if os.path.exists(cmdfile): 00301 os.system('rm -rf '+cmdfile) 00302 00303 importevla(asdm=self.asdm, vis=msname, scans='2',online=True, shadow=False, flagzero=False, 00304 applyflags=False,savecmds=True, flagbackup=False) 00305 00306 # No flags were applied 00307 res = flagdata(vis=msname, mode='summary') 00308 self.assertEqual(res['flagged'],0) 00309 00310 # Apply only row 213 using flagcmd 00311 # The command in row 213 is the following: 00312 # antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199' 00313 # spw='EVLA_X#A0C0#0' correlation='LL,LR,RL 00314 flagcmd(vis=msname, action='apply', tablerows=213) 00315 00316 # Check flags. RR should no be flagged 00317 res = flagdata(vis=msname, mode='summary') 00318 self.assertEqual(res['correlation']['RR']['flagged'],0,'RR should not be flagged') 00319 self.assertEqual(res['correlation']['LL']['flagged'],29440) 00320 self.assertEqual(res['correlation']['LR']['flagged'],29440) 00321 self.assertEqual(res['correlation']['RL']['flagged'],29440) 00322 self.assertEqual(res['antenna']['ea06']['flagged'],88320) 00323 self.assertEqual(res['antenna']['ea07']['flagged'],3840,'Only a few baselines should be flagged') 00324 self.assertEqual(res['antenna']['ea08']['flagged'],3840,'Only a few baselines should be flagged') 00325 00326 # Check output file existence 00327 self.assertTrue(os.path.exists(cmdfile)) 00328 00329 # Check file content 00330 ff = open(cmdfile,'r') 00331 cmdlist = ff.readlines() 00332 ncmds = cmdlist.__len__() 00333 ff.close() 00334 self.assertEqual(ncmds, 214, 'Only Online cmds should have been saved to file') 00335 00336 # Unapply row 213 and apply it in flagdata using the file 00337 # TO DO : after row selection is available in file 00338 00339 def test_apply5(self): 00340 '''importevla: Apply only shadow flags''' 00341 msname = 'xosro_shadow.ms' 00342 importevla(asdm=self.asdm, vis=msname,online=False, shadow=True, flagzero=False, 00343 applyflags=True,savecmds=False, flagbackup=False) 00344 00345 # This data set doesn't have shadow. Maybe change later! 00346 res = flagdata(vis=msname, mode='summary') 00347 self.assertEqual(res['flagged'],0,'There are shadowed antenna in this data set') 00348 00349 00350 00351 def test_savepars(self): 00352 '''importevla: save the flag commands and do not apply''' 00353 msname = 'notapplied.ms' 00354 cmdfile = msname.replace('.ms','_cmd.txt') 00355 if os.path.exists(msname): 00356 os.system('rm -rf '+msname) 00357 if os.path.exists(cmdfile): 00358 os.system('rm -rf '+cmdfile) 00359 00360 importevla(asdm=self.asdm, vis=msname,scans='11~13',online=True,flagzero=True,shadow=True,savecmds=True, 00361 applyflags=False,flagbackup=False) 00362 00363 # Check flags 00364 res = flagdata(vis=msname, mode='summary') 00365 self.assertEqual(res['flagged'],0,'No flags should have been applied') 00366 00367 # Check output file existence 00368 self.assertTrue(os.path.exists(cmdfile)) 00369 00370 # Check file content 00371 ff = open(cmdfile,'r') 00372 cmdlist = ff.readlines() 00373 ncmds = cmdlist.__len__() 00374 ff.close() 00375 self.assertEqual(ncmds, 216, 'Online, shadow and clip zeros should be saved to file') 00376 00377 # Apply flags using flagdata 00378 flagdata(vis=msname, mode='list', inpfile=cmdfile) 00379 00380 res = flagdata(vis=msname, mode='summary') 00381 self.assertEqual(res['flagged'],6090624) 00382 00383 00384 class cleanup(unittest.TestCase): 00385 00386 def setUp(self): 00387 pass 00388 00389 def tearDown(self): 00390 os.system('rm -rf *applied*ms*') 00391 os.system('rm -rf *online*ms*') 00392 os.system('rm -rf *zeros*ms*') 00393 os.system('rm -rf *xosro*.ms*') 00394 # pass 00395 00396 def test1a(self): 00397 '''Importevla: Cleanup''' 00398 pass 00399 00400 def suite(): 00401 return [importevla_test, cleanup] 00402 00403
1.8.0
