|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 ############################################################################# 00002 # $Id:$ 00003 # Test Name: # 00004 # Regression Test Script for ASDM version 1.2, 1.3 import to MS # 00005 # and the "inverse filler" task exportasdm 00006 # # 00007 # Rationale for Inclusion: # 00008 # The conversion of ASDM to MS and back needs to be verified. # 00009 # # 00010 # Features tested: # 00011 # 1) Is the import performed without raising exceptions # 00012 # 2) Do all expected tables exist # 00013 # 3) Can the MS be opened # 00014 # 4) Do the tables contain expected values # 00015 # 5) Is exportasdm performed without raising exceptions # 00016 # 6) Is the created ASDM well-formed (XML) and complete # 00017 # 7) Can the resulting ASDM be reimported without raising exceptions # 00018 # 8) Does it have the same number of integrations as the original # 00019 # # 00020 # Input data: # 00021 # one dataset for the filler of ASDM 1.0 # 00022 # one simulated MS dataset # 00023 # # 00024 ############################################################################# 00025 import os 00026 import sys 00027 import shutil 00028 from __main__ import default 00029 from tasks import * 00030 from taskinit import * 00031 import unittest 00032 00033 myname = 'asdm-import_regression' 00034 00035 # default ASDM dataset name 00036 myasdm_dataset_name = 'uid___X5f_X18951_X1' 00037 myms_dataset_name = 'M51.ms' 00038 00039 # name of the resulting MS 00040 msname = myasdm_dataset_name+'.ms' 00041 00042 # name of the exported ASDM 00043 asdmname = myms_dataset_name+'.asdm' 00044 00045 # name of the reimported MS 00046 reimp_msname = 'reimported-'+myms_dataset_name 00047 00048 def checktable(thename, theexpectation): 00049 global msname, myname 00050 tb.open(msname+"/"+thename) 00051 if thename == "": 00052 thename = "MAIN" 00053 for mycell in theexpectation: 00054 print myname, ": comparing ", mycell 00055 value = tb.getcell(mycell[0], mycell[1]) 00056 # see if value is array 00057 try: 00058 isarray = value.__len__ 00059 except: 00060 # it's not an array 00061 # zero tolerance? 00062 if mycell[3] == 0: 00063 in_agreement = (value == mycell[2]) 00064 else: 00065 in_agreement = ( abs(value - mycell[2]) < mycell[3]) 00066 else: 00067 # it's an array 00068 # zero tolerance? 00069 if mycell[3] == 0: 00070 in_agreement = (value == mycell[2]).all() 00071 else: 00072 in_agreement = (abs(value - mycell[2]) < mycell[3]).all() 00073 if not in_agreement: 00074 print myname, ": Error in MS subtable", thename, ":" 00075 print " column ", mycell[0], " row ", mycell[1], " contains ", value 00076 print " expected value is ", mycell[2] 00077 tb.close() 00078 return False 00079 tb.close() 00080 print myname, ": table ", thename, " as expected." 00081 return True 00082 00083 ######################### 00084 00085 def verify_asdm(asdmname, withPointing): 00086 print "Verifying asdm ", asdmname 00087 if(not os.path.exists(asdmname)): 00088 print "asdm ", asdmname, " doesn't exist." 00089 raise Exception 00090 # test for the existence of all obligatory tables 00091 allTables = [ "Antenna.xml", 00092 "ASDM.xml", 00093 # "CalData.xml", 00094 # "CalDelay.xml", 00095 # "CalReduction.xml", 00096 "ConfigDescription.xml", 00097 "CorrelatorMode.xml", 00098 "DataDescription.xml", 00099 "ExecBlock.xml", 00100 "Feed.xml", 00101 "Field.xml", 00102 #"FocusModel.xml", 00103 #"Focus.xml", 00104 "Main.xml", 00105 "PointingModel.xml", 00106 "Polarization.xml", 00107 "Processor.xml", 00108 "Receiver.xml", 00109 "SBSummary.xml", 00110 "Scan.xml", 00111 "Source.xml", 00112 "SpectralWindow.xml", 00113 "State.xml", 00114 "Station.xml", 00115 "Subscan.xml", 00116 "SwitchCycle.xml" 00117 ] 00118 isOK = True 00119 for fileName in allTables: 00120 filePath = asdmname+'/'+fileName 00121 if(not os.path.exists(filePath)): 00122 print "ASDM table file ", filePath, " doesn't exist." 00123 isOK = False 00124 else: 00125 # test if well formed 00126 rval = os.system('xmllint --noout '+filePath) 00127 if(rval !=0): 00128 print "Table ", filePath, " is not a well formed XML document." 00129 isOK = False 00130 00131 print "Note: xml validation not possible since ASDM DTDs (schemas) not yet online." 00132 00133 if(not os.path.exists(asdmname+"/ASDMBinary")): 00134 print "ASDM binary directory "+asdmname+"/ASDMBinary doesn't exist." 00135 isOK = False 00136 00137 if(withPointing and not os.path.exists(asdmname+"/Pointing.bin")): 00138 print "ASDM binary file "+asdmname+"/Pointing.bin doesn't exist." 00139 isOK = False 00140 00141 if (not isOK): 00142 raise Exception 00143 00144 # Setup for different data importing 00145 class test_base(unittest.TestCase): 00146 def setUp_m51(self): 00147 res = None 00148 if(os.path.exists(myasdm_dataset_name)): 00149 shutil.rmtree(myasdm_dataset_name) 00150 00151 datapath=os.environ.get('CASAPATH').split()[0]+'/data/regression/asdm-import/input/' 00152 shutil.copytree(datapath + myasdm_dataset_name, myasdm_dataset_name) 00153 datapath=os.environ.get('CASAPATH').split()[0]+'/data/regression/exportasdm/input/' 00154 shutil.copytree(datapath + myms_dataset_name, myms_dataset_name) 00155 default(importasdm) 00156 00157 def setUp_xosro(self): 00158 # self.asdm = 'X_osro_scan1' 00159 self.asdm = 'X_osro_013.55979.93803716435' 00160 datapath=os.environ.get('CASAPATH').split()[0]+'/data/regression/unittest/flagdata/' 00161 if(not os.path.lexists(self.asdm)): 00162 os.system('ln -s '+datapath+self.asdm +' '+self.asdm) 00163 00164 default(importasdm) 00165 00166 ########################### 00167 # beginning of actual test 00168 class asdm_import1(test_base): 00169 00170 def setUp(self): 00171 self.setUp_m51() 00172 00173 def tearDown(self): 00174 shutil.rmtree(myasdm_dataset_name) 00175 shutil.rmtree(myms_dataset_name) 00176 shutil.rmtree(msname,ignore_errors=True) 00177 shutil.rmtree(msname+'.flagversions',ignore_errors=True) 00178 00179 00180 def test1(self): 00181 '''Asdm-import: Test good v1.2 input with filler v3 and inverse filler v3 ''' 00182 retValue = {'success': True, 'msgs': "", 'error_msgs': '' } 00183 00184 self.res = importasdm(myasdm_dataset_name, useversion='v3') 00185 self.assertEqual(self.res, None) 00186 print myname, ": Success! Now checking output ..." 00187 mscomponents = set(["table.dat", 00188 "table.f0", 00189 "table.f1", 00190 "table.f2", 00191 "table.f3", 00192 "table.f4", 00193 "table.f5", 00194 "table.f6", 00195 "table.f7", 00196 "table.f8", 00197 "ANTENNA/table.dat", 00198 "DATA_DESCRIPTION/table.dat", 00199 "FEED/table.dat", 00200 "FIELD/table.dat", 00201 "FLAG_CMD/table.dat", 00202 "HISTORY/table.dat", 00203 "OBSERVATION/table.dat", 00204 "POINTING/table.dat", 00205 "POLARIZATION/table.dat", 00206 "PROCESSOR/table.dat", 00207 "SOURCE/table.dat", 00208 "SPECTRAL_WINDOW/table.dat", 00209 "STATE/table.dat", 00210 "SYSCAL/table.dat", 00211 "ANTENNA/table.f0", 00212 "DATA_DESCRIPTION/table.f0", 00213 "FEED/table.f0", 00214 "FIELD/table.f0", 00215 "FLAG_CMD/table.f0", 00216 "HISTORY/table.f0", 00217 "OBSERVATION/table.f0", 00218 "POINTING/table.f0", 00219 "POLARIZATION/table.f0", 00220 "PROCESSOR/table.f0", 00221 "SOURCE/table.f0", 00222 "SPECTRAL_WINDOW/table.f0", 00223 "STATE/table.f0", 00224 "SYSCAL/table.f0" 00225 ]) 00226 for name in mscomponents: 00227 if not os.access(msname+"/"+name, os.F_OK): 00228 print myname, ": Error ", msname+"/"+name, "doesn't exist ..." 00229 retValue['success']=False 00230 retValue['error_msgs']=retValue['error_msgs']+msname+'/'+name+' does not exist' 00231 else: 00232 print myname, ": ", name, "present." 00233 print myname, ": MS exists. All tables present. Try opening as MS ..." 00234 try: 00235 ms.open(msname) 00236 except: 00237 print myname, ": Error Cannot open MS table", msname 00238 retValue['success']=False 00239 retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+msname 00240 else: 00241 ms.close() 00242 print myname, ": OK. Checking tables in detail ..." 00243 00244 # check main table first 00245 name = "" 00246 # col name, row number, expected value, tolerance 00247 expected = [ 00248 ['UVW', 42, [ 0., 0., 0. ], 1E-8], 00249 ['EXPOSURE', 42, 1.008, 0], 00250 ['DATA', 42, [ [10.5526886+0.0j] ], 1E-7] 00251 ] 00252 results = checktable(name, expected) 00253 if not results: 00254 retValue['success']=False 00255 retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed' 00256 else: 00257 retValue['success']=True 00258 00259 expected = [ 00260 # old values using TAI ['UVW', 638, [-65.07623467, 1.05534109, -33.65801386], 1E-8], 00261 ['UVW', 638, [-65.14758508, 1.13423277, -33.51712451], 1E-7], 00262 ['EXPOSURE', 638, 1.008, 0], 00263 ['DATA', 638, [ [0.00362284+0.00340279j] ], 1E-8] 00264 ] 00265 results = checktable(name, expected) 00266 if not results: 00267 retValue['success']=False 00268 retValue['error_msgs']=retValue['error_msgs']+'Check of table MAIN failed' 00269 else: 00270 retValue['success']=True 00271 00272 name = "ANTENNA" 00273 expected = [ ['OFFSET', 1, [ 0., 0., 0.], 0], 00274 ['POSITION', 1, [2202242.5520, -5445215.1570, -2485305.0920], 0.0001], 00275 ['DISH_DIAMETER',1, 12.0, 0] 00276 ] 00277 results = checktable(name, expected) 00278 if not results: 00279 retValue['success']=False 00280 retValue['error_msgs']=retValue['error_msgs']+'Check of table ANTENNA failed' 00281 else: 00282 retValue['success']=True 00283 00284 name = "POINTING" 00285 expected = [ ['DIRECTION', 10, [[ 1.94681283],[ 1.19702955]], 1E-8], 00286 ['INTERVAL', 10, 0.048, 0], 00287 ['TARGET', 10, [[ 1.94681283], [ 1.19702955]], 1E-8], 00288 ['TIME', 10, 4758823736.016000, 1E-6], 00289 ['TIME_ORIGIN', 10, 0., 0], 00290 ['POINTING_OFFSET', 10, [[ 0.],[ 0.]], 0], 00291 ['ENCODER', 10, [ 1.94851533, 1.19867576], 1E-8 ] 00292 ] 00293 results = checktable(name, expected) 00294 if not results: 00295 retValue['success']=False 00296 retValue['error_msgs']=retValue['error_msgs']+'Check of table POINTING failed' 00297 else: 00298 retValue['success']=True 00299 00300 self.assertTrue(retValue['success'],retValue['error_msgs']) 00301 00302 myvis = myms_dataset_name 00303 os.system('rm -rf exportasdm-output.asdm myinput.ms') 00304 os.system('cp -R ' + myvis + ' myinput.ms') 00305 default('exportasdm') 00306 try: 00307 print "\n>>>> Test of exportasdm v3: input MS is ", myvis 00308 print "(a simulated input MS with pointing table)" 00309 rval = exportasdm( 00310 vis = 'myinput.ms', 00311 asdm = 'exportasdm-output.asdm', 00312 archiveid="S002", 00313 apcorrected=False, 00314 useversion='v3' 00315 ) 00316 print "rval is ", rval 00317 if not rval: 00318 raise Exception 00319 os.system('rm -rf '+asdmname+'; mv exportasdm-output.asdm '+asdmname) 00320 verify_asdm(asdmname, True) 00321 except: 00322 print myname, ': *** Unexpected error exporting MS to ASDM, regression failed ***' 00323 raise 00324 00325 try: 00326 print "Reimporting the created ASDM (v3)...." 00327 importasdm(asdm=asdmname, vis=reimp_msname, wvr_corrected_data='no', useversion='v3') 00328 print "Testing existence of reimported MS ...." 00329 if(not os.path.exists(reimp_msname)): 00330 print "MS ", reimp_msname, " doesn't exist." 00331 raise Exception 00332 print "Testing equivalence of the original and the reimported MS." 00333 tb.open(myms_dataset_name) 00334 nrowsorig = tb.nrows() 00335 print "Original MS contains ", nrowsorig, "integrations." 00336 tb.close() 00337 tb.open(reimp_msname) 00338 nrowsreimp = tb.nrows() 00339 tb.close() 00340 print "Reimported MS contains ", nrowsreimp, "integrations." 00341 if(not nrowsreimp==nrowsorig): 00342 print "Numbers of integrations disagree." 00343 raise Exception 00344 except: 00345 print myname, ': *** Unexpected error reimporting the exported ASDM, regression failed ***' 00346 raise 00347 00348 class asdm_import2(test_base): 00349 00350 def setUp(self): 00351 self.setUp_xosro() 00352 00353 def tearDown(self): 00354 os.system('rm -rf '+self.asdm) 00355 00356 def test_CAS4532(self): 00357 '''importasdm CAS-4532: white spaces on Antenna.xml''' 00358 # The X_osro_scan1/Antenna.xml and SpectralWindow.xml 00359 # contain white spaces between some of the contents and 00360 # the tags. This should not cause any error in the XML 00361 # parser from fh.readXML 00362 import flaghelper as fh 00363 00364 flagcmddict = fh.readXML(self.asdm, 0.0) 00365 self.assertTrue(flagcmddict, 'Some XML file may contain white spaces not handled by readXML') 00366 00367 self.assertEqual(flagcmddict.keys().__len__(),214) 00368 00369 def suite(): 00370 return [asdm_import1,asdm_import2] 00371 00372
1.8.0
