|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import os 00002 import shutil 00003 import time 00004 import unittest 00005 import testhelper as th 00006 import partitionhelper as ph 00007 from tasks import * 00008 from taskinit import * 00009 from __main__ import default 00010 from parallel.parallel_task_helper import ParallelTaskHelper 00011 00012 00013 ''' Unit Tests for task partition''' 00014 00015 # jagonzal (CAS-4287): Add a cluster-less mode to by-pass parallel processing for MMSs as requested 00016 if os.environ.has_key('BYPASS_SEQUENTIAL_PROCESSING'): 00017 ParallelTaskHelper.bypassParallelProcessing(1) 00018 00019 00020 class test_base(unittest.TestCase): 00021 00022 def setUp_ngc4826(self): 00023 00024 datapath = os.environ.get('CASAPATH').split()[0] + \ 00025 "/data/regression/unittest/gaincal/" 00026 00027 # Input MS contain 4 obsID, 11 scans and 4 spws 00028 self.prefix = 'ngc4826' 00029 self.msfile = self.prefix + '.ms' 00030 00031 # Output files 00032 self.mmsfile = self.prefix + '.mms' 00033 00034 fpath = os.path.join(datapath,self.msfile) 00035 if os.path.lexists(fpath): 00036 shutil.copytree(fpath, self.msfile, symlinks=True) 00037 else: 00038 self.fail('Data does not exist -> '+fpath) 00039 00040 default('partition') 00041 00042 def setUp_fourants(self): 00043 00044 datapath = os.environ.get('CASAPATH').split()[0] + \ 00045 "/data/regression/unittest/flagdata/" 00046 00047 # Input MS contain 2 scans and 16 spws. It has all data columns 00048 self.prefix = 'Four_ants_3C286' 00049 self.msfile = self.prefix + '.ms' 00050 00051 # Output files 00052 self.mmsfile = self.prefix + '.mms' 00053 00054 fpath = os.path.join(datapath,self.msfile) 00055 if os.path.lexists(fpath): 00056 shutil.copytree(fpath, self.msfile, symlinks=True) 00057 else: 00058 self.fail('Data does not exist -> '+fpath) 00059 00060 default('partition') 00061 00062 00063 class partition_test1(test_base): 00064 00065 def setUp(self): 00066 self.setUp_ngc4826() 00067 00068 def tearDown(self): 00069 shutil.rmtree(self.msfile, ignore_errors=True) 00070 shutil.rmtree(self.mmsfile, ignore_errors=True) 00071 shutil.rmtree(self.msfile+'.flagversions', ignore_errors=True) 00072 shutil.rmtree(self.mmsfile+'.flagversions', ignore_errors=True) 00073 00074 def test_nomms(self): 00075 '''Partition: Create a normal MS with createmms=False''' 00076 partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False) 00077 time.sleep(10) 00078 00079 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00080 00081 # Compare both tables. Ignore the DATA column and compare it in next line 00082 self.assertTrue(th.compTables(self.msfile, self.mmsfile, 00083 ['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','DATA'])) 00084 00085 # Compare the DATA column 00086 self.assertTrue(th.compVarColTables(self.msfile,self.mmsfile,'DATA')) 00087 00088 def test_default(self): 00089 '''Partition: create an MMS with default values''' 00090 partition(vis=self.msfile, outputvis=self.mmsfile) 00091 time.sleep(10) 00092 00093 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00094 00095 # Take the dictionary and compare with original MS 00096 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00097 00098 # Compare nrows of all scans 00099 slist = ph.getMMSScans(thisdict) 00100 00101 for s in slist: 00102 mmsN = ph.getMMSScanNrows(thisdict, s) 00103 msN = ph.getScanNrows(self.msfile, s) 00104 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00105 %(s, mmsN, msN)) 00106 00107 # Compare spw IDs 00108 for s in slist: 00109 mms_spw = ph.getSpwIds(self.mmsfile, s) 00110 ms_spw = ph.getSpwIds(self.msfile, s) 00111 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00112 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) 00113 00114 # TO DO: Compare both table using compTables when sorting in partition is fixed 00115 self.assertTrue(th.compTables(self.msfile, self.mmsfile, 00116 ['FLAG','FLAG_CATEGORY','TIME_CENTROID', 00117 'WEIGHT_SPECTRUM','DATA'])) 00118 00119 # Compare the DATA column 00120 # self.assertTrue(th.compVarColTables(self.msfile,self.mmsfile,'DATA')) 00121 00122 00123 def test_scan_selection(self): 00124 '''Partition: create an MMS using scan selection''' 00125 partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan', scan='1,2,3,11') 00126 time.sleep(10) 00127 00128 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00129 00130 # Take the dictionary and compare with original MS 00131 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00132 00133 # Compare nrows of all scans in selection 00134 slist = ph.getMMSScans(thisdict) 00135 self.assertEqual(slist.__len__(), 4) 00136 00137 for s in slist: 00138 mmsN = ph.getMMSScanNrows(thisdict, s) 00139 msN = ph.getScanNrows(self.msfile, s) 00140 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00141 %(s, mmsN, msN)) 00142 00143 # Compare spw IDs 00144 for s in slist: 00145 mms_spw = ph.getSpwIds(self.mmsfile, s) 00146 ms_spw = ph.getSpwIds(self.msfile, s) 00147 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00148 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) 00149 00150 def test_spw_separation(self): 00151 '''Partition: create an MMS separated by spws with observation selection''' 00152 # NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation 00153 # IDs, therefore I need to selection by obs id in partition 00154 partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', observation='2') 00155 time.sleep(10) 00156 00157 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00158 00159 # Take the dictionary and compare with original MS 00160 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00161 00162 # Dictionary with selection to compare with original MS 00163 mysel = {'observation':'2'} 00164 00165 # Compare nrows of all scans in MMS and MS 00166 slist = ph.getMMSScans(thisdict) 00167 for s in slist: 00168 mmsN = ph.getMMSScanNrows(thisdict, s) 00169 msN = ph.getScanNrows(self.msfile, s, selection=mysel) 00170 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00171 %(s, mmsN, msN)) 00172 00173 # Compare spw IDs 00174 for s in slist: 00175 mms_spw = ph.getSpwIds(self.mmsfile, s) 00176 ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) 00177 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00178 'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN)) 00179 00180 00181 def test_spw_selection(self): 00182 '''Partition: create an MMS separated by spws with spw=2,4 selection''' 00183 partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', spw='2,4') 00184 time.sleep(10) 00185 00186 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00187 00188 # Take the dictionary and compare with original MS 00189 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00190 00191 # Dictionary with selection to compare with original MS 00192 mysel = {'spw':'2,4'} 00193 00194 # Compare nrows of all scans in selection 00195 slist = ph.getMMSScans(thisdict) 00196 for s in slist: 00197 mmsN = ph.getMMSScanNrows(thisdict, s) 00198 msN = ph.getScanNrows(self.msfile, s, selection=mysel) 00199 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00200 %(s, mmsN, msN)) 00201 00202 # Compare spw IDs 00203 for s in slist: 00204 mms_spw = ph.getSpwIds(self.mmsfile, s) 00205 ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) 00206 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00207 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw)) 00208 00209 # def addCleanup(self, function, *args, **kwargs): 00210 00211 class partition_test2(test_base): 00212 00213 def setUp(self): 00214 self.setUp_fourants() 00215 00216 def tearDown(self): 00217 shutil.rmtree(self.msfile, ignore_errors=True) 00218 shutil.rmtree(self.mmsfile, ignore_errors=True) 00219 shutil.rmtree(self.msfile+'.flagversions', ignore_errors=True) 00220 shutil.rmtree(self.mmsfile+'.flagversions', ignore_errors=True) 00221 00222 # The followin test fails in the OSX platforms if the full MS is used to 00223 # create the MMS. It does not fail in partition, but in the ms.getscansummary() 00224 # methods. Check this soon 00225 def test_sepaxis(self): 00226 '''Partition: separationaxis=both''' 00227 partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='both') 00228 # partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='both') 00229 time.sleep(10) 00230 00231 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00232 00233 # Dictionary with selection to compare with original MS 00234 mysel = {'spw':'0~11'} 00235 00236 # Take the dictionary and compare with original MS 00237 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00238 00239 # Compare nrows of all scans in selection 00240 slist = ph.getMMSScans(thisdict) 00241 self.assertEqual(slist.__len__(), 2) 00242 for s in slist: 00243 mmsN = ph.getMMSScanNrows(thisdict, s) 00244 msN = ph.getScanNrows(self.msfile, s, selection=mysel) 00245 # msN = ph.getScanNrows(self.msfile, s) 00246 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00247 %(s, mmsN, msN)) 00248 00249 # Compare spw IDs 00250 for s in slist: 00251 mms_spw = ph.getSpwIds(self.mmsfile, s) 00252 ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) 00253 # ms_spw = ph.getSpwIds(self.msfile, s) 00254 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00255 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw)) 00256 00257 00258 def test_all_columns(self): 00259 '''Partition: datacolumn=all''' 00260 partition(vis=self.msfile, outputvis=self.mmsfile, datacolumn='all') 00261 time.sleep(10) 00262 00263 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00264 00265 # Take the dictionary and compare with original MS 00266 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00267 00268 # Compare nrows of all scans in selection 00269 slist = ph.getMMSScans(thisdict) 00270 self.assertEqual(slist.__len__(), 2) 00271 for s in slist: 00272 mmsN = ph.getMMSScanNrows(thisdict, s) 00273 msN = ph.getScanNrows(self.msfile, s) 00274 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00275 %(s, mmsN, msN)) 00276 00277 # Compare spw IDs 00278 for s in slist: 00279 mms_spw = ph.getSpwIds(self.mmsfile, s) 00280 ms_spw = ph.getSpwIds(self.msfile, s) 00281 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00282 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw)) 00283 00284 def test_scan_spw(self): 00285 '''Partition: separationaxis=scan with spw selection''' 00286 partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan', 00287 spw='1~4,10,11') 00288 time.sleep(10) 00289 00290 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00291 00292 # Take the dictionary and compare with original MS 00293 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00294 00295 # Dictionary with selection to compare with original MS 00296 mysel = {'spw':'1~4,10,11'} 00297 00298 # Compare nrows of all scans in selection 00299 slist = ph.getMMSScans(thisdict) 00300 for s in slist: 00301 mmsN = ph.getMMSScanNrows(thisdict, s) 00302 msN = ph.getScanNrows(self.msfile, s, selection=mysel) 00303 self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' 00304 %(s, mmsN, msN)) 00305 00306 # Compare spw IDs 00307 for s in slist: 00308 mms_spw = ph.getSpwIds(self.mmsfile, s) 00309 ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel) 00310 self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\ 00311 'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw)) 00312 00313 00314 def test_numsubms(self): 00315 '''Partition: small numsubms value''' 00316 # There are 16 spws; we want only 6 sub-MSs. 00317 partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', 00318 numsubms=6) 00319 time.sleep(10) 00320 00321 self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test') 00322 00323 # Take the dictionary and compare with original MS 00324 thisdict = listpartition(vis=self.mmsfile, createdict=True) 00325 00326 # Check the number of sub-MSs 00327 mmslist = [] 00328 klist = thisdict.keys() 00329 for kk in klist: 00330 mmslist.append(thisdict[kk]['MS']) 00331 00332 nsubms = mmslist.__len__() 00333 self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs') 00334 00335 # Check that spw list is the same in MS and MMS 00336 spwlist = ph.getMMSSpwIds(thisdict) 00337 00338 # Reference list of scans in MS 00339 slist = ph.getScanList(self.msfile) 00340 setlist = set([]) 00341 for s in slist: 00342 ms_spw = ph.getSpwIds(self.msfile, s) 00343 for a in ms_spw: 00344 setlist.add(a) 00345 00346 self.assertEqual(list(setlist), spwlist) 00347 00348 def test_flagversions(self): 00349 '''Partition: check that the .flagversions is created''' 00350 00351 # Run partition and create the .flagversions 00352 partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True) 00353 self.assertTrue(os.path.exists(self.mmsfile+'.flagversions')) 00354 00355 # Check that the number of backups in MMS is correct 00356 aflocal = casac.agentflagger() 00357 aflocal.open(self.mmsfile) 00358 nv = aflocal.getflagversionlist() 00359 aflocal.done() 00360 self.assertEqual(len(nv), 3) 00361 00362 # Run flagdata on MMS to check if it works well. 00363 flagdata(vis=self.mmsfile, mode='unflag', flagbackup=True) 00364 00365 # Check that the number of backups in MMS is correct 00366 aflocal = casac.agentflagger() 00367 aflocal.open(self.mmsfile) 00368 nvref = aflocal.getflagversionlist() 00369 aflocal.done() 00370 self.assertEqual(len(nvref), 4) 00371 00372 def test_flagsrestore(self): 00373 '''Partition: check that we can restore the flags''' 00374 # Delete any flagversions 00375 if os.path.exists(self.msfile+'.flagversions'): 00376 shutil.rmtree(self.msfile+'.flagversions') 00377 00378 # Unflag the MS 00379 flagdata(vis=self.msfile, mode='unflag', flagbackup=False) 00380 00381 # Run partition and create the .flagversions 00382 partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True) 00383 self.assertTrue(os.path.exists(self.mmsfile+'.flagversions')) 00384 00385 # Flag spw=9 and then spw=7 in the MMS 00386 flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True) 00387 flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True) 00388 00389 # There should be flags in spw=7 and 9 in MMS 00390 res = flagdata(vis=self.mmsfile, mode='summary') 00391 self.assertEqual(res['flagged'],549888) 00392 00393 # Restore the original flags (there should be none) 00394 flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1') 00395 res = flagdata(vis=self.mmsfile, mode='summary') 00396 self.assertEqual(res['flagged'],0) 00397 00398 00399 def suite(): 00400 return [partition_test1, partition_test2] 00401 00402 00403 00404 00405 00406 00407 00408 00409 00410 00411 00412 00413 00414 00415 00416 00417 00418
1.8.0
