|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import numpy 00002 import pprint 00003 import os, re, shutil 00004 from tasks import * 00005 from taskinit import * 00006 from locatescript import copydata 00007 00008 def description(): 00009 return "Tests time and channel averaging with split." 00010 00011 # The different spw selection strings are designed to test different paths 00012 # through split, not MSSelection. 00013 # Typically channel averaging does not change the # of rows. 00014 expected = { 00015 # Input MS 00016 'labelled_by_time+ichan.ms': { # Has weight spectrum (some rows altered) 00017 # Time averaging width in s. 00018 'tav': 20, 00019 # Channel averaging width in channels. 00020 'cav': 2, 00021 # The rest of the keys at this level are chanselstrs. 00022 '': { 00023 # Test type 00024 'nrows_aft_tavg': 1079, 00025 'nrows_aft_cavg': 1079, 00026 'datshp': (1, 4, 1079), 00027 'cells': { 00028 #T or C (col, row) val, or (val, tolerance) 00029 'tav': {('FLAG', 0): [False for i in xrange(8)], 00030 ('WEIGHT', 0): (589824.0, 0.1), 00031 ('EXPOSURE', 0): 18.874367952346802, 00032 ('DATA', 1): (numpy.array([[0.0+0.j, 4.5+1.j, 00033 4.5+2.j, 4.5+3.j, 00034 4.5+4.j, 4.5+5.j, 00035 0.0+0.j, 0.0+0.j]]), 0.005), 00036 ('WEIGHT_SPECTRUM', 1): (numpy.array([[0., 43690.668, 87381.336, 00037 131072.03125, 174762.671875, 00038 218453.359375, 0., 0.]]), 0.05), 00039 ('TIME', 1): (4715114710.4857559, 0.01), 00040 ('EXPOSURE', 1): 20.971519947052002, 00041 ('FLAG', 1): [True, False, False, False, 00042 False, False, True, True], 00043 ('WEIGHT', 1): (655360.0, 0.1), 00044 ('SIGMA', 1): 0.0012352647, 00045 ('WEIGHT', 2): (45.0, 0.05), 00046 ('WEIGHT_SPECTRUM', 2): (numpy.array([[5.625, 5.625, 5.625, 5.625, 00047 5.625, 5.625, 5.625, 5.625]]), 00048 0.001), 00049 ('FLAG', 3): [True for i in xrange(8)], 00050 ('WEIGHT_SPECTRUM', 4): (numpy.array([[16384.0, 8192.0, 00051 8192.0, 8192.0, 00052 8192.0, 8192.0, 00053 8192.0, 8192.0]]), 0.1), 00054 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00055 }, 00056 'cav': {('FLAG', 0): [False for i in xrange(4)], 00057 ('WEIGHT', 0): (589824.0, 0.1), 00058 ('EXPOSURE', 0): 18.874367952346802, 00059 ('FEED1', 0): 0, 00060 ('FEED2', 9): 0, 00061 ('DATA', 1): (numpy.array([[4.5+1.j, 4.5+2.6j, 00062 4.5+4.5555556j, 0.0+0.j]]), 00063 0.005), 00064 ('WEIGHT_SPECTRUM', 1): (numpy.array([[43690.668, 218453.375, 00065 393216.03125, 0.]]), 00066 0.05), 00067 ('TIME', 1): (4715114710.4857559, 0.01), 00068 ('EXPOSURE', 1): 20.971519947052002, 00069 ('ARRAY_ID', 1): 0, 00070 ('FLAG', 1): [False, False, False, True], 00071 ('WEIGHT', 1): (655360.0, 0.1), 00072 ('SIGMA', 1): 0.0008734640, 00073 ('WEIGHT', 2): (45.0, 0.05), 00074 ('WEIGHT_SPECTRUM', 2): (numpy.array([[11.25, 11.25, 11.25, 11.25]]), 00075 0.01), 00076 ('FLAG', 3): [True for i in xrange(4)], 00077 ('WEIGHT_SPECTRUM', 4): (numpy.array([[24576.0, 16384.0, 00078 16384.0, 16384.0]]), 0.1), 00079 ('ANTENNA2', 5): 7 # Baseline 0-6 is dropped 00080 }}}, 00081 '0': { 00082 'nrows_aft_tavg': 269, 00083 'nrows_aft_cavg': 269, 00084 'datshp': (1, 4, 269), 00085 'cells': { 00086 #T or C (col, row) val, or (val, tolerance) 00087 'tav': {('FLAG', 0): [False for i in xrange(8)], 00088 ('WEIGHT', 0): (589824.0, 1.0), 00089 ('EXPOSURE', 0): 18.874367952346802, 00090 ('DATA', 1): (numpy.array([[0.0+0.j, 4.5+1.j, 00091 4.5+2.j, 4.5+3.j, 00092 4.5+4.j, 4.5+5.j, 00093 0.0+0.j, 0.0+0.j]]), 0.005), 00094 ('WEIGHT_SPECTRUM', 1): (numpy.array([[0.0, 43690.66796875, 00095 87381.3359375, 131072.03125, 00096 174762.671875, 218453.359375, 00097 0., 0.]]), 0.05), 00098 ('TIME', 1): (4715114710.4857559, 0.01), 00099 ('EXPOSURE', 1): 20.971519947052002, 00100 ('FLAG', 1): [True, False, False, False, 00101 False, False, True, True], 00102 ('WEIGHT', 1): (655360.0, 0.1), 00103 ('SIGMA', 1): 0.0012352647, 00104 ('WEIGHT', 2): (45.0, 0.05), 00105 ('WEIGHT_SPECTRUM', 2): (numpy.array([[5.625, 5.625, 5.625, 5.625, 00106 5.625, 5.625, 5.625, 5.625]]), 00107 0.01), 00108 ('FLAG', 3): [True for i in xrange(8)], 00109 ('WEIGHT_SPECTRUM', 4): (numpy.array([[16384.004, 8192.0, 00110 8192.0, 8192.0, 00111 8192.0, 8192.0, 00112 8192.0, 8192.0]]), 0.1), 00113 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00114 }, 00115 'cav': {('FLAG', 0): [False for i in xrange(4)], 00116 ('WEIGHT', 0): (589824.0, 0.1), 00117 ('EXPOSURE', 0): 18.874367952346802, 00118 ('FEED1', 0): 0, 00119 ('FEED2', 9): 0, 00120 ('DATA', 1): (numpy.array([[4.5+1.j, 4.5+2.6j, 00121 4.5+4.5555556j, 0.0+0.j]]), 00122 0.005), 00123 ('WEIGHT_SPECTRUM', 1): (numpy.array([[43690.66796875, 218453.375, 00124 393216.03125, 0.]]), 00125 0.05), 00126 ('TIME', 1): (4715114710.4857559, 0.01), 00127 ('EXPOSURE', 1): 20.971519947052002, 00128 ('ARRAY_ID', 1): 0, 00129 ('FLAG', 1): [False, False, False, True], 00130 ('WEIGHT', 1): (655360.0, 0.1), 00131 ('SIGMA', 1): 0.000873464, 00132 ('WEIGHT', 2): (45.0, 0.05), 00133 ('WEIGHT_SPECTRUM', 2): (numpy.array([[11.25, 11.25, 00134 11.25, 11.25]]), 0.01), 00135 ('FLAG', 3): [True for i in xrange(4)], 00136 ('WEIGHT_SPECTRUM', 4): (numpy.array([[24576.0, 16384.0, 00137 16384.0, 16384.0]]), 0.1), 00138 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00139 }}}, 00140 '2': { # spw 2 didn't get the manual flagging that spw 0 did. 00141 'nrows_aft_tavg': 270, 00142 'nrows_aft_cavg': 270, 00143 'datshp': (1, 4, 270), 00144 'cells': { 00145 #T or C (col, row) val, or (val, tolerance) 00146 'tav': {('FLAG', 0): [False for i in xrange(8)], 00147 ('WEIGHT', 0): (589823.9375, 0.1), 00148 ('EXPOSURE', 0): 18.874367952346802, 00149 ('DATA', 1): (numpy.array([[4.5+0.j, 4.5+1.j, 00150 4.5+2.j, 4.5+3.j, 00151 4.5+4.j, 4.5+5.j, 00152 4.5+6.j, 4.5+7.j]]), 0.005), 00153 ('WEIGHT_SPECTRUM', 1): (numpy.array([[81920.0 for i in xrange(8)]]), 00154 0.1), 00155 ('TIME', 1): (4715114710.4857559, 0.01), 00156 ('EXPOSURE', 1): 20.971519947052002, 00157 ('FLAG', 1): [False for i in xrange(8)], 00158 ('WEIGHT', 1): (655360.0, 0.1), 00159 ('SIGMA', 1): 0.0012352647, 00160 ('WEIGHT', 2): (655359.9375, 0.05), 00161 ('ANTENNA2', 5): 6 # Baseline 0-6 is OK in spw 2. 00162 }, 00163 'cav': {('FLAG', 0): [False for i in xrange(4)], 00164 ('WEIGHT', 0): (589823.9375, 1.0), 00165 ('EXPOSURE', 0): 18.874367952346802, 00166 ('FEED1', 0): 0, 00167 ('FEED2', 9): 0, 00168 ('DATA', 1): (numpy.array([[4.5+0.5j, 4.5+2.5j, 00169 4.5+4.5j, 4.5+6.5j]]), 0.005), 00170 ('WEIGHT_SPECTRUM', 1): (numpy.array([[163840.0 for i in xrange(4)]]), 00171 0.1), 00172 ('TIME', 1): (4715114710.4857559, 0.01), 00173 ('EXPOSURE', 1): 20.971519947052002, 00174 ('ARRAY_ID', 1): 0, 00175 ('WEIGHT', 1): (655360.0, 0.1), 00176 ('SIGMA', 1): 0.000873464, 00177 ('WEIGHT', 2): (655359.9375, 0.05) 00178 }}}, 00179 '0~2:0~3': { 00180 'nrows_aft_tavg': 808, 00181 'nrows_aft_cavg': 808, 00182 'datshp': (1, 2, 808), 00183 'cells': { 00184 #T or C (col, row) val, or (val, tolerance) 00185 'tav': {('FLAG', 0): [False for i in xrange(4)], 00186 ('WEIGHT', 0): (294912.0, 0.1), 00187 ('EXPOSURE', 0): 18.8744, 00188 ('DATA', 1): (numpy.array([[0.0+0.j, 4.5+1.j, 00189 4.5+2.j, 4.5+3.j]]), 0.005), 00190 ('TIME', 1): (4715114710.4857559, 0.01), 00191 ('TIME_CENTROID', 1): (4715114710.4857559, 0.01), 00192 ('EXPOSURE', 1): 20.971519947052002, 00193 ('WEIGHT', 1): (327680.0, 0.1), 00194 ('SIGMA', 1): 0.001746928, 00195 ('WEIGHT_SPECTRUM', 1): (numpy.array([0.0, 54613.34, 00196 109226.68, 163840.]), 00197 0.01), 00198 ('WEIGHT', 2): (22.5, 0.1), 00199 ('WEIGHT_SPECTRUM', 4): (numpy.array([[16384.0, 8192.0, 00200 8192.0, 8192.0]]), 0.005), 00201 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00202 }, 00203 'cav': {('FLAG', 0): [False, False], 00204 ('WEIGHT', 0): (294912.0, 1.0), 00205 ('EXPOSURE', 0): 18.8744, 00206 ('FEED1', 0): 0, 00207 ('FEED2', 9): 0, 00208 ('DATA', 1): (numpy.array([[4.5+1.j, 4.5+2.6j]]), 0.005), 00209 ('WEIGHT_SPECTRUM', 1): (numpy.array([[54613.33984375, 273066.6875]]), 00210 0.05), 00211 ('TIME', 1): (4715114710.4857559, 0.01), 00212 ('EXPOSURE', 1): 20.971519947052002, 00213 ('ARRAY_ID', 1): 0, 00214 ('FLAG', 1): [False, False], 00215 ('WEIGHT', 1): (327680.0, 0.1), 00216 ('SIGMA', 1): 0.001235264635, 00217 ('WEIGHT', 2): (22.5, 0.1), 00218 ('WEIGHT_SPECTRUM', 2): (numpy.array([[11.25, 11.25]]), 00219 0.01), 00220 ('FLAG', 3): [True, True], # wt invalid, but wtsp valid 00221 ('WEIGHT_SPECTRUM', 4): (numpy.array([[24576.0, 16384.0]]), 0.1), 00222 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00223 }}}, 00224 '0,2:0~3': { 00225 'nrows_aft_tavg': 539, 00226 'nrows_aft_cavg': 539, 00227 'datshp': set([(1, 4, 1), (1, 2, 1)]), 00228 'cells': { 00229 #T or C (col, row) val, or (val, tolerance) 00230 'tav': {('FLAG', 0): [False for i in xrange(8)], 00231 ('WEIGHT', 0): (589824.0, 0.1), 00232 ('EXPOSURE', 0): 18.874367952346802, 00233 ('DATA', 1): (numpy.array([[0.0+0.j, 4.5+1.j, 4.5+2.j, 4.5+3.j, 00234 4.5+4.j, 4.5+5.j, 0.0+0.j, 0+0.j]]), 00235 0.005), 00236 ('WEIGHT_SPECTRUM', 1): (numpy.array([[0.0, 43690.668, 00237 87381.336, 131072.03125, 00238 174762.672, 218453.359, 00239 0., 0.]]), 00240 0.05), 00241 ('TIME', 1): (4715114710.4857559, 0.01), 00242 ('EXPOSURE', 1): 20.971519947052002, 00243 ('WEIGHT', 1): (655360.0, 0.1), 00244 ('SIGMA', 1): 0.0012352647, 00245 ('WEIGHT', 2): (45.0, 0.05), 00246 ('WEIGHT_SPECTRUM', 2): (numpy.array([[5.625, 5.625, 5.625, 5.625, 00247 5.625, 5.625, 5.625, 5.625]]), 00248 0.01), 00249 ('FLAG', 3): [True for i in xrange(8)], # wt invalid, but wtsp valid 00250 ('WEIGHT_SPECTRUM', 4): (numpy.array([[16384.0, 8192.0, 00251 8192.0, 8192.0, 00252 8192.0, 8192.0, 00253 8192.0, 8192.0]]), 0.005), 00254 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00255 ('DATA_DESC_ID', 44): 0, 00256 ('DATA_DESC_ID', 134): 1 00257 }, 00258 'cav': {('FLAG', 0): [False for i in xrange(4)], 00259 ('WEIGHT', 0): (589824.0, 0.1), 00260 ('EXPOSURE', 0): 18.874368, 00261 ('FEED1', 0): 0, 00262 ('FEED2', 9): 0, 00263 ('DATA', 1): (numpy.array([[4.5+1.0j, 4.5+2.6j, 00264 4.5+4.5555553j, 0.0+0.j]]), 0.005), 00265 ('DATA', 81): (numpy.array([[14.4541+0.5j, 14.4541+2.5j, 00266 14.4541+4.5j, 14.4541+6.5j]]), 00267 0.005), 00268 ('DATA', 192): (numpy.array([[14.462124+0.5j, 14.462124+2.5j]]), 00269 0.005), 00270 ('WEIGHT_SPECTRUM', 1): (numpy.array([[43690.668, 218453.375, 00271 393216.03125, 0.]]), 00272 0.05), 00273 ('TIME', 1): (4715114710.4857559, 0.01), 00274 ('EXPOSURE', 1): 20.971519947, 00275 ('ARRAY_ID', 1): 0, 00276 ('WEIGHT', 1): (655360.0, 0.1), 00277 ('SIGMA', 1): 0.000873464, 00278 ('WEIGHT', 2): (45.0, 0.05), 00279 ('WEIGHT_SPECTRUM', 2): (numpy.array([[11.25, 11.25, 11.25, 11.25]]), 00280 0.01), 00281 ('FLAG', 3): [True for i in xrange(4)], # wt invalid, but wtsp valid 00282 ('WEIGHT_SPECTRUM', 4): (numpy.array([[24576.00585938, 16384.00390625, 00283 16384.00390625, 16384.00390625]]), 00284 0.05), 00285 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00286 ('DATA_DESC_ID', 134): 1, 00287 ('DATA', 134): (numpy.array([[5.0+0.5j, 5.0+2.5j]]), 0.005) 00288 } 00289 } 00290 }, 00291 '0:4~7': { # Tests channel selection not starting at 0, 00292 'nrows_aft_tavg': 269, # with a nontrivial WEIGHT_SPECTRUM. 00293 'nrows_aft_cavg': 269, 00294 'datshp': (1, 2, 269), 00295 'cells': { 00296 #T or C (col, row) val, or (val, tolerance) 00297 'tav': {('FLAG', 0): [False for i in xrange(4)], 00298 ('WEIGHT', 0): (294912.0, 0.1), 00299 ('EXPOSURE', 0): 18.874367952346802, 00300 ('DATA', 1): (numpy.array([[4.500+4.j, 4.500+5.j, 00301 0.000+0.j, 0.000+0.j]]), 00302 0.005), 00303 ('WEIGHT_SPECTRUM', 1): (numpy.array([[145635.5625, 182044.46875, 00304 0., 0.]]), 0.05), 00305 ('TIME', 1): (4715114710.4857559, 0.01), 00306 ('EXPOSURE', 1): 20.971519947052002, 00307 ('WEIGHT', 1): (327680.0, 0.1), 00308 ('SIGMA', 1): 0.001746928, 00309 ('WEIGHT', 2): (22.5, 0.05), 00310 ('WEIGHT_SPECTRUM', 2): (numpy.array([[5.625, 5.625, 5.625, 5.625]]), 00311 0.01), 00312 ('FLAG', 3): [True for i in xrange(4)], 00313 ('WEIGHT_SPECTRUM', 4): (numpy.array([[8192.00195312, 8192.00195312, 00314 8192.00195312, 8192.00195312]]), 00315 0.005), 00316 ('ANTENNA2', 5): 7, # Baseline 0-6 is dropped 00317 ('DATA_DESC_ID', 44): 0, 00318 ('DATA_DESC_ID', 134): 0 00319 }, 00320 'cav': {('FLAG', 0): [False, False], 00321 ('WEIGHT', 0): (294912.0, 0.1), 00322 ('EXPOSURE', 0): 18.874368, 00323 ('FEED1', 0): 0, 00324 ('FEED2', 9): 0, 00325 ('DATA', 1): (numpy.array([[4.5+4.5556j, 0.0+0.j]]), 0.005), 00326 ('DATA', 81): (numpy.array([[14.454+4.5j, 00327 14.454+6.5j]]), 0.005), 00328 ('DATA', 192): (numpy.array([[43.48377+4.5j, 00329 43.48377+6.5j]]), 0.005), 00330 ('WEIGHT_SPECTRUM', 1): (numpy.array([[327680.03125, 0.0]]), 00331 0.05), 00332 ('TIME', 1): (4715114710.4857559, 0.01), 00333 ('EXPOSURE', 1): 20.971519947, 00334 ('ARRAY_ID', 1): 0, 00335 ('WEIGHT', 1): (327680.0, 0.1), 00336 ('SIGMA', 1): 0.001235264635, 00337 ('WEIGHT', 2): (22.5, 0.1), 00338 ('WEIGHT_SPECTRUM', 2): (numpy.array([[11.25, 11.25]]), 0.01), 00339 ('FLAG', 3): [True, True], 00340 ('ANTENNA2', 5): 7, 00341 ('DATA_DESC_ID', 134): 0, 00342 ('DATA', 134): (numpy.array([[33.58345+4.5j, 00343 33.58345+6.5j]]), 0.005) 00344 } 00345 } 00346 } 00347 } 00348 } 00349 00350 def isquantized(val): 00351 """ 00352 Returns whether or not val has a type with quantized values, i.e. 00353 would 1.1 * val's type differ from val's? 00354 """ 00355 # try: 1.1 * val ... except: might be more general, but could be expensive. 00356 retval = True 00357 if isinstance(val, (float, complex)): 00358 retval = False 00359 elif hasattr(val, 'issubdtype'): 00360 if numpy.issubdtype(val.dtype, float) or numpy.issubdtype(val.dtype, complex): 00361 retval = False 00362 return retval 00363 00364 00365 def data(): 00366 """As I understand it, this must return the filenames of needed input data.""" 00367 00368 # Just in case order matters. 00369 inplist = expected.keys() 00370 inplist.sort() 00371 00372 #return ['split/' + inpms for inpms in inplist] 00373 return inplist 00374 00375 def run(fetch=False): 00376 00377 #####fetch data 00378 if fetch: 00379 for f in data( ): 00380 copydata( f, os.getcwd( ) ) 00381 00382 input_mses = data() 00383 badcells = {} 00384 for input_ms in input_mses: 00385 tbin = str(expected[input_ms]['tav']) + 's' 00386 cbin = expected[input_ms]['cav'] 00387 00388 spwsels = expected[input_ms].keys() 00389 spwsels.remove('tav') 00390 spwsels.remove('cav') 00391 spwsels.sort() # Prob. not needed. 00392 for spwsel in spwsels: 00393 tavms = re.sub(r'.ms$', '', input_ms) + '_%s_timebin%s.ms' % (spwsel, tbin) 00394 #print "run: tavms =", tavms 00395 outputms = re.sub(r'.ms$', '', tavms) + '_width%d.ms' % cbin 00396 #print "run: outputms =", outputms 00397 00398 for oms in (tavms, outputms): # Get rid of past problems. 00399 if os.path.isdir(oms): 00400 shutil.rmtree(oms) 00401 00402 # The actual run. 00403 try: 00404 (nrows_aft_tavg, 00405 nrows_aft_cavg, 00406 shp_aft_cavg) = time_then_chan_avg(input_ms, tbin, cbin, 00407 outms=outputms, 00408 chanselstr=spwsel, 00409 zaptemp=False) 00410 except Exception, e: 00411 raise Exception, "Error (%s) running time_then_chan_avg()." % e 00412 00413 ###### Test # of rows. 00414 if nrows_aft_tavg != expected[input_ms][spwsel]['nrows_aft_tavg']: 00415 raise Exception, """ 00416 The number of rows after time averaging (%d) does not match 00417 the expected number (%d) for input MS %s with selection string "%s" 00418 """ % (nrows_aft_tavg, expected[input_ms][spwsel]['nrows_aft_tavg'], 00419 input_ms, spwsel) 00420 00421 if nrows_aft_cavg != expected[input_ms][spwsel]['nrows_aft_cavg']: 00422 raise Exception, """ 00423 The number of rows after channel averaging (%d) does not match 00424 the expected number (%d) for input MS %s with selection string "%s" 00425 """ % (nrows_aft_cavg, expected[input_ms][spwsel]['nrows_aft_cavg'], 00426 input_ms, spwsel) 00427 00428 if shp_aft_cavg != expected[input_ms][spwsel]['datshp']: 00429 raise Exception, """ 00430 The data shape (%s) after channel averaging does not match 00431 the expectation (%s) for input MS %s with selection string "%s" 00432 """ % (shp_aft_cavg, expected[input_ms][spwsel]['datshp'], 00433 input_ms, spwsel) 00434 00435 ###### Compare averaged data with expectation, and rm the averaged 00436 ###### data if it passes. 00437 outputmses = {'tav': tavms, 'cav': outputms} 00438 for avtype, avms in outputmses.iteritems(): 00439 expcells = expected[input_ms][spwsel]['cells'][avtype] 00440 try: 00441 tb.open(avms) 00442 except Exception, e: 00443 raise Exception, "Error (%s) opening %s." % (e, avms) 00444 try: 00445 for ((col, row), expent) in expcells.iteritems(): 00446 usetol = True 00447 if isinstance(expent, tuple): 00448 tol = expent[1] 00449 expent = expent[0] 00450 elif not isquantized(expent): 00451 tol = 0.005 * expent 00452 else: 00453 usetol = False 00454 00455 gotval = tb.getcell(col, row) 00456 if hasattr(gotval, 'flatten') and gotval.size == 1: 00457 gotval = gotval.flatten()[0] 00458 if usetol: 00459 closeenough = (abs(gotval - expent) < tol) 00460 else: 00461 closeenough = (gotval == expent) 00462 if hasattr(closeenough, 'all'): 00463 closeenough = closeenough.all() 00464 if not closeenough: 00465 if not badcells.has_key(avms): 00466 badcells[avms] = {} 00467 badcells[avms][(col, row)] = (gotval, expent) 00468 except Exception, e: 00469 raise Exception, "Error (%s) checking %s's cell %s." % (e, avms, 00470 (col, row)) 00471 finally: 00472 tb.close() 00473 00474 if not badcells.has_key(avms): 00475 shutil.rmtree(avms) 00476 if badcells: 00477 pp = pprint.PrettyPrinter(indent=0, width=80) 00478 pp.pprint(badcells) 00479 raise Exception, "There were unexpected values in the averaged MSes. Check the log." 00480 00481 print '' 00482 print 'Regression PASSED' 00483 print '' 00484 00485 return [] 00486 00487 00488 def time_then_chan_avg(inms, tbin, chanbin, outms="", zaptemp=True, 00489 zaporig=False, chanselstr="", datacolstr='corrected'): 00490 """ 00491 Bins time and frequency in 1.x passes. First inms is time averaged to 00492 in_timebin<tbin>.ms by split with timebin = tbin. Then in_timebin<tbin>.ms 00493 (much smaller than inms!) is channel averaged to outms by split with width 00494 = chanbin. 00495 00496 outms: Name to use for the output MS. Defaults to inms - ".ms" + 00497 "_" + chanselstr + "_timebin" + tbin + "_width" + chanbin + ".ms". 00498 00499 zaptemp: whether or not to delete the intermediate ms. 00500 zaporig: whether or not to delete the original ms. 00501 00502 chanselstr: An spw:chan;step channel selection string corresponding to 00503 split's spw. Note that it is an error to select fewer 00504 than chanbin channels in any of the selected input spws. 00505 Defaulting to averaging all the channels would be reasonable in 00506 that case, but it would make it hard to detect more serious 00507 errors. Not that you should rely on casapy to detect your bad 00508 selection strings... 00509 """ 00510 casalog.origin('time_then_chan_avg') 00511 inms_root = inms.rpartition(".ms")[0] 00512 if not inms_root: 00513 inms_root = inms 00514 inms_root = os.path.basename(inms_root) # Keep outms in local directory. 00515 troot = inms_root + '_' + chanselstr + "_timebin" + tbin 00516 tms = troot + ".ms" 00517 00518 funnyshapes = False 00519 00520 try: 00521 # Do time averaging. 00522 if (chanselstr.find(':') > -1) and ((chanselstr.find(',') > -1) or 00523 (chanselstr.find(';') > -1)): 00524 funnyshapes = True 00525 00526 ## print 'tbin =', tbin 00527 ## print 'tavspw =', chanselstr 00528 ## print 'tavdc =', datacolstr 00529 ## print 'inms =', inms 00530 casalog.post("Time averaging to " + tms) 00531 split(vis=inms, outputvis=tms, timebin=tbin, spw=chanselstr, 00532 datacolumn=datacolstr) 00533 ms.open(tms) 00534 nrows_aft_tavg = ms.nrow() 00535 ms.close() 00536 except Exception, e: 00537 casalog.post( 00538 """Error from split(vis='%s', outputvis='%s', 00539 timebin='%s', spw='%s', datacolumn='%s').""" % (inms, tms, 00540 tbin, 00541 chanselstr, 00542 datacolstr), 00543 'SEVERE') 00544 if os.path.isdir(tms): 00545 casalog.post("\t%s has been left on disk for your inspection." % tms, 00546 'SEVERE') 00547 raise e 00548 00549 if outms == "": 00550 outms = "%s_width%d.ms" % (troot, chanbin) 00551 try: 00552 # Do channel averaging. 00553 casalog.post("Channel averaging to " + outms) 00554 if datacolstr.lower() not in ['all', 'float_data', 'lag_data']: 00555 datacolstr = 'data' 00556 split(vis=tms, outputvis=outms, datacolumn=datacolstr, width=chanbin) 00557 tb.open(outms) 00558 if funnyshapes: 00559 dat = tb.getvarcol('DATA') 00560 nrows_aft_cavg = len(dat) 00561 shape_aft_cavg = set([d.shape for d in dat.itervalues()]) 00562 else: 00563 dat = tb.getcol('DATA') 00564 shape_aft_cavg = dat.shape 00565 nrows_aft_cavg = shape_aft_cavg[-1] 00566 tb.close() 00567 except Exception, e: 00568 casalog.post( 00569 """Error from split(vis='%s', outputvis='%s', 00570 datacolumn='data', width=%d, spw='%s').""" % (tms, outms, 00571 chanbin, 00572 chanselstr), 00573 'SEVERE') 00574 raise e 00575 if zaptemp: 00576 shutil.rmtree(tms) 00577 if zaporig: 00578 casalog.post("Following your instruction to shutil.rmtree(" + inms + ")", 'WARN') 00579 shutil.rmtree(inms) 00580 return nrows_aft_tavg, nrows_aft_cavg, shape_aft_cavg 00581 00582 00583 def disk_usage(pat): 00584 """ 00585 Returns a dictionary, keyed by path, of the disk usage in kilobytes for 00586 each path matching the shell filename pattern pat. 00587 """ 00588 retval = {} 00589 dudu = os.popen('du -sk ' + pat) 00590 for outline in dudu: 00591 try: 00592 size, path = outline.split() # newline dumped. 00593 size = int(size) 00594 retval[path] = size 00595 except Exception, e: 00596 print "Error (%s) getting size and path from %s." % (e, outline) 00597 return retval
1.8.0
