|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import os 00002 from taskinit import * 00003 import flaghelper as fh 00004 00005 debug = False 00006 00007 00008 def importevla( 00009 asdm=None, 00010 vis=None, 00011 ocorr_mode=None, 00012 compression=None, 00013 asis=None, 00014 scans=None, 00015 verbose=None, 00016 overwrite=None, 00017 online=None, 00018 tbuff=None, 00019 flagzero=None, 00020 flagpol=None, 00021 shadow=None, 00022 tolerance=None, 00023 addantenna=None, 00024 applyflags=None, 00025 savecmds=None, 00026 outfile=None, 00027 flagbackup=None, 00028 ): 00029 """ Convert a Science Data Model (SDM) dataset into a CASA Measurement Set (MS) 00030 ....This version is under development and is geared to handling EVLA specific flag and 00031 ....system files, and is otherwise equivalent to importasdm. 00032 .... 00033 ....Keyword arguments: 00034 ....asdm -- Name of input SDM file (directory) 00035 ........default: none; example: asdm='TOSR0001_sb1308595_1.55294.83601028935' 00036 00037 ....""" 00038 00039 # Python script 00040 # Origninator: Steven T. Myers 00041 # Written (3.0.1) STM 2010-03-11 modify importasdm to include flagging from xml 00042 # Vers1.0 (3.0.1) STM 2010-03-16 add tbuff argument 00043 # Vers2.0 (3.0.1) STM 2010-03-29 minor improvements 00044 # Vers3.0 (3.0.2) STM 2010-04-13 add flagzero, doshadow 00045 # Vers4.0 (3.0.2) STM 2010-04-20 add flagpol 00046 # Vers5.0 (3.0.2) STM 2010-05-27 combine flagzero clips 00047 # Vers6.0 (3.1.0) STM 2010-07-01 flagbackup option 00048 # Vers7.0 (3.1.0) STM 2010-08-18 remove corr_mode,wvr_corrected_data,singledish,antenna 00049 # Vers7.1 (3.1.0) STM 2010-10-07 remove time_sampling, srt 00050 # Vers7.1 (3.1.0) STM 2010-10-07 use helper functions, flagger tool, fill FLAG_CMD 00051 # Vers7.2 (3.1.0) STM 2010-10-29 minor modifications to defaults and messages 00052 # Vers8.0 (3.2.0) STM 2010-11-23 tbuff not sub-par of applyflags=T 00053 # Vers8.1 (3.2.0) STM 2010-12-01 prec=9 on timestamps 00054 # Vers8.2 (3.2.0) MKH 2010-12-06 added scan selection 00055 # Vers8.3 (3.2.0) GAM 2011-01-18 added switchedpower option (sw power gain/tsys) 00056 # Vers8.4 (3.2.0) STM 2011-03-24 fix casalog.post line-length bug 00057 # Vers8.5 (3.4.0) STM 2011-12-08 new readflagxml for new Flag.xml format 00058 # Vers8.6 (3.4.0) STM 2011-02-22 full handling of new Flag.xml ant+spw+pol flags 00059 # Vers9.0 (3.4.0) SMC 2012-03-13 ported to use the new flagger tool (agentflagger) 00060 # 00061 00062 # Create local versions of the flagger and ms tools 00063 aflocal = casac.agentflagger() 00064 mslocal = casac.ms() 00065 00066 # 00067 try: 00068 casalog.origin('importevla') 00069 casalog.post('You are using importevla v9.0 SMC Updated 2012-03-13' 00070 ) 00071 viso = '' 00072 casalog.post('corr_mode is forcibly set to all.') 00073 if len(vis) > 0: 00074 viso = vis 00075 else: 00076 viso = asdm + '.ms' 00077 vis = asdm 00078 corr_mode = 'all' 00079 wvr_corrected_data = 'no' 00080 singledish = False 00081 srt = 'all' 00082 time_sampling = 'all' 00083 showversion = True 00084 execute_string = 'asdm2MS --icm "' + corr_mode + '" --isrt "' \ 00085 + srt + '" --its "' + time_sampling + '" --ocm "' \ 00086 + ocorr_mode + '" --wvr-corrected-data "' \ 00087 + wvr_corrected_data + '" --asis "' + asis + '" --scans "' \ 00088 + scans + '" --logfile "' + casalog.logfile() + '"' 00089 if showversion: 00090 casalog.post('asdm2MS --revision --logfile "' 00091 + casalog.logfile() + '"') 00092 os.system('asdm2MS --revision --logfile "' 00093 + casalog.logfile() + '"') 00094 if compression: 00095 execute_string = execute_string + ' --compression' 00096 if verbose: 00097 execute_string = execute_string + ' --verbose' 00098 if not overwrite and os.path.exists(viso): 00099 raise Exception, \ 00100 'You have specified and existing ms and have indicated you do not wish to overwrite it' 00101 # 00102 # If viso+".flagversions" then process differently depending on the value of overwrite.. 00103 # 00104 dotFlagversion = viso + '.flagversions' 00105 if os.path.exists(dotFlagversion): 00106 if overwrite: 00107 casalog.post("Found '" + dotFlagversion 00108 + "' . It'll be deleted before running the filler." 00109 ) 00110 os.system('rm -rf %s' % dotFlagversion) 00111 else: 00112 casalog.post("Found '%s' but can't overwrite it." 00113 % dotFlagversion) 00114 raise Exception, "Found '%s' but can't overwrite it." \ 00115 % dotFlagversion 00116 00117 execute_string = execute_string + ' ' + asdm + ' ' + viso 00118 casalog.post('Running the asdm2MS standalone invoked as:') 00119 # Print execute_string 00120 casalog.post(execute_string) 00121 00122 # Catch the return status and exit on failure 00123 ret_status = os.system(execute_string) 00124 if ret_status != 0: 00125 casalog.post('asdm2MS failed to execute with exit error '+str(ret_status), 'SEVERE') 00126 raise Exception, 'ASDM conversion error, please check if it is a valid ASDM.' 00127 00128 if compression: 00129 visover = viso 00130 viso = visover.replace('.ms','.compressed.ms') 00131 if flagbackup: 00132 ok = af.open(viso) 00133 ok = af.saveflagversion('Original', 00134 comment='Original flags on import', merge='save') 00135 ok = af.done() 00136 print 'Backed up original flag column to ' + viso \ 00137 + '.flagversions' 00138 casalog.post('Backed up original flag column to ' + viso 00139 + '.flagversions') 00140 else: 00141 casalog.post('Warning: will not back up original flag column' 00142 , 'WARN') 00143 # 00144 # ============================= 00145 # Begin EVLA specific code here 00146 # ============================= 00147 nflags = 0 00148 00149 # All flag cmds 00150 allflags = {} 00151 00152 if os.access(asdm + '/Flag.xml', os.F_OK): 00153 # Find (and copy) Flag.xml 00154 print ' Found Flag.xml in SDM, copying to MS' 00155 casalog.post('Found Flag.xml in SDM, copying to MS') 00156 os.system('cp -rf ' + asdm + '/Flag.xml ' + viso + '/') 00157 # Find (and copy) Antenna.xml 00158 if os.access(asdm + '/Antenna.xml', os.F_OK): 00159 print ' Found Antenna.xml in SDM, copying to MS' 00160 casalog.post('Found Antenna.xml in SDM, copying to MS') 00161 os.system('cp -rf ' + asdm + '/Antenna.xml ' + viso 00162 + '/') 00163 else: 00164 raise Exception, 'Failed to find Antenna.xml in SDM' 00165 # Find (and copy) SpectralWindow.xml 00166 if os.access(asdm + '/SpectralWindow.xml', os.F_OK): 00167 print ' Found SpectralWindow.xml in SDM, copying to MS' 00168 casalog.post('Found SpectralWindow.xml in SDM, copying to MS' 00169 ) 00170 os.system('cp -rf ' + asdm + '/SpectralWindow.xml ' 00171 + viso + '/') 00172 else: 00173 raise Exception, \ 00174 'Failed to find SpectralWindow.xml in SDM' 00175 # 00176 # Parse Flag.xml into flag dictionary 00177 # 00178 if online: 00179 flago = fh.readXML(asdm, tbuff) 00180 onlinekeys = flago.keys() 00181 00182 nkeys = onlinekeys.__len__() 00183 nflags += nkeys 00184 allflags = flago.copy() 00185 00186 casalog.post('Created %s commands for online flags'%str(nflags)) 00187 00188 else: 00189 if online: 00190 # print 'ERROR: No Flag.xml in SDM' 00191 casalog.post('ERROR: No Flag.xml in SDM', 'SEVERE') 00192 else: 00193 # print 'WARNING: No Flag.xml in SDM' 00194 casalog.post('WARNING: No Flag.xml in SDM', 'WARN') 00195 00196 if flagzero or shadow: 00197 # Get overall MS time range for later use (if needed) 00198 (ms_startmjds, ms_endmjds, ms_starttime, ms_endtime) = \ 00199 getmsmjds(viso) 00200 00201 # Now add zero and shadow flags 00202 if flagzero: 00203 flagz = {} 00204 # clip zero data 00205 # NOTE: currently hard-wired to RL basis 00206 # assemble into flagging commands and add to myflagd 00207 flagz['time'] = 0.5 * (ms_startmjds + ms_endmjds) 00208 flagz['interval'] = ms_endmjds - ms_startmjds 00209 flagz['level'] = 0 00210 flagz['severity'] = 0 00211 flagz['type'] = 'FLAG' 00212 flagz['applied'] = False 00213 flagz['antenna'] = '' 00214 flagz['mode'] = 'clip' 00215 00216 # Flag cross-hands too 00217 if flagpol: 00218 flagz['reason'] = 'CLIP_ZERO_ALL' 00219 flagz['command'] = \ 00220 'mode=clip clipzeros=True correlation=ABS_ALL' 00221 flagz['id'] = 'ZERO_ALL' 00222 allflags[nflags] = flagz.copy() 00223 nflags += 1 00224 nflagz = 1 00225 else: 00226 00227 flagz['reason'] = 'CLIP_ZERO_RR' 00228 flagz['command'] = \ 00229 'mode=clip clipzeros=True correlation=ABS_RR' 00230 flagz['id'] = 'ZERO_RR' 00231 allflags[nflags] = flagz.copy() 00232 nflags += 1 00233 00234 flagz['reason'] = 'CLIP_ZERO_LL' 00235 flagz['command'] = \ 00236 'mode=clip clipzeros=True correlation=ABS_LL' 00237 flagz['id'] = 'ZERO_LL' 00238 allflags[nflags] = flagz.copy() 00239 nflags += 1 00240 nflagz = 2 00241 00242 casalog.post('Created %s command(s) to clip zeros'%str(nflagz)) 00243 00244 if shadow: 00245 flagh = {} 00246 # flag shadowed data 00247 flagh['time'] = 0.5 * (ms_startmjds + ms_endmjds) 00248 flagh['interval'] = ms_endmjds - ms_startmjds 00249 flagh['level'] = 0 00250 flagh['severity'] = 0 00251 flagh['type'] = 'FLAG' 00252 flagh['applied'] = False 00253 flagh['antenna'] = '' 00254 flagh['mode'] = 'shadow' 00255 flagh['reason'] = 'SHADOW' 00256 00257 scmd = 'mode=shadow tolerance=' + str(tolerance) 00258 00259 if type(addantenna) == str: 00260 if addantenna != '': 00261 # it's a filename, create a dictionary 00262 antdict = fh.readAntennaList(addantenna) 00263 scmd = scmd +' addantenna='+str(antdict) 00264 00265 elif type(addantenna) == dict: 00266 if addantenna != {}: 00267 scmd = scmd +' addantenna='+str(addantenna) 00268 00269 flagh['command'] = scmd 00270 00271 flagh['id'] = 'SHADOW' 00272 allflags[nflags] = flagh.copy() 00273 nflags += 1 00274 00275 casalog.post('Created 1 command to flag shadowed data') 00276 00277 # List of rows to save 00278 allkeys = allflags.keys() 00279 00280 # Apply the flags 00281 if applyflags: 00282 if nflags > 0: 00283 00284 # Open the MS and attach the tool 00285 aflocal.open(viso) 00286 00287 # Create a loose union 00288 # flaglist = [] 00289 # for k in myflagd.keys(): 00290 # cmdline = myflagd[k]['command'] 00291 # flaglist.append(cmdline) 00292 # 00293 # unionpars = fh.getUnion(mslocal, viso, allflags) 00294 00295 # Select the data 00296 aflocal.selectdata() 00297 00298 # Setup the agent's parameters 00299 saved_list = fh.setupAgent(aflocal, allflags, [], True, True) 00300 00301 # Initialize the agents 00302 aflocal.init() 00303 00304 # Backup the flags. Already done in the beginning of the task 00305 # if flagbackup: 00306 # fh.backupFlags(aflocal=aflocal, msfile='', prename='importevla') 00307 00308 # Run the tool 00309 stats = aflocal.run(True, True) 00310 00311 casalog.post('Applied %s flag commands to data'%str(nflags)) 00312 00313 # Destroy the tool 00314 aflocal.done() 00315 00316 # Save the flags to FLAG_CMD and update the APPLIED column 00317 fh.writeFlagCmd(viso, allflags, allkeys, True, '', '') 00318 00319 else: 00320 00321 casalog.post('There are no flags to apply') 00322 00323 else : 00324 casalog.post('Will not apply flags (applyflags=False), use flagcmd to apply') 00325 if nflags > 0: 00326 fh.writeFlagCmd(viso, allflags, allkeys, False, '', '') 00327 00328 00329 # Save the flag commads to an ASCII file 00330 if savecmds: 00331 00332 if nflags > 0: 00333 # Save the cmds to a file 00334 if outfile == '': 00335 # Save to standard filename 00336 outfile = viso.replace('.ms','_cmd.txt') 00337 00338 fh.writeFlagCmd(viso, allflags, allkeys, False, '', outfile) 00339 00340 casalog.post('Saved %s flag commands to %s'%(nflags,outfile)) 00341 00342 else: 00343 casalog.post('There are no flag commands to save') 00344 00345 00346 except Exception, instance: 00347 00348 casalog.post('%s' % instance, 'ERROR') 00349 00350 # write history 00351 mslocal.open(viso, nomodify=False) 00352 mslocal.writehistory(message='taskname = importevla', 00353 origin='importevla') 00354 mslocal.writehistory(message='asdm = "' + str(asdm) + '"', 00355 origin='importevla') 00356 mslocal.writehistory(message='vis = "' + str(viso) + '"', 00357 origin='importevla') 00358 if flagzero: 00359 mslocal.writehistory(message='flagzero = T"', 00360 origin='importevla') 00361 if flagpol: 00362 mslocal.writehistory(message='flagpol = T"', 00363 origin='importevla') 00364 if shadow: 00365 mslocal.writehistory(message='shadow = T"', 00366 origin='importevla') 00367 mslocal.writehistory(message='tolerance = "' + str(tolerance) 00368 + '"', origin='importevla') 00369 mslocal.writehistory(message='addantenna = "' 00370 + str(addantenna) + '"', 00371 origin='importevla') 00372 if applyflags: 00373 mslocal.writehistory(message='applyflags = T"', 00374 origin='importevla') 00375 00376 mslocal.close() 00377 00378 00379 # =============================================================================== 00380 00381 00382 def getmsmjds(vis): 00383 # Get start and end times from MS, return in mjds 00384 # this might take too long for large MS 00385 # NOTE: could also use values from OBSERVATION table col TIME_RANGE 00386 mslocal2 = casac.ms() 00387 success = True 00388 ms_time1 = '' 00389 ms_time2 = '' 00390 ms_startmjds = 0.0 00391 ms_endmjds = 0.0 00392 try: 00393 mslocal2.open(vis) 00394 timd = mslocal2.range(['time']) 00395 mslocal2.close() 00396 except: 00397 success = False 00398 print 'Error opening MS ' + vis 00399 if success: 00400 ms_startmjds = timd['time'][0] 00401 ms_endmjds = timd['time'][1] 00402 t = qa.quantity(ms_startmjds, 's') 00403 t1sd = t['value'] 00404 ms_time1 = qa.time(t, form='ymd', prec=9)[0] 00405 t = qa.quantity(ms_endmjds, 's') 00406 t2sd = t['value'] 00407 ms_time2 = qa.time(t, form='ymd', prec=9)[0] 00408 casalog.post('MS spans timerange ' + ms_time1 + ' to ' 00409 + ms_time2) 00410 else: 00411 print 'WARNING: Could not open vis as MS to find times' 00412 casalog.post('WARNING: Could not open vis as MS to find times') 00413 return (ms_startmjds, ms_endmjds, ms_time1, ms_time2) 00414
1.8.0
