|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import os 00002 import commands 00003 import math 00004 import pdb 00005 import shutil 00006 import string 00007 import time 00008 import ast 00009 from taskinit import * 00010 00011 ''' 00012 A set of helper functions for the tasks tflagdata and flagcmd. 00013 I/O functions: 00014 readFile 00015 readXML 00016 makeDict 00017 readAntennaList 00018 writeAntennaList 00019 writeFlagCmd 00020 writeRflagThresholdFile 00021 00022 Parameter handling 00023 compressSelectionList 00024 fixType 00025 getLinePars 00026 getNumPar 00027 getReason 00028 getSelectionPars 00029 getUnion 00030 purgeEmptyPars 00031 purgeParameter 00032 setupAgent 00033 00034 Others 00035 backupFlags 00036 convertDictToString 00037 convertStringToDict 00038 extractAntennaInfo 00039 extractRflagOutputFromSummary 00040 00041 ''' 00042 ###some helper tools 00043 tblocal = tbtool() 00044 00045 00046 debug = False 00047 00048 ####################################################### 00049 # 00050 # Reading functions 00051 # 00052 ####################################################### 00053 def isCalTable(msname): 00054 '''Check if a file is a cal table 00055 msname --> filename 00056 Return 1 for cal table, 0 for and 2 for MMS''' 00057 00058 try: 00059 tblocal.open(msname) 00060 except: 00061 raise ValueError, "Unable to open MS %s" % msname 00062 00063 tbinfo = tblocal.info() 00064 tblocal.close() 00065 retval = 0 00066 00067 if tbinfo['type'] == 'Calibration': 00068 retval = 1 00069 00070 elif tbinfo['type'] == 'Measurement Set': 00071 # MMS type 00072 if tbinfo['subType'] == 'CONCATENATED': 00073 retval = 2 00074 else: 00075 # MS type 00076 retval = 0 00077 else: 00078 retval = 0 00079 00080 00081 return retval 00082 00083 00084 def readFile(inputfile): 00085 '''Read in the lines from an input file 00086 inputfile --> file in disk with a list of strings per line 00087 00088 Returns a list. Blank lines are skipped. 00089 ''' 00090 00091 flagfile = inputfile 00092 00093 if (type(flagfile) == str) & os.path.exists(flagfile): 00094 try: 00095 ff = open(flagfile, 'r') 00096 except: 00097 raise Exception, 'Error opening file ' + flagfile 00098 else: 00099 raise Exception, \ 00100 'ASCII file not found - please verify the name' 00101 00102 # 00103 # Parse file 00104 try: 00105 cmdlist = [] 00106 for line in ff: 00107 cmd = line.rstrip() 00108 cmdlist.append(cmd) 00109 00110 except: 00111 raise Exception, 'Error reading lines from file ' \ 00112 + flagtable 00113 00114 ff.close() 00115 00116 return cmdlist 00117 00118 # jagonzal (CAS-4119): Use absolute paths for input files to ensure that the engines find them 00119 def addAbsPath(input_file): 00120 '''Read in the lines from an input file 00121 inputfile --> file in disk with a list of strings per line 00122 00123 Re-writes the file changing relative file names to absolute file names 00124 ''' 00125 00126 output_file = input_file + ".tmp" 00127 if (type(input_file) == str) & os.path.exists(input_file): 00128 try: 00129 input_file_id = open(input_file, 'r') 00130 except: 00131 raise Exception, 'Error opening file ' + input_file 00132 00133 try: 00134 output_file_id = open(output_file, 'w') 00135 except: 00136 output_file_id.close() 00137 raise Exception, 'Error opening file ' + output_file 00138 else: 00139 raise Exception, \ 00140 'ASCII file not found - please verify the name' 00141 00142 # 00143 # Parse file 00144 try: 00145 for line in input_file_id: 00146 cmd = line.rstrip() 00147 cmd_pars = cmd.split(" ") 00148 for par in cmd_pars: 00149 if ((par.count("inpfile") > 0) or 00150 (par.count("outfile") > 0) or 00151 (par.count("addantenna") > 0) or 00152 (par.count("timedev") > 0) or 00153 (par.count("freqdev") > 0)): 00154 par = par.split("=") 00155 file_local = par[1] 00156 if file_local.count("'")>0: 00157 file_local = file_local.split("'") 00158 file_local = file_local[1] 00159 elif file_local.count('"')>0: 00160 file_local = file_local.split('"') 00161 file_local = file_local[1] 00162 else: 00163 continue 00164 file_abs = os.path.abspath(file_local) 00165 cmd = cmd.replace(file_local,file_abs) 00166 00167 output_file_id.write(cmd+"\n") 00168 00169 except: 00170 input_file_id.close() 00171 output_file_id.close() 00172 raise Exception, 'Error reading lines from file ' \ 00173 + flagtable 00174 00175 input_file_id.close() 00176 output_file_id.close() 00177 os.rename(output_file, input_file) 00178 00179 def makeDict(cmdlist, myreason='any'): 00180 '''Make a dictionary compatible with a FLAG_CMD table structure 00181 and select by reason if any is given 00182 00183 cmdlist --> list of parameters to go into the COMMAND column 00184 myreason --> reason to select from 00185 00186 Returns a dictionary with the the selected rows with the following 00187 structure and default values: 00188 00189 flagd['row'] = 00190 flagd['applied'] = False 00191 flagd['command'] = '' 00192 flagd['interval'] = 0.0 00193 flagd['level'] = 0 00194 flagd['reason'] = '' 00195 flagd['severity'] = 0 00196 flagd['time'] = 0.0 00197 flagd['type'] = 'FLAG' 00198 00199 ''' 00200 00201 if cmdlist.__len__() == 0: 00202 raise Exception, 'Empty list of commands' 00203 00204 # select by these reasons 00205 myreaslist = [] 00206 if type(myreason) == str: 00207 if myreason != 'any': 00208 myreaslist.append(myreason) 00209 elif type(myreason) == list: 00210 myreaslist = myreason 00211 else: 00212 casalog.post('Cannot read reason; it contains unknown variable types', 00213 'ERROR') 00214 return 00215 00216 if debug: 00217 print "reason in selection" 00218 print myreaslist 00219 00220 # List of reasons in input file 00221 nrows = cmdlist.__len__() 00222 00223 # If any selection was requested 00224 reaslist = [] 00225 for i in range(nrows): 00226 command = cmdlist[i] 00227 if command.__contains__('reason'): 00228 reaslist.append(getReason(command)) 00229 else: 00230 # so that cmdlist and reaslist have the same sizes 00231 reaslist.append('NoReasonToMatch') 00232 00233 if debug: 00234 print "reason in input" 00235 print reaslist 00236 00237 00238 # Defaults for columns 00239 applied = False 00240 interval = 0.0 00241 level = 0 00242 severity = 0 00243 coltime = 0.0 00244 coltype = 'FLAG' 00245 00246 myflagd = {} 00247 ncmds = 0 00248 rowlist = range(nrows) 00249 00250 # If select by reason is requested 00251 if myreaslist.__len__() > 0: 00252 rowl = [] 00253 00254 # If selection matches what is in input line 00255 for j in range(reaslist.__len__()): 00256 if myreaslist.count(reaslist[j]) > 0: 00257 rowl.append(j) 00258 rowlist = rowl 00259 00260 try: 00261 for i in rowlist: 00262 flagd = {} 00263 reason = '' 00264 00265 command = cmdlist[i] 00266 if reaslist[i] != 'NoReasonToMatch': 00267 reason = reaslist[i] 00268 00269 # Skip comment lines 00270 if command.startswith('#'): 00271 continue 00272 if command == '': 00273 casalog.post('Ignoring empty command line', 'WARN') 00274 continue 00275 if command.__contains__('summary'): 00276 casalog.post('Mode summary is not allowed in list operation', 'WARN') 00277 continue 00278 00279 # If shadow, remove the addantenna dictionary 00280 if command.__contains__('shadow') and command.__contains__('addantenna'): 00281 i0 = command.rfind('addantenna') 00282 if command[i0+11] == '{': 00283 # It is a dictionary. Remove it from line 00284 i1 = command.rfind('}') 00285 antpar = command[i0+11:i1+1] 00286 temp = command[i0:i1+1] 00287 newcmd = command.replace(temp,'') 00288 antpardict = convertStringToDict(antpar) 00289 flagd['addantenna'] = antpardict 00290 command = newcmd 00291 00292 flagd['row'] = str(i) 00293 flagd['applied'] = applied 00294 flagd['reason'] = reason 00295 00296 # Remove reason from command line 00297 command = command.rstrip() 00298 newline = purgeParameter(command, 'reason') 00299 00300 # Remove any empty parameter (CAS-4015) 00301 command = purgeEmptyPars(newline) 00302 command = command.strip() 00303 00304 flagd['command'] = command 00305 flagd['interval'] = interval 00306 flagd['level'] = level 00307 flagd['severity'] = severity 00308 flagd['time'] = coltime 00309 flagd['type'] = coltype 00310 # Insert into main dictionary 00311 myflagd[ncmds] = flagd 00312 ncmds += 1 00313 00314 except: 00315 raise Exception, 'Cannot create dictionary' 00316 00317 casalog.post(':makeDict::myflagd=%s'%myflagd,'DEBUG') 00318 00319 return myflagd 00320 00321 00322 00323 def readXML(sdmfile, mytbuff): 00324 ''' 00325 # readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses 00326 # into returned dictionary as flag command strings 00327 # sdmfile (string) path to SDM containing Antenna.xml and Flag.xml 00328 # mytbuff (float) time interval (start and end) padding (seconds) 00329 # 00330 # Usage: myflags = readflagxml(sdmfile,tbuff) 00331 # 00332 # Dictionary structure: 00333 # fid : 'id' (string) 00334 # 'mode' (string) flag mode ('online') 00335 # 'antenna' (string) 00336 # 'timerange' (string) 00337 # 'reason' (string) 00338 # 'time' (float) in mjd seconds 00339 # 'interval' (float) in mjd seconds 00340 # 'command' (string) string (for COMMAND col in FLAG_CMD) 00341 # 'type' (string) 'FLAG' / 'UNFLAG' 00342 # 'applied' (bool) set to True here on read-in 00343 # 'level' (int) set to 0 here on read-in 00344 # 'severity' (int) set to 0 here on read-in 00345 # 00346 # Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA 00347 # Updated STM 2012-02-14 handle spectral window indices, names, IDs 00348 # Updated STM 2012-02-21 handle polarization types 00349 # 00350 # Mode to use for spectral window selection in commands: 00351 # spwmode = 0 none (flag all spw) 00352 # spwmode = 1 use name 00353 # spwmode = -1 use index (counting rows in SpectralWindow.xml) 00354 # 00355 # Mode to use for polarization selection in commands: 00356 # polmode = 0 none (flag all pols/corrs) 00357 # polmode = 1 use polarization type 00358 # 00359 # CURRENT DEFAULT: Use spw names, flag pols 00360 ''' 00361 00362 spwmode = 1 00363 polmode = 1 00364 00365 # 00366 try: 00367 from xml.dom import minidom 00368 except ImportError, e: 00369 print 'failed to load xml.dom.minidom:\n', e 00370 exit(1) 00371 00372 if type(mytbuff) != float: 00373 casalog.post('Found incorrect type for tbuff','SEVERE') 00374 exit(1) 00375 # mytbuff = 1.0 00376 00377 # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends) 00378 flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK) 00379 antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK) 00380 spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK) 00381 if not flagexist: 00382 casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE') 00383 exit(1) 00384 if not antexist: 00385 casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE' 00386 ) 00387 exit(1) 00388 if not spwexist: 00389 casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 00390 'WARN') 00391 00392 # construct look-up dictionary of name vs. id from Antenna.xml 00393 xmlants = minidom.parse(sdmfile + '/Antenna.xml') 00394 antdict = {} 00395 rowlist = xmlants.getElementsByTagName('row') 00396 for rownode in rowlist: 00397 rowname = rownode.getElementsByTagName('name') 00398 ant = str(rowname[0].childNodes[0].nodeValue) 00399 rowid = rownode.getElementsByTagName('antennaId') 00400 # CAS-4532: remove spaces between content and tags 00401 antid = str(rowid[0].childNodes[0].nodeValue).strip() 00402 antdict[antid] = ant 00403 casalog.post('Found ' + str(rowlist.length) 00404 + ' antennas in Antenna.xml') 00405 00406 # construct look-up dictionary of name vs. id from SpectralWindow.xml 00407 if spwexist: 00408 xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml') 00409 spwdict = {} 00410 rowlist = xmlspws.getElementsByTagName('row') 00411 ispw = 0 00412 for rownode in rowlist: 00413 rowid = rownode.getElementsByTagName('spectralWindowId') 00414 # CAS-4532: remove spaces between content and tags 00415 spwid = str(rowid[0].childNodes[0].nodeValue).strip() 00416 spwdict[spwid] = {} 00417 spwdict[spwid]['index'] = ispw 00418 # SMC: 6/3/2012 ALMA SDM does not have name 00419 if rownode.getElementsByTagName('name'): 00420 rowname = rownode.getElementsByTagName('name') 00421 spw = str(rowname[0].childNodes[0].nodeValue) 00422 spwdict[spwid]['name'] = spw 00423 else: 00424 spwmode = -1 00425 00426 # rowid = rownode.getElementsByTagName('spectralWindowId') 00427 # spwid = str(rowid[0].childNodes[0].nodeValue) 00428 # spwdict[spwid] = {} 00429 # spwdict[spwid]['index'] = ispw 00430 ispw += 1 00431 casalog.post('Found ' + str(rowlist.length) 00432 + ' spw in SpectralWindow.xml') 00433 00434 # report chosen spw and pol modes 00435 if spwmode > 0: 00436 casalog.post('Will construct spw flags using names') 00437 elif spwmode < 0: 00438 casalog.post('Will construct spw flags using table indices') 00439 else: 00440 casalog.post('') 00441 # 00442 if polmode == 0: 00443 casalog.post('Will not set polarization dependent flags (flag all corrs)' 00444 ) 00445 else: 00446 casalog.post('Will construct polarization flags using polarizationType' 00447 ) 00448 00449 # now read Flag.xml into dictionary row by row 00450 xmlflags = minidom.parse(sdmfile + '/Flag.xml') 00451 flagdict = {} 00452 rowlist = xmlflags.getElementsByTagName('row') 00453 nrows = rowlist.length 00454 newsdm = -1 00455 newspw = -1 00456 newpol = -1 00457 for fid in range(nrows): 00458 rownode = rowlist[fid] 00459 rowfid = rownode.getElementsByTagName('flagId') 00460 fidstr = str(rowfid[0].childNodes[0].nodeValue) 00461 flagdict[fid] = {} 00462 flagdict[fid]['id'] = fidstr 00463 rowid = rownode.getElementsByTagName('antennaId') 00464 antid = rowid[0].childNodes[0].nodeValue 00465 # check if there is a numAntenna specified (new format) 00466 rownant = rownode.getElementsByTagName('numAntenna') 00467 antname = '' 00468 if rownant.__len__() > 0: 00469 xid = antid.split() 00470 nant = int(rownant[0].childNodes[0].nodeValue) 00471 if newsdm < 0: 00472 casalog.post('Found numAntenna=' + str(nant) 00473 + ' must be a new style SDM') 00474 newsdm = 1 00475 if nant > 0: 00476 for ia in range(nant): 00477 aid = xid[2 + ia] 00478 ana = antdict[aid] 00479 if antname == '': 00480 antname = ana 00481 else: 00482 antname += ',' + ana 00483 else: 00484 # numAntenna = 0 means flag all antennas 00485 antname = '' 00486 else: 00487 if newsdm < 0: 00488 casalog.post('No numAntenna entry found, must be a old style SDM' 00489 ) 00490 newsdm = 0 00491 nant = 1 00492 aid = antid 00493 ana = antdict[aid] 00494 antname = ana 00495 # start and end times in mjd ns 00496 rowstart = rownode.getElementsByTagName('startTime') 00497 start = int(rowstart[0].childNodes[0].nodeValue) 00498 startmjds = float(start) * 1.0E-9 - mytbuff 00499 t = qa.quantity(startmjds, 's') 00500 starttime = qa.time(t, form='ymd', prec=9)[0] 00501 rowend = rownode.getElementsByTagName('endTime') 00502 end = int(rowend[0].childNodes[0].nodeValue) 00503 endmjds = float(end) * 1.0E-9 + mytbuff 00504 t = qa.quantity(endmjds, 's') 00505 endtime = qa.time(t, form='ymd', prec=9)[0] 00506 # time and interval for FLAG_CMD use 00507 times = 0.5 * (startmjds + endmjds) 00508 intervs = endmjds - startmjds 00509 flagdict[fid]['time'] = times 00510 flagdict[fid]['interval'] = intervs 00511 # reasons 00512 rowreason = rownode.getElementsByTagName('reason') 00513 reas = str(rowreason[0].childNodes[0].nodeValue) 00514 # Replace any white space with underscores 00515 reason = reas.replace(' ','_') 00516 # NEW SDM ADDITIONS 2011-11-01 00517 rownspw = rownode.getElementsByTagName('numSpectralWindow') 00518 spwstring = '' 00519 if spwmode != 0 and rownspw.__len__() > 0: 00520 nspw = int(rownspw[0].childNodes[0].nodeValue) 00521 # has a new-style spw specification 00522 if newspw < 0: 00523 if not spwexist: 00524 casalog.post('Cannot open ' + sdmfile 00525 + '/SpectralWindow.xml', 'SEVERE') 00526 exit(1) 00527 casalog.post('Found SpectralWindow=' + str(nspw) 00528 + ' must be a new style SDM') 00529 newspw = 1 00530 if nspw > 0: 00531 rowspwid = \ 00532 rownode.getElementsByTagName('spectralWindowId') 00533 spwids = rowspwid[0].childNodes[0].nodeValue 00534 xspw = spwids.split() 00535 for isp in range(nspw): 00536 spid = str(xspw[2 + isp]) 00537 if spwmode > 0: 00538 spstr = spwdict[spid]['name'] 00539 else: 00540 spstr = str(spwdict[spid]['index']) 00541 if spwstring == '': 00542 spwstring = spstr 00543 else: 00544 spwstring += ',' + spstr 00545 polstring = '' 00546 rownpol = rownode.getElementsByTagName('numPolarizationType') 00547 if polmode != 0 and rownpol.__len__() > 0: 00548 npol = int(rownpol[0].childNodes[0].nodeValue) 00549 # has a new-style pol specification 00550 if newpol < 0: 00551 casalog.post('Found numPolarizationType=' + str(npol) 00552 + ' must be a new style SDM') 00553 newpol = 1 00554 if npol > 0: 00555 rowpolid = \ 00556 rownode.getElementsByTagName('polarizationType') 00557 polids = rowpolid[0].childNodes[0].nodeValue 00558 xpol = polids.split() 00559 for ipol in range(npol): 00560 polid = str(xpol[2 + ipol]) 00561 if polstring == '': 00562 polstring = polid 00563 else: 00564 polstring += ',' + polid 00565 # 00566 # Construct antenna name and timerange and reason strings 00567 flagdict[fid]['antenna'] = antname 00568 timestr = starttime + '~' + endtime 00569 flagdict[fid]['timerange'] = timestr 00570 flagdict[fid]['reason'] = reason 00571 # Construct command strings (per input flag) 00572 cmd = "antenna='" + antname + "' timerange='" + timestr + "'" 00573 if spwstring != '': 00574 cmd += " spw='" + spwstring + "'" 00575 flagdict[fid]['spw'] = spwstring 00576 # if polstring != '': 00577 # cmd += " poln='" + polstring + "'" 00578 # flagdict[fid]['poln'] = polstring 00579 if polstring != '': 00580 # Write the poln translation in correlation 00581 if polstring.count('R')>0: 00582 if polstring.count('L')>0: 00583 corr = 'RR,RL,LR,LL' 00584 else: 00585 corr = 'RR,RL,LR' 00586 elif polstring.count('L')>0: 00587 corr = 'LL,LR,RL' 00588 elif polstring.count('X')>0: 00589 if polstring.count('Y')>0: 00590 corr = 'XX,XY,YX,YY' 00591 else: 00592 corr = 'XX,XY,YX' 00593 elif polstring.count('Y')>0: 00594 corr = 'YY,YX,XY' 00595 00596 cmd += " correlation='" + corr + "'" 00597 # flagdict[fid]['poln'] = polstring 00598 flagdict[fid]['command'] = cmd 00599 # 00600 flagdict[fid]['type'] = 'FLAG' 00601 flagdict[fid]['applied'] = False 00602 flagdict[fid]['level'] = 0 00603 flagdict[fid]['severity'] = 0 00604 flagdict[fid]['mode'] = 'online' 00605 00606 flags = {} 00607 if rowlist.length > 0: 00608 flags = flagdict 00609 casalog.post('Found ' + str(rowlist.length) 00610 + ' flags in Flag.xml') 00611 else: 00612 casalog.post('No valid flags found in Flag.xml') 00613 00614 # return the dictionary for later use 00615 return flags 00616 00617 00618 def readFlagCmdTable(msfile,myflagrows=[],useapplied=True,myreason='any'): 00619 '''Read flag commands from rows of the FLAG_CMD table of msfile 00620 If useapplied=False then include only rows with APPLIED=False 00621 If myreason is anything other than '', then select on that''' 00622 00623 # 00624 # Return flagcmd structure: 00625 # 00626 # The flagcmd key is the integer row number from FLAG_CMD 00627 # 00628 # Dictionary structure: 00629 # key : 'id' (string) 00630 # 'mode' (string) flag mode '','clip','shadow','quack' 00631 # 'antenna' (string) 00632 # 'timerange' (string) 00633 # 'reason' (string) 00634 # 'time' (float) in mjd seconds 00635 # 'interval' (float) in mjd seconds 00636 # 'cmd' (string) string (for COMMAND col in FLAG_CMD) 00637 # 'type' (string) 'FLAG' / 'UNFLAG' 00638 # 'applied' (bool) set to True here on read-in 00639 # 'level' (int) set to 0 here on read-in 00640 # 'severity' (int) set to 0 here on read-in 00641 00642 # Open and read columns from FLAG_CMD 00643 mstable = msfile + '/FLAG_CMD' 00644 00645 # Note, tb.getcol doesn't allow random row access, read all 00646 00647 try: 00648 tb.open(mstable) 00649 f_time = tb.getcol('TIME') 00650 f_interval = tb.getcol('INTERVAL') 00651 f_type = tb.getcol('TYPE') 00652 f_reas = tb.getcol('REASON') 00653 f_level = tb.getcol('LEVEL') 00654 f_severity = tb.getcol('SEVERITY') 00655 f_applied = tb.getcol('APPLIED') 00656 f_cmd = tb.getcol('COMMAND') 00657 tb.close() 00658 except: 00659 casalog.post('Error reading table ' + mstable, 'ERROR') 00660 raise Exception 00661 00662 nrows = f_time.__len__() 00663 00664 myreaslist = [] 00665 00666 # Parse myreason 00667 if type(myreason) == str: 00668 if myreason != 'any': 00669 myreaslist.append(myreason) 00670 elif type(myreason) == list: 00671 myreaslist = myreason 00672 else: 00673 casalog.post('Cannot read reason; it contains unknown variable types' 00674 , 'ERROR') 00675 return 00676 00677 myflagcmd = {} 00678 if nrows > 0: 00679 nflagd = 0 00680 if myflagrows.__len__() > 0: 00681 rowlist = myflagrows 00682 else: 00683 rowlist = range(nrows) 00684 # Prune rows if needed 00685 if not useapplied: 00686 rowl = [] 00687 for i in rowlist: 00688 if not f_applied[i]: 00689 rowl.append(i) 00690 rowlist = rowl 00691 if myreaslist.__len__() > 0: 00692 rowl = [] 00693 for i in rowlist: 00694 if myreaslist.count(f_reas[i]) > 0: 00695 rowl.append(i) 00696 rowlist = rowl 00697 00698 for i in rowlist: 00699 flagd = {} 00700 cmd = f_cmd[i] 00701 if cmd == '': 00702 casalog.post('Ignoring empty COMMAND string', 'WARN') 00703 continue 00704 00705 # Extract antenna and timerange strings from cmd 00706 antstr = '' 00707 timstr = '' 00708 00709 flagd['id'] = str(i) 00710 flagd['antenna'] = '' 00711 flagd['mode'] = '' 00712 flagd['time'] = f_time[i] 00713 flagd['interval'] = f_interval[i] 00714 flagd['type'] = f_type[i] 00715 flagd['reason'] = f_reas[i] 00716 flagd['level'] = f_level[i] 00717 flagd['severity'] = f_severity[i] 00718 flagd['applied'] = f_applied[i] 00719 00720 # If shadow, remove the addantenna dictionary 00721 if cmd.__contains__('shadow') \ 00722 and cmd.__contains__('addantenna'): 00723 i0 = cmd.rfind('addantenna') 00724 if cmd[i0 + 11] == '{': 00725 # It is a dictionary. Remove it from line 00726 i1 = cmd.rfind('}') 00727 antpar = cmd[i0 + 11:i1 + 1] 00728 temp = cmd[i0:i1 + 1] 00729 newcmd = cmd.replace(temp, '') 00730 antpardict = fh.convertStringToDict(antpar) 00731 flagd['addantenna'] = antpardict 00732 cmd = newcmd 00733 00734 flagd['command'] = cmd 00735 00736 keyvlist = cmd.split() 00737 if keyvlist.__len__() > 0: 00738 for keyv in keyvlist: 00739 try: 00740 (xkey, val) = keyv.split('=') 00741 except: 00742 print 'Error: not key=value pair for ' + keyv 00743 break 00744 xval = val 00745 # strip quotes from value 00746 if xval.count("'") > 0: 00747 xval = xval.strip("'") 00748 if xval.count('"') > 0: 00749 xval = xval.strip('"') 00750 00751 if xkey == 'mode': 00752 flagd['mode'] = xval 00753 elif xkey == 'timerange': 00754 timstr = xval 00755 elif xkey == 'antenna': 00756 flagd['antenna'] = xval 00757 elif xkey == 'id': 00758 flagd['id'] = xval 00759 00760 # STM 2010-12-08 Do not put timerange if not in command 00761 # if timstr=='': 00762 # .... # Construct timerange from time,interval 00763 # .... centertime = f_time[i] 00764 # .... interval = f_interval[i] 00765 # .... startmjds = centertime - 0.5*interval 00766 # .... t = qa.quantity(startmjds,'s') 00767 # .... starttime = qa.time(t,form="ymd",prec=9) 00768 # .... endmjds = centertime + 0.5*interval 00769 # .... t = qa.quantity(endmjds,'s') 00770 # .... endtime = qa.time(t,form="ymd",prec=9) 00771 # .... timstr = starttime+'~'+endtime 00772 flagd['timerange'] = timstr 00773 # Keep original key index, might need this later 00774 myflagcmd[i] = flagd 00775 nflagd += 1 00776 casalog.post('Read ' + str(nflagd) 00777 + ' rows from FLAG_CMD table in ' + msfile) 00778 else: 00779 casalog.post('FLAG_CMD table in %s is empty, no flags extracted' 00780 % msfile, 'WARN') 00781 00782 return myflagcmd 00783 00784 #def getUnion(mslocal, vis, cmddict): 00785 def getUnion(vis, cmddict): 00786 '''Get a dictionary of a union of all selection parameters from a list of lines: 00787 vis --> MS 00788 cmddict --> dictionary of parameters and values (par=val) such as the one 00789 returned by makeDict() 00790 ''' 00791 00792 # Dictionary of parameters to return 00793 dicpars = { 00794 'field':'', 00795 'scan':'', 00796 'antenna':'', 00797 'spw':'', 00798 'timerange':'', 00799 'correlation':'', 00800 'intent':'', 00801 'feed':'', 00802 'array':'', 00803 'uvrange':'', 00804 'observation':'' 00805 } 00806 00807 # Strings for each parameter 00808 scans = '' 00809 fields = '' 00810 ants = '' 00811 times = '' 00812 corrs = '' 00813 ints = '' 00814 feeds = '' 00815 arrays = '' 00816 uvs = '' 00817 spws = '' 00818 obs = '' 00819 00820 nrows = cmddict.keys().__len__() 00821 # nrows = cmdlist.__len__() 00822 00823 # for i in range(nrows): 00824 for k in cmddict.keys(): 00825 # cmdline = cmdlist[i] 00826 cmdline = cmddict[k]['command'] 00827 00828 # Skip if it is a comment line 00829 if cmdline.startswith('#'): 00830 break 00831 00832 # split by white space 00833 keyvlist = cmdline.split() 00834 if keyvlist.__len__() > 0: 00835 00836 # Split by '=' 00837 for keyv in keyvlist: 00838 00839 (xkey,xval) = keyv.split('=') 00840 00841 # Remove quotes 00842 if type(xval) == str: 00843 if xval.count("'") > 0: 00844 xval = xval.strip("'") 00845 if xval.count('"') > 0: 00846 xval = xval.strip('"') 00847 00848 # Check which parameter 00849 if xkey == "scan": 00850 scans += xval + ',' 00851 00852 elif xkey == "field": 00853 fields += xval + ',' 00854 00855 elif xkey == "antenna": 00856 ants += xval + ';' 00857 00858 elif xkey == "timerange": 00859 times += xval + ',' 00860 00861 elif xkey == "correlation": 00862 corrs += xval + ',' 00863 00864 elif xkey == "intent": 00865 ints += xval + ',' 00866 00867 elif xkey == "feed": 00868 feeds += xval + ',' 00869 00870 elif xkey == "array": 00871 arrays += xval + ',' 00872 00873 elif xkey == "uvrange": 00874 uvs += xval + ',' 00875 00876 elif xkey == "spw": 00877 spws += xval + ',' 00878 00879 elif xkey == "observation": 00880 obs += xval + ',' 00881 00882 00883 # Strip out the extra comma at the end 00884 scans = scans.rstrip(',') 00885 fields = fields.rstrip(',') 00886 ants = ants.rstrip(';') 00887 times = times.rstrip(',') 00888 corrs = corrs.rstrip(',') 00889 ints = ints.rstrip(',') 00890 feeds = feeds.rstrip(',') 00891 arrays = arrays.rstrip(',') 00892 uvs = uvs.rstrip(',') 00893 spws = spws.rstrip(',') 00894 obs = obs.rstrip(',') 00895 00896 dicpars['scan'] = scans 00897 dicpars['field'] = fields 00898 # Antennas are handled better within the framework. 00899 dicpars['antenna'] = '' 00900 # Times are handled better within the framework. 00901 dicpars['timerange'] = '' 00902 # Correlations should be handled only by the agents 00903 dicpars['correlation'] = '' 00904 dicpars['intent'] = ints 00905 dicpars['feed'] = feeds 00906 dicpars['array'] = arrays 00907 dicpars['uvrange'] = uvs 00908 dicpars['spw'] = spws 00909 dicpars['observation'] = obs 00910 00911 00912 # Compress the selection list to reduce MSSelection parsing time. 00913 # 'field','spw','antenna' strings in dicpars will be modified in-place. 00914 compressSelectionList(vis,dicpars); 00915 00916 # Real number of input lines 00917 # Get the number of occurrences of each parameter 00918 npars = getNumPar(cmddict) 00919 nlines = nrows - npars['comment'] 00920 00921 # Make the union. 00922 for k,v in npars.iteritems(): 00923 if k != 'comment': 00924 if v < nlines: 00925 dicpars[k] = '' 00926 00927 00928 uniondic = dicpars.copy() 00929 # Remove empty parameters from the dictionary 00930 for k,v in dicpars.iteritems(): 00931 if v == '': 00932 uniondic.pop(k) 00933 00934 return uniondic 00935 00936 00937 def getNumPar(cmddict): 00938 '''Get the number of occurrences of all parameter keys 00939 cmdlist --> list of strings with parameters and values 00940 ''' 00941 00942 # nrows = cmdlist.__len__() 00943 00944 # Dictionary of number of occurrences to return 00945 npars = { 00946 'field':0, 00947 'scan':0, 00948 'antenna':0, 00949 'spw':0, 00950 'timerange':0, 00951 'correlation':0, 00952 'intent':0, 00953 'feed':0, 00954 'array':0, 00955 'uvrange':0, 00956 'comment':0, 00957 'observation':0 00958 } 00959 00960 ci = 0 # count the number of lines with comments (starting with a #) 00961 si = 0 # count the number of lines with scan 00962 fi = 0 # count the number of lines with field 00963 ai = 0 # count the number of lines with antenna 00964 ti = 0 # count the number of lines with timerange 00965 coi = 0 # count the number of lines with correlation 00966 ii = 0 # count the number of lines with intent 00967 fei = 0 # count the number of lines with feed 00968 ari = 0 # count the number of lines with array 00969 ui = 0 # count the number of lines with uvrange 00970 pi = 0 # count the number of lines with spw 00971 oi = 0 # count the number of lines with observation 00972 00973 # for i in range(nrows): 00974 for k in cmddict.keys(): 00975 # cmdline = cmdlist[i] 00976 cmdline = cmddict[k]['command'] 00977 00978 if cmdline.startswith('#'): 00979 ci += 1 00980 npars['comment'] = ci 00981 continue 00982 00983 # split by white space 00984 keyvlist = cmdline.split() 00985 if keyvlist.__len__() > 0: 00986 00987 # Split by '=' 00988 for keyv in keyvlist: 00989 00990 # Skip if it is a comment character # 00991 if keyv.count('#') > 0: 00992 break 00993 00994 (xkey,xval) = keyv.split('=') 00995 00996 # Remove quotes 00997 if type(xval) == str: 00998 if xval.count("'") > 0: 00999 xval = xval.strip("'") 01000 if xval.count('"') > 0: 01001 xval = xval.strip('"') 01002 # Remove blanks 01003 xval=xval.replace(' ',''); 01004 01005 # Check which parameter, if not empty 01006 if xval != "": 01007 if xkey == "scan": 01008 si += 1 01009 npars['scan'] = si 01010 01011 elif xkey == "field": 01012 fi += 1 01013 npars['field'] = fi 01014 01015 elif xkey == "antenna": 01016 ai += 1 01017 npars['antenna'] = ai 01018 01019 elif xkey == "timerange": 01020 ti += 1 01021 npars['timerange'] = ti 01022 01023 elif xkey == "correlation": 01024 coi += 1 01025 npars['correlation'] = coi 01026 01027 elif xkey == "intent": 01028 ii += 1 01029 npars['intent'] = ii 01030 01031 elif xkey == "feed": 01032 fei += 1 01033 npars['feed'] = fei 01034 01035 elif xkey == "array": 01036 ari += 1 01037 npars['array'] = ari 01038 arrays += xval + ',' 01039 01040 elif xkey == "uvrange": 01041 ui += 1 01042 npars['uvrange'] = ui 01043 01044 elif xkey == "spw": 01045 pi += 1 01046 npars['spw'] = pi 01047 01048 elif xkey == "observation": 01049 oi += 1 01050 npars['observation'] = oi 01051 01052 01053 return npars 01054 01055 01056 #def compressSelectionList(mslocal=None, vis='',dicpars={}): 01057 def compressSelectionList(vis='',dicpars={}): 01058 """ 01059 - Find a loose union of data-selection parameters, to reduce the MSSelection parsing load. 01060 - This compressed selection list is only meant to be used with af.selectdata(), because 01061 further selections are handled internally. 01062 - Use MSSelection in its 'onlyparse=True' mode to gather a list of fields, spws, antennas 01063 touched by the selection list. These are the only keys for which MSSelection does not 01064 need to parse the MS, and will cover the expensive cases where complicated antenna 01065 and spw expressions can slow MSSelection down. 01066 """ 01067 from numpy import unique; 01068 01069 # mslocal = casac.ms() 01070 ms.open(vis, nomodify=False) 01071 try: 01072 indices = ms.msseltoindex(vis=vis,field=dicpars['field'], spw=dicpars['spw'],baseline=dicpars['antenna']); 01073 finally: 01074 ms.close() 01075 01076 c_field = str(list(unique(indices['field']))).strip('[]'); 01077 c_spw = str(list(unique(indices['spw']))).strip('[]'); 01078 c_antenna = str(list( unique( list(indices['antenna1']) + list(indices['antenna2']) ) ) ).strip('[]'); 01079 01080 dicpars['field'] = c_field; 01081 dicpars['spw'] = c_spw; 01082 dicpars['antenna'] = c_antenna; 01083 01084 # Programmer note : Other selection parameters that can be compressed accurately 01085 # from MS subtable information alone (no need to parse the main table) are 01086 # 'array', 'observationid', 'state(or intent)'. They are currently un-available 01087 # via ms.msseltoindex() and therefore not used here yet. 01088 01089 return; 01090 01091 01092 01093 def writeFlagCmd(msfile, myflags, vrows, applied, add_reason, outfile): 01094 ''' 01095 Writes the flag commands to FLAG_CMD or to an ASCII file 01096 01097 msfile --> MS 01098 myflags --> dictionary of commands read from inputfile (from readFromTable, etc.) 01099 vrows --> list of valid rows from myflags dictionary to save 01100 applied --> value to update APPLIED column of FLAG_CMD 01101 add_reason --> reason to add to output (replace input reason, if any) 01102 outfile --> if not empty, save to it 01103 Returns the number of commands written to output 01104 ''' 01105 01106 nadd = 0 01107 try: 01108 import pylab as pl 01109 except ImportError, e: 01110 print 'failed to load pylab:\n', e 01111 exit(1) 01112 01113 # append to a file 01114 if outfile != '': 01115 ffout = open(outfile, 'a') 01116 01117 try: 01118 01119 for key in myflags.keys(): 01120 # Remove leading and trailing white spaces 01121 cmdline = myflags[key]['command'].strip() 01122 01123 # Add addantenna parameter back 01124 if myflags[key].__contains__('addantenna'): 01125 addantenna = myflags[key]['addantenna'] 01126 cmdline = cmdline + ' addantenna=' + str(addantenna) 01127 01128 reason = myflags[key]['reason'] 01129 01130 # There is no reason in input 01131 if reason == '': 01132 # Add new reason to output 01133 if add_reason != '': 01134 print >> ffout, '%s reason=\'%s\'' %(cmdline, add_reason) 01135 else: 01136 print >> ffout, '%s' %cmdline 01137 01138 # There is reason in input 01139 else: 01140 # Output reason is empty 01141 if add_reason == '': 01142 print >> ffout, '%s reason=\'%s\'' %(cmdline, reason) 01143 01144 else: 01145 # Replace input reason with new reason 01146 print >> ffout, '%s reason=\'%s\'' %(cmdline, add_reason) 01147 01148 except: 01149 raise Exception, 'Error writing lines to file ' \ 01150 + outfile 01151 ffout.close() 01152 return 01153 01154 # Append new commands to existing table 01155 if vrows.__len__() > 0: 01156 # Extract flags from dictionary into list 01157 tim_list = [] 01158 intv_list = [] 01159 cmd_list = [] 01160 reas_list = [] 01161 typ_list = [] 01162 sev_list = [] 01163 lev_list = [] 01164 app_list = [] 01165 01166 # Only write valid rows that have been applied to MS 01167 01168 for key in vrows: 01169 # do not write line with summary mode 01170 command = myflags[key]['command'] 01171 if command.__contains__('summary'): 01172 continue 01173 01174 # Add addantenna back 01175 if myflags[key].__contains__('addantenna'): 01176 addantenna = myflags[key]['addantenna'] 01177 command = command + ' addantenna=' + str(addantenna) 01178 01179 cmd_list.append(command) 01180 tim_list.append(myflags[key]['time']) 01181 intv_list.append(myflags[key]['interval']) 01182 if add_reason != '': 01183 reas_list.append(add_reason) 01184 else: 01185 reas_list.append(myflags[key]['reason']) 01186 typ_list.append(myflags[key]['type']) 01187 sev_list.append(myflags[key]['severity']) 01188 lev_list.append(myflags[key]['level']) 01189 app_list.append(applied) 01190 01191 01192 # Save to FLAG_CMD table 01193 nadd = cmd_list.__len__() 01194 01195 mstable = msfile + '/FLAG_CMD' 01196 try: 01197 tblocal.open(mstable, nomodify=False) 01198 except: 01199 raise Exception, 'Error opening FLAG_CMD table ' + mstable 01200 nrows = int(tblocal.nrows()) 01201 casalog.post('There are ' + str(nrows) 01202 + ' rows already in FLAG_CMD', 'DEBUG') 01203 # add blank rows 01204 tblocal.addrows(nadd) 01205 # now fill them in 01206 tblocal.putcol('TIME', pl.array(tim_list), startrow=nrows, nrow=nadd) 01207 tblocal.putcol('INTERVAL', pl.array(intv_list), startrow=nrows, 01208 nrow=nadd) 01209 tblocal.putcol('REASON', pl.array(reas_list), startrow=nrows, 01210 nrow=nadd) 01211 tblocal.putcol('COMMAND', pl.array(cmd_list), startrow=nrows, 01212 nrow=nadd) 01213 # Other columns 01214 tblocal.putcol('TYPE', pl.array(typ_list), startrow=nrows, nrow=nadd) 01215 tblocal.putcol('SEVERITY', pl.array(sev_list), startrow=nrows, 01216 nrow=nadd) 01217 tblocal.putcol('LEVEL', pl.array(lev_list), startrow=nrows, 01218 nrow=nadd) 01219 tblocal.putcol('APPLIED', pl.array(app_list), startrow=nrows, 01220 nrow=nadd) 01221 tblocal.close() 01222 01223 casalog.post('Saved ' + str(nadd) + ' rows to FLAG_CMD') 01224 01225 else: 01226 casalog.post('Saved zero rows to FLAG_CMD; no flags found') 01227 01228 return nadd 01229 01230 01231 def getReason(cmdline): 01232 '''Get the reason values from a line with strings 01233 cmdline --> a string with parameters 01234 returns a string with reason values. 01235 ''' 01236 01237 reason = '' 01238 01239 # Skip comment lines 01240 if cmdline.startswith('#'): 01241 return reason 01242 01243 # Split by white space 01244 keyvlist = cmdline.split() 01245 if keyvlist.__len__() > 0: 01246 01247 for keyv in keyvlist: 01248 01249 # Split by '=' 01250 (xkey,xval) = keyv.split('=') 01251 01252 # Remove quotes 01253 if type(xval) == str: 01254 if xval.count("'") > 0: 01255 xval = xval.strip("'") 01256 if xval.count('"') > 0: 01257 xval = xval.strip('"') 01258 01259 # Check if reason is in 01260 if xkey == "reason": 01261 reason = xval 01262 break; 01263 01264 01265 return reason 01266 01267 01268 def getLinePars(cmdline, mlist=[]): 01269 '''Get a dictionary of all selection parameters from a line: 01270 cmdline --> a string with parameters 01271 mlist --> a list of mode's parameters to add to the output dictionary 01272 01273 Returns a dictionary. 01274 ''' 01275 01276 # Dictionary of parameters to return 01277 dicpars = {} 01278 01279 # Skip comment lines 01280 if cmdline.startswith('#'): 01281 return dicpars 01282 01283 # split by white space 01284 keyvlist = cmdline.split() 01285 if keyvlist.__len__() > 0: 01286 01287 # Split by '=' 01288 for keyv in keyvlist: 01289 01290 (xkey,xval) = keyv.split('=') 01291 01292 # Remove quotes 01293 if type(xval) == str: 01294 if xval.count("'") > 0: 01295 xval = xval.strip("'") 01296 if xval.count('"') > 0: 01297 xval = xval.strip('"') 01298 01299 # Check which parameter 01300 if xkey == "scan": 01301 dicpars['scan'] = xval 01302 01303 elif xkey == "field": 01304 dicpars['field'] = xval 01305 01306 elif xkey == "antenna": 01307 dicpars['antenna'] = xval 01308 01309 elif xkey == "timerange": 01310 dicpars['timerange'] = xval 01311 01312 elif xkey == "correlation": 01313 dicpars['correlation'] = xval.upper() 01314 01315 elif xkey == "intent": 01316 dicpars['intent'] = xval 01317 01318 elif xkey == "feed": 01319 dicpars['feed'] = xval 01320 01321 elif xkey == "array": 01322 dicpars['array'] = xval 01323 01324 elif xkey == "uvrange": 01325 dicpars['uvrange'] = xval 01326 01327 elif xkey == "spw": 01328 dicpars['spw'] = xval 01329 01330 elif xkey == "observation": 01331 dicpars['observation'] = xval 01332 01333 elif xkey == "mode": 01334 if xval == 'manualflag': 01335 xval = 'manual' 01336 dicpars['mode'] = xval 01337 01338 elif mlist != []: 01339 # Any parameters requested for this mode? 01340 for m in mlist: 01341 if xkey == m: 01342 dicpars[m] = xval 01343 01344 casalog.post(':getLinePars::dicpars=%s'%dicpars, 'DEBUG') 01345 01346 return dicpars 01347 01348 def getSelectionPars(cmdline): 01349 '''Get a dictionary of all selection parameters from a line: 01350 cmdline --> a string with parameters 01351 ''' 01352 01353 # Dictionary of parameters to return 01354 dicpars = {} 01355 01356 # Skip comment lines 01357 if cmdline.startswith('#'): 01358 return dicpars 01359 01360 # split by white space 01361 keyvlist = cmdline.split() 01362 if keyvlist.__len__() > 0: 01363 01364 # Split by '=' 01365 for keyv in keyvlist: 01366 01367 (xkey,xval) = keyv.split('=') 01368 01369 # Remove quotes 01370 if type(xval) == str: 01371 if xval.count("'") > 0: 01372 xval = xval.strip("'") 01373 if xval.count('"') > 0: 01374 xval = xval.strip('"') 01375 01376 # Check which parameter 01377 if xkey == "scan": 01378 dicpars['scan'] = xval 01379 01380 elif xkey == "field": 01381 dicpars['field'] = xval 01382 01383 elif xkey == "antenna": 01384 dicpars['antenna'] = xval 01385 01386 elif xkey == "timerange": 01387 dicpars['timerange'] = xval 01388 01389 # Correlation will be handled by the agent 01390 elif xkey == "correlation": 01391 dicpars['correlation'] = '' 01392 01393 elif xkey == "intent": 01394 dicpars['intent'] = xval 01395 01396 elif xkey == "feed": 01397 dicpars['feed'] = xval 01398 01399 elif xkey == "array": 01400 dicpars['array'] = xval 01401 01402 elif xkey == "uvrange": 01403 dicpars['uvrange'] = xval 01404 01405 elif xkey == "spw": 01406 dicpars['spw'] = xval 01407 01408 elif xkey == "observation": 01409 dicpars['observation'] = xval 01410 01411 01412 return dicpars 01413 01414 01415 def readNtime(params): 01416 '''Check the value and units of ntime 01417 params --> dictionary of agent's parameters ''' 01418 01419 newtime = 0.0 01420 01421 if params.has_key('ntime'): 01422 ntime = params['ntime'] 01423 01424 # Verify the ntime value 01425 if type(ntime) == float or type(ntime) == int: 01426 if ntime <= 0: 01427 raise Exception, 'Parameter ntime cannot be < = 0' 01428 else: 01429 # units are seconds 01430 newtime = float(ntime) 01431 01432 elif type(ntime) == str: 01433 if ntime == 'scan': 01434 # iteration time step is a scan 01435 newtime = 0.0 01436 else: 01437 # read the units from the string 01438 qtime = qa.quantity(ntime) 01439 01440 if qtime['unit'] == 'min': 01441 # convert to seconds 01442 qtime = qa.convert(qtime, 's') 01443 elif qtime['unit'] == '': 01444 qtime['unit'] = 's' 01445 01446 # check units 01447 if qtime['unit'] == 's': 01448 newtime = qtime['value'] 01449 else: 01450 casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN') 01451 01452 params['ntime'] = float(newtime) 01453 01454 01455 def fixType(params): 01456 '''Give correct types to non-string parameters 01457 The types are defined in the XML file of the task tflagdata''' 01458 01459 # manual parameter 01460 if params.has_key('autocorr'): 01461 params['autocorr'] = eval(params['autocorr'].capitalize()) 01462 01463 # quack parameters 01464 if params.has_key('quackmode') and not params['quackmode'] in ['beg' 01465 , 'endb', 'end', 'tail']: 01466 raise Exception, \ 01467 "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \ 01468 % params['quackmode'] 01469 if params.has_key('quackinterval'): 01470 params['quackinterval'] = float(params['quackinterval']) 01471 if params.has_key('quackincrement'): 01472 if type(params['quackincrement']) == str: 01473 params['quackincrement'] = eval(params['quackincrement'].capitalize()) 01474 01475 # clip parameters 01476 if params.has_key('clipminmax'): 01477 value01 = params['clipminmax'] 01478 # turn string into [min,max] range 01479 value0 = value01.lstrip('[') 01480 value = value0.rstrip(']') 01481 r = value.split(',') 01482 rmin = float(r[0]) 01483 rmax = float(r[1]) 01484 params['clipminmax'] = [rmin, rmax] 01485 if params.has_key('clipoutside'): 01486 if type(params['clipoutside']) == str: 01487 params['clipoutside'] = eval(params['clipoutside'].capitalize()) 01488 else: 01489 params['clipoutside'] = params['clipoutside'] 01490 if params.has_key('channelavg'): 01491 params['channelavg'] = eval(params['channelavg'].capitalize()) 01492 if params.has_key('clipzeros'): 01493 params['clipzeros'] = eval(params['clipzeros'].capitalize()) 01494 01495 01496 # shadow parameter 01497 if params.has_key('tolerance'): 01498 params['tolerance'] = float(params['tolerance']) 01499 01500 # elevation parameters 01501 if params.has_key('lowerlimit'): 01502 params['lowerlimit'] = float(params['lowerlimit']) 01503 if params.has_key('upperlimit'): 01504 params['upperlimit'] = float(params['upperlimit']) 01505 01506 # extend parameters 01507 if params.has_key('extendpols'): 01508 params['extendpols'] = eval(params['extendpols'].capitalize()) 01509 if params.has_key('growtime'): 01510 params['growtime'] = float(params['growtime']) 01511 if params.has_key('growfreq'): 01512 params['growfreq'] = float(params['growfreq']) 01513 if params.has_key('growaround'): 01514 params['growaround'] = eval(params['growaround'].capitalize()) 01515 if params.has_key('flagneartime'): 01516 params['flagneartime'] = eval(params['flagneartime'].capitalize()) 01517 if params.has_key('flagnearfreq'): 01518 params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize()) 01519 01520 # tfcrop parameters 01521 if params.has_key('combinescans'): 01522 params['combinescans'] = eval(params['combinescans'].capitalize()) 01523 if params.has_key('timecutoff'): 01524 params['timecutoff'] = float(params['timecutoff']) 01525 if params.has_key('freqcutoff'): 01526 params['freqcutoff'] = float(params['freqcutoff']) 01527 if params.has_key('maxnpieces'): 01528 params['maxnpieces'] = int(params['maxnpieces']) 01529 if params.has_key('halfwin'): 01530 params['halfwin'] = int(params['halfwin']) 01531 01532 # rflag parameters 01533 if params.has_key('winsize'): 01534 params['winsize'] = int(params['winsize']); 01535 if params.has_key('timedev'): 01536 timepar = params['timedev'] 01537 try: 01538 timepar = eval(timepar) 01539 except Exception: 01540 timepar = readRFlagThresholdFile(params['timedev'],'timedev'); 01541 params['timedev'] = timepar 01542 if params.has_key('freqdev'): 01543 freqpar = params['freqdev'] 01544 try: 01545 freqpar = eval(freqpar) 01546 except Exception: 01547 freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev'); 01548 params['freqdev'] = freqpar 01549 if params.has_key('timedevscale'): 01550 params['timedevscale'] = float(params['timedevscale']); 01551 if params.has_key('freqdevscale'): 01552 params['freqdevscale'] = float(params['freqdevscale']); 01553 if params.has_key('spectralmin'): 01554 params['spectralmin'] = float(params['spectralmin']); 01555 if params.has_key('spectralmax'): 01556 params['spectralmax'] = float(params['spectralmax']); 01557 01558 01559 01560 def purgeEmptyPars(cmdline): 01561 '''Remove empty parameters from a string: 01562 cmdline --> a string with parameters 01563 01564 returns a string containing only parameters with values 01565 ''' 01566 newstr = '' 01567 01568 # split by white space 01569 keyvlist = cmdline.split() 01570 if keyvlist.__len__() > 0: 01571 01572 # Split by '=' 01573 for keyv in keyvlist: 01574 01575 (xkey,xval) = keyv.split('=') 01576 01577 # Remove quotes 01578 if type(xval) == str: 01579 if xval.count("'") > 0: 01580 xval = xval.strip("'") 01581 if xval.count('"') > 0: 01582 xval = xval.strip('"') 01583 01584 # Write only parameters with values 01585 if xval == '': 01586 continue 01587 else: 01588 newstr = newstr+xkey+'='+xval+' ' 01589 01590 else: 01591 casalog.post('String of parameters is empty','WARN') 01592 01593 return newstr 01594 01595 01596 def purgeParameter(cmdline, par): 01597 '''Remove parameter from a string: 01598 cmdline --> a string with a parameter to be removed 01599 par --> the parameter to be removed from the string 01600 01601 returns a string containing the remaining parameters 01602 ''' 01603 01604 newstr = '' 01605 01606 # split by white space 01607 keyvlist = cmdline.split() 01608 if keyvlist.__len__() > 0: 01609 01610 # Split by '=' 01611 for keyv in keyvlist: 01612 01613 (xkey,xval) = keyv.split('=') 01614 01615 # Remove quotes 01616 if type(xval) == str: 01617 if xval.count("'") > 0: 01618 xval = xval.strip("'") 01619 if xval.count('"') > 0: 01620 xval = xval.strip('"') 01621 01622 # Write only parameters with values 01623 if xkey == par: 01624 continue 01625 else: 01626 newstr = newstr+xkey+'=' + xval+ ' ' 01627 01628 else: 01629 casalog.post('String of parameters is empty','WARN') 01630 01631 return newstr 01632 01633 def setupAgent(aflocal, myflagcmd, myrows, apply, writeflags, display=''): 01634 ''' Setup the parameters of each agent and call the agentflagger tool 01635 01636 myflagcmd --> it is a dictionary coming from readFromTable, readFile, etc. 01637 myrows --> selected rows to apply/unapply flags 01638 apply --> it's a boolean to control whether to apply or unapply the flags 01639 writeflags --> used by mode=rflag only 01640 display --> used by mode='rflag only''' 01641 01642 01643 if not myflagcmd.__len__() >0: 01644 casalog.post('There are no flag cmds in list', 'SEVERE') 01645 return 01646 01647 # Parameters for each mode 01648 manualpars = ['autocorr'] 01649 unflagpars = [] 01650 clippars = ['clipminmax', 'clipoutside','datacolumn', 'channelavg', 'clipzeros'] 01651 quackpars = ['quackinterval','quackmode','quackincrement'] 01652 shadowpars = ['tolerance', 'addantenna'] 01653 elevationpars = ['lowerlimit','upperlimit'] 01654 tfcroppars = ['ntime','combinescans','datacolumn','timecutoff','freqcutoff', 01655 'timefit','freqfit','maxnpieces','flagdimension','usewindowstats','halfwin'] 01656 extendpars = ['ntime','combinescans','extendpols','growtime','growfreq','growaround', 01657 'flagneartime','flagnearfreq'] 01658 rflagpars = ['winsize','timedev','freqdev','timedevscale','freqdevscale','spectralmax','spectralmin'] 01659 01660 01661 # dictionary of successful command lines to save to outfile 01662 savelist = {} 01663 01664 # Setup the agent for each input line 01665 for key in myflagcmd.keys(): 01666 cmdline = myflagcmd[key]['command'] 01667 applied = myflagcmd[key]['applied'] 01668 interval = myflagcmd[key]['interval'] 01669 level = myflagcmd[key]['level'] 01670 reason = myflagcmd[key]['reason'] 01671 severity = myflagcmd[key]['severity'] 01672 coltime = myflagcmd[key]['time'] 01673 coltype = myflagcmd[key]['type'] 01674 if debug: 01675 print 'cmdline for key%s'%key 01676 print '%s'%cmdline 01677 print 'applied is %s'%applied 01678 01679 if cmdline.startswith('#'): 01680 continue 01681 01682 modepars = {} 01683 parslist = {} 01684 mode = '' 01685 valid = True 01686 addantenna = {} 01687 01688 # Get the specific parameters for the mode 01689 if cmdline.__contains__('mode'): 01690 if cmdline.__contains__('manual'): 01691 mode = 'manual' 01692 modepars = getLinePars(cmdline,manualpars) 01693 elif cmdline.__contains__('clip'): 01694 mode = 'clip' 01695 modepars = getLinePars(cmdline,clippars) 01696 elif cmdline.__contains__('quack'): 01697 mode = 'quack' 01698 modepars = getLinePars(cmdline,quackpars) 01699 elif cmdline.__contains__('shadow'): 01700 mode = 'shadow' 01701 modepars = getLinePars(cmdline,shadowpars) 01702 01703 # Get addantenna dictionary 01704 if myflagcmd[key].__contains__('addantenna'): 01705 addantenna = myflagcmd[key]['addantenna'] 01706 modepars['addantenna'] = addantenna 01707 else: 01708 # Get antenna filename 01709 if (modepars.__contains__('addantenna')): 01710 ant_par = modepars['addantenna'] 01711 01712 # It must be a string 01713 if (type(ant_par) == str and ant_par != ''): 01714 antennafile = modepars['addantenna'] 01715 addantenna = readAntennaList(antennafile) 01716 modepars['addantenna'] = addantenna 01717 01718 elif cmdline.__contains__('elevation'): 01719 mode = 'elevation' 01720 modepars = getLinePars(cmdline,elevationpars) 01721 elif cmdline.__contains__('tfcrop'): 01722 mode = 'tfcrop' 01723 modepars = getLinePars(cmdline,tfcroppars) 01724 elif cmdline.__contains__('extend'): 01725 mode = 'extend' 01726 modepars = getLinePars(cmdline,extendpars) 01727 elif cmdline.__contains__('unflag'): 01728 mode = 'unflag' 01729 modepars = getLinePars(cmdline,unflagpars) 01730 elif cmdline.__contains__('rflag'): 01731 mode = 'rflag' 01732 modepars = getLinePars(cmdline,rflagpars) 01733 01734 ##### 01735 ### According to 'shadow' file handling, this code should be here... 01736 ### but, it's already done inside fixType. 01737 ##### 01738 #if( type(modepars['timedev']) == str and writeflags == True): 01739 # timedev = readRFlagThresholdFile(modepars['timedev'],'timedev') 01740 # modepars['timedev'] = timedev 01741 #if( type(modepars['freqdev']) == str and writeflags == True): 01742 # freqdev = readRFlagThresholdFile(modepars['freqdev'],'freqdev') 01743 # modepars['freqdev'] = freqdev 01744 01745 # Add the writeflags and display parameters 01746 modepars['writeflags'] = writeflags 01747 modepars['display'] = display 01748 else: 01749 # Unknown mode, ignore it 01750 casalog.post('Ignoring unknown mode', 'WARN') 01751 valid = False 01752 01753 else: 01754 # No mode means manual 01755 mode = 'manual' 01756 cmdline = cmdline+' mode=manual' 01757 modepars = getLinePars(cmdline,manualpars) 01758 01759 01760 # Read ntime 01761 readNtime(modepars) 01762 01763 # Cast the correct type to non-string parameters 01764 fixType(modepars) 01765 01766 # Add the apply/unapply parameter to dictionary 01767 modepars['apply'] = apply 01768 01769 # Unapply selected rows only and re-apply the other rows with APPLIED=True 01770 if not apply and myrows.__len__() > 0: 01771 if key in myrows: 01772 modepars['apply'] = False 01773 elif not applied: 01774 casalog.post("Skipping this %s"%modepars,"DEBUG") 01775 continue 01776 elif applied: 01777 modepars['apply'] = True 01778 valid = False 01779 01780 # Keep only cmds that overlap with the unapply cmds 01781 # TODO later 01782 01783 # Hold the name of the agent and the cmd row number 01784 agent_name = mode.capitalize()+'_'+str(key) 01785 modepars['name'] = agent_name 01786 01787 # Remove the data selection parameters if there is only one agent for performance reasons. 01788 # Explanation: if only one agent exists and the data selection parameters are parsed to it, 01789 # it will have to go through the entire MS and check if each data selection given to the agent 01790 # matches what the user asked in the selected data. 01791 01792 # Only correlation, antenna and timerange will go to the agent 01793 # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too 01794 if myflagcmd.__len__() == 1: 01795 sellist=['scan','field','intent','feed','array','uvrange','observation'] 01796 for k in sellist: 01797 if modepars.has_key(k): 01798 modepars.pop(k) 01799 01800 casalog.post('Parsing parameters of mode %s in row %s'%(mode,key), 'DEBUG') 01801 casalog.post('%s'%modepars, 'DEBUG') 01802 if debug: 01803 print 'Parsing parameters of mode %s in row %s'%(mode,key) 01804 print modepars 01805 01806 # Parse the dictionary of parameters to the tool 01807 if (not aflocal.parseagentparameters(modepars)): 01808 casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,key), 'WARN') 01809 continue 01810 01811 # Save the dictionary of valid agents 01812 if valid: 01813 # add this command line to list to save in outfile 01814 parslist['row'] = key 01815 parslist['command'] = cmdline 01816 parslist['applied'] = applied 01817 parslist['interval'] = interval 01818 parslist['level'] = level 01819 parslist['reason'] = reason 01820 parslist['severity'] = severity 01821 parslist['time'] = coltime 01822 parslist['type'] = coltype 01823 if addantenna != {}: 01824 parslist['addantenna'] = addantenna 01825 savelist[key] = parslist 01826 01827 if debug: 01828 casalog.post('Dictionary of valid commands to save') 01829 casalog.post('%s'%savelist) 01830 01831 return savelist 01832 01833 01834 def backupFlags(aflocal=None, msfile='', prename='flagbackup'): 01835 '''Create a backup of the FLAG column 01836 01837 aflocal local version of the agentflagger tool or 01838 msfile name of MS/cal table to backup 01839 prename prefix for name of flag backup file 01840 01841 If msfile is given, aflocal will not be used 01842 01843 Create names like this: 01844 flags.flagcmd_1, 01845 flags.flagdata_1, 01846 01847 Generally <task>_<i>, where i is the smallest 01848 integer giving a name, which does not already exist''' 01849 01850 if msfile != '': 01851 # open msfile and attach it to tool 01852 aflocal = aftool() 01853 aflocal.open(msfile) 01854 01855 elif aflocal == None: 01856 casalog.post('Need an MS or a copy of the agentflagger tool to create a backup','WARN') 01857 return 01858 01859 prefix = prename 01860 try: 01861 existing = aflocal.getflagversionlist(printflags=False) 01862 01863 # remove comments from strings 01864 existing = [x[0:x.find(' : ')] for x in existing] 01865 i = 1 01866 while True: 01867 versionname = prefix + '_' + str(i) 01868 01869 if not versionname in existing: 01870 break 01871 else: 01872 i = i + 1 01873 01874 time_string = str(time.strftime('%Y-%m-%d %H:%M:%S')) 01875 01876 casalog.post('Saving current flags to ' + versionname, 'DEBUG') 01877 01878 aflocal.saveflagversion(versionname=versionname, 01879 comment='Flags autosave on ' + time_string, merge='replace') 01880 finally: 01881 if msfile != '': 01882 aflocal.done() 01883 01884 return 01885 01886 01887 #### 01888 #### Set of functions to handle antenna information for shadowing. 01889 #### 01890 #### - extractAntennaInfo : Extract info into a returned dictionary (and text file) 01891 #### - antListWrite : Write the dictionary to a text file 01892 #### - antListRead : Read the text file and return a dictionary 01893 #### 01894 #### Example : 01895 #### alist = extractAntennaInfo(msname='../Data/shadowtest.ms', 01896 #### antnamelist=['VLA1','VLA15'], outfile='ttt.txt') 01897 #### 01898 #### alist = antListRead('ttt.txt'); 01899 #### 01900 ################################### 01901 #### Example output text file ( 'ttt.txt' ) 01902 ################################### 01903 # 01904 #name = VLA1 01905 #diameter = 25.0 01906 #position = [-1601144.961466915, -5041998.0197185818, 3554864.76811967] 01907 #name = VLA15 01908 #diameter = 25.0 01909 #position = [-1601556.245351332, -5041990.7252590274, 3554684.6464035073] 01910 # 01911 ################################### 01912 #### Example output dictionary ( alist ) 01913 ################################### 01914 # 01915 #{'0': {'diameter ': 25.0, 01916 # 'name ': 'VLA1', 01917 # 'position ': [-1601144.961466915, 01918 # -5041998.0197185818, 01919 # 3554864.7681196001]}, 01920 # '1': {'diameter ': 25.0, 01921 # 'name ': 'VLA15', 01922 # 'position ': [-1601556.245351332, 01923 # -5041990.7252590274, 01924 # 3554684.6464035069]}} 01925 # 01926 ################################## 01927 01928 def extractAntennaInfo(msname='', antnamelist=[], outfile=''): 01929 """ 01930 Function to extract antenna names, positions, and diameters 01931 for a specified subset of the ANTENNA subtable of the specified MS. 01932 - It writes a text file, which can be sent as input for shadow in the task 01933 - It also returns a dictionary, which can be sent as input for shadow in the tool. 01934 This dictionary can also be given as input in the task. 01935 01936 msname : name of MS 01937 antennalist : list of strings (antenna names). Names must match exactly. Case insensitive. 01938 outfile : name of text file. Will be overwritten if exists. If outfile='', no output file is written 01939 01940 Always returns a dictionary containing the same info as in the file 01941 01942 Example : 01943 antinfo = extractAntennaInfo(msname='xxx.ms',antnamelist=['vla1','vla2'],outfile='out.txt'); 01944 """ 01945 ## Check that the MS exists 01946 if(not os.path.exists(msname)): 01947 print "Cannot find MS : ", msname; 01948 return False; 01949 01950 ## If outfile exists, delete it 01951 if(os.path.exists(outfile)): 01952 print "Replacing existing file : ", outfile; 01953 rmcmd = "rm -rf "+outfile; 01954 os.system(rmcmd); 01955 01956 ## Convert input antenna names to upper-case 01957 newants=[]; 01958 for ants in antnamelist: 01959 newants.append( ants.upper() ); 01960 antnamelist = newants; 01961 01962 ## Read antenna subtable of input MS 01963 tblocal.open(msname+'/ANTENNA'); 01964 a_position = (tblocal.getcol('POSITION')).transpose(); 01965 a_dish_diameter = tblocal.getcol('DISH_DIAMETER'); 01966 a_name = tblocal.getcol('NAME'); 01967 tblocal.close(); 01968 01969 ## Pick out only selected antennas from this list, and make a dictionary 01970 antlist = {}; 01971 counter=0; 01972 for antid in range(0, len(a_name)): 01973 if (a_name[antid]).upper() in antnamelist: 01974 antlist[str(counter)] = { 'name':a_name[antid] , 'position': list(a_position[antid]) , 'diameter': a_dish_diameter[antid] } ; 01975 counter=counter+1; 01976 01977 ## Open a new file and write this info into it, if requested 01978 if(outfile != ''): 01979 print "Making new file : ", outfile; 01980 writeAntennaList(outfile, antlist); 01981 ## always return the dictionary anyway. 01982 return antlist; 01983 01984 ############################################## 01985 def writeAntennaList(outfile='', antlist={}): 01986 """ 01987 Save the antlist dictionary as a text file 01988 """ 01989 ofile = file(outfile, 'w'); 01990 for apid in sorted(antlist): 01991 apars = antlist[apid]; 01992 ofile.write("name=" + str(apars['name']) + '\n'); 01993 ofile.write("diameter=" + str(apars['diameter'])+'\n'); 01994 ofile.write("position=" + str((apars['position']))+'\n'); 01995 ofile.close(); 01996 01997 ############################################## 01998 def readAntennaList(infile=''): 01999 """ 02000 Read the antlist text file and return a dictionary 02001 02002 A return value of empty {} indicates an error (or, empty file). 02003 02004 The file needs to have 3 entries per antenna, on separate lines. 02005 The diameter and position are in units of meters, with positions in ITRF. 02006 Multiple antennas can be specified by repeating these three lines. 02007 Blank lines are allowed. 02008 Lines can be commented with '#' as the first character. 02009 02010 Example : 02011 name = ea05 02012 diameter = 25.0 02013 position = [-1601144.96146691, -5041998.01971858, 3554864.76811967] 02014 02015 """ 02016 02017 if (type(infile) == str) & os.path.exists(infile): 02018 try: 02019 ifile = file(infile,'r'); 02020 except: 02021 raise Exception, 'Error opening file ' + infile 02022 02023 thelist = ifile.readlines(); 02024 ifile.close(); 02025 else: 02026 raise Exception, \ 02027 'File %s not found - please verify the name'%infile 02028 02029 cleanlist=[]; 02030 for aline in thelist: 02031 if(len(aline)>5 and aline[0] != '#'): 02032 cleanlist.append(aline.rstrip()); 02033 02034 #print 'Found ' + str(len(cleanlist)) + ' valid lines out of ' + str(len(thelist)); 02035 02036 if( len(cleanlist) > 0 and len(cleanlist) % 3 != 0 ): 02037 print "\nThe file needs to have 3 entries per antenna, on separate lines. For example :" 02038 print "name=ea05" 02039 print "diameter=25.0"; 02040 print "position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]"; 02041 print "\n"; 02042 print "The diameter and position are in units of meters, with positions in ITRF"; 02043 return False; 02044 02045 antlist={}; 02046 counter=0; 02047 for aline in range(0,len(cleanlist),3): 02048 antdict = {}; 02049 for row in range(0,3): 02050 pars = cleanlist[aline+row].split("="); 02051 #print aline, row, pars 02052 if(len(pars) != 2): 02053 print 'Error in parsing : ', cleanlist[aline+row]; 02054 return {}; 02055 else: 02056 if(pars[0].count('name') > 0 ): 02057 antdict[pars[0].rstrip()] = str(pars[1].rsplit()[0]); 02058 if(pars[0].count('diameter') > 0 ): 02059 antdict[pars[0].rstrip()] = float(pars[1]); 02060 if(pars[0].count('position') > 0 ): 02061 plist = pars[1][1:-2].replace('[','').split(','); 02062 if(len(plist) != 3): 02063 print 'Error in parsing : ', cleanlist[aline+row] 02064 return {}; 02065 else: 02066 qlist=[]; 02067 for ind in range(0,3): 02068 qlist.append(float(plist[ind])); 02069 antdict[pars[0].rstrip()] = qlist; 02070 antlist[str(counter)] = antdict; 02071 counter = counter+1; 02072 02073 return antlist; 02074 02075 ################################################ 02076 # 02077 # Function to pull out RFLAG thresholds from the returned report dictionary. 02078 # 02079 def writeRFlagThresholdFile(rflag_thresholds={},timedevfile='', freqdevfile='',agent_id=0): 02080 """ 02081 Extract the RFLAG output thresholds from the threshold dictionary 02082 Return them as arrays, and optionally, write them into a file. 02083 """ 02084 # Decide the output file name. 02085 if( type(timedevfile) == str and timedevfile != '' and timedevfile.count('[')==0 and (not timedevfile.replace('.','').isdigit() ) ): 02086 toutfile = timedevfile 02087 else: 02088 toutfile = 'rflag_output_thresholds_timedev'+str(agent_id)+'.txt' 02089 02090 # Decide the output file name. 02091 if( type(freqdevfile) == str and freqdevfile != '' and freqdevfile.count('[')==0 and (not freqdevfile.replace('.','').isdigit() ) ): 02092 foutfile = freqdevfile 02093 else: 02094 foutfile = 'rflag_output_thresholds_freqdev'+str(agent_id)+'.txt' 02095 02096 # save rflag output in file, and print them everywhere. 02097 casalog.post("Saving RFlag_"+str(agent_id)+" output in : " + toutfile + " and " + foutfile, 'INFO') 02098 02099 ofiletime = file(toutfile, 'w'); 02100 ofilefreq = file(foutfile, 'w'); 02101 # Construct dictionary from what needs to be stored. 02102 timedict = {'name':rflag_thresholds['name'] , 'timedev': (rflag_thresholds['timedev']).tolist()} 02103 freqdict = {'name':rflag_thresholds['name'] , 'freqdev': (rflag_thresholds['freqdev']).tolist()} 02104 timestr = convertDictToString(timedict) 02105 freqstr = convertDictToString(freqdict) 02106 # Write to file 02107 ofiletime.write(timestr + '\n'); 02108 ofilefreq.write(freqstr + '\n'); 02109 # Send to logger 02110 casalog.post("RFlag_"+str(agent_id)+" output timedev written to " + toutfile + " : " + timestr, 'INFO'); 02111 casalog.post("RFlag_"+str(agent_id)+" output freqdev written to " + foutfile + " : " + freqstr, 'INFO'); 02112 # End filed 02113 ofiletime.write('\n'); 02114 ofiletime.close(); 02115 ofilefreq.write('\n'); 02116 ofilefreq.close(); 02117 # Returne timedev, freqdev contents. This is for savepars later. 02118 return timedict['timedev'], freqdict['freqdev'] 02119 02120 ############################################## 02121 def readRFlagThresholdFile(infile='',inkey=''): 02122 """ 02123 Read the input RFlag threshold file, and return dictionaries. 02124 """ 02125 if(infile==''): 02126 return []; 02127 02128 if ( not os.path.exists(infile) ): 02129 print 'Cannot find file : ', infile 02130 return []; 02131 02132 ifile = file(infile,'r'); 02133 thelist = ifile.readlines(); 02134 ifile.close(); 02135 02136 cleanlist=[]; 02137 for aline in thelist: 02138 if(len(aline)>5 and aline[0] != '#'): 02139 cleanlist.append(aline.rstrip().rstrip('\n')) 02140 02141 threshlist={}; 02142 for aline in range(0,len(cleanlist)): 02143 threshlist[str(aline)] = convertStringToDict(cleanlist[aline]); 02144 if threshlist[str(aline)].has_key(inkey): 02145 devthreshold = threshlist[str(aline)][inkey] 02146 02147 # return only the last one. There should be only one anyway. 02148 return devthreshold 02149 02150 ############################################## 02151 ## Note - replace all arrays by lists before coming here. 02152 def convertDictToString(indict={}): 02153 '''Convert dictionary to string''' 02154 thestr = str(indict); 02155 # Remove newlines and spaces from this string. 02156 thestr = thestr.replace('\n',''); 02157 thestr = thestr.replace(' ',''); 02158 return thestr; 02159 ############################################## 02160 def convertStringToDict(instr=''): 02161 '''Convert string to dictionary''' 02162 instr = instr.replace('\n',''); 02163 try: 02164 thedict = ast.literal_eval(instr) 02165 except Exception, instance: 02166 casalog.post("*** Error converting string %s to dictionary : \'%s\'" % (instr,instance),'ERROR'); 02167 return thedict; 02168 ############################################## 02169 02170 def extractRFlagOutputFromSummary(mode,summary_stats_list, flagcmd): 02171 """ 02172 Function to pull out 'rflag' output from the long dictionary, and 02173 (1) write the output files with thresholds. If the filename is specified, use it. 02174 If filename is not specified, make one up. 02175 (2) modify entries in 'cmdline' so that it is ready for savepars. 02176 This is to ensure that 'savepars' saves the contents of the threshold-files 02177 and not just the file-names. It has to save it in the form that tflagdata 02178 accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user 02179 need not keep track of threshold text files if they use 'savepars' with action='apply'. 02180 """ 02181 if type(summary_stats_list) is dict: 02182 nreps = summary_stats_list['nreport'] 02183 for rep in range(0,nreps): 02184 repname = 'report'+str(rep) 02185 if summary_stats_list[repname]['type'] == "rflag": 02186 # Pull out the rflag threshold dictionary. This has a 'name' in it. 02187 rflag_thresholds = summary_stats_list[repname] 02188 ##print rflag_thresholds 02189 # Get the rflag id, to later construct a 'name' from to match the above. 02190 rflagid = 0 02191 if mode=='list': 02192 rflagid = int( rflag_thresholds['name'].replace('Rflag_','') ) 02193 # Go through the flagcmd list, to find the 'rflags'..... 02194 for key in flagcmd.keys(): 02195 cmdline = flagcmd[key]['command']; 02196 if cmdline.__contains__('rflag'): 02197 # Check for match between input flagcmd and output threshold, via the rflag id 02198 if(key==rflagid): 02199 # If timedev,freqdev are missing from cmdline, add empty ones. 02200 if( not cmdline.__contains__('timedev=') ): # aah. don't confuse it with timedevscale 02201 cmdline = cmdline + " timedev=[] "; 02202 if( not cmdline.__contains__('freqdev=') ): 02203 cmdline = cmdline + " freqdev=[] "; 02204 # Pull out timedev, freqdev strings from flagcmd 02205 rflagpars = getLinePars(cmdline , ['timedev','freqdev','writeflags']); 02206 ##print "cmdline : ", cmdline 02207 ##print "rflagpars : ", rflagpars 02208 # Write RFlag thresholds to these file names. 02209 newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, rflagpars['timedev'], rflagpars['freqdev'], rflagid) 02210 ## Modify the flagcmd string, so that savepars sees the contents of the file 02211 if( rflagpars['timedev'].__contains__('[') ): 02212 oldstring = 'timedev='+str(rflagpars['timedev']) 02213 newstring = 'timedev='+str(newtimedev).replace(' ','') 02214 ##print "time : replacing " , oldstring , newstring 02215 cmdline = cmdline.replace( oldstring, newstring ); 02216 if( rflagpars['freqdev'].__contains__('[') ): 02217 oldstring = 'freqdev='+str(rflagpars['freqdev']) 02218 newstring = 'freqdev='+str(newfreqdev).replace(' ','') 02219 ##print "freq : replacing " , oldstring , newstring 02220 cmdline = cmdline.replace( oldstring, newstring ); 02221 # Remove writeflags from the cmd to prevent it from going into savepars 02222 oldstring = 'writeflags='+str(rflagpars['writeflags']) 02223 cmdline = cmdline.replace( oldstring, "" ); 02224 02225 flagcmd[key]['command'] = cmdline; 02226 02227 02228 02229 ##############################################
1.8.0
