|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 # 00002 # This file was generated using xslt from its XML file 00003 # 00004 # Copyright 2008, Associated Universities Inc., Washington DC 00005 # 00006 import sys 00007 import os 00008 from casac import * 00009 import string 00010 import time 00011 import inspect 00012 import gc 00013 import numpy 00014 from odict import odict 00015 from task_partition import partition 00016 from task_partition import casalog 00017 00018 class partition_pg_: 00019 __name__ = "partition" 00020 00021 def __init__(self) : 00022 self.__bases__ = (partition_pg_,) 00023 self.__doc__ = self.__call__.__doc__ 00024 00025 00026 def __call__(self, vis=None, outputvis=None, createmms=None, separationaxis=None, numsubms=None, datacolumn=None, calmsselection=None, calmsname=None, calfield=None, calscan=None, calintent=None, field=None, spw=None, antenna=None, timebin=None, combine=None, timerange=None, scan=None, scanintent=None, array=None, uvrange=None, observation=None, async=None): 00027 00028 """Experimental task to produce multi-MSs using parallelism 00029 Experimental version for use with parallelization framework. 00030 Do not use this for standard analysis... yet. 00031 00032 Partition is a task to create a multi-MS out of an MS. General selection 00033 parameters are included, and one or all of the various data columns 00034 (DATA, LAG_DATA and/or FLOAT_DATA, and possibly MODEL_DATA and/or 00035 CORRECTED_DATA) can be selected. It can also be used to create a normal 00036 MS, split based on the given data selection parameters. 00037 00038 The partition task creates a multi-MS using the parallelization framework. 00039 If a cluster is not present, it will create a default cluster based on the 00040 resources of the system. One can create a simple_cluster prior to running 00041 partition by doing the following. 00042 00043 from simple_cluster import * 00044 sc = simple_cluster() 00045 sc.init_cluster('cluster-config.txt', 'test') 00046 00047 The file 'cluster-config.txt' contains information on the machine that will 00048 be used for the cluster. Please see the help of simple_cluster for more information. 00049 00050 A multi-MS is structured to have a reference MS on the top directory and a 00051 sub-directory called SUBMSS, which contain each partitioned sub-MS. The 00052 reference MS contains links to the sub-tables of the first sub-MS. The other 00053 sub-MSs contain a copy of the sub-tables each. A multi-MS looks like this in disk. 00054 00055 ls ngc5921.mms 00056 ANTENNA FLAG_CMD POLARIZATION SPECTRAL_WINDOW table.dat 00057 DATA_DESCRIPTION HISTORY PROCESSOR STATE table.info 00058 FEED OBSERVATION SORTED_TABLE SUBMSS WEATHER 00059 FIELD POINTING SOURCE SYSCAL 00060 00061 ls ngc5921.mms/SUBMSS/ 00062 ngc5921.0000.ms/ ngc5921t.0002.ms/ ngc5921.0004.ms/ ngc5921.0006.ms/ 00063 ngc5921.0001.ms/ ngc5921.0003.ms/ ngc5921.0005.ms/ 00064 00065 Inside casapy, one can use the task listpartition to list the information 00066 from a multi-MS. 00067 00068 00069 Keyword arguments: 00070 vis -- Name of input visibility file 00071 default: none; example: vis='ngc5921.ms' 00072 00073 outputvis -- Name of output visibility file 00074 default: none; example: outputvis='ngc5921.mms' 00075 00076 createmms -- Create a multi-MS as the output. 00077 default: True 00078 If False, it will work like the split task and create a 00079 normal MS, split according to the given data selection parameters. 00080 Note that, when this parameter is set to False, a simple_cluster 00081 will not be created. 00082 00083 separationaxis -- Axis to do parallelization across. 00084 default: 'scan' 00085 Options: 'scan', 'spw', 'both' 00086 The 'both' option will try to partition in both scan and spw axes. 00087 00088 numsubms -- The number of sub-MSs to create. 00089 default: 64 00090 00091 00092 datacolumn -- Which data column to use when partitioning. 00093 default='data'; example: datacolumn='data' 00094 Options: 'data', 'model', 'corrected', 'all', 00095 'float_data', 'lag_data', 'float_data,data', and 00096 'lag_data,data'. 00097 N.B.: 'all' = whichever of the above that are present. 00098 Otherwise the selected column will go to DATA (or 00099 FLOAT_DATA) in the output. 00100 00101 calmsselection -- Method by which the calibration scans will be identified 00102 when creating a separate calibration MS. 00103 default:'none' 00104 Options: 'none', 'auto', 'manual'. 00105 00106 'auto' -- 00107 calmsname -- Name of output measurement set. 00108 default = ' ' 00109 00110 'manual' 00111 calmsname -- Name of output measurement set. 00112 default: '' 00113 calfield -- Field selection for calibration MS. 00114 default: '' 00115 calscans -- Scan selection for calibration MS. 00116 default: '' 00117 calintent -- Scan intent selection for calibration MS. 00118 default: '' 00119 00120 --- Data selection parameters (see help par.selectdata for more detailed 00121 information) 00122 00123 field -- Select field using field id(s) or field name(s). 00124 [run listobs to obtain the list iof d's or names] 00125 default: ''=all fields If field string is a non-negative 00126 integer, it is assumed to be a field index 00127 otherwise, it is assumed to be a field name 00128 field='0~2'; field ids 0,1,2 00129 field='0,4,5~7'; field ids 0,4,5,6,7 00130 field='3C286,3C295'; fields named 3C286 and 3C295 00131 field = '3,4C*'; field id 3, all names starting with 4C 00132 00133 spw -- Select spectral window/channels 00134 default: ''=all spectral windows and channels 00135 spw='0~2,4'; spectral windows 0,1,2,4 (all channels) 00136 spw='<2'; spectral windows less than 2 (i.e. 0,1) 00137 spw='0:5~61'; spw 0, channels 5 to 61 00138 spw='0,10,3:3~45'; spw 0,10 all channels, spw 3 - chans 3 to 45. 00139 spw='0~2:2~6'; spw 0,1,2 with channels 2 through 6 in each. 00140 spw = '*:3~64' channels 3 through 64 for all sp id's 00141 spw = ' :3~64' will NOT work. 00142 spw = '*:0;60~63' channel 0 and channels 60 to 63 for all IFs 00143 ';' needed to separate different channel ranges in one spw 00144 spw='0:0~10;15~60'; spectral window 0 with channels 0-10,15-60 00145 spw='0:0~10,1:20~30,2:1;2;4'; spw 0, channels 0-10, 00146 spw 1, channels 20-30, and spw 2, channels, 1, 2 and 4 00147 00148 antenna -- Select data based on antenna/baseline 00149 default: '' (all) 00150 Non-negative integers are assumed to be antenna indices, and 00151 anything else is taken as an antenna name. 00152 00153 Examples: 00154 antenna='5&6': baseline between antenna index 5 and index 6. 00155 antenna='VA05&VA06': baseline between VLA antenna 5 and 6. 00156 antenna='5&6;7&8': baselines 5-6 and 7-8 00157 antenna='5': all baselines with antenna 5 00158 antenna='5,6,10': all baselines including antennas 5, 6, or 10 00159 antenna='5,6,10&': all baselines with *only* antennas 5, 6, or 00160 10. (cross-correlations only. Use && 00161 to include autocorrelations, and &&& 00162 to get only autocorrelations.) 00163 antenna='!ea03,ea12,ea17': all baselines except those that 00164 include EVLA antennas ea03, ea12, or 00165 ea17. 00166 00167 timebin -- Interval width for time averaging. 00168 default: '0s' or '-1s' (no averaging) 00169 Example: timebin='30s' 00170 '10' means '10s' 00171 00172 combine -- Let time bins span changes in scan and/or state. 00173 default = '' (separate time bins by both of the above) 00174 combine = 'scan': Can be useful when the scan number 00175 goes up with each integration, 00176 as in many WSRT MSes. 00177 combine = ['scan', 'state']: disregard scan and state 00178 numbers when time averaging. 00179 combine = 'state,scan': Same as above. 00180 00181 timerange -- Select data based on time range: 00182 default = '' (all); examples, 00183 timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' 00184 Note: if YYYY/MM/DD is missing date, timerange defaults to the 00185 first day in the dataset 00186 timerange='09:14:0~09:54:0' picks 40 min on first day 00187 timerange='25:00:00~27:30:00' picks 1 hr to 3 hr 30min 00188 on next day 00189 timerange='09:44:00' data within one integration of time 00190 timerange='>10:24:00' data after this time 00191 00192 array -- (Sub)array number range 00193 default: ''=all 00194 00195 uvrange -- Select data within uvrange (default units meters) 00196 default: ''=all; example: 00197 uvrange='0~1000klambda'; uvrange from 0-1000 kilo-lambda 00198 uvrange='>4klambda';uvranges greater than 4 kilo-lambda 00199 uvrange='0~1000km'; uvrange in kilometers 00200 00201 scan -- Scan number range 00202 default: ''=all 00203 00204 observation -- Select by observation ID(s) 00205 default: ''=all 00206 00207 00208 00209 """ 00210 a=inspect.stack() 00211 stacklevel=0 00212 for k in range(len(a)): 00213 if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 00214 stacklevel=k 00215 break 00216 myf=sys._getframe(stacklevel).f_globals 00217 myf['__last_task'] = 'partition' 00218 myf['taskname'] = 'partition' 00219 ### 00220 myf['update_params'](func=myf['taskname'],printtext=False) 00221 ### 00222 ### 00223 #Handle globals or user over-ride of arguments 00224 # 00225 function_signature_defaults=dict(zip(self.__call__.func_code.co_varnames,self.__call__.func_defaults)) 00226 useLocalDefaults = False 00227 00228 for item in function_signature_defaults.iteritems(): 00229 key,val = item 00230 keyVal = eval(key) 00231 if (keyVal == None): 00232 #user hasn't set it - use global/default 00233 pass 00234 else: 00235 #user has set it - use over-ride 00236 if (key != 'self') : 00237 useLocalDefaults = True 00238 #myf[key]=keyVal 00239 00240 myparams = {} 00241 if useLocalDefaults : 00242 for item in function_signature_defaults.iteritems(): 00243 key,val = item 00244 keyVal = eval(key) 00245 exec('myparams[key] = keyVal') 00246 if (keyVal == None): 00247 exec('myparams[key] = '+ key + ' = self.itsdefault(key)') 00248 keyVal = eval(key) 00249 if(type(keyVal) == dict) : 00250 exec('myparams[key] = ' + key + ' = keyVal[len(keyVal)-1][\'value\']') 00251 00252 else : 00253 uselessvariable = None 00254 myparams['vis'] = vis = myf['vis'] 00255 myparams['outputvis'] = outputvis = myf['outputvis'] 00256 myparams['createmms'] = createmms = myf['createmms'] 00257 myparams['separationaxis'] = separationaxis = myf['separationaxis'] 00258 myparams['numsubms'] = numsubms = myf['numsubms'] 00259 myparams['datacolumn'] = datacolumn = myf['datacolumn'] 00260 myparams['calmsselection'] = calmsselection = myf['calmsselection'] 00261 myparams['calmsname'] = calmsname = myf['calmsname'] 00262 myparams['calfield'] = calfield = myf['calfield'] 00263 myparams['calscan'] = calscan = myf['calscan'] 00264 myparams['calintent'] = calintent = myf['calintent'] 00265 myparams['field'] = field = myf['field'] 00266 myparams['spw'] = spw = myf['spw'] 00267 myparams['antenna'] = antenna = myf['antenna'] 00268 myparams['timebin'] = timebin = myf['timebin'] 00269 myparams['combine'] = combine = myf['combine'] 00270 myparams['timerange'] = timerange = myf['timerange'] 00271 myparams['scan'] = scan = myf['scan'] 00272 myparams['scanintent'] = scanintent = myf['scanintent'] 00273 myparams['array'] = array = myf['array'] 00274 myparams['uvrange'] = uvrange = myf['uvrange'] 00275 myparams['observation'] = observation = myf['observation'] 00276 00277 00278 result = None 00279 00280 # 00281 # The following is work around to avoid a bug with current python translation 00282 # 00283 mytmp = {} 00284 00285 mytmp['vis'] = vis 00286 mytmp['outputvis'] = outputvis 00287 mytmp['createmms'] = createmms 00288 mytmp['separationaxis'] = separationaxis 00289 mytmp['numsubms'] = numsubms 00290 mytmp['datacolumn'] = datacolumn 00291 mytmp['calmsselection'] = calmsselection 00292 mytmp['calmsname'] = calmsname 00293 mytmp['calfield'] = calfield 00294 mytmp['calscan'] = calscan 00295 mytmp['calintent'] = calintent 00296 mytmp['field'] = field 00297 mytmp['spw'] = spw 00298 mytmp['antenna'] = antenna 00299 mytmp['timebin'] = timebin 00300 mytmp['combine'] = combine 00301 mytmp['timerange'] = timerange 00302 mytmp['scan'] = scan 00303 mytmp['scanintent'] = scanintent 00304 mytmp['array'] = array 00305 mytmp['uvrange'] = uvrange 00306 mytmp['observation'] = observation 00307 pathname='file:///'+os.environ.get('CASAPATH').split()[0]+'/share/xml/' 00308 trec = casac.utils().torecord(pathname+'partition.xml') 00309 00310 casalog.origin('partition') 00311 if not trec.has_key('partition') or not casac.utils().verify(mytmp, trec['partition']) : 00312 return False 00313 00314 00315 try : 00316 casalog.post('') 00317 casalog.post('##########################################') 00318 casalog.post('##### Begin Task: partition #####') 00319 casalog.post('') 00320 result = partition(vis, outputvis, createmms, separationaxis, numsubms, datacolumn, calmsselection, calmsname, calfield, calscan, calintent, field, spw, antenna, timebin, combine, timerange, scan, scanintent, array, uvrange, observation) 00321 casalog.post('') 00322 casalog.post('##### End Task: partition #####') 00323 casalog.post('##########################################') 00324 00325 00326 # saveinputs for individule engine has no use 00327 # saveinputs should alos be removed from casa_in_py.py 00328 # 00329 # 00330 # saveinputs = myf['saveinputs'] 00331 # saveinputs('partition', 'partition.last', myparams) 00332 # 00333 # 00334 except Exception, instance: 00335 #print '**** Error **** ',instance 00336 pass 00337 00338 gc.collect() 00339 return result 00340 # 00341 # 00342 ## 00343 # def paramgui(self, useGlobals=True): 00344 # """ 00345 # Opens a parameter GUI for this task. If useGlobals is true, then any relevant global parameter settings are used. 00346 # """ 00347 # import paramgui 00348 # 00349 # a=inspect.stack() 00350 # stacklevel=0 00351 # for k in range(len(a)): 00352 # if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 00353 # stacklevel=k 00354 # break 00355 # myf = sys._getframe(stacklevel).f_globals 00356 # 00357 # if useGlobals: 00358 # paramgui.setGlobals(myf) 00359 # else: 00360 # paramgui.setGlobals({}) 00361 # 00362 # paramgui.runTask('partition', myf['_ip']) 00363 # paramgui.setGlobals({}) 00364 # 00365 # 00366 # 00367 # 00368 def defaults(self, param=None): 00369 a=inspect.stack() 00370 stacklevel=0 00371 for k in range(len(a)): 00372 if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 00373 stacklevel=k 00374 break 00375 myf=sys._getframe(stacklevel).f_globals 00376 a = odict() 00377 a['vis'] = '' 00378 a['outputvis'] = '' 00379 a['createmms'] = True 00380 a['datacolumn'] = 'data' 00381 a['calmsselection'] = 'none' 00382 a['field'] = '' 00383 a['spw'] = '' 00384 a['antenna'] = '' 00385 a['timebin'] = '0s' 00386 a['timerange'] = '' 00387 a['scan'] = '' 00388 a['scanintent'] = '' 00389 a['array'] = '' 00390 a['uvrange'] = '' 00391 a['observation'] = '' 00392 00393 a['async']=False 00394 a['timebin'] = { 00395 0:odict([{'notvalue':'0s'}, {'combine':''}])} 00396 a['createmms'] = { 00397 0:odict([{'value':True}, {'separationaxis':'scan'}, {'numsubms':64}]), 00398 1:{'value':False}} 00399 a['calmsselection'] = { 00400 0:{'value':'none'}, 00401 1:odict([{'value':'auto'}, {'calmsname':''}]), 00402 2:odict([{'value':'manual'}, {'calmsname':''}, {'calfield':''}, {'calscan':''}, {'calintent':''}])} 00403 00404 ### This function sets the default values but also will return the list of 00405 ### parameters or the default value of a given parameter 00406 if(param == None): 00407 myf['__set_default_parameters'](a) 00408 elif(param == 'paramkeys'): 00409 return a.keys() 00410 else: 00411 if(a.has_key(param)): 00412 #if(type(a[param]) == dict) : 00413 # return a[param][len(a[param])-1]['value'] 00414 #else : 00415 return a[param] 00416 00417 00418 # 00419 # 00420 def check_params(self, param=None, value=None): 00421 a=inspect.stack() 00422 stacklevel=0 00423 for k in range(len(a)): 00424 if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 00425 stacklevel=k 00426 break 00427 myf=sys._getframe(stacklevel).f_globals 00428 00429 # print 'param:', param, 'value:', value 00430 try : 00431 if str(type(value)) != "<type 'instance'>" : 00432 value0 = value 00433 value = myf['cu'].expandparam(param, value) 00434 matchtype = False 00435 if(type(value) == numpy.ndarray): 00436 if(type(value) == type(value0)): 00437 myf[param] = value.tolist() 00438 else: 00439 #print 'value:', value, 'value0:', value0 00440 #print 'type(value):', type(value), 'type(value0):', type(value0) 00441 myf[param] = value0 00442 if type(value0) != list : 00443 matchtype = True 00444 else : 00445 myf[param] = value 00446 value = myf['cu'].verifyparam({param:value}) 00447 if matchtype: 00448 value = False 00449 except Exception, instance: 00450 #ignore the exception and just return it unchecked 00451 myf[param] = value 00452 return value 00453 00454 # 00455 # 00456 def description(self, key='partition', subkey=None): 00457 desc={'partition': 'Experimental task to produce multi-MSs using parallelism', 00458 'vis': 'Name of input measurement set', 00459 'outputvis': 'Name of output measurement set', 00460 'createmms': 'Should this create a multi-MS output', 00461 'separationaxis': 'Axis to do parallelization across(scan,spw,both)', 00462 'numsubms': 'The number of SubMSs to create', 00463 'datacolumn': 'Which data column(s) to split out', 00464 'calmsselection': 'Cal Data Selection (\'none\', \'auto\', \'manual\')', 00465 'calmsname': 'Name of output measurement set', 00466 'calfield': 'Field Selection for calibration ms', 00467 'calscan': 'Select data by scan numbers', 00468 'calintent': 'Select data by scan intent', 00469 'field': 'Select field using ID(s) or name(s)', 00470 'spw': 'Select spectral window/channels', 00471 'antenna': 'Select data based on antenna/baseline', 00472 'timebin': 'Bin width for time averaging', 00473 'combine': 'Let time bins span changes in scan and/or state', 00474 'timerange': 'Select data by time range', 00475 'scan': 'Select data by scan numbers', 00476 'scanintent': 'Select data by scan intent', 00477 'array': 'Select (sub)array(s) by array ID number', 00478 'uvrange': 'Select data by baseline length', 00479 'observation': 'Select by observation ID(s)', 00480 00481 'async': 'If true the taskname must be started using partition(...)' 00482 } 00483 00484 # 00485 # Set subfields defaults if needed 00486 # 00487 00488 if(desc.has_key(key)) : 00489 return desc[key] 00490 00491 def itsdefault(self, paramname) : 00492 a = {} 00493 a['vis'] = '' 00494 a['outputvis'] = '' 00495 a['createmms'] = True 00496 a['separationaxis'] = 'scan' 00497 a['numsubms'] = 64 00498 a['datacolumn'] = 'data' 00499 a['calmsselection'] = 'none' 00500 a['calmsname'] = '' 00501 a['calfield'] = '' 00502 a['calscan'] = '' 00503 a['calintent'] = '' 00504 a['field'] = '' 00505 a['spw'] = '' 00506 a['antenna'] = '' 00507 a['timebin'] = '0s' 00508 a['combine'] = '' 00509 a['timerange'] = '' 00510 a['scan'] = '' 00511 a['scanintent'] = '' 00512 a['array'] = '' 00513 a['uvrange'] = '' 00514 a['observation'] = '' 00515 00516 if a.has_key(paramname) : 00517 return a[paramname] 00518 partition_pg = partition_pg_()
1.8.0
