|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import os 00002 import shutil 00003 import stat 00004 import time 00005 from taskinit import * 00006 import partitionhelper as ph 00007 from parallel.parallel_task_helper import ParallelTaskHelper 00008 00009 def virtualconcat(vislist,concatvis,freqtol,dirtol,respectname, 00010 visweightscale,keepcopy,copypointing): 00011 """ 00012 Concatenate visibility data sets creating a multi-MS. 00013 00014 Combine the input datasets into a multi-MS. 00015 NOTE: The input datasets are moved into the multi-MS and may be modified 00016 to account for subtable reference changes. 00017 If none of the input MSs have any scratch columns, none are created. 00018 Otherwise scratch columns are created and initialized in those MSs 00019 which don't have a complete set. 00020 00021 00022 Keyword arguments: 00023 vis -- Name of input visibility files (MS) 00024 default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms'] 00025 concatvis -- Name of the output visibility file 00026 default: none; example: concatvis='src2.ms' 00027 freqtol -- Frequency shift tolerance for considering data as the same spwid 00028 default: '' means always combine 00029 example: freqtol='10MHz' will not combine spwid unless they are 00030 within 10 MHz 00031 dirtol -- Direction shift tolerance for considering data as the same field 00032 default: ;; means always combine 00033 example: dirtol='1.arcsec' will not combine data for a field unless 00034 their phase center is less than 1 arcsec. 00035 respectname -- If true, fields with a different name are not merged even if their 00036 direction agrees (within dirtol) 00037 default: True 00038 visweightscale -- list of the weight scales to be applied to the individual MSs 00039 default: [] (don't modify weights, equivalent to setting scale to 1 for each MS) 00040 keepcopy -- If true, a copy of the input MSs is kept in their original place. 00041 default: false 00042 copypointing -- If true, the POINTING table information will be present in the output. 00043 If false, the result is an empty POINTING table. 00044 default: True 00045 00046 """ 00047 00048 ### 00049 #Python script 00050 00051 tempdir = '' 00052 originalvis = vislist 00053 try: 00054 casalog.origin('virtualconcat') 00055 t = tbtool() 00056 m = mstool() 00057 00058 #break the reference between vis and vislist as we modify vis 00059 if(type(vislist)==str): 00060 vis=[vislist] 00061 else: 00062 vis=list(vislist) 00063 #dto. for concavis 00064 theconcatvis = concatvis 00065 00066 doweightscale = False 00067 if(len(visweightscale)>0): 00068 if (len(visweightscale) != len(vis)): 00069 raise Exception, 'parameter visweightscale must have same number of elements as parameter vis' 00070 for factor in visweightscale: 00071 if factor<0.: 00072 raise Exception, 'parameter visweightscale must only contain positive numbers' 00073 elif factor!=1.: 00074 doweightscale=True 00075 00076 # if there are MMSs among the input, make their constituents the new input 00077 mmslist = [] 00078 ismaster = [] 00079 for elvis in vis: 00080 ismaster.append(True) # may be revised later 00081 if(ParallelTaskHelper.isParallelMS(elvis)): 00082 mmslist.append(elvis) 00083 if len(mmslist)>0: 00084 casalog.post('*** The following input measurement sets are multi-MSs', 'INFO') 00085 for mname in mmslist: 00086 casalog.post('*** '+mname, 'INFO') 00087 oldvis = vis 00088 oldvisweightscale = visweightscale 00089 vis = [] 00090 visweightscale = [] 00091 ismaster = [] # reset ismaster 00092 i = 0 00093 for elvis in oldvis: 00094 if elvis in mmslist: # append the subMSs individually 00095 m.open(elvis) 00096 mses = m.getreferencedtables() 00097 m.close() 00098 mses.sort() 00099 mastername = os.path.basename(os.path.dirname(os.path.realpath(elvis+'/ANTENNA'))) 00100 for mname in mses: 00101 #print 'subms: ', mname 00102 vis.append(mname) 00103 if doweightscale: 00104 visweightscale.append(oldvisweightscale[i]) 00105 if os.path.basename(mname) == mastername: 00106 ismaster.append(True) 00107 else: 00108 ismaster.append(False) 00109 else: 00110 vis.append(elvis) 00111 if doweightscale: 00112 visweightscale.append(oldvisweightscale[i]) 00113 ismaster.append(True) 00114 i += 1 00115 00116 00117 if((type(concatvis)!=str) or (len(concatvis.split()) < 1)): 00118 raise Exception, 'parameter concatvis is invalid' 00119 00120 if(vis.count(concatvis) > 0): 00121 vis.remove(concatvis) 00122 00123 if(os.path.exists(concatvis)): 00124 raise Exception, 'The output MMS must not yet exist.' 00125 00126 if keepcopy: 00127 tempdir = 'concat_tmp_'+str(time.time()) 00128 os.mkdir(tempdir) 00129 for elvis in originalvis: 00130 shutil.move(elvis,tempdir) # keep timestamps and permissions 00131 shutil.copytree(tempdir+'/'+elvis, elvis, True) # symlinks=True 00132 00133 if not copypointing: # delete the rows of all pointing tables 00134 casalog.post('*** copypointing==False: resulting MMS will have empty POINTING table.', 'INFO') 00135 tmptabname = 'TMPPOINTING'+str(time.time()) 00136 tmptabname2 = 'TMPPOINTING2'+str(time.time()) 00137 shutil.rmtree(tmptabname, ignore_errors=True) 00138 shutil.rmtree(tmptabname2, ignore_errors=True) 00139 shutil.move(vis[0]+'/POINTING', tmptabname) 00140 t.open(tmptabname) 00141 if(t.nrows()>0): 00142 t.copy(newtablename=tmptabname2, deep=False, valuecopy=True, norows=True) 00143 t.close() 00144 shutil.rmtree(tmptabname, ignore_errors=True) 00145 else: # the POINTING table is already empty 00146 t.close() 00147 casalog.post('*** Input POINTING table was already empty.', 'INFO') 00148 shutil.move(tmptabname, tmptabname2) 00149 00150 for i in range(len(vis)): # replace the POINTING tables by the empty one 00151 os.system('rm -rf '+vis[i]+'/POINTING') 00152 shutil.copytree(tmptabname2, vis[i]+'/POINTING') 00153 shutil.rmtree(tmptabname2, ignore_errors=True) 00154 00155 if(len(vis) >0): # (note: in case len is 1, we only copy, essentially) 00156 theconcatvis = vis[0] 00157 if(len(vis)==1): 00158 shutil.copytree(vis[0], concatvis, True) 00159 vis.remove(vis[0]) 00160 00161 # Determine if scratch columns should be considered at all 00162 # by checking if any of the MSs has them. 00163 00164 considerscrcols = False 00165 needscrcols = [] 00166 if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))): 00167 00168 # check if all scratch columns are present 00169 t.open(theconcatvis) 00170 if(t.colnames().count('CORRECTED_DATA')==1 00171 or t.colnames().count('MODEL_DATA')==1): 00172 considerscrcols = True # there are scratch columns 00173 00174 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 00175 or t.colnames().count('MODEL_DATA')==0) 00176 t.close() 00177 else: 00178 raise Exception, 'Visibility data set '+theconcatvis+' not found - please verify the name' 00179 00180 for elvis in vis : ###Oh no Elvis does not exist Mr Bill 00181 if(not os.path.exists(elvis)): 00182 raise Exception, 'Visibility data set '+elvis+' not found - please verify the name' 00183 00184 # check if all scratch columns are present 00185 t.open(elvis) 00186 if(t.colnames().count('CORRECTED_DATA')==1 00187 or t.colnames().count('MODEL_DATA')==1): 00188 considerscrcols = True # there are scratch columns 00189 00190 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 00191 or t.colnames().count('MODEL_DATA')==0) 00192 t.close() 00193 00194 # start actual work, file existence has already been checked 00195 00196 if(considerscrcols and needscrcols[0]): 00197 # create scratch cols 00198 casalog.post('creating scratch columns in '+theconcatvis , 'INFO') 00199 cb.open(theconcatvis) # calibrator-open creates scratch columns 00200 cb.close() 00201 00202 # scale the weights of the first MS in the chain 00203 if doweightscale: 00204 wscale = visweightscale[0] 00205 if(wscale==1.): 00206 casalog.post('Will leave the weights for this MS unchanged.', 'INFO') 00207 else: 00208 casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO') 00209 t.open(theconcatvis, nomodify=False) 00210 for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']: 00211 if (colname in t.colnames()) and (t.iscelldefined(colname,0)): 00212 for j in xrange(0,t.nrows()): 00213 a = t.getcell(colname, j) 00214 a *= wscale 00215 t.putcell(colname, j, a) 00216 t.close() 00217 00218 m.open(theconcatvis,nomodify=False) 00219 mmsmembers = [theconcatvis] 00220 00221 auxfile = 'concat_aux_'+str(time.time()) 00222 00223 i = 0 00224 for elvis in vis : 00225 i = i + 1 00226 00227 mmsmembers.append(elvis) 00228 casalog.post('adding '+elvis+' to multi-MS '+concatvis, 'INFO') 00229 00230 wscale = 1. 00231 if doweightscale: 00232 wscale = visweightscale[i] 00233 if(wscale==1.): 00234 casalog.post('Will leave the weights for this MS unchanged.', 'INFO') 00235 else: 00236 casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO') 00237 00238 if(considerscrcols and needscrcols[i]): 00239 # create scratch cols 00240 casalog.post('creating scratch columns for '+elvis, 'INFO') 00241 cb.open(elvis) # calibrator-open creates scratch columns 00242 cb.close() 00243 00244 m.virtconcatenate(msfile=elvis, 00245 auxfilename=auxfile, 00246 freqtol=freqtol,dirtol=dirtol,respectname=respectname, 00247 weightscale=wscale) 00248 #end for 00249 os.remove(auxfile) 00250 00251 m.writehistory(message='taskname=virtualconcat',origin='virtualconcat') 00252 m.writehistory(message='vis = "'+str(vis)+'"',origin='virtualconcat') 00253 m.writehistory(message='concatvis = "'+str(concatvis)+'"',origin='virtualconcat') 00254 m.writehistory(message='freqtol = "'+str(freqtol)+'"',origin='virtualconcat') 00255 m.writehistory(message='dirtol = "'+str(dirtol)+'"',origin='virtualconcat') 00256 m.writehistory(message='respectname = "'+str(respectname)+'"',origin='virtualconcat') 00257 m.writehistory(message='visweightscale = "'+str(visweightscale)+'"',origin='virtualconcat') 00258 00259 m.close() 00260 00261 # concatenate the POINTING tables 00262 masterptable = mmsmembers[0]+'/POINTING' 00263 ptablemembers = [] 00264 if os.path.exists(masterptable) and copypointing: 00265 casalog.post('Concatenating the POINTING tables ...', 'INFO') 00266 i = 0 00267 for i in xrange(len(mmsmembers)): 00268 ptable = mmsmembers[i]+'/POINTING' 00269 if ismaster[i] and os.path.exists(ptable): 00270 casalog.post(' '+ptable, 'INFO') 00271 shutil.move(ptable, ptable+str(i)) 00272 ptablemembers.append(ptable+str(i)) 00273 #end for 00274 t.createmultitable(masterptable, ptablemembers, 'SUBTBS') 00275 # endif 00276 00277 ph.makeMMS(concatvis, mmsmembers, 00278 True, # copy subtables from first to all other members 00279 ['POINTING']) # excluding POINTING which will be linked 00280 00281 # remove the remaining "hulls" of the emptied input MMSs (if there are any) 00282 for elvis in mmslist: 00283 shutil.rmtree(elvis) 00284 00285 if keepcopy: 00286 for elvis in originalvis: 00287 shutil.move(tempdir+'/'+elvis, elvis) 00288 os.rmdir(tempdir) 00289 00290 except Exception, instance: 00291 print '*** Error ***',instance 00292 if keepcopy and tempdir!='': 00293 print "Restoring original MSs ..." 00294 for elvis in originalvis: 00295 if os.path.exists(tempdir+'/'+elvis): 00296 shutil.rmtree(elvis) 00297 shutil.move(tempdir+'/'+elvis, elvis) 00298 os.rmdir(tempdir) 00299 raise Exception, instance 00300
1.8.0
