|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 import os 00002 import shutil 00003 import stat 00004 import time 00005 from taskinit import * 00006 from parallel.parallel_task_helper import ParallelTaskHelper 00007 00008 def concat(vislist,concatvis,freqtol,dirtol,respectname,timesort,copypointing, 00009 visweightscale,createmms): 00010 """concatenate visibility datasets 00011 The list of data sets given in the vis argument are concatenated into an output 00012 data set in concatvis. If concatvis already exists (e.g., it is the same as the 00013 first input data set), then the other input data sets will be appended to the 00014 concatvis data set. There is no limit to the number of input data sets. 00015 00016 If none of the input data sets have any scratch columns (model and corrected 00017 columns), none are created in the concatvis. Otherwise these columns are 00018 created on output and initialized to their default value (1 in model column, 00019 data in corrected column) for those data with no input columns. 00020 00021 Spectral windows for each data set with the same chanelization, and within a 00022 specified frequency tolerance of another data set will be combined into one 00023 spectral window. 00024 00025 A field position in one data set that is within a specified direction tolerance 00026 of another field position in any other data set will be combined into one 00027 field. The field names need not be the same---only their position is used. 00028 00029 Each appended dataset is assigned a new observation id. 00030 00031 Keyword arguments: 00032 vis -- Name of input visibility files to be combined 00033 default: none; example: vis = ['src2.ms','ngc5921.ms','ngc315.ms'] 00034 concatvis -- Name of visibility file that will contain the concatenated data 00035 note: if this file exits on disk then the input files are 00036 added to this file. Otherwise the new file contains 00037 the concatenated data. Be careful here when concatenating to 00038 an existing file. 00039 default: none; example: concatvis='src2.ms' 00040 example: concatvis='outvis.ms' 00041 00042 other examples: 00043 concat(vis=['src2.ms','ngc5921.ms'], concatvis='src2.ms') 00044 will concatenate 'ngc5921.ms' into 'src2.ms', and the original 00045 src2.ms is lost 00046 00047 concat(vis=['src2.ms','ngc5921.ms'], concatvis='out.ms') 00048 will concatenate 'ngc5921.ms' and 'src2.ms' into a file named 00049 'out.ms'; the original 'ngc5921.ms' and 'src2.ms' are untouched. 00050 00051 concat(vis=['v1.ms','v2.ms'], concatvis = 'vall.ms') 00052 then 00053 concat(vis=['v3.ms','v4.ms'], concatvis = 'vall.ms') 00054 vall.ms will contains v1.ms+v2.ms+v3.ms+v4.ms 00055 00056 Note: run flagmanager to save flags in the concatvis 00057 00058 freqtol -- Frequency shift tolerance for considering data to be in the same 00059 spwid. The number of channels must also be the same. 00060 default: '' do not combine unless frequencies are equal 00061 example: freqtol='10MHz' will not combine spwid unless they are 00062 within 10 MHz. 00063 Note: This option is useful to conbine spectral windows with very slight 00064 frequency differences caused by Doppler tracking, for example. 00065 00066 dirtol -- Direction shift tolerance for considering data as the same field 00067 default: '' means always combine. 00068 example: dirtol='1.arcsec' will not combine data for a field unless 00069 their phase center differ by less than 1 arcsec. If the field names 00070 are different in the input data sets, the name in the output data 00071 set will be the first relevant data set in the list. 00072 00073 respectname -- If true, fields with a different name are not merged even if their 00074 direction agrees (within dirtol) 00075 default: False 00076 00077 timesort -- If true, the output visibility table will be sorted in time. 00078 default: false. Data in order as read in. 00079 example: timesort=true 00080 Note: There is no constraint on data that is simultaneously observed for 00081 more than one field; for example multi-source correlation of VLBA data. 00082 00083 copypointing -- Make a proper copy of the POINTING subtable (can be time consuming). 00084 If False, the result is an empty POINTING table. 00085 default: True 00086 00087 visweightscale -- The weights of the individual MSs will be scaled in the concatenated 00088 output MS by the factors in this list. Useful for handling heterogeneous arrays. 00089 Use plotms to inspect the "Wt" column as a reference for determining the scaling 00090 factors. See the cookbook for more details. 00091 example: [1.,3.,3.] - scale the weights of the second and third MS by a factor 3. 00092 default: [] (empty list) - no scaling 00093 00094 createmms -- disabled. Please use task virtualconcat. 00095 default: False 00096 00097 """ 00098 00099 ### 00100 #Python script 00101 try: 00102 casalog.origin('concat') 00103 t = tbtool() 00104 m = mstool() 00105 00106 #break the reference between vis and vislist as we modify vis 00107 if(type(vislist)==str): 00108 vis=[vislist] 00109 else: 00110 vis=list(vislist) 00111 #dto. for concavis 00112 theconcatvis = concatvis 00113 00114 # warn if there are MMSs 00115 mmslist = [] 00116 for elvis in vis : ###Oh no Elvis does not exist Mr Bill 00117 if(ParallelTaskHelper.isParallelMS(elvis)): 00118 mmslist.append(elvis) 00119 if len(mmslist)>0: 00120 if (vis[0] == mmslist[0]): 00121 casalog.post('*** The first input MS is a multi-MS to which no row can be added. Cannot proceed.', 'WARN') 00122 casalog.post('*** Please use virtualconcat or convert the first input MS to a normal MS using split.', 'WARN') 00123 raise Exception, 'Cannot append to a multi-MS. Please use virtualconcat.' 00124 00125 casalog.post('*** The following input measurement sets are multi-MSs', 'INFO') 00126 for mname in mmslist: 00127 casalog.post('*** '+mname, 'INFO') 00128 casalog.post('*** Use virtualconcat to produce a single multi-MS from several multi-MSs.', 'INFO') 00129 00130 00131 00132 doweightscale = False 00133 if(len(visweightscale)>0): 00134 if (len(visweightscale) != len(vis)): 00135 raise Exception, 'parameter visweightscale must have same number of elements as parameter vis' 00136 for factor in visweightscale: 00137 if factor<0.: 00138 raise Exception, 'parameter visweightscale must only contain positive numbers' 00139 elif factor!=1.: 00140 doweightscale=True 00141 00142 if(createmms): 00143 raise Exception, 'createmms disabled. Please use task virtualconcat from now on.' 00144 00145 if((type(concatvis)!=str) or (len(concatvis.split()) < 1)): 00146 raise Exception, 'parameter concatvis is invalid' 00147 if(vis.count(concatvis) > 0): 00148 vis.remove(concatvis) 00149 00150 if(os.path.exists(concatvis)): 00151 casalog.post('Will be concatenating into the existing ms '+concatvis , 'WARN') 00152 else: 00153 if(len(vis) >0): # (note: in case len is 1, we only copy, essentially) 00154 casalog.post('copying '+vis[0]+' to '+theconcatvis , 'INFO') 00155 shutil.copytree(vis[0], theconcatvis) 00156 # note that the resulting copy is writable even if the original was read-only 00157 vis.remove(vis[0]) 00158 00159 if not copypointing: # remove the rows from the POINTING table of the first MS 00160 casalog.post('*** copypointing==False: resulting MS will have empty POINTING table.', 'INFO') 00161 tmptabname = 'TMPPOINTING'+str(time.time()) 00162 shutil.rmtree(tmptabname, ignore_errors=True) 00163 shutil.move(theconcatvis+'/POINTING', tmptabname) 00164 t.open(tmptabname) 00165 if(t.nrows()>0): 00166 t.copy(newtablename=theconcatvis+'/POINTING', deep=False, valuecopy=True, norows=True) 00167 t.close() 00168 shutil.rmtree(tmptabname, ignore_errors=True) 00169 else: # the POINTING table is already empty 00170 casalog.post('*** Input POINTING table was already empty.', 'INFO') 00171 shutil.move(tmptabname, theconcatvis+'/POINTING') 00172 t.close() 00173 00174 # Determine if scratch columns should be considered at all 00175 # by checking if any of the MSs has them. 00176 00177 considerscrcols = False 00178 needscrcols = [] 00179 if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))): 00180 00181 # check if all scratch columns are present 00182 t.open(theconcatvis) 00183 if(t.colnames().count('CORRECTED_DATA')==1 00184 or t.colnames().count('MODEL_DATA')==1): 00185 considerscrcols = True # there are scratch columns 00186 00187 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 00188 or t.colnames().count('MODEL_DATA')==0) 00189 t.close() 00190 else: 00191 raise Exception, 'Visibility data set '+theconcatvis+' not found - please verify the name' 00192 00193 00194 for elvis in vis : ###Oh no Elvis does not exist Mr Bill 00195 if(not os.path.exists(elvis)): 00196 raise Exception, 'Visibility data set '+elvis+' not found - please verify the name' 00197 00198 # check if all scratch columns are present 00199 t.open(elvis) 00200 if(t.colnames().count('CORRECTED_DATA')==1 00201 or t.colnames().count('MODEL_DATA')==1): 00202 considerscrcols = True # there are scratch columns 00203 00204 needscrcols.append(t.colnames().count('CORRECTED_DATA')==0 00205 or t.colnames().count('MODEL_DATA')==0) 00206 t.close() 00207 00208 # start actual work, file existence has already been checked 00209 i = 0 00210 if(considerscrcols and needscrcols[i]): 00211 # create scratch cols 00212 casalog.post('creating scratch columns in '+theconcatvis , 'INFO') 00213 cb.open(theconcatvis) # calibrator-open creates scratch columns 00214 cb.close() 00215 00216 # scale the weights of the first MS in the chain 00217 if doweightscale: 00218 wscale = visweightscale[i] 00219 if(wscale==1.): 00220 casalog.post('Will leave the weights for this MS unchanged.', 'INFO') 00221 else: 00222 casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO') 00223 t.open(theconcatvis, nomodify=False) 00224 for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']: 00225 if (colname in t.colnames()) and (t.iscelldefined(colname,0)): 00226 for j in xrange(0,t.nrows()): 00227 a = t.getcell(colname, j) 00228 a *= wscale 00229 t.putcell(colname, j, a) 00230 t.close() 00231 00232 # determine handling switch value 00233 handlingswitch = 0 00234 if not copypointing: 00235 handlingswitch = 2 00236 00237 m.open(theconcatvis,nomodify=False) 00238 mmsmembers = [theconcatvis] 00239 00240 for elvis in vis : 00241 i = i + 1 00242 destms = "" 00243 casalog.post('concatenating '+elvis+' into '+theconcatvis , 'INFO') 00244 00245 wscale = 1. 00246 if doweightscale: 00247 wscale = visweightscale[i] 00248 if(wscale==1.): 00249 casalog.post('Will leave the weights for this MS unchanged.', 'INFO') 00250 else: 00251 casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO') 00252 00253 if(considerscrcols and needscrcols[i]): 00254 if(ParallelTaskHelper.isParallelMS(elvis)): 00255 raise Exception, 'Cannot create scratch columns in a multi-MS. Use virtualconcat.' 00256 else: 00257 # create scratch cols 00258 casalog.post('creating scratch columns for '+elvis+' (original MS unchanged)', 'INFO') 00259 tempname = elvis+'_with_scrcols' 00260 shutil.rmtree(tempname, ignore_errors=True) 00261 shutil.copytree(elvis, tempname) 00262 cb.open(tempname) # calibrator-open creates scratch columns 00263 cb.close() 00264 # concatenate copy instead of original file 00265 m.concatenate(msfile=tempname,freqtol=freqtol,dirtol=dirtol,respectname=respectname, 00266 weightscale=wscale,handling=handlingswitch, 00267 destmsfile=destms) 00268 shutil.rmtree(tempname, ignore_errors=True) 00269 else: 00270 m.concatenate(msfile=elvis,freqtol=freqtol,dirtol=dirtol,respectname=respectname, 00271 weightscale=wscale,handling=handlingswitch, 00272 destmsfile=destms) 00273 00274 if timesort: 00275 casalog.post('Sorting main table by TIME ...', 'INFO') 00276 m.timesort() 00277 00278 m.writehistory(message='taskname=concat',origin='concat') 00279 m.writehistory(message='vis = "'+str(vis)+'"',origin='concat') 00280 m.writehistory(message='concatvis = "'+str(concatvis)+'"',origin='concat') 00281 m.writehistory(message='freqtol = "'+str(freqtol)+'"',origin='concat') 00282 m.writehistory(message='dirtol = "'+str(dirtol)+'"',origin='concat') 00283 m.writehistory(message='respectname = "'+str(respectname)+'"',origin='concat') 00284 m.writehistory(message='copypointing = "'+str(copypointing)+'"',origin='concat') 00285 m.writehistory(message='visweightscale = "'+str(visweightscale)+'"',origin='concat') 00286 00287 m.close() 00288 00289 return True 00290 00291 except Exception, instance: 00292 print '*** Error ***',instance 00293 raise Exception, instance 00294
1.8.0
