|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 # 00002 # This file was generated using xslt from its XML file 00003 # 00004 # Copyright 2008, Associated Universities Inc., Washington DC 00005 # 00006 import sys 00007 import os 00008 from casac import * 00009 import string 00010 import time 00011 import inspect 00012 import gc 00013 import numpy 00014 from odict import odict 00015 from task_flagdata import flagdata 00016 from task_flagdata import casalog 00017 00018 class flagdata_pg_: 00019 __name__ = "flagdata" 00020 00021 def __init__(self) : 00022 self.__bases__ = (flagdata_pg_,) 00023 self.__doc__ = self.__call__.__doc__ 00024 00025 00026 def __call__(self, vis=None, mode=None, autocorr=None, inpfile=None, reason=None, spw=None, field=None, antenna=None, uvrange=None, timerange=None, correlation=None, scan=None, intent=None, array=None, observation=None, feed=None, clipminmax=None, datacolumn=None, clipoutside=None, channelavg=None, clipzeros=None, quackinterval=None, quackmode=None, quackincrement=None, tolerance=None, addantenna=None, lowerlimit=None, upperlimit=None, ntime=None, combinescans=None, timecutoff=None, freqcutoff=None, timefit=None, freqfit=None, maxnpieces=None, flagdimension=None, usewindowstats=None, halfwin=None, winsize=None, timedev=None, freqdev=None, timedevscale=None, freqdevscale=None, spectralmax=None, spectralmin=None, extendpols=None, growtime=None, growfreq=None, growaround=None, flagneartime=None, flagnearfreq=None, minrel=None, maxrel=None, minabs=None, maxabs=None, spwchan=None, spwcorr=None, basecnt=None, action=None, display=None, flagbackup=None, savepars=None, cmdreason=None, outfile=None, async=None): 00027 00028 """All-purpose flagging task based on data-selections and flagging modes/algorithms. 00029 00030 This task can flag a Measurement Set or a calibration table. It has 00031 two main types of operation. One type will read the parameters from 00032 the interface and flag using any of the various available modes. The 00033 other type will read the commands from a text file or a Python list of 00034 strings, containing a list of flag commands (each line containing data 00035 selection parameters and any parameter specific for the mode being 00036 requested). Please see examples at the end of this help. 00037 00038 It is also possible to only save the parameters set in the interface 00039 without flagging. The parameters can be saved in the FLAG_CMD sub-table 00040 or in a text file. Note that when saving to an external file, the parameters 00041 will be appended to the given file. 00042 00043 The available flagging modes are: manual, clip, shadow, quack, elevation, 00044 tfcrop, rflag, extend, unflag and summary. 00045 00046 The current flags can be automatically backed up before applying new 00047 flags if the parameter flagbackup is set. Previous flag versions can be 00048 recovered using the flagmanager task. 00049 00050 NOTE on flagging calibration tables. 00051 ----------------------------------- 00052 00053 When the input is a calibration table, the modes 'elevation' and 'shadow' 00054 will be disabled. Data selection for calibration tables is limited to field, 00055 antenna and spw at the moment. It is only possible to save the parameters 00056 to an external file. 00057 00058 The task will flag a subset of data based on the following modes of operation: 00059 00060 list = list of flagging commands to apply to MS/cal table 00061 manual = flagging based on specific selection parameters 00062 clip = clip data according to values 00063 quack = remove/keep specific time range at scan beginning/end 00064 shadow = remove antenna-shadowed data 00065 elevation = remove data below/above given elevations 00066 tfcrop = automatic identification of outliers on the time-freq plane 00067 rflag = automatic detection of outliers based on sliding-window RMS filters 00068 extend = extend and/or grow flags beyond what the basic algorithms detect 00069 summary = report the amount of flagged data 00070 unflag = unflag the specified data 00071 00072 Keyword arguments: 00073 vis -- Name of input visibility file or calibration table. 00074 default: '' (none) 00075 example1: vis='uid___A002_X2a5c2f_X54.ms' or 00076 example2: vis='cal-X54.B1' 00077 00078 Any flagging will only be applied to the specified selection. 00079 00080 antenna -- Select data based on baseline 00081 default: '' (all); example: antenna='DV04&DV06' baseline DV04-DV06 00082 antenna='DV04&DV06;DV07&DV10' #baselines DV04-DV06 and DV07-DV10 00083 antenna='DV06' # all cross-correlation baselines between antenna DV06 and 00084 all other available antennas 00085 antenna='DV04,DV06' # all baselines with antennas DV04 and DV06 00086 antenna='DV06&&DV06' # only the auto-correlation baselines for antenna DV06 00087 antenna='DV04&&*' # cross and auto-correlation baselines between antenna DV04 00088 and all other available antennas 00089 antenna='0~2&&&' # only the auto-correlation baselines for antennas 00090 in range 0~2 00091 00092 spw -- Select data based on spectral window and channels 00093 default: '' (all); example: spw='1' 00094 spw='<2' #spectral windows less than 2 00095 spw='>1' #spectral windows greater than 1 00096 spw='1:0~10' # first 10 channels from spw 1 00097 spw='1:0~5;120~128' # multiple separated channel chunks. 00098 00099 Note : For modes clip, tfcrop and rflag, channel-ranges can be excluded 00100 from flagging by selecting ranges such as spw='0:0~5;10~63'. This is a way to 00101 protect known spectral-lines from being flagged by the autoflag algorithms. 00102 00103 correlation -- Correlation types or expression. 00104 default: '' (all correlations) 00105 For modes clip, tfcrop or rflag, the default means ABS_ALL. If 00106 the input is cal table that does not contain a complex data column, 00107 the default will fall back to REAL_ALL. 00108 example: correlation='XX,YY' or 00109 options: Any of 'ABS', 'ARG', 'REAL', 'IMAG', 'NORM' followed by 00110 any of 'ALL', 'I', 'XX', 'YY', 'RR', 'LL', 'Sol1', 'Sol2', 'WVR' 00111 'WVR' refers to the water vapour radiometer of ALMA data. 00112 example: correlation='REAL_XX,XY' 00113 -->correlation selection is not supported for modes other than clip, tfcrop or 00114 rflag in cal tables. 00115 00116 Note that the operators ABS,ARG,REAL, etc. are written only once as the first value. 00117 if more than one correlation is given, the operator will be applied to all of them. 00118 00119 The expression is used only in modes clip, tfcrop and rflag. 00120 00121 field -- Select data based on field id(s) or name(s) 00122 default: '' (all); example: field='1' 00123 field='0~2' # field ids inclusive from 0 to 2 00124 field='3C*' # all field names starting with 3C 00125 00126 uvrange -- Select data within uvrange (default units meters) 00127 default: '' (all); example: 00128 uvrange='0~1000klambda'; uvrange from 0-1000 kilo-lambda 00129 uvrange='>4klamda';uvranges greater than 4 kilo-lambda 00130 uvrange='0~1000km'; uvrange in kilometers 00131 -->uvrange selection is not supported for cal tables. 00132 00133 timerange -- Select data based on time range: 00134 default = '' (all); example, 00135 timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' 00136 Note: YYYY/MM/DD can be dropped as needed: 00137 timerange='09:14:0~09:54:0' # this time range 00138 timerange='09:44:00' # data within one integration of time 00139 timerange='>10:24:00' # data after this time 00140 timerange='09:44:00+00:13:00' #data 13 minutes after time 00141 -->timerange selection is not supported for cal tables. 00142 00143 scan -- Select data based on scan number 00144 default: '' (all); example: scan='>3' 00145 -->scan selection is not supported for cal tables. 00146 00147 intent -- Select data based on observation intent 00148 default: '' (all); example: intent='*CAL*,*BAND*' 00149 -->intent selection is not supported for cal tables. 00150 00151 array -- Selection based on the antenna array 00152 default: '' (all); 00153 -->array selection is not supported for cal tables. 00154 00155 observation -- Selection based on the observation ID 00156 default: '' (all); example: observation='1' or observation=1 00157 -->observation selection is not supported for cal tables. 00158 00159 feed -- Selection based on the feed - NOT IMPLEMENTED YET 00160 00161 00162 mode -- Mode of operation. 00163 options: 'list', 'manual','clip','quack','shadow','elevation', 'tfcrop', 'extend', 00164 'unflag', 'summary' 00165 default: 'manual' 00166 00167 list -- Flag according to the data selection and flag commands specified in the input list. 00168 The input list may come from a text file or from a Python list of strings. 00169 Each input line may contain data selection parameters and any parameter 00170 specific to the mode given in the line. Default values will be used for 00171 the parameters that are not present in the line. Each line will be taken 00172 as a command to the task. If data is pre-selected using any of the selection 00173 parameters, then flagging will apply only to that subset of the MS. 00174 00175 The summary mode is not allowed in the input list. 00176 00177 For optimization, the task will create a union of the data selection parameters 00178 present in the list and select only that portion of the MS. 00179 00180 NOTE: the flag commands will be applied only when action='apply'. If 00181 action='calculate' the flags will be calculated, but not applied. 00182 This is useful if display is set to something other than 'none'. If 00183 action='' or 'none', the flag commands will not be applied either. 00184 An empty action is useful only to save the parameters of the list 00185 to a file or to the FLAG_CMD sub-table. 00186 00187 inpfile -- Input ASCII file or a Python list of command strings. 00188 default: '' 00189 options: [] or '' with filename. 00190 Example1: the following commands can be saved into a file and given 00191 to the task (e.g. save it to flags.txt). 00192 00193 scan='1~3' mode='manual' 00194 mode='clip' clipminmax=[0,2] correlation='ABS_XX' clipoutside=False 00195 spw='9' mode='tfcrop' correlation='ABS_YY' ntime=51.0 00196 mode='extend' extendpols=True 00197 00198 flagdata(vis,mode='list',inpfile='flags.txt') 00199 00200 Example2: the same commands can be given in a Python list on the command line 00201 to the task. 00202 cmd=["scan='1~3' mode='manual'", 00203 "mode='clip' clipminmax=[0,2] correlation='ABS_XX' clipoutside=False", 00204 "spw='9' mode='tfcrop' correlation='ABS_YY' ntime=51.0", 00205 "mode='extend' extendpols=True"] 00206 00207 flagdata(vis,mode='list',inpfile=cmd) 00208 00209 reason -- select flag commands based on REASON(s) 00210 default: 'any' (all flags regardless of reason) 00211 can be a string, or list of strings 00212 example: reason='FOCUS_ERROR' 00213 reason=['FOCUS_ERROR','SUBREFLECTOR_ERROR'] 00214 00215 NOTE: what is within the string is literally 00216 matched, e.g. reason='' matches only blank reasons, 00217 and reason = 'FOCUS_ERROR,SUBREFLECTOR_ERROR' 00218 matches this compound reason string only 00219 00220 See the syntax for writing flag commands at the end of this help. 00221 00222 00223 manual -- Flag according to the data selection specified. 00224 This is the default mode (used when the mode is not specified). 00225 00226 autocorr -- Flag only the auto-correlations. Note that this parameter is only 00227 active when set to True. If set to False it does NOT mean "do not 00228 flag auto-correlations". 00229 default: False 00230 options: True,False 00231 00232 clip -- Clip data according to values of the following subparameters. The polarization 00233 expression is given by the correlation parameter. 00234 00235 channelavg -- Average data over (selected) channels? Channel 00236 selections are taken into account. Flagged channels 00237 are excluded from the average. The average is done 00238 after applying the specified clipexpr, i.e. scalar 00239 averaging. 00240 default: False 00241 options: True,False 00242 00243 clipminmax -- Range of data (Jy) that will NOT be flagged 00244 default: []; it will flag only NaN and Infs. 00245 example: [0.0,1.5] 00246 It will always flag the NaN/Inf data, even when a range is specified. 00247 00248 datacolumn -- Column to use for clipping. 00249 default: 'DATA' 00250 options: 'DATA', 'CORRECTED','MODEL', 'RESIDUAL, 'RESIDUAL_DATA', 00251 'FPARAM', 'CPARAM', 'SNR'. 00252 00253 NOTE: by default for calibration tables, the framework will try to use FPARAM; 00254 if that does not exist, it will try to use CPARAM. If none of them exist, it 00255 will give an error and exit. 00256 00257 clipoutside -- Clip OUTSIDE the range? 00258 default: True 00259 example: False; flag data WITHIN the range. 00260 00261 clipzeros -- Clip zero-value data. 00262 default: False 00263 00264 quack -- Option to remove specified part of scan beginning/end 00265 00266 quackinterval -- Time in seconds from scan beginning/end to flag. Make time slightly 00267 smaller than the desired time. 00268 default: 0.0 00269 quackmode -- Quack mode 00270 default: 'beg' 00271 options: 'beg' ==> beginning of scan 00272 'endb' ==> end of scan. 00273 'tail' ==> all but beginning of scan 00274 'end' ==> all but end of scan. 00275 quackincrement -- Quack incrementally in time? 00276 default: False 00277 False ==> the quack interval is counted from the 00278 beginning of the scan 00279 True ==> the quack interval is counted from the 00280 first unflagged data in the scan 00281 00282 shadow -- Option to flag data of shadowed antennas. This mode is not available 00283 for cal tables. 00284 00285 All antennas in the antenna-subtable of the MS (and the corresponding 00286 diameters) will be considered for shadow-flag calculations. 00287 For a given timestep, an antenna is flagged if any of its baselines 00288 (projected onto the uv-plane) is shorter than radius_1 + radius_2 - tolerance. 00289 The value of 'w' is used to determine which antenna is behind the other. 00290 The phase-reference center is used for antenna-pointing direction. 00291 00292 tolerance -- Amount of shadowing allowed (or tolerated), in meters. 00293 A positive number allows antennas to overlap in projection 00294 A negative number forces antennas apart in projection 00295 Zero implies a distance of radius_1+radius_2 between antenna centers. 00296 default: 0.0 00297 00298 addantenna -- It can be either a file name with additional antenna names, positions 00299 and diameters, or a Python dictionary with the same information. 00300 You can use the flaghelper functions to create the dictionary from a file. 00301 default: '' 00302 00303 To create a dictionary inside casapy. 00304 > import flaghelper as fh 00305 > antdic = fh.readAntennaList(antfile) 00306 00307 Where antfile is a text file in disk that contains information such as: 00308 name=VLA01 00309 diameter=25.0 00310 position=[-1601144.96146691, -5041998.01971858, 3554864.76811967] 00311 name=VLA02 00312 diameter=25.0 00313 position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178] 00314 00315 00316 elevation -- Option to flag based on antenna elevation. This mode is not available 00317 for cal tables. 00318 00319 lowerlimit -- Lower limiting elevation in degrees. Data coming from a baseline 00320 where one or both antennas were pointing at a strictly lower elevation 00321 (as function of time), will be flagged. 00322 default: 0.0 00323 00324 upperlimit -- Upper limiting elevation in degrees. Data coming from a baseline 00325 where one or both antennas were pointing at a strictly higher elevation 00326 (as function of time), will be flagged. 00327 default: 90.0 00328 00329 tfcrop -- Flag using the TFCrop autoflag algorithm. The polarization 00330 expression is given by the correlation parameter. 00331 00332 For each field, spw, timerange (specified by ntime), and baseline, 00333 (1) Average visibility amplitudes along time dimension to 00334 form an average spectrum 00335 (2) Calculate a robust piece-wise polynomial fit for the band-shape 00336 at the base of RFI spikes. Calculate 'stddev' of (data - fit). 00337 (3) Flag points deviating from the fit by more than N-stddev 00338 (4) Repeat (1-3) along the other dimension. 00339 00340 This algorithm is designed to operate on un-calibrated data (step (2)), 00341 as well as calibrated data. It is usually helpful to extend the flags along 00342 time, frequency, and correlation in a second step (via mode='extend) 00343 00344 ntime -- Timerange (in seconds or minutes) over which to buffer data before 00345 running the algorithm. 00346 options: 'scan' or any other float value or string containing the units. 00347 default: 'scan' 00348 example: '1.5min' 00349 : 1.2 (taken in seconds) 00350 The dataset will be iterated through in time-chunks defined here. 00351 00352 combinescans -- accumulate data across scans. 00353 default: False 00354 This parameter is relevant only ntime is specified as a time-interval (not 'scan') 00355 00356 datacolumn -- Column to use for flagging. 00357 default: 'DATA' 00358 options: 'DATA', 'CORRECTED','MODEL', 'RESIDUAL, 'RESIDUAL_DATA', 00359 'FPARAM', 'CPARAM', 'SNR'. 00360 00361 NOTE: by default for calibration tables, the framework will try to use FPARAM; 00362 if that does not exist, it will try to use CPARAM. If none of them exist, it 00363 will give an error and exit. 00364 00365 00366 timecutoff -- Flag threshold in time. Flag all data-points further than N-stddev 00367 from the fit. This threshold catches time-varying RFI spikes ( 00368 narrow and broad-band), but will not catch RFI that is persistent in time. 00369 default: 4.0 00370 Flagging is done in upto 5 iterations. The stddev calculation is adaptive and 00371 converges to a value that reflects only the data and no RFI. At each iteration, 00372 the same relative threshold is applied to detect flags. (Step (3) of the algorithm). 00373 00374 freqcutoff -- Flag threshold in frequency. Flag all data-points further than N-stddev 00375 from the fit. 00376 default: 3.0 00377 Same as timecutoff, but along the frequency-dimension. This threshold catches 00378 narrow-band RFI that may or may not be persistent in time. 00379 00380 timefit -- Fitting function for the time direction 00381 default: 'line' 00382 options: 'line', 'poly' 00383 A 'line' fit is a robust straight-line fit across the entire timerange (defined 00384 by 'ntime'). 00385 A 'poly' fit is a robust piece-wise polynomial fit across the timerange. 00386 Note: A robust fit is computed in upto 5 iterations. At each iteration, the stddev 00387 between the data and the fit is computed, values beyond N-stddev are flagged, 00388 and the fit and stddev are re-calculated with the remaining points. 00389 This stddev calculation is adaptive, and converges to a value that reflects 00390 only the data and no RFI. It also provides a varying set of flagging thresholds, 00391 that allows deep flagging only when the fit best represents the true data. 00392 Choose 'poly' only if the visibilities are expected to vary significantly over the 00393 timerange selected by 'ntime', or if there is a lot of strong but intermittent RFI. 00394 00395 freqfit -- Fitting function for the frequency direction 00396 default: 'poly' 00397 options: 'line','poly' 00398 Same as for the 'timefit' parameter. 00399 Choose 'line' only if you are operating on bandpass-corrected data, or residuals, 00400 and expect that the bandshape is linear. The 'poly' option works better on 00401 uncalibrated bandpasses with narrow-band RFI spikes. 00402 00403 maxnpieces -- Maxinum number of pieces to allow in the piecewise-polynomial fits 00404 default: 7 00405 options: 1 - 9 00406 This parameter is used only if 'timefit' or 'freqfit' are chosen as 'poly'. 00407 If there is significant broad-band RFI, reduce this number. Using too many 00408 pieces could result in the RFI being fitted in the 'clean' bandpass. 00409 In later stages of the fit, a third-order polynomial is fit per piece, so 00410 for best results, please ensure that nchan/maxnpieces is at-least 10. 00411 00412 flagdimension -- Choose the directions along which to perform flagging 00413 default: 'freqtime'; first flag along frequency, and then along time 00414 options: 'time', 'freq', 'timefreq', 'freqtime' 00415 For most cases, 'freqtime' or 'timefreq' are appropriate, and differences 00416 between these choices are apparant only if RFI in one dimension is 00417 significantly stronger than the other. The goal is to flag the dominant RFI first. 00418 If there are very few (less than 5) channels of data, then choose 'time'. 00419 Similarly for 'freq'. 00420 00421 usewindowstats -- Use sliding-window statistics to find additional flags. 00422 default: 'none' 00423 options: 'none', 'sum', 'std', 'both' 00424 Note: This is experimental ! 00425 The 'sum' option chooses to flag a point, if the mean-value in a 00426 window centered on that point deviates from the fit by more than 00427 N-stddev/2.0. 00428 Note: stddev is calculated between the data and fit as explained in Step (2). 00429 This option is an attempt to catch broad-band or 00430 time-persistent RFI that the above polynomial fits will mistakenly fit 00431 as the clean band. It is an approximation to the sumThreshold 00432 method found to be effective by Offringa et.al (2010) for LOFAR data. 00433 The 'std' option chooses to flag a point, if the 'local' stddev calculated 00434 in a window centered on that point is larger than N-stddev/2.0. 00435 This option is an attempt to catch noisy RFI that is not excluded in the 00436 polynomial fits, and which increases the global stddev, and results in 00437 fewer flags (based on the N-stddev threshold). 00438 00439 halfwin -- Half width of sliding window to use with 'usewindowstats' 00440 default: 1 (a 3-point window size) 00441 options: 1,2,3 00442 Note: This is experimental ! 00443 00444 rflag -- Detect outliers based on the RFLAG algorithm (ref. E.Greisen, AIPS, 2011). 00445 The polarization expression is given by the correlation parameter. 00446 00447 Iterate through the data in chunks of time. For each chunk, calculate local 00448 statistics, and apply flags based on user supplied (or auto-calculated) thresholds. 00449 00450 Step 1 : Time analysis (for each channel) 00451 -- calculate local rms of real and imag visibilities, within a sliding time window 00452 -- calculate the median rms across time windows, deviations of local rms from 00453 this median, and the median deviation 00454 -- flag if local rms is larger than timedevscale x (medianRMS + medianDev) 00455 00456 Step 2 : Spectral analysis (for each time) 00457 -- calculate avg of real and imag visibilities and their rms across channels 00458 -- calculate the deviation of each channel from this avg, and the median-deviation 00459 -- flag if deviation is larger than freqdevscale x medianDev 00460 00461 Example usage : 00462 00463 (1) Calculate thresholds automatically per scan, and use them to find flags. 00464 Specify scale-factor for time-analysis thresholds, use default for frequency. 00465 00466 flagdata('my.ms', mode='rflag',spw='9',timedevscale=4.0) 00467 00468 (2) Supply noise-estimates to be used with default scale-factors. 00469 00470 flagdata(vis='my.ms', mode='rflag', spw='9', timedev=0.1, freqdev=0.5, action='calculate) 00471 00472 (3) Two-passes. This replicates the usage pattern in AIPS. 00473 00474 -- The first pass saves commands in an output text files, with auto-calculated 00475 thresholds. Thresholds are returned from rflag only when action='calculate'. 00476 The user can edit this file before doing the second pass, 00477 but the python-dictionary structure must be preserved. 00478 00479 -- The second pass applies these commands (action='apply'). 00480 00481 flagdata(vis='my.ms', mode='rflag', spw='9,10', timedev='tdevfile.txt', 00482 freqdev='fdevfile.txt', action='calculate') 00483 flagdata(vis='my.ms', mode='rflag', spw='9,10', timedev='tdevfile.txt', 00484 freqdev='fdevfile.txt', action='apply') 00485 00486 With action='calculate', display='report' will produce diagnostic plots 00487 showing data-statistics and thresholds (the same thresholds as those 00488 written out to 'tdevfile.txt' and 'fdevfile.txt'). 00489 00490 Note : The rflag algorithm was originally developed by Eric Greisen in 00491 AIPS (31DEC11). 00492 AIPS documentation : Section E.5 of the AIPS cookbook 00493 (Appendix E : Special Considerations for EVLA data calibration and imaging in AIPS, 00494 http://www.aips.nrao.edu/cook.html#CEE ) 00495 00496 Note : Since this algorithm operates with two passes through each 00497 chunk of data (time and freq axes), some data points get flagged 00498 twice. This can affect the flag-percentage estimate printed in the 00499 logger at runtime. An accurate estimate can be obtained via the 00500 summary mode. 00501 00502 ntime -- Timerange (in seconds or minutes) over which to buffer data before running the algorithm. 00503 options: 'scan' or any other float value or string containing the units. 00504 default: 'scan' 00505 example: '1.5min' 00506 : 1.2 (taken in seconds) 00507 The dataset will be iterated through in time-chunks defined here. 00508 00509 combinescans -- accumulate data across scans. 00510 default: False 00511 It only makes sense when ntime not equal 'scan'. 00512 00513 datacolumn -- Column to use for flagging. 00514 default: 'DATA' 00515 options: 'DATA', 'CORRECTED','MODEL', 'RESIDUAL, 'RESIDUAL_DATA', 00516 'FPARAM', 'CPARAM', 'SNR'. 00517 00518 NOTE: by default for calibration tables, the framework will try to use FPARAM; 00519 if that does not exist, it will try to use CPARAM. If none of them exist, it 00520 will give an error and exit. 00521 00522 00523 winsize -- number of timesteps in the sliding time window ( fparm(1) in AIPS ). 00524 default: 3 00525 00526 timedev -- time-series noise estimate ( noise in AIPS ). 00527 default: [] 00528 Examples : 00529 timedev = 0.5 : Use this noise-estimate to calculate flags. Do not recalculate. 00530 timedev = [ [1,9,0.2], [1,10,0.5] ] : Use noise-estimate of 0.2 for field 1, 00531 spw 9, and noise-estimate of 0.5 for field 1, spw 10. 00532 timedev = [] : Auto-calculate noise estimates. 00533 00534 00535 freqdev -- spectral noise estimate ( scutoff in AIPS ). 00536 This step depends on having a relatively-flat bandshape. 00537 Same parameter-options as 'timedev'. 00538 default: [] 00539 00540 00541 timedevscale -- For Step 1 (time analysis), flag a point if local rms around it 00542 is larger than 'timedevscale' x 'timedev' ( fparm(0) in AIPS ) 00543 default: 5.0 00544 00545 freqdevscale -- For Step 2 (spectral analysis), flag a point if local rms around it 00546 is larger than 'freqdevscale' x 'freqdev' ( fparm(10) in AIPS ) 00547 default: 5.0 00548 00549 spectralmax -- Flag whole spectrum if 'freqdev' is greater than spectralmax ( fparm(6) in AIPS ) 00550 default: 1E6 00551 00552 spectralmin -- Flag whole spectrum if 'freqdev' is less than spectralmin ( fparm(5) in AIPS ) 00553 default: 0.0 00554 00555 00556 00557 extend -- Extend and/or grow flags beyond what the basic algorithms detect. 00558 This mode will extend the accumulated flags available in the MS, 00559 regardless of which algorithm created them. 00560 00561 Extensions will apply only within the selected data, according to the settings 00562 of extendpols,growtime,growfreq,growaround, flagneartime,flagnearfreq. 00563 00564 Note : Runtime summary counts in the logger can sometimes report larger 00565 flag percentages than what is actually flagged. This is because 00566 extensions onto already-flagged data-points are counted as new flags. 00567 An accurate flag count can be obtained via the summary mode. 00568 00569 ntime -- Timerange (in seconds or minutes) over which to buffer data before running 00570 the algorithm. 00571 options: 'scan' or any other float value or string containing the units. 00572 default: 'scan' 00573 example: '1.5min' 00574 : 1.2 (taken in seconds) 00575 The dataset will be iterated through in time-chunks defined here. 00576 00577 combinescans -- accumulate data across scans. 00578 default: False 00579 It only makes sense when ntime not equal 'scan'. 00580 00581 extendpols -- Extend flags to all selected correlations 00582 default: True 00583 options: True/False 00584 For example, to extend flags from RR to only RL and LR, a data-selection 00585 of correlation='RR,LR,RL' is required along with extendpols=True. 00586 00587 growtime -- For any channel, flag the entire timerange in the current 2D chunk ( 00588 set by 'ntime') if more than X% of the timerange is already flagged. 00589 default: 50.0 00590 options: 0.0 - 100.0 00591 This option catches the low-intensity parts of time-persistent RFI. 00592 00593 growfreq -- For any timestep, flag all channels in the current 2D chunk (set by 00594 data-selection) if more than X% of the channels are already flagged. 00595 default: 50.0 00596 options: 0.0 - 100.0 00597 This option catches broad-band RFI that is partially identified by earlier steps. 00598 00599 growaround -- Flag a point based on the number of flagged points around it. 00600 default: False 00601 options: True/False 00602 For every un-flagged point on the 2D time/freq plane, if more than four 00603 surrounding points are already flagged, flag that point. 00604 This option catches some wings of strong RFI spikes. 00605 00606 flagneartime -- Flag points before and after every flagged one, in the time-direction. 00607 default: False 00608 options: True/False 00609 Note: This can result in excessive flagging. 00610 00611 flagnearfreq -- Flag points before and after every flagged one, in the frequency-direction 00612 default: False 00613 options: True/False 00614 This option allows flagging of wings in the spectral response of strong RFI. 00615 Note: This can result in excessive flagging. 00616 00617 00618 unflag -- Unflag according to the data selection specified. 00619 00620 summary -- List the number of rows and data points flagged. 00621 00622 minrel -- Minimum number of flags (relative) to include in histogram 00623 default: 0.0 00624 00625 maxrel -- Maximum number of flags (relative) to include in histogram 00626 default: 1.0 00627 00628 minabs -- Minimum number of flags (absolute, inclusive) to include in histogram 00629 default: 0 00630 00631 maxabs -- Maximum number of flags (absolute, inclusive) to include in histogram 00632 To indicate infinity, use any negative number. 00633 default: -1 00634 00635 spwchan -- list the number of flags per spw and per channel. 00636 default: False 00637 00638 spwcorr -- list the number of flags per spw and per correlation. 00639 default: False 00640 00641 basecnt -- list the number of flags per baseline 00642 default: False 00643 00644 In summary mode, the task returns a dictionary of flagging statistics. 00645 Example: 00646 00647 s = flagdata(..., mode='summary') 00648 00649 Then s will be a dictionary which contains 00650 s['total'] : total number of data 00651 s['flagged'] : amount of flagged data 00652 00653 00654 00655 action -- Action to perform in MS/cal table or in the input list of parameters. 00656 options: 'none', 'apply','calculate' 00657 default: 'apply' 00658 00659 'apply' -- Apply the flags to the MS. 00660 00661 display -- Display data and/or end-of-MS reports at run-time. 00662 default: 'none' 00663 options: 'none', 'data', 'report', 'both' 00664 00665 'none' --> It will not display anything. 00666 00667 'data' --> display data and flags per-chunk at run-time, within an interactive GUI. 00668 00669 This option opens a GUI to show the 2D time-freq planes of 00670 the data with old and new flags, for all correlations per baseline. 00671 -- The GUI allows stepping through all baselines (prev/next) in 00672 the current chunk (set by 'ntime'), and stepping to the next-chunk. 00673 -- The 'flagdata' task can be quit from the GUI, in case it becomes 00674 obvious that the current set of parameters is just wrong. 00675 -- There is an option to stop the display but continue flagging. 00676 00677 'report' --> displays end-of-MS reports on the screen. 00678 00679 'both' --> displays data per chunk and end-of-MS reports on the screen 00680 00681 flagbackup -- Automatically backup flags before running the tool. 00682 Flagversion names are chosen automatically, and are based on the 00683 mode being used. 00684 default: True 00685 options: True/False 00686 00687 'calculate' -- Only calculate the flags but do not write them to the MS. This is 00688 useful if used together with the display to analyse the results before 00689 writing to the MS. 00690 00691 display -- Display data and/or end-of-MS reports at run-time. See extended description 00692 above. 00693 default: 'none' 00694 options: 'none', 'data', 'report', 'both' 00695 00696 ' ' -- When set to empty, the underlying tool will not be executed and no flags 00697 will be produced. No data selection will be done either. This is useful 00698 when used together with the parameter savepars to only save the current 00699 parameters (or list of parameters) to the FLAG_CMD sub-table or to an 00700 external file. 00701 00702 00703 savepars -- Save the current parameters to the FLAG_CMD table of the MS or to an output text file. 00704 Note that when display is set to anything other than 'none', savepars 00705 will be disabled. This is done because in an interactive mode, the user 00706 may skip data which may invalidate the initial input parameters and there 00707 is no way to save the interactive commands. 00708 default: False 00709 options: True/False 00710 00711 cmdreason -- A string containing a reason to save to the FLAG_CMD table or to an 00712 output text file given by the outfile sub-parameter. If the input 00713 contains any reason, they will be replaced with this one. At the 00714 moment it is not possible to add more than one reason. 00715 default: ' '; no reason will be added to output. 00716 example: cmdreason='CLIP_ZEROS' 00717 00718 outfile -- Name of output file to save the current parameters. 00719 default: ' '; it will save the parameters to the FLAG_CMD table of the MS. 00720 example: outfile='flags.txt' will save the parameters in a text file. 00721 00722 00723 ---- EXAMPLES ---- 00724 00725 NOTE: The vector mode of the previous flagdata task can be achieved with this task by using it 00726 with mode='list' and the commands given in a list in inpmode=[]. Example: 00727 00728 flagdata('my.ms', inpmode='list', inpfile=["mode='clip' clipzeros=True","mode='shadow']) 00729 00730 00731 1) Manually flag scans 1~3 and save the parameters to the FLAG_CMD sub-table. 00732 00733 flagdata('my.ms', scan='1~3, mode='manual', savepars=True) 00734 00735 2) Save the parameters to a file that is open in append mode. 00736 00737 flagdata('my.ms', scan='1~3, mode='manual', savepars=True, outfile='flags.txt') 00738 00739 3a) Flag all the commands given in the Python list of strings. 00740 00741 cmd = ["scan='1~3' mode='manual'", 00742 "spw='9' mode='tfcrop' correlation='ABS_RR,LL' ntime=51.0", 00743 "mode='extend' extendpols=True"] 00744 00745 flagdata('my.ms', mode='list', inpfile=cmd) 00746 00747 3b) Flag all the commands given in the file called flags.txt. 00748 00749 > cat flags.txt 00750 scan='1~3' mode='manual' 00751 spw='9' mode='tfcrop' correlation='ABS_RR,LL' ntime=51.0 00752 mode='extend' extendpols=True 00753 00754 flagdata('my.ms', mode='list', inpfile='flags.txt') 00755 00756 4) Display the data and flags per-chunk and do not write flags to the MS. 00757 00758 flagdata('my.ms', mode='list', inpfile='flags.txt', action='calculate', display='data') 00759 00760 5) Flag all the antennas except antenna=5. 00761 00762 flagdata(vis='my.ms', antenna='!5', mode='manual) 00763 00764 6) Clip the NaN in the data. An empty clipminmax will flag only NaN. 00765 00766 flagdata('my.ms', mode='clip') 00767 00768 7) Clip only the water vapour radiometer data. 00769 00770 flagdata('my.ms',mode='clip',clipminmax=[0,50], correlation='ABS_WVR') 00771 00772 8) Clip only zero-value data. 00773 00774 flagdata('my.ms',mode='clip',clipzeros=True) 00775 00776 9a) Flag only auto-correlations using the autocorr parameter. 00777 00778 flagdata('my.ms', autocorr=True) 00779 00780 9b) Flag only auto-correlations using the antenna selection. 00781 00782 flagdata('my.ms', mode='manual', antenna='*&&&') 00783 00784 10a) Flag based on selected reasons from a file. 00785 00786 > cat flags.txt 00787 scan='1~3' mode='manual' reason='MYREASON' 00788 spw='9' mode='clip' clipzeros=True reason='CLIPZEROS' 00789 mode='manual' scan='4' reason='MYREASON' 00790 00791 flagdata('my.ms', mode='list', inpfile='flags.txt', reason='MYREASON') 00792 00793 10b) The same result of 10a can be achieved using the task flagcmd. 00794 00795 flagcmd('my.ms', inpmode='file', inpfile='flags.txt', action='apply', reason='MYREASON') 00796 00797 11) Automatic flagging using 'rflag', using auto-thresholds, and specifying 00798 a threshold scale-factor to use for flagging. 00799 00800 flagdata('my.ms', mode='rflag',spw='9',timedevscale=4.0,action='apply') 00801 00802 12) Save the interface parameters to the FLAG_CMD sub-table of the MS. Add a reason 00803 to the flag command. This cmdreason will be added to the REASON column of the 00804 FLAG_CMD sub-table. Apply flags in flagcmd. 00805 00806 flagdata('my.ms', mode='clip',channelavg=False, clipminmax=[30., 60.], spw='0:0~10', 00807 correlation='ABS_XX,XY', action='', savepars=True, cmdreason='CLIPXX_XY') 00808 00809 > Select based on the reason. 00810 flagcmd('my.ms', action='apply', reason='CLIPXX_XY') 00811 00812 13) Flag antennas that are shadowed by antennas not present in the MS. 00813 00814 > Create a text file with information about the antennas. 00815 > cat ant.txt 00816 name=VLA01 00817 diameter=25.0 00818 position=[-1601144.96146691, -5041998.01971858, 3554864.76811967] 00819 name=VLA02 00820 diameter=25.0 00821 position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178] 00822 name=VLA09 00823 diameter=25.0 00824 position=[-1601197.2182404203, -5041974.3604805721, 3554875.1995636248] 00825 name=VLA10 00826 diameter=25.0 00827 position=[-1601227.3367843349,-5041975.7011900628,3554859.1642644769] 00828 00829 00830 flagdata('my.vis', mode='shadow', tolerance=10.0, addantenna='ant.txt') 00831 00832 The antenna information can also be given as a Python dictionary. To create the 00833 dictionary using the flaghelper functions, do the following inside casapy: 00834 00835 > import flaghelper as fh 00836 > antdic = fh.readAntennaList(antfile) 00837 00838 flagdata('my.vis', mode='shadow', tolerance=10.0, addantenna=antdic) 00839 00840 14) Apply the online flags that come from importasdm. 00841 00842 > In importasdm, save the online flags to a file. 00843 importasdm('myasdm', 'asdm.ms', process_flags=True, savecmds=True, outfile='online_flags.txt') 00844 00845 > You can edit the online_flags.txt to add other flagging commands or apply it directly. 00846 flagdata('asdm.ms', mode='list', inpfile='online_flags.txt') 00847 00848 > The same result can be achieved using the task flagcmd. 00849 flagcmd('asdm.ms', inpmode='file', inpfile='online_flags.txt', action='apply') 00850 00851 15) Clip zero data from a calibration table. 00852 00853 flagdata('cal-X54.B1', mode='clip', clipzeros=True, datacolumn='CPARAM') 00854 00855 16) Clip data with SNR <4.0. 00856 00857 flagdata('cal-X54.B1', mode='clip', clipminmax=[0.0,4.0], clipoutside=False, datacolumn='SNR') 00858 00859 00860 ---- SYNTAX FOR COMMANDS GIVEN IN A FILE or LIST OF STRINGS ---- 00861 00862 Basic Syntax Rules 00863 00864 Commands are strings (which may contain internal "strings") consisting of 00865 KEY=VALUE pairs separated by whitespace. 00866 00867 NOTE: There should be no whitespace between KEY=VALUE or within each KEY or 00868 VALUE, since the simple parser first breaks command lines on whitespace, 00869 then on "=". 00870 00871 Use only white spaces to separate the parameters (no commas). 00872 00873 Each key should only appear once on a given command line/string. 00874 00875 There is an implicit "mode" for each command, with the default 00876 being 'manual' if not given. 00877 00878 Comment lines can start with '#' and will be ignored. 00879 00880 Example: 00881 00882 scan='1~3' mode='manual' 00883 # this line will ignored 00884 spw='9' mode='tfcrop' correlation='ABS_XX,YY' ntime=51.0 00885 mode='extend' extendpols=True 00886 scan='1~3,10~12' mode='quack' quackinterval=1.0 00887 00888 00889 """ 00890 a=inspect.stack() 00891 stacklevel=0 00892 for k in range(len(a)): 00893 if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 00894 stacklevel=k 00895 break 00896 myf=sys._getframe(stacklevel).f_globals 00897 myf['__last_task'] = 'flagdata' 00898 myf['taskname'] = 'flagdata' 00899 ### 00900 myf['update_params'](func=myf['taskname'],printtext=False) 00901 ### 00902 ### 00903 #Handle globals or user over-ride of arguments 00904 # 00905 function_signature_defaults=dict(zip(self.__call__.func_code.co_varnames,self.__call__.func_defaults)) 00906 useLocalDefaults = False 00907 00908 for item in function_signature_defaults.iteritems(): 00909 key,val = item 00910 keyVal = eval(key) 00911 if (keyVal == None): 00912 #user hasn't set it - use global/default 00913 pass 00914 else: 00915 #user has set it - use over-ride 00916 if (key != 'self') : 00917 useLocalDefaults = True 00918 #myf[key]=keyVal 00919 00920 myparams = {} 00921 if useLocalDefaults : 00922 for item in function_signature_defaults.iteritems(): 00923 key,val = item 00924 keyVal = eval(key) 00925 exec('myparams[key] = keyVal') 00926 if (keyVal == None): 00927 exec('myparams[key] = '+ key + ' = self.itsdefault(key)') 00928 keyVal = eval(key) 00929 if(type(keyVal) == dict) : 00930 exec('myparams[key] = ' + key + ' = keyVal[len(keyVal)-1][\'value\']') 00931 00932 else : 00933 uselessvariable = None 00934 myparams['vis'] = vis = myf['vis'] 00935 myparams['mode'] = mode = myf['mode'] 00936 myparams['autocorr'] = autocorr = myf['autocorr'] 00937 myparams['inpfile'] = inpfile = myf['inpfile'] 00938 myparams['reason'] = reason = myf['reason'] 00939 myparams['spw'] = spw = myf['spw'] 00940 myparams['field'] = field = myf['field'] 00941 myparams['antenna'] = antenna = myf['antenna'] 00942 myparams['uvrange'] = uvrange = myf['uvrange'] 00943 myparams['timerange'] = timerange = myf['timerange'] 00944 myparams['correlation'] = correlation = myf['correlation'] 00945 myparams['scan'] = scan = myf['scan'] 00946 myparams['intent'] = intent = myf['intent'] 00947 myparams['array'] = array = myf['array'] 00948 myparams['observation'] = observation = myf['observation'] 00949 myparams['feed'] = feed = myf['feed'] 00950 myparams['clipminmax'] = clipminmax = myf['clipminmax'] 00951 myparams['datacolumn'] = datacolumn = myf['datacolumn'] 00952 myparams['clipoutside'] = clipoutside = myf['clipoutside'] 00953 myparams['channelavg'] = channelavg = myf['channelavg'] 00954 myparams['clipzeros'] = clipzeros = myf['clipzeros'] 00955 myparams['quackinterval'] = quackinterval = myf['quackinterval'] 00956 myparams['quackmode'] = quackmode = myf['quackmode'] 00957 myparams['quackincrement'] = quackincrement = myf['quackincrement'] 00958 myparams['tolerance'] = tolerance = myf['tolerance'] 00959 myparams['addantenna'] = addantenna = myf['addantenna'] 00960 myparams['lowerlimit'] = lowerlimit = myf['lowerlimit'] 00961 myparams['upperlimit'] = upperlimit = myf['upperlimit'] 00962 myparams['ntime'] = ntime = myf['ntime'] 00963 myparams['combinescans'] = combinescans = myf['combinescans'] 00964 myparams['timecutoff'] = timecutoff = myf['timecutoff'] 00965 myparams['freqcutoff'] = freqcutoff = myf['freqcutoff'] 00966 myparams['timefit'] = timefit = myf['timefit'] 00967 myparams['freqfit'] = freqfit = myf['freqfit'] 00968 myparams['maxnpieces'] = maxnpieces = myf['maxnpieces'] 00969 myparams['flagdimension'] = flagdimension = myf['flagdimension'] 00970 myparams['usewindowstats'] = usewindowstats = myf['usewindowstats'] 00971 myparams['halfwin'] = halfwin = myf['halfwin'] 00972 myparams['winsize'] = winsize = myf['winsize'] 00973 myparams['timedev'] = timedev = myf['timedev'] 00974 myparams['freqdev'] = freqdev = myf['freqdev'] 00975 myparams['timedevscale'] = timedevscale = myf['timedevscale'] 00976 myparams['freqdevscale'] = freqdevscale = myf['freqdevscale'] 00977 myparams['spectralmax'] = spectralmax = myf['spectralmax'] 00978 myparams['spectralmin'] = spectralmin = myf['spectralmin'] 00979 myparams['extendpols'] = extendpols = myf['extendpols'] 00980 myparams['growtime'] = growtime = myf['growtime'] 00981 myparams['growfreq'] = growfreq = myf['growfreq'] 00982 myparams['growaround'] = growaround = myf['growaround'] 00983 myparams['flagneartime'] = flagneartime = myf['flagneartime'] 00984 myparams['flagnearfreq'] = flagnearfreq = myf['flagnearfreq'] 00985 myparams['minrel'] = minrel = myf['minrel'] 00986 myparams['maxrel'] = maxrel = myf['maxrel'] 00987 myparams['minabs'] = minabs = myf['minabs'] 00988 myparams['maxabs'] = maxabs = myf['maxabs'] 00989 myparams['spwchan'] = spwchan = myf['spwchan'] 00990 myparams['spwcorr'] = spwcorr = myf['spwcorr'] 00991 myparams['basecnt'] = basecnt = myf['basecnt'] 00992 myparams['action'] = action = myf['action'] 00993 myparams['display'] = display = myf['display'] 00994 myparams['flagbackup'] = flagbackup = myf['flagbackup'] 00995 myparams['savepars'] = savepars = myf['savepars'] 00996 myparams['cmdreason'] = cmdreason = myf['cmdreason'] 00997 myparams['outfile'] = outfile = myf['outfile'] 00998 00999 01000 result = None 01001 01002 # 01003 # The following is work around to avoid a bug with current python translation 01004 # 01005 mytmp = {} 01006 01007 mytmp['vis'] = vis 01008 mytmp['mode'] = mode 01009 mytmp['autocorr'] = autocorr 01010 mytmp['inpfile'] = inpfile 01011 mytmp['reason'] = reason 01012 mytmp['spw'] = spw 01013 mytmp['field'] = field 01014 mytmp['antenna'] = antenna 01015 mytmp['uvrange'] = uvrange 01016 mytmp['timerange'] = timerange 01017 mytmp['correlation'] = correlation 01018 mytmp['scan'] = scan 01019 mytmp['intent'] = intent 01020 mytmp['array'] = array 01021 mytmp['observation'] = observation 01022 mytmp['feed'] = feed 01023 mytmp['clipminmax'] = clipminmax 01024 mytmp['datacolumn'] = datacolumn 01025 mytmp['clipoutside'] = clipoutside 01026 mytmp['channelavg'] = channelavg 01027 mytmp['clipzeros'] = clipzeros 01028 mytmp['quackinterval'] = quackinterval 01029 mytmp['quackmode'] = quackmode 01030 mytmp['quackincrement'] = quackincrement 01031 mytmp['tolerance'] = tolerance 01032 mytmp['addantenna'] = addantenna 01033 mytmp['lowerlimit'] = lowerlimit 01034 mytmp['upperlimit'] = upperlimit 01035 mytmp['ntime'] = ntime 01036 mytmp['combinescans'] = combinescans 01037 mytmp['timecutoff'] = timecutoff 01038 mytmp['freqcutoff'] = freqcutoff 01039 mytmp['timefit'] = timefit 01040 mytmp['freqfit'] = freqfit 01041 mytmp['maxnpieces'] = maxnpieces 01042 mytmp['flagdimension'] = flagdimension 01043 mytmp['usewindowstats'] = usewindowstats 01044 mytmp['halfwin'] = halfwin 01045 mytmp['winsize'] = winsize 01046 mytmp['timedev'] = timedev 01047 mytmp['freqdev'] = freqdev 01048 mytmp['timedevscale'] = timedevscale 01049 mytmp['freqdevscale'] = freqdevscale 01050 mytmp['spectralmax'] = spectralmax 01051 mytmp['spectralmin'] = spectralmin 01052 mytmp['extendpols'] = extendpols 01053 mytmp['growtime'] = growtime 01054 mytmp['growfreq'] = growfreq 01055 mytmp['growaround'] = growaround 01056 mytmp['flagneartime'] = flagneartime 01057 mytmp['flagnearfreq'] = flagnearfreq 01058 mytmp['minrel'] = minrel 01059 mytmp['maxrel'] = maxrel 01060 mytmp['minabs'] = minabs 01061 mytmp['maxabs'] = maxabs 01062 mytmp['spwchan'] = spwchan 01063 mytmp['spwcorr'] = spwcorr 01064 mytmp['basecnt'] = basecnt 01065 mytmp['action'] = action 01066 mytmp['display'] = display 01067 mytmp['flagbackup'] = flagbackup 01068 mytmp['savepars'] = savepars 01069 mytmp['cmdreason'] = cmdreason 01070 mytmp['outfile'] = outfile 01071 pathname='file:///'+os.environ.get('CASAPATH').split()[0]+'/share/xml/' 01072 trec = casac.utils().torecord(pathname+'flagdata.xml') 01073 01074 casalog.origin('flagdata') 01075 if not trec.has_key('flagdata') or not casac.utils().verify(mytmp, trec['flagdata']) : 01076 return False 01077 01078 01079 try : 01080 casalog.post('') 01081 casalog.post('##########################################') 01082 casalog.post('##### Begin Task: flagdata #####') 01083 casalog.post('') 01084 result = flagdata(vis, mode, autocorr, inpfile, reason, spw, field, antenna, uvrange, timerange, correlation, scan, intent, array, observation, feed, clipminmax, datacolumn, clipoutside, channelavg, clipzeros, quackinterval, quackmode, quackincrement, tolerance, addantenna, lowerlimit, upperlimit, ntime, combinescans, timecutoff, freqcutoff, timefit, freqfit, maxnpieces, flagdimension, usewindowstats, halfwin, winsize, timedev, freqdev, timedevscale, freqdevscale, spectralmax, spectralmin, extendpols, growtime, growfreq, growaround, flagneartime, flagnearfreq, minrel, maxrel, minabs, maxabs, spwchan, spwcorr, basecnt, action, display, flagbackup, savepars, cmdreason, outfile) 01085 casalog.post('') 01086 casalog.post('##### End Task: flagdata #####') 01087 casalog.post('##########################################') 01088 01089 01090 # saveinputs for individule engine has no use 01091 # saveinputs should alos be removed from casa_in_py.py 01092 # 01093 # 01094 # saveinputs = myf['saveinputs'] 01095 # saveinputs('flagdata', 'flagdata.last', myparams) 01096 # 01097 # 01098 except Exception, instance: 01099 #print '**** Error **** ',instance 01100 pass 01101 01102 gc.collect() 01103 return result 01104 # 01105 # 01106 ## 01107 # def paramgui(self, useGlobals=True): 01108 # """ 01109 # Opens a parameter GUI for this task. If useGlobals is true, then any relevant global parameter settings are used. 01110 # """ 01111 # import paramgui 01112 # 01113 # a=inspect.stack() 01114 # stacklevel=0 01115 # for k in range(len(a)): 01116 # if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 01117 # stacklevel=k 01118 # break 01119 # myf = sys._getframe(stacklevel).f_globals 01120 # 01121 # if useGlobals: 01122 # paramgui.setGlobals(myf) 01123 # else: 01124 # paramgui.setGlobals({}) 01125 # 01126 # paramgui.runTask('flagdata', myf['_ip']) 01127 # paramgui.setGlobals({}) 01128 # 01129 # 01130 # 01131 # 01132 def defaults(self, param=None): 01133 a=inspect.stack() 01134 stacklevel=0 01135 for k in range(len(a)): 01136 if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 01137 stacklevel=k 01138 break 01139 myf=sys._getframe(stacklevel).f_globals 01140 a = odict() 01141 a['vis'] = '' 01142 a['mode'] = 'manual' 01143 a['action'] = 'apply' 01144 a['savepars'] = False 01145 01146 a['async']=False 01147 a['mode'] = { 01148 0:odict([{'value':'manual'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'autocorr':False}]), 01149 1:odict([{'value':'manualflag'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'autocorr':False}]), 01150 2:odict([{'value':'list'}, {'inpfile':''}, {'reason':'any'}]), 01151 3:odict([{'value':'clip'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'datacolumn':'DATA'}, {'clipminmax':[]}, {'clipoutside':True}, {'channelavg':False}, {'clipzeros':False}]), 01152 4:odict([{'value':'quack'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'quackinterval':0.0}, {'quackmode':'beg'}, {'quackincrement':False}]), 01153 5:odict([{'value':'shadow'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'tolerance':0.0}, {'addantenna':''}]), 01154 6:odict([{'value':'elevation'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'lowerlimit':0.0}, {'upperlimit':90.0}]), 01155 7:odict([{'value':'tfcrop'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'ntime':'scan'}, {'combinescans':False}, {'datacolumn':'DATA'}, {'timecutoff':4.0}, {'freqcutoff':3.0}, {'timefit':'line'}, {'freqfit':'poly'}, {'maxnpieces':7}, {'flagdimension':'freqtime'}, {'usewindowstats':'none'}, {'halfwin':1}]), 01156 8:odict([{'value':'rflag'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'ntime':'scan'}, {'combinescans':False}, {'datacolumn':'DATA'}, {'winsize':3}, {'timedev':''}, {'freqdev':''}, {'timedevscale':5.0}, {'freqdevscale':5.0}, {'spectralmax':1E6}, {'spectralmin':0.0}]), 01157 9:odict([{'value':'extend'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'ntime':'scan'}, {'combinescans':False}, {'extendpols':True}, {'growtime':50.0}, {'growfreq':50.0}, {'growaround':False}, {'flagneartime':False}, {'flagnearfreq':False}]), 01158 10:odict([{'value':'unflag'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}]), 01159 11:odict([{'value':'summary'}, {'field':''}, {'spw':''}, {'antenna':''}, {'timerange':''}, {'correlation':''}, {'scan':''}, {'intent':''}, {'array':''}, {'uvrange':''}, {'observation':''}, {'feed':''}, {'minrel':0.0}, {'maxrel':1.0}, {'minabs':0}, {'maxabs':-1}, {'spwchan':False}, {'spwcorr':False}, {'basecnt':False}])} 01160 a['action'] = { 01161 0:odict([{'value':'apply'}, {'display':''}, {'flagbackup':True}]), 01162 1:odict([{'value':'calculate'}, {'display':''}]), 01163 2:{'value':''}, 01164 3:{'value':'none'}} 01165 a['savepars'] = { 01166 0:{'value':False}, 01167 1:odict([{'value':True}, {'cmdreason':''}, {'outfile':''}])} 01168 01169 ### This function sets the default values but also will return the list of 01170 ### parameters or the default value of a given parameter 01171 if(param == None): 01172 myf['__set_default_parameters'](a) 01173 elif(param == 'paramkeys'): 01174 return a.keys() 01175 else: 01176 if(a.has_key(param)): 01177 #if(type(a[param]) == dict) : 01178 # return a[param][len(a[param])-1]['value'] 01179 #else : 01180 return a[param] 01181 01182 01183 # 01184 # 01185 def check_params(self, param=None, value=None): 01186 a=inspect.stack() 01187 stacklevel=0 01188 for k in range(len(a)): 01189 if (string.find(a[k][1], 'ipython console') > 0) or (string.find(a[k][1], '<string>') >= 0): 01190 stacklevel=k 01191 break 01192 myf=sys._getframe(stacklevel).f_globals 01193 01194 # print 'param:', param, 'value:', value 01195 try : 01196 if str(type(value)) != "<type 'instance'>" : 01197 value0 = value 01198 value = myf['cu'].expandparam(param, value) 01199 matchtype = False 01200 if(type(value) == numpy.ndarray): 01201 if(type(value) == type(value0)): 01202 myf[param] = value.tolist() 01203 else: 01204 #print 'value:', value, 'value0:', value0 01205 #print 'type(value):', type(value), 'type(value0):', type(value0) 01206 myf[param] = value0 01207 if type(value0) != list : 01208 matchtype = True 01209 else : 01210 myf[param] = value 01211 value = myf['cu'].verifyparam({param:value}) 01212 if matchtype: 01213 value = False 01214 except Exception, instance: 01215 #ignore the exception and just return it unchecked 01216 myf[param] = value 01217 return value 01218 01219 # 01220 # 01221 def description(self, key='flagdata', subkey=None): 01222 desc={'flagdata': 'All-purpose flagging task based on data-selections and flagging modes/algorithms.', 01223 'vis': 'Name of MS file or calibration table to flag', 01224 'mode': 'Flagging mode (list/manual/clip/shadow/quack/\ 01225 elevation/tfcrop/rflag/extend/unflag/summary)', 01226 'autocorr': 'Flag auto-correlations', 01227 'inpfile': 'Input ASCII file or Python list of strings with flag commands.', 01228 'reason': 'Select by REASON types', 01229 'spw': 'Spectral-window/frequency/channel: \'\' ==> all, spw=\'0:17~19\'', 01230 'field': 'Field names or field index numbers: \'\' ==> all, field=\'0~2,3C286\'', 01231 'antenna': 'Antenna/baselines: \'\' ==> all, antenna =\'3,VA04\'', 01232 'uvrange': 'UV range: \'\' ==> all; uvrange =\'0~100klambda\', default units=meters', 01233 'timerange': 'Time range: \'\' ==> all,timerange=\'09:14:0~09:54:0\'', 01234 'correlation': 'Correlation: \'\' ==> all, correlation=\'XX,YY\'', 01235 'scan': 'Scan numbers: \'\' ==> all', 01236 'intent': 'Observation intent: \'\' ==> all, intent=\'CAL*POINT*\'', 01237 'array': '(Sub)array numbers: \'\' ==> all', 01238 'observation': 'Observation ID: \'\' ==> all', 01239 'feed': 'Multi-feed numbers: Not yet implemented', 01240 'clipminmax': 'Range to use for clipping', 01241 'datacolumn': 'Data column on which to operate (data,corrected,model,residual)', 01242 'clipoutside': 'Clip outside the range, or within it', 01243 'channelavg': 'Average over channels (scalar average)', 01244 'clipzeros': 'Clip zero-value data', 01245 'quackinterval': 'Quack n seconds from scan beginning or end', 01246 'quackmode': 'Quack mode. \'beg\' ==> first n seconds of scan.\'endb\' ==> last n seconds of scan. \'end\' ==> all but first n seconds of scan. \'tail\' ==> all but last n seconds of scan.', 01247 'quackincrement': 'Flag incrementally in time?', 01248 'tolerance': 'Amount of shadow allowed (in meters)', 01249 'addantenna': 'File name or dictionary with additional antenna names, positions and diameters', 01250 'lowerlimit': 'Lower limiting elevation (in degrees)', 01251 'upperlimit': 'Upper limiting elevation (in degrees)', 01252 'ntime': 'Time-range to use for each chunk (in seconds or minutes)', 01253 'combinescans': 'Accumulate data across scans.', 01254 'timecutoff': 'Flagging thresholds in units of deviation from the fit', 01255 'freqcutoff': 'Flagging thresholds in units of deviation from the fit', 01256 'timefit': 'Fitting function for the time direction (poly/line)', 01257 'freqfit': 'Fitting function for the frequency direction (poly/line)', 01258 'maxnpieces': 'Number of pieces in the polynomial-fits (for \'freqfit\' or \'timefit\' = \'poly\')', 01259 'flagdimension': 'Dimensions along which to calculate fits (freq/time/freqtime/timefreq)', 01260 'usewindowstats': 'Calculate additional flags using sliding window statistics (none,sum,std,both)', 01261 'halfwin': 'Half-width of sliding window to use with \'usewindowstats\' (1,2,3).', 01262 'winsize': 'Number of timesteps in the sliding time window [aips:fparm(1)]', 01263 'timedev': 'Time-series noise estimate [aips:noise]', 01264 'freqdev': 'Spectral noise estimate [aips:scutoff]', 01265 'timedevscale': 'Threshold scaling for timedev [aips:fparm(9)]', 01266 'freqdevscale': 'Threshold scaling for freqdev [aips:fparm(10)]', 01267 'spectralmax': 'Flag whole spectrum if freqdev is greater than spectralmax [aips:fparm(6)]', 01268 'spectralmin': 'Flag whole spectrum if freqdev is less than spectralmin [aips:fparm(5)] ', 01269 'extendpols': 'If any correlation is flagged, flag all correlations', 01270 'growtime': 'Flag all \'ntime\' integrations if more than X% of the timerange is flagged (0-100)', 01271 'growfreq': 'Flag all selected channels if more than X% of the frequency range is flagged(0-100)', 01272 'growaround': 'Flag data based on surrounding flags', 01273 'flagneartime': 'Flag one timestep before and after a flagged one (True/False)', 01274 'flagnearfreq': 'Flag one channel before and after a flagged one (True/False)', 01275 'minrel': 'minimum number of flags (relative)', 01276 'maxrel': 'maximum number of flags (relative)', 01277 'minabs': 'minimum number of flags (absolute)', 01278 'maxabs': 'maximum number of flags (absolute). Use a negative value to indicate infinity.', 01279 'spwchan': 'Print summary of channels per spw', 01280 'spwcorr': 'Print summary of correlation per spw', 01281 'basecnt': 'Print summary counts per baseline', 01282 'action': 'Action to perform in MS and/or in inpfile (none/apply/calculate)', 01283 'display': 'Display data and/or end-of-MS reports at runtime (data/report/both).', 01284 'flagbackup': 'Back up the state of flags before the run', 01285 'savepars': 'Save the current parameters to the FLAG_CMD table or to a file', 01286 'cmdreason': 'Reason to save to output file or to FLAG_CMD table.', 01287 'outfile': 'Name of output file to save current parameters. If empty, save to FLAG_CMD', 01288 01289 'async': 'If true the taskname must be started using flagdata(...)' 01290 } 01291 01292 # 01293 # Set subfields defaults if needed 01294 # 01295 01296 if(desc.has_key(key)) : 01297 return desc[key] 01298 01299 def itsdefault(self, paramname) : 01300 a = {} 01301 a['vis'] = '' 01302 a['mode'] = 'manual' 01303 a['autocorr'] = False 01304 a['inpfile'] = '' 01305 a['reason'] = 'any' 01306 a['spw'] = '' 01307 a['field'] = '' 01308 a['antenna'] = '' 01309 a['uvrange'] = '' 01310 a['timerange'] = '' 01311 a['correlation'] = '' 01312 a['scan'] = '' 01313 a['intent'] = '' 01314 a['array'] = '' 01315 a['observation'] = '' 01316 a['feed'] = '' 01317 a['clipminmax'] = [] 01318 a['datacolumn'] = 'DATA' 01319 a['clipoutside'] = True 01320 a['channelavg'] = False 01321 a['clipzeros'] = False 01322 a['quackinterval'] = 1.0 01323 a['quackmode'] = 'beg' 01324 a['quackincrement'] = False 01325 a['tolerance'] = 0.0 01326 a['addantenna'] = '' 01327 a['lowerlimit'] = 0.0 01328 a['upperlimit'] = 90.0 01329 a['ntime'] = 'scan' 01330 a['combinescans'] = False 01331 a['timecutoff'] = 4.0 01332 a['freqcutoff'] = 3.0 01333 a['timefit'] = 'line' 01334 a['freqfit'] = 'poly' 01335 a['maxnpieces'] = 7 01336 a['flagdimension'] = 'freqtime' 01337 a['usewindowstats'] = 'none' 01338 a['halfwin'] = 1 01339 a['winsize'] = 3 01340 a['timedev'] = '' 01341 a['freqdev'] = '' 01342 a['timedevscale'] = 5.0 01343 a['freqdevscale'] = 5.0 01344 a['spectralmax'] = 1E6 01345 a['spectralmin'] = 0.0 01346 a['extendpols'] = True 01347 a['growtime'] = 50.0 01348 a['growfreq'] = 50.0 01349 a['growaround'] = False 01350 a['flagneartime'] = False 01351 a['flagnearfreq'] = False 01352 a['minrel'] = 0.0 01353 a['maxrel'] = 1.0 01354 a['minabs'] = 0 01355 a['maxabs'] = -1 01356 a['spwchan'] = False 01357 a['spwcorr'] = False 01358 a['basecnt'] = False 01359 a['action'] = 'apply' 01360 a['display'] = '' 01361 a['flagbackup'] = True 01362 a['savepars'] = False 01363 a['cmdreason'] = '' 01364 a['outfile'] = '' 01365 01366 if a.has_key(paramname) : 01367 return a[paramname] 01368 flagdata_pg = flagdata_pg_()
1.8.0
