flagdata.py

Go to the documentation of this file.

#
# This file was generated using xslt from its XML file
#
# Copyright 2009, Associated Universities Inc., Washington DC
#
import sys
import os
from  casac import *
import string
from taskinit import casalog
#from taskmanager import tm
import task_flagdata
def flagdata(vis='', mode='manual', autocorr=False, inpfile='', reason='any', spw='', field='', antenna='', uvrange='', timerange='', correlation='', scan='', intent='', array='', observation='', feed='', clipminmax=[], datacolumn='DATA', clipoutside=True, channelavg=False, clipzeros=False, quackinterval=1.0, quackmode='beg', quackincrement=False, tolerance=0.0, addantenna='', lowerlimit=0.0, upperlimit=90.0, ntime='scan', combinescans=False, timecutoff=4.0, freqcutoff=3.0, timefit='line', freqfit='poly', maxnpieces=7, flagdimension='freqtime', usewindowstats='none', halfwin=1, winsize=3, timedev='', freqdev='', timedevscale=5.0, freqdevscale=5.0, spectralmax=1E6, spectralmin=0.0, extendpols=True, growtime=50.0, growfreq=50.0, growaround=False, flagneartime=False, flagnearfreq=False, minrel=0.0, maxrel=1.0, minabs=0, maxabs=-1, spwchan=False, spwcorr=False, basecnt=False, action='apply', display='', flagbackup=True, savepars=False, cmdreason='', outfile=''):

        """All-purpose flagging task based on data-selections and flagging modes/algorithms.

            This task can flag a Measurement Set or a calibration table. It has 
            two main types of operation. One type will read the parameters from 
            the interface and flag using any of the various available modes. The 
            other type will read the commands from a text file or a Python list of 
            strings, containing a list of flag commands (each line containing data 
            selection parameters and any parameter specific for the mode being 
            requested). Please see examples at the end of this help.
            
            It is also possible to only save the parameters set in the interface 
            without flagging. The parameters can be saved in the FLAG_CMD sub-table 
            or in a text file. Note that when saving to an external file, the parameters 
            will be appended to the given file.
            
            The available flagging modes are: manual, clip, shadow, quack, elevation, 
            tfcrop, rflag, extend, unflag and summary.

            The current flags can be automatically backed up before applying new 
            flags if the parameter flagbackup is set. Previous flag versions can be 
            recovered using the flagmanager task.
            
            NOTE on flagging calibration tables.
            -----------------------------------
            
            When the input is a calibration table, the modes 'elevation' and 'shadow'
            will be disabled. Data selection for calibration tables is limited to field,
            antenna and spw at the moment. It is only possible to save the parameters 
            to an external file. 
            
The task will flag a subset of data based on the following modes of operation:

    list        = list of flagging commands to apply to MS/cal table
    manual      = flagging based on specific selection parameters
    clip        = clip data according to values
    quack       = remove/keep specific time range at scan beginning/end
    shadow      = remove antenna-shadowed data
    elevation   = remove data below/above given elevations
    tfcrop      = automatic identification of outliers on the time-freq plane
    rflag       = automatic detection of outliers based on sliding-window RMS filters
    extend      = extend and/or grow flags beyond what the basic algorithms detect
    summary     = report the amount of flagged data
    unflag      = unflag the specified data

    Keyword arguments:
    vis -- Name of input visibility file or calibration table.
        default: '' (none) 
        example1: vis='uid___A002_X2a5c2f_X54.ms' or
        example2: vis='cal-X54.B1'

        Any flagging will only be applied to the specified selection.

              antenna -- Select data based on baseline
                    default: '' (all); example: antenna='DV04&DV06' baseline DV04-DV06
                    antenna='DV04&DV06;DV07&DV10' #baselines DV04-DV06 and DV07-DV10
                    antenna='DV06' # all cross-correlation baselines between antenna DV06 and 
                            all other available antennas
                    antenna='DV04,DV06' # all baselines with antennas DV04 and DV06
                    antenna='DV06&&DV06' # only the auto-correlation baselines for antenna DV06
                    antenna='DV04&&*' # cross and auto-correlation baselines between antenna DV04
                                             and all other available antennas
                    antenna='0~2&&&' # only the auto-correlation baselines for antennas 
                                                   in range 0~2

              spw -- Select data based on spectral window and channels
                    default: '' (all); example: spw='1'
                    spw='<2' #spectral windows less than 2
                    spw='>1' #spectral windows greater than 1
                    spw='1:0~10' # first 10 channels from spw 1
                    spw='1:0~5;120~128' # multiple separated channel chunks.

                    Note : For modes clip, tfcrop and rflag, channel-ranges can be excluded 
                    from flagging by selecting ranges such as spw='0:0~5;10~63'. This is a way to
                    protect known spectral-lines from being flagged by the autoflag algorithms.

              correlation -- Correlation types or expression.
                    default: '' (all correlations)
                            For modes clip, tfcrop or rflag, the default means ABS_ALL. If
                            the input is cal table that does not contain a complex data column,
                            the default will fall back to REAL_ALL.
                    example: correlation='XX,YY' or
                    options: Any of 'ABS', 'ARG', 'REAL', 'IMAG', 'NORM' followed by
                             any of 'ALL', 'I', 'XX', 'YY', 'RR', 'LL', 'Sol1', 'Sol2', 'WVR'
                             'WVR' refers to the water vapour radiometer of ALMA data.
                    example: correlation='REAL_XX,XY'
                  -->correlation selection is not supported for modes other than clip, tfcrop or
                      rflag in cal tables.      
                    
                    Note that the operators ABS,ARG,REAL, etc. are written only once as the first value.
                    if more than one correlation is given, the operator will be applied to all of them.
                    
                    The expression is used only in modes clip, tfcrop and rflag.

              field -- Select data based on field id(s) or name(s)
                    default: '' (all); example: field='1'
                    field='0~2' # field ids inclusive from 0 to 2
                    field='3C*' # all field names starting with 3C

              uvrange -- Select data within uvrange (default units meters)
                    default: '' (all); example:
                    uvrange='0~1000klambda'; uvrange from 0-1000 kilo-lambda
                    uvrange='>4klamda';uvranges greater than 4 kilo-lambda
                    uvrange='0~1000km'; uvrange in kilometers
                  -->uvrange selection is not supported for cal tables.                          

              timerange  -- Select data based on time range:
                    default = '' (all); example,
                    timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
                    Note: YYYY/MM/DD can be dropped as needed:
                    timerange='09:14:0~09:54:0' # this time range
                    timerange='09:44:00' # data within one integration of time
                    timerange='>10:24:00' # data after this time
                    timerange='09:44:00+00:13:00' #data 13 minutes after time
                  -->timerange selection is not supported for cal tables.      

              scan -- Select data based on scan number
                    default: '' (all); example: scan='>3'
                  -->scan selection is not supported for cal tables.                          

              intent -- Select data based on observation intent
                    default: '' (all); example: intent='*CAL*,*BAND*'
                  -->intent selection is not supported for cal tables.                          

              array -- Selection based on the antenna array
                    default: '' (all); 
                  -->array selection is not supported for cal tables.                          

              observation -- Selection based on the observation ID
                    default: '' (all); example: observation='1' or observation=1
                  -->observation selection is not supported for cal tables.                          

              feed -- Selection based on the feed - NOT IMPLEMENTED YET


    mode -- Mode of operation.
        options: 'list', 'manual','clip','quack','shadow','elevation', 'tfcrop', 'extend',
                 'unflag', 'summary'
        default: 'manual'

        list -- Flag according to the data selection and flag commands specified in the input list.
                The input list may come from a text file or from a Python list of strings.
                Each input line may contain data selection parameters and any parameter
                specific to the mode given in the line. Default values will be used for
                the parameters that are not present in the line. Each line will be taken 
                as a command to the task. If data is pre-selected using any of the selection 
                parameters, then flagging will apply only to that subset of the MS.
                  
                The summary mode is not allowed in the input list.
                  
                For optimization, the task will create a union of the data selection parameters
                present in the list and select only that portion of the MS.

                NOTE: the flag commands will be applied only when action='apply'. If
                      action='calculate' the flags will be calculated, but not applied.
                      This is useful if display is set to something other than 'none'. If
                      action='' or 'none', the flag commands will not be applied either.
                      An empty action is useful only to save the parameters of the list
                      to a file or to the FLAG_CMD sub-table.
                  
               inpfile -- Input ASCII file or a Python list of command strings.
                  default: ''
                  options: [] or '' with filename.
                  Example1: the following commands can be saved into a file and given
                            to the task (e.g. save it to flags.txt).
                  
                     scan='1~3' mode='manual'
                     mode='clip' clipminmax=[0,2] correlation='ABS_XX' clipoutside=False
                     spw='9' mode='tfcrop' correlation='ABS_YY' ntime=51.0
                     mode='extend' extendpols=True

                    flagdata(vis,mode='list',inpfile='flags.txt')

                  Example2: the same commands can be given in a Python list on the command line
                             to the task.
                     cmd=["scan='1~3' mode='manual'", 
                          "mode='clip' clipminmax=[0,2] correlation='ABS_XX' clipoutside=False",
                          "spw='9' mode='tfcrop' correlation='ABS_YY' ntime=51.0",
                          "mode='extend' extendpols=True"]

                     flagdata(vis,mode='list',inpfile=cmd)                    
        
               reason -- select flag commands based on REASON(s) 
                   default: 'any' (all flags regardless of reason)
                            can be a string, or list of strings
                   example: reason='FOCUS_ERROR'
                            reason=['FOCUS_ERROR','SUBREFLECTOR_ERROR']
    
                   NOTE: what is within the string is literally
                   matched, e.g. reason='' matches only blank reasons, 
                   and reason = 'FOCUS_ERROR,SUBREFLECTOR_ERROR'
                   matches this compound reason string only
                     
                  See the syntax for writing flag commands at the end of this help.                  


        manual -- Flag according to the data selection specified.
                  This is the default mode (used when the mode is not specified).
                  
                autocorr -- Flag only the auto-correlations. Note that this parameter is only
                            active when set to True. If set to False it does NOT mean "do not
                            flag auto-correlations".
                    default: False
                    options: True,False
                  
        clip -- Clip data according to values of the following subparameters. The polarization
                expression is given by the correlation parameter.
                    
               channelavg -- Average data over (selected) channels? Channel
                            selections are taken into account. Flagged channels
                            are excluded from the average. The average is done
                            after applying the specified clipexpr, i.e. scalar
                            averaging.
                    default: False
                    options: True,False
                    
               clipminmax -- Range of data (Jy) that will NOT be flagged
                    default: []; it will flag only NaN and Infs.
                    example: [0.0,1.5]
                    It will always flag the NaN/Inf data, even when a range is specified.
                    
               datacolumn -- Column to use for clipping.
                    default: 'DATA'
                    options: 'DATA', 'CORRECTED','MODEL', 'RESIDUAL, 'RESIDUAL_DATA',
                             'FPARAM', 'CPARAM', 'SNR'.
                             
                    NOTE: by default for calibration tables, the framework will try to use FPARAM; 
                          if that does not exist, it will try to use CPARAM. If none of them exist, it
                          will give an error and exit.
                    
               clipoutside -- Clip OUTSIDE the range?
                    default: True
                    example: False; flag data WITHIN the range.

               clipzeros -- Clip zero-value data.
                    default: False

        quack -- Option to remove specified part of scan beginning/end

               quackinterval -- Time in seconds from scan beginning/end to flag. Make time slightly 
                                 smaller than the desired time.
                    default: 0.0
               quackmode -- Quack mode
                    default: 'beg'
                    options: 'beg'  ==> beginning of scan
                             'endb' ==> end of scan. 
                             'tail' ==> all but beginning of scan
                             'end'  ==> all but end of scan.
               quackincrement -- Quack incrementally in time?
                    default: False
                             False  ==> the quack interval is counted from the
                                        beginning of the scan
                             True   ==> the quack interval is counted from the
                                        first unflagged data in the scan

        shadow -- Option to flag data of shadowed antennas. This mode is not available
                    for cal tables.
                    
                   All antennas in the antenna-subtable of the MS (and the corresponding
                   diameters) will be considered for shadow-flag calculations. 
                   For a given timestep, an antenna is flagged if any of its baselines 
                   (projected onto the uv-plane) is shorter than  radius_1 + radius_2 - tolerance. 
                   The value of 'w' is used to determine which antenna is behind the other.
                   The phase-reference center is used for antenna-pointing direction. 
                     
               tolerance -- Amount of shadowing allowed (or tolerated), in meters. 
                        A positive number allows antennas to overlap in projection
                        A negative number forces antennas apart in projection
                        Zero implies a distance of radius_1+radius_2 between antenna centers.
                   default: 0.0

               addantenna -- It can be either a file name with additional antenna names, positions 
                         and diameters, or a Python dictionary with the same information. 
                         You can use the flaghelper functions to create the dictionary from a file.  
                   default: ''
                   
                   To create a dictionary inside casapy.
                   > import flaghelper as fh
                   > antdic = fh.readAntennaList(antfile)
                   
                   Where antfile is a text file in disk that contains information such as:
                    name=VLA01
                    diameter=25.0
                    position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]
                    name=VLA02
                    diameter=25.0
                    position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178]
                  

        elevation -- Option to flag based on antenna elevation. This mode is not available
                      for cal tables.
                 
               lowerlimit -- Lower limiting elevation in degrees. Data coming from a baseline
                             where one or both antennas were pointing at a strictly lower elevation
                             (as function of time), will be flagged.
                   default: 0.0
                   
               upperlimit -- Upper limiting elevation in degrees. Data coming from a baseline
                             where one or both antennas were pointing at a strictly higher elevation
                             (as function of time), will be flagged.
                   default: 90.0

        tfcrop -- Flag using the TFCrop autoflag algorithm. The polarization
                  expression is given by the correlation parameter.

                 For each field, spw, timerange (specified by ntime), and baseline, 
                 (1) Average visibility amplitudes along time dimension to 
                       form an average spectrum
                 (2) Calculate a robust piece-wise polynomial fit for the band-shape 
                      at the base of RFI spikes. Calculate 'stddev' of (data - fit).
                 (3) Flag points deviating from the fit by more than N-stddev
                 (4) Repeat (1-3) along the other dimension.

                 This algorithm is designed to operate on un-calibrated data (step (2)), 
                 as well as calibrated data.  It is usually helpful to extend the flags along
                 time, frequency, and correlation in a second step (via mode='extend)

               ntime -- Timerange (in seconds or minutes) over which to buffer data before 
                        running the algorithm. 
                    options: 'scan' or any other float value or string containing the units.
                    default: 'scan'
                    example: '1.5min'
                           : 1.2 (taken in seconds)
                        The dataset will be iterated through in time-chunks defined here.
                
               combinescans -- accumulate data across scans.
                    default: False
                        This parameter is relevant only ntime is specified as a time-interval (not 'scan')

               datacolumn -- Column to use for flagging.
                    default: 'DATA'
                    options: 'DATA', 'CORRECTED','MODEL', 'RESIDUAL, 'RESIDUAL_DATA',
                             'FPARAM', 'CPARAM', 'SNR'.
                             
                    NOTE: by default for calibration tables, the framework will try to use FPARAM; 
                          if that does not exist, it will try to use CPARAM. If none of them exist, it
                          will give an error and exit.
                    

               timecutoff -- Flag threshold in time. Flag all data-points further than N-stddev 
                             from the fit. This threshold catches time-varying RFI spikes (
                             narrow and broad-band), but will not catch RFI that is persistent in time.
                    default: 4.0
                          Flagging is done in upto 5 iterations. The stddev calculation is adaptive and 
                          converges to a value that reflects only the data and no RFI. At each iteration,
                          the same relative threshold is applied to detect flags. (Step (3) of the algorithm).

               freqcutoff -- Flag threshold in frequency. Flag all data-points further than N-stddev 
                             from the fit.
                    default: 3.0
                        Same as timecutoff, but along the frequency-dimension. This threshold catches
                        narrow-band RFI that may or may not be persistent in time. 

               timefit --  Fitting function for the time direction
                    default: 'line'
                    options: 'line', 'poly'
                      A 'line' fit is a robust straight-line fit across the entire timerange (defined 
                      by 'ntime').
                      A 'poly' fit is a robust piece-wise polynomial fit across the timerange.
                      Note: A robust fit is computed in upto 5 iterations. At each iteration, the stddev
                            between the data and the fit is computed, values beyond N-stddev are flagged,
                            and the fit and stddev are re-calculated with the remaining points.
                            This stddev calculation is adaptive, and converges to a value that reflects 
                            only the data and no RFI.  It also provides a varying set of flagging thresholds,
                            that allows deep flagging only when the fit best represents the true data.
                      Choose 'poly' only if the visibilities are expected to vary significantly over the
                      timerange selected by 'ntime', or if there is a lot of strong but intermittent RFI.

               freqfit -- Fitting function for the frequency direction
                    default: 'poly'
                    options: 'line','poly'
                        Same as for the 'timefit' parameter.
                        Choose 'line' only if you are operating on bandpass-corrected data, or residuals, 
                        and expect that the bandshape is linear. The 'poly' option works better on 
                        uncalibrated bandpasses with narrow-band RFI spikes.   

               maxnpieces -- Maxinum number of pieces to allow in the piecewise-polynomial fits 
                    default: 7
                    options: 1 - 9
                        This parameter is used only if 'timefit' or 'freqfit' are chosen as 'poly'.
                        If there is significant broad-band RFI, reduce this number. Using too many
                        pieces could result in the RFI being fitted in the 'clean' bandpass.    
                        In later stages of the fit, a third-order polynomial is fit per piece, so 
                        for best results, please ensure that nchan/maxnpieces is at-least 10.

               flagdimension -- Choose the directions along which to perform flagging
                    default: 'freqtime'; first flag along frequency, and then along time 
                    options: 'time', 'freq', 'timefreq', 'freqtime'
                         For most cases, 'freqtime' or 'timefreq' are appropriate, and differences
                         between these choices are apparant only if RFI in one dimension is 
                         significantly stronger than the other. The goal is to flag the dominant RFI first.
                         If there are very few (less than 5) channels of data, then choose 'time'.
                         Similarly for 'freq'.

               usewindowstats -- Use sliding-window statistics to find additional flags.
                    default: 'none' 
                    options: 'none', 'sum', 'std', 'both'   
                         Note: This is experimental !
                           The 'sum' option chooses to flag a point, if the mean-value in a 
                           window centered on that point deviates from the fit by more than
                           N-stddev/2.0. 
                         Note: stddev is calculated between the data and fit as explained in Step (2).
                            This option is an attempt to catch broad-band or 
                            time-persistent RFI  that the above polynomial fits will mistakenly fit
                            as the clean band. It is an approximation to the sumThreshold
                            method found to be effective by Offringa et.al (2010) for LOFAR data. 
                            The 'std' option chooses to flag a point, if the 'local' stddev calculated
                            in a window centered on that point is larger than N-stddev/2.0. 
                            This option is an attempt to catch noisy RFI that is not excluded in the
                            polynomial fits, and which increases the global stddev, and results in 
                            fewer flags (based on the N-stddev threshold).
                                                                
                halfwin -- Half width of sliding window to use with 'usewindowstats'
                    default: 1  (a 3-point window size)
                    options: 1,2,3
                        Note: This is experimental !

        rflag -- Detect outliers based on the RFLAG algorithm (ref. E.Greisen, AIPS, 2011).
                 The polarization expression is given by the correlation parameter.

                 Iterate through the data in chunks of time.  For each chunk, calculate local
                 statistics, and apply flags based on user supplied (or auto-calculated) thresholds.

                 Step 1 : Time analysis (for each channel)
                   -- calculate local rms of real and imag visibilities, within a sliding time window
                   -- calculate the median rms across time windows, deviations of local rms from
                      this median, and the median deviation 
                   -- flag if local rms is larger than timedevscale x (medianRMS + medianDev)

                 Step 2 : Spectral analysis (for each time)
                   -- calculate avg of real and imag visibilities and their rms across channels
                   -- calculate the deviation of each channel from this avg, and the median-deviation
                   -- flag if deviation is larger than freqdevscale x medianDev

                 Example usage : 

                  (1) Calculate thresholds automatically per scan, and use them to find flags.
                      Specify scale-factor for time-analysis thresholds, use default for frequency.

                      flagdata('my.ms', mode='rflag',spw='9',timedevscale=4.0)
 
                  (2) Supply noise-estimates to be used with default scale-factors. 
  
                      flagdata(vis='my.ms', mode='rflag', spw='9', timedev=0.1, freqdev=0.5, action='calculate)

                  (3) Two-passes. This replicates the usage pattern in AIPS.

                      -- The first pass saves commands in an output text files, with auto-calculated 
                         thresholds. Thresholds are returned from rflag only when action='calculate'. 
                         The user can edit this file before doing the second pass, 
                         but the python-dictionary structure must be preserved.

                      -- The second pass applies these commands (action='apply').

                          flagdata(vis='my.ms', mode='rflag', spw='9,10', timedev='tdevfile.txt', 
                                    freqdev='fdevfile.txt', action='calculate')
                          flagdata(vis='my.ms', mode='rflag', spw='9,10', timedev='tdevfile.txt', 
                                    freqdev='fdevfile.txt', action='apply')

                        With action='calculate', display='report' will produce diagnostic plots 
                        showing data-statistics and thresholds (the same thresholds as those
                        written out to 'tdevfile.txt' and 'fdevfile.txt'). 
           
                        Note : The rflag algorithm was originally developed by Eric Greisen in 
                                AIPS (31DEC11).
                        AIPS documentation : Section E.5 of the AIPS cookbook
                        (Appendix E : Special Considerations for EVLA data calibration and imaging in AIPS,
                        http://www.aips.nrao.edu/cook.html#CEE )

                        Note : Since this algorithm operates with two passes through each
                           chunk of data (time and freq axes), some data points get flagged
                           twice. This can affect the flag-percentage estimate printed in the
                           logger at runtime.  An accurate estimate can be obtained via the
                           summary mode.

               ntime -- Timerange (in seconds or minutes) over which to buffer data before running the algorithm.
                     options: 'scan' or any other float value or string containing the units.
                     default: 'scan'
                     example: '1.5min'
                            : 1.2 (taken in seconds)
                        The dataset will be iterated through in time-chunks defined here.
                
               combinescans -- accumulate data across scans.
                      default: False
                          It only makes sense when ntime not equal 'scan'.                

               datacolumn -- Column to use for flagging.
                    default: 'DATA'
                    options: 'DATA', 'CORRECTED','MODEL', 'RESIDUAL, 'RESIDUAL_DATA',
                             'FPARAM', 'CPARAM', 'SNR'.
                             
                    NOTE: by default for calibration tables, the framework will try to use FPARAM; 
                          if that does not exist, it will try to use CPARAM. If none of them exist, it
                          will give an error and exit.
                    

               winsize -- number of timesteps in the sliding time window ( fparm(1) in AIPS ). 
                    default: 3
                    
               timedev -- time-series noise estimate ( noise in AIPS ). 
                    default: []
                    Examples : 
                         timedev = 0.5 : Use this noise-estimate to calculate flags. Do not recalculate.
                         timedev = [ [1,9,0.2], [1,10,0.5] ] :  Use noise-estimate of 0.2 for field 1, 
                                   spw 9, and noise-estimate of 0.5 for field 1, spw 10.
                         timedev = [] : Auto-calculate noise estimates.
                
                    
               freqdev -- spectral noise estimate ( scutoff in AIPS ).
                          This step depends on having a relatively-flat bandshape.
                          Same parameter-options as 'timedev'. 
                    default: []
                     

               timedevscale -- For Step 1 (time analysis), flag a point if local rms around it 
                               is larger than  'timedevscale' x 'timedev'   ( fparm(0) in AIPS )
                    default: 5.0
                    
               freqdevscale -- For Step 2 (spectral analysis), flag a point if local rms around it
                               is larger than 'freqdevscale' x 'freqdev'     ( fparm(10) in AIPS )
                    default: 5.0

               spectralmax -- Flag whole spectrum if 'freqdev' is greater than spectralmax ( fparm(6) in AIPS )
                    default: 1E6
                    
               spectralmin -- Flag whole spectrum if 'freqdev' is less than spectralmin ( fparm(5) in AIPS )
                    default: 0.0



        extend -- Extend and/or grow flags beyond what the basic algorithms detect.
                        This mode will extend the accumulated flags available in the MS,
                        regardless of which algorithm created them.
                         
                        Extensions will apply only within the selected data, according to the settings 
                        of extendpols,growtime,growfreq,growaround, flagneartime,flagnearfreq.
                        
                        Note : Runtime summary counts in the logger can sometimes report larger
                            flag percentages than what is actually flagged. This is because 
                            extensions onto already-flagged data-points are counted as new flags. 
                            An accurate flag count can be obtained via the summary mode.

               ntime -- Timerange (in seconds or minutes) over which to buffer data before running 
                        the algorithm.
                     options: 'scan' or any other float value or string containing the units.
                     default: 'scan'
                     example: '1.5min'
                            : 1.2 (taken in seconds)
                        The dataset will be iterated through in time-chunks defined here.
                
               combinescans -- accumulate data across scans.
                      default: False
                          It only makes sense when ntime not equal 'scan'.                

               extendpols -- Extend flags to all selected correlations
                      default: True
                      options: True/False
                         For example, to extend flags from RR to only RL and LR, a data-selection
                         of correlation='RR,LR,RL' is required along with extendpols=True. 

               growtime -- For any channel, flag the entire timerange in the current 2D chunk (
                           set by 'ntime') if more than X% of the timerange is already flagged.
                      default: 50.0
                      options: 0.0 - 100.0
                         This option catches the low-intensity parts of time-persistent RFI.

               growfreq -- For any timestep, flag all channels in the current 2D chunk (set by 
                           data-selection) if more than X% of the channels are already flagged.
                      default: 50.0
                      options: 0.0 - 100.0
                         This option catches broad-band RFI that is partially identified by earlier steps.

               growaround -- Flag a point based on the number of flagged points around it.
                      default: False
                      options: True/False
                            For every un-flagged point on the 2D time/freq plane, if more than four
                            surrounding points are already flagged, flag that point. 
                            This option catches some wings of strong RFI spikes.

               flagneartime -- Flag points before and after every flagged one, in the time-direction.
                      default: False
                      options: True/False
                      Note: This can result in excessive flagging.

               flagnearfreq -- Flag points before and after every flagged one, in the frequency-direction
                    default: False
                    options: True/False
                    This option allows flagging of wings in the spectral response of strong RFI.
                    Note: This can result in excessive flagging.


        unflag -- Unflag according to the data selection specified.

        summary -- List the number of rows and data points flagged.
                  
               minrel -- Minimum number of flags (relative) to include in histogram
                    default: 0.0

               maxrel -- Maximum number of flags (relative) to include in histogram
                    default: 1.0

               minabs -- Minimum number of flags (absolute, inclusive) to include in histogram
                    default: 0

               maxabs -- Maximum number of flags (absolute, inclusive) to include in histogram
                          To indicate infinity, use any negative number.
                    default: -1

               spwchan -- list the number of flags per spw and per channel.
                    default: False

               spwcorr -- list the number of flags per spw and per correlation.
                    default: False

               basecnt -- list the number of flags per baseline
                    default: False

                    In summary mode, the task returns a dictionary of flagging statistics.
                    Example:

                        s = flagdata(..., mode='summary')

                     Then s will be a dictionary which contains
                    s['total']   : total number of data
                    s['flagged'] : amount of flagged data
                    
        

    action -- Action to perform in MS/cal table or in the input list of parameters. 
        options: 'none', 'apply','calculate'
        default: 'apply'

        'apply' -- Apply the flags to the MS.
        
               display -- Display data and/or end-of-MS reports at run-time.        
                   default: 'none'
                   options: 'none', 'data', 'report', 'both'

                     'none' --> It will not display anything.

                     'data' --> display data and flags per-chunk at run-time, within an interactive GUI.

                      This option opens a GUI to show the 2D time-freq planes of
                      the data with old and new flags, for all correlations per baseline.
                      -- The GUI allows stepping through all baselines (prev/next) in
                      the current chunk (set by 'ntime'), and stepping to the next-chunk.
                      -- The 'flagdata' task can be quit from the GUI, in case it becomes
                      obvious that the current set of parameters is just wrong.
                      -- There is an option to stop the display but continue flagging.

                    'report' --> displays end-of-MS reports on the screen.

                    'both' --> displays data per chunk and end-of-MS reports on the screen

               flagbackup -- Automatically backup flags before running the tool.
                            Flagversion names are chosen automatically, and are based on the
                            mode being used.
                   default: True
                   options: True/False                    
        
        'calculate' -- Only calculate the flags but do not write them to the MS. This is
                     useful if used together with the display to analyse the results before
                     writing to the MS.

               display -- Display data and/or end-of-MS reports at run-time. See extended description
                          above. 
                   default: 'none'
                   options: 'none', 'data', 'report', 'both'
                   
        ' ' -- When set to empty, the underlying tool will not be executed and no flags
                will be produced. No data selection will be done either. This is useful
                when used together with the parameter savepars to only save the current
                parameters (or list of parameters) to the FLAG_CMD sub-table or to an
                external file.  
                 

    savepars -- Save the current parameters to the FLAG_CMD table of the MS or to an output text file.
                Note that when display is set to anything other than 'none', savepars
                will be disabled. This is done because in an interactive mode, the user
                may skip data which may invalidate the initial input parameters and there
                is no way to save the interactive commands.
                default: False
                options: True/False

                cmdreason -- A string containing a reason to save to the FLAG_CMD table or to an 
                             output text file given by the outfile sub-parameter. If the input 
                             contains any reason, they will be replaced with this one. At the 
                             moment it is not possible to add more than one reason.
                    default: ' '; no reason will be added to output.
                    example: cmdreason='CLIP_ZEROS'
                
                outfile -- Name of output file to save the current parameters.
                    default: ' '; it will save the parameters to the FLAG_CMD table of the MS.
                    example: outfile='flags.txt' will save the parameters in a text file.
                    
    
---- EXAMPLES ----

    NOTE: The vector mode of the previous flagdata task can be achieved with this task by using it
          with mode='list' and the commands given in a list in inpmode=[]. Example:
    
       flagdata('my.ms', inpmode='list', inpfile=["mode='clip' clipzeros=True","mode='shadow'])
        

    1) Manually flag scans 1~3 and save the parameters to the FLAG_CMD sub-table.
    
        flagdata('my.ms', scan='1~3, mode='manual', savepars=True)
            
    2) Save the parameters to a file that is open in append mode.

        flagdata('my.ms', scan='1~3, mode='manual', savepars=True, outfile='flags.txt')
            
    3a) Flag all the commands given in the Python list of strings.
    
        cmd = ["scan='1~3' mode='manual'",
               "spw='9' mode='tfcrop' correlation='ABS_RR,LL' ntime=51.0",
               "mode='extend' extendpols=True"]
    
        flagdata('my.ms', mode='list', inpfile=cmd)

    3b) Flag all the commands given in the file called flags.txt.
    
        > cat flags.txt
        scan='1~3' mode='manual'
        spw='9' mode='tfcrop' correlation='ABS_RR,LL' ntime=51.0
        mode='extend' extendpols=True
    
        flagdata('my.ms', mode='list', inpfile='flags.txt')

    4) Display the data and flags per-chunk and do not write flags to the MS.
    
        flagdata('my.ms', mode='list', inpfile='flags.txt', action='calculate', display='data')
        
    5) Flag all the antennas except antenna=5.
    
        flagdata(vis='my.ms', antenna='!5', mode='manual)
            
    6) Clip the NaN in the data. An empty clipminmax will flag only NaN.
    
        flagdata('my.ms', mode='clip')
        
    7) Clip only the water vapour radiometer data.
    
        flagdata('my.ms',mode='clip',clipminmax=[0,50], correlation='ABS_WVR')

    8) Clip only zero-value data.

        flagdata('my.ms',mode='clip',clipzeros=True)
        
    9a) Flag only auto-correlations using the autocorr parameter.
       
        flagdata('my.ms', autocorr=True)
            
    9b) Flag only auto-correlations using the antenna selection.
       
        flagdata('my.ms', mode='manual', antenna='*&&&')

    10a) Flag based on selected reasons from a file.
       
        > cat flags.txt
        scan='1~3' mode='manual' reason='MYREASON'
        spw='9' mode='clip' clipzeros=True reason='CLIPZEROS'
        mode='manual' scan='4' reason='MYREASON'
        
        flagdata('my.ms', mode='list', inpfile='flags.txt', reason='MYREASON')

    10b) The same result of 10a can be achieved using the task flagcmd.

        flagcmd('my.ms', inpmode='file', inpfile='flags.txt', action='apply', reason='MYREASON')
            
    11) Automatic flagging using 'rflag', using auto-thresholds, and specifying 
          a threshold scale-factor to use for flagging.

        flagdata('my.ms', mode='rflag',spw='9',timedevscale=4.0,action='apply')

    12) Save the interface parameters to the FLAG_CMD sub-table of the MS. Add a reason
        to the flag command. This cmdreason will be added to the REASON column of the
        FLAG_CMD sub-table. Apply flags in flagcmd.

        flagdata('my.ms', mode='clip',channelavg=False, clipminmax=[30., 60.], spw='0:0~10', 
                  correlation='ABS_XX,XY', action='', savepars=True, cmdreason='CLIPXX_XY')

        > Select based on the reason.
        flagcmd('my.ms', action='apply', reason='CLIPXX_XY')

    13) Flag antennas that are shadowed by antennas not present in the MS.

        > Create a text file with information about the antennas.
        > cat ant.txt
          name=VLA01
          diameter=25.0
          position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]
          name=VLA02
          diameter=25.0
          position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178]
          name=VLA09
          diameter=25.0
          position=[-1601197.2182404203, -5041974.3604805721, 3554875.1995636248]
          name=VLA10
          diameter=25.0
          position=[-1601227.3367843349,-5041975.7011900628,3554859.1642644769]    
           

         flagdata('my.vis', mode='shadow', tolerance=10.0, addantenna='ant.txt')

        The antenna information can also be given as a Python dictionary. To create the
        dictionary using the flaghelper functions, do the following inside casapy:

             > import flaghelper as fh
             > antdic = fh.readAntennaList(antfile)

         flagdata('my.vis', mode='shadow', tolerance=10.0, addantenna=antdic)

    14) Apply the online flags that come from importasdm.

        > In importasdm, save the online flags to a file.
         importasdm('myasdm', 'asdm.ms', process_flags=True, savecmds=True, outfile='online_flags.txt')

        > You can edit the online_flags.txt to add other flagging commands or apply it directly.
         flagdata('asdm.ms', mode='list', inpfile='online_flags.txt')

        > The same result can be achieved using the task flagcmd.
         flagcmd('asdm.ms', inpmode='file', inpfile='online_flags.txt', action='apply')

    15) Clip zero data from a calibration table.
    
        flagdata('cal-X54.B1', mode='clip', clipzeros=True, datacolumn='CPARAM')
        
    16) Clip data with SNR <4.0.
    
        flagdata('cal-X54.B1', mode='clip', clipminmax=[0.0,4.0], clipoutside=False, datacolumn='SNR')
    

---- SYNTAX FOR COMMANDS GIVEN IN A FILE or LIST OF STRINGS ----

    Basic Syntax Rules
        
          Commands are strings (which may contain internal "strings") consisting of
          KEY=VALUE pairs separated by whitespace. 

          NOTE: There should be no whitespace between KEY=VALUE or within each KEY or 
          VALUE, since the simple parser first breaks command lines on whitespace, 
          then on "=". 

          Use only white spaces to separate the parameters (no commas).
        
          Each key should only appear once on a given command line/string.
        
          There is an implicit "mode" for each command, with the default
          being 'manual' if not given.

          Comment lines can start with '#' and will be ignored.

          Example:
        
          scan='1~3' mode='manual'
          # this line will ignored
          spw='9' mode='tfcrop' correlation='ABS_XX,YY' ntime=51.0
          mode='extend' extendpols=True
          scan='1~3,10~12' mode='quack' quackinterval=1.0
             

        """

#
#    The following is work around to avoid a bug with current python translation
#
        mytmp = {}

        mytmp['vis'] = vis
        mytmp['mode'] = mode
        mytmp['autocorr'] = autocorr
        mytmp['inpfile'] = inpfile
        mytmp['reason'] = reason
        mytmp['spw'] = spw
        mytmp['field'] = field
        mytmp['antenna'] = antenna
        mytmp['uvrange'] = uvrange
        mytmp['timerange'] = timerange
        mytmp['correlation'] = correlation
        mytmp['scan'] = scan
        mytmp['intent'] = intent
        mytmp['array'] = array
        mytmp['observation'] = observation
        mytmp['feed'] = feed
        mytmp['clipminmax'] = clipminmax
        mytmp['datacolumn'] = datacolumn
        mytmp['clipoutside'] = clipoutside
        mytmp['channelavg'] = channelavg
        mytmp['clipzeros'] = clipzeros
        mytmp['quackinterval'] = quackinterval
        mytmp['quackmode'] = quackmode
        mytmp['quackincrement'] = quackincrement
        mytmp['tolerance'] = tolerance
        mytmp['addantenna'] = addantenna
        mytmp['lowerlimit'] = lowerlimit
        mytmp['upperlimit'] = upperlimit
        mytmp['ntime'] = ntime
        mytmp['combinescans'] = combinescans
        mytmp['timecutoff'] = timecutoff
        mytmp['freqcutoff'] = freqcutoff
        mytmp['timefit'] = timefit
        mytmp['freqfit'] = freqfit
        mytmp['maxnpieces'] = maxnpieces
        mytmp['flagdimension'] = flagdimension
        mytmp['usewindowstats'] = usewindowstats
        mytmp['halfwin'] = halfwin
        mytmp['winsize'] = winsize
        mytmp['timedev'] = timedev
        mytmp['freqdev'] = freqdev
        mytmp['timedevscale'] = timedevscale
        mytmp['freqdevscale'] = freqdevscale
        mytmp['spectralmax'] = spectralmax
        mytmp['spectralmin'] = spectralmin
        mytmp['extendpols'] = extendpols
        mytmp['growtime'] = growtime
        mytmp['growfreq'] = growfreq
        mytmp['growaround'] = growaround
        mytmp['flagneartime'] = flagneartime
        mytmp['flagnearfreq'] = flagnearfreq
        mytmp['minrel'] = minrel
        mytmp['maxrel'] = maxrel
        mytmp['minabs'] = minabs
        mytmp['maxabs'] = maxabs
        mytmp['spwchan'] = spwchan
        mytmp['spwcorr'] = spwcorr
        mytmp['basecnt'] = basecnt
        mytmp['action'] = action
        mytmp['display'] = display
        mytmp['flagbackup'] = flagbackup
        mytmp['savepars'] = savepars
        mytmp['cmdreason'] = cmdreason
        mytmp['outfile'] = outfile
        pathname='file:///'+os.environ.get('CASAPATH').split()[0]+'/share/xml/'
        trec = casac.utils().torecord(pathname+'flagdata.xml')

        casalog.origin('flagdata')
        if trec.has_key('flagdata') and casac.utils().verify(mytmp, trec['flagdata']) :
            result = task_flagdata.flagdata(vis, mode, autocorr, inpfile, reason, spw, field, antenna, uvrange, timerange, correlation, scan, intent, array, observation, feed, clipminmax, datacolumn, clipoutside, channelavg, clipzeros, quackinterval, quackmode, quackincrement, tolerance, addantenna, lowerlimit, upperlimit, ntime, combinescans, timecutoff, freqcutoff, timefit, freqfit, maxnpieces, flagdimension, usewindowstats, halfwin, winsize, timedev, freqdev, timedevscale, freqdevscale, spectralmax, spectralmin, extendpols, growtime, growfreq, growaround, flagneartime, flagnearfreq, minrel, maxrel, minabs, maxabs, spwchan, spwcorr, basecnt, action, display, flagbackup, savepars, cmdreason, outfile)

        else :
          result = False
        return result