|
casa
$Rev:20696$
|
|
casa
$Rev:20696$
|
00001 ###Base class for casa test scripts 00002 ###More complicated tests may inherit from this 00003 import os 00004 import string 00005 import commands 00006 import shutil 00007 import sys 00008 from casac import * 00009 import inspect 00010 import re 00011 #from tasks import * # execfile 00012 #from taskinit import casalog 00013 00014 ia=casac.image() 00015 ms=casac.ms() 00016 tb=casac.table() 00017 00018 class testbase : 00019 def __init__(self, workdir=None): 00020 if workdir == None: 00021 workdir=os.getcwd() 00022 self.dataBaseDirectory=[workdir+'/Data'] 00023 self.resultDirectory=workdir+'/Results' 00024 self.workingDirectory=workdir 00025 self.scriptRepository=workdir+'/Scripts' 00026 self.testsToRun=[] 00027 self.testList={} 00028 self.notest=False 00029 00030 def createDirs(self): 00031 if os.access(self.resultDirectory, os.F_OK) is False: 00032 print self.resultDirectory+' does not exist, creating it' 00033 os.makedirs(self.resultDirectory) 00034 if os.access(self.workingDirectory, os.F_OK) is False: 00035 print self.workingDirectory+' does not exist, creating it' 00036 os.makedirs(self.workingDirectory) 00037 00038 def setDataBaseDir(self, dir=['./Data']): 00039 if type(dir) != type(["directory", "list"]): 00040 raise TypeError, type(dir) 00041 self.dataBaseDirectory=dir 00042 00043 def setResultDir(self, dir='./Results'): 00044 self.resultDirectory=dir 00045 00046 def setWorkingDir(self, dir='./Temporaire'): 00047 self.workingDirectory=dir 00048 00049 def setScriptsDir(self, dir='./Scripts'): 00050 self.scriptRepository=dir 00051 00052 def cleanup(self): 00053 if os.path.isdir(self.workingDirectory): 00054 #print 'Removing '+ self.workingDirectory 00055 shutil.rmtree(self.workingDirectory) 00056 00057 def locateTests(self, tests=None): 00058 theF=[] 00059 theFiles=[] 00060 if tests is None : 00061 # This does not work, need to search also for DIR/name.py 00062 # Also, searching in .../tests/<name>/regression.py is 00063 # obsolete 00064 raise Exception, "Unsupported!" 00065 00066 theFiles=os.listdir(self.scriptRepository) 00067 tempy=[] 00068 ###locate only the *.py files 00069 for leF in theFiles: 00070 if os.path.isdir(self.scriptRepository + '/tests/' + leF) and \ 00071 os.path.isfile(self.scriptRepository + '/tests/' + leF + '/regression.py'): 00072 tempy.append(leF + '/regression.py') 00073 theFiles=tempy 00074 else: 00075 for leF in tests : 00076 leScript=self.searchscript(leF) 00077 if(leScript != ''): 00078 theFiles.append(leScript) 00079 00080 self.testsToRun=[] 00081 for k in range(len(theFiles)): 00082 #self.testsToRun.append(string.split(theFiles[k],'.py')[0]) 00083 self.testsToRun.append(theFiles[k]) 00084 print "tests to run = ", self.testsToRun 00085 return len(self.testsToRun) 00086 00087 def getTest(self, testnamek, testName): 00088 #Copy the script to the working dir 00089 if testnamek[0:6] == 'tests/': 00090 shutil.copy(self.scriptRepository+'/'+testnamek, 00091 self.workingDirectory+'/'+testName+'.py') 00092 shutil.copy(self.scriptRepository+'/'+testnamek, \ 00093 self.workingDirectory+'/') 00094 00095 def testname(self, ind=0): 00096 if(len(self.testsToRun)==0): 00097 print 'No tests defined yet' 00098 return '' 00099 return self.testsToRun[ind] 00100 00101 def getDescription(self, testName, testId): 00102 leFile=self.testsToRun[testId] 00103 00104 if leFile[0:6] == 'tests/': 00105 leTest = __import__(testName) 00106 reload(leTest) 00107 try: 00108 # Fails if module does not 00109 # define the description() function 00110 desc = leTest.description() 00111 desc = re.sub('\s+', ' ', desc) 00112 00113 except: 00114 desc = None 00115 else: 00116 desc = None 00117 return desc 00118 00119 def runtests(self, testName, testId=0, dry=False): 00120 try: 00121 leFile=self.testsToRun[testId] 00122 00123 if leFile[0:6] == 'tests/': 00124 print "Import", leFile 00125 leTest = __import__(testName) 00126 #print " imported" 00127 allData=leTest.data() 00128 # check if there is a doCopy function and use it if possible 00129 try: 00130 doCopy=leTest.doCopy() 00131 print doCopy 00132 if (len(doCopy) != len(allData)): 00133 doCopy=[-1] 00134 print 'Error: doCopy function supplied list of incorrect length.' 00135 else: 00136 print 'Using doCopy function to determine whether to copy or link input data.' 00137 except: 00138 doCopy=[-1] 00139 00140 print "Required data =", allData 00141 for leIndex, leData in enumerate(allData) : 00142 theData=self.locatedata(leData) 00143 if(theData != ''): 00144 os.system('rm -rf '+ self.workingDirectory+'/'+leData) 00145 if(os.path.isdir(theData)): 00146 if(doCopy[0] == -1 or doCopy[leIndex] != 0): 00147 shutil.copytree(theData, self.workingDirectory+'/'+leData) 00148 else: 00149 os.system('ln -sf '+theData+' '+self.workingDirectory+'/'+leData) 00150 if(os.path.isfile(theData)): 00151 if(doCopy[0] == -1 or doCopy[leIndex] != 0): 00152 shutil.copy(theData, self.workingDirectory+'/'+leData) 00153 else: 00154 os.system('ln -sf '+theData+' '+self.workingDirectory+'/'+leData) 00155 if not dry: 00156 theImages=leTest.run() 00157 else: 00158 # ngc5921redux 00159 theImages = ['ngc5921_regression/ngc5921.clean.image'] 00160 00161 # h121 00162 theImages = ['nrao150.3mm.image', 'h121.co10.image', '0224b.3mm.image', 'h121b.co10.image', 'h121all.3mm.image', 'h121c.co10.image' ] 00163 00164 00165 del leTest 00166 00167 if type(theImages) == None: 00168 raise Exception, "Illegal return value from run()" 00169 00170 leResult=[] 00171 for leImage in theImages : 00172 leResult.append(self.workingDirectory+'/'+leImage) 00173 self.defineQualityTestList(leResult) 00174 return leResult, theImages 00175 else: 00176 print "execfile", leFile 00177 if dry: 00178 return [], [] 00179 a=inspect.stack() 00180 stacklevel=0 00181 for k in range(len(a)): 00182 if (string.find(a[k][1], 'ipython console') > 0): 00183 stacklevel=k 00184 break 00185 gl=sys._getframe(stacklevel).f_globals 00186 00187 # Execute the test from a generated exec-<test>.py 00188 # which does the following 00189 # regstate=True 00190 # execfile(leFile) 00191 # if not regstate: 00192 # raise Exception ... 00193 # 00194 # This is a workaround: Changes in regstate 00195 # do not propagate to here if the regression 00196 # script is run directly with execfile() from 00197 # here. But an exeption propagates 00198 00199 fd=open('exec-'+leFile, 'w') 00200 print >> fd, "regstate=True" # used in regression scripts to signal error 00201 print >> fd, "execfile('"+leFile+"')" 00202 print >> fd, "print 'regstate =', regstate" 00203 print >> fd, "if not regstate:" 00204 print >> fd, " raise Exception, 'regstate = False'" 00205 fd.close() 00206 00207 # What we really want, but regstate doesn't propagate: 00208 # execfile(leFile, gl) 00209 execfile('./exec-'+leFile, gl) 00210 00211 return [], [] # no product images known 00212 except: 00213 self.notest=True 00214 #print >> sys.stderr, "Error running test:", sys.exc_info()[0] 00215 raise 00216 00217 def defineQualityTestList(self,theResult): 00218 self.testList.clear() 00219 for k in range(len(theResult)) : 00220 if(os.path.isdir(theResult[k])): 00221 # previous developer said: 00222 # Good chance its an image for now till i know whether its a caltable or ms 00223 # actually check for ms, if not assume image 00224 tb.open(theResult[k]) 00225 tabkwords=tb.keywordnames() 00226 tb.done() 00227 if('MS_VERSION' in tabkwords): 00228 ms.open(theResult[k]) 00229 self.testList[theResult[k]]=[] 00230 self.testList[theResult[k]].append('ms') 00231 ms.done() 00232 elif('coords' in tabkwords): 00233 ## table is an image 00234 self.testList[theResult[k]]=[] 00235 self.testList[theResult[k]].append('simple') 00236 ia.open(theResult[k]) 00237 shp=ia.shape() 00238 ## Moment images do not have a spectral axis... 00239 # much assumes the "canonical" CASA axis order. 00240 # imageTest will adddegaxes to any im it opens (2011/12/15) 00241 mycs=ia.coordsys() 00242 findstok=mycs.findcoordinate("stokes") 00243 findspec=mycs.findcoordinate("spectral") 00244 if findspec[0]: 00245 spix=findspec[1] 00246 if(shp[spix]>5): 00247 print 'spectral shape= ', shp[spix],' ', theResult[k], k 00248 self.testList[theResult[k]].append('cube') 00249 if findstok[0]: 00250 kpix=findstok[1] 00251 if(shp[kpix]==1): 00252 self.testList[theResult[k]].append('pol1') 00253 elif( shp[kpix]==2): 00254 self.testList[theResult[k]].append('pol2') 00255 elif(shp[kpix]==4): 00256 self.testList[theResult[k]].append('pol4') 00257 ia.close() 00258 00259 def whatQualityTest(self): 00260 if(self.notest): 00261 return [] 00262 return self.testList 00263 00264 def searchscript(self, testname): 00265 scriptdir=self.scriptRepository 00266 print "searching for script", testname, "in ",scriptdir 00267 testName=string.lower(testname) 00268 00269 # search for DIR/tests/<name>.py 00270 if os.path.isdir(scriptdir+'/tests/'): 00271 allScripts=os.listdir(scriptdir+'/tests/') 00272 else: 00273 allScripts=[] 00274 print "allScripts = ", allScripts 00275 theScript='' 00276 numOfScript=0 00277 for scr in allScripts : 00278 #if(string.find(scr,testname)>=0): 00279 if(scr == testname+'.py'): 00280 print scr, testname 00281 #if (self.ispythonscript(scr)): 00282 theScript = 'tests/'+scr 00283 numOfScript +=1 00284 00285 # search for DIR/<name>.py 00286 if os.path.isdir(scriptdir): 00287 allScripts=os.listdir(scriptdir) 00288 else: 00289 allScripts=[] 00290 print "allScripts = ", allScripts 00291 for scr in allScripts: 00292 #print scriptdir, scr, testname 00293 if (scr == testname + '.py'): 00294 theScript = scr 00295 numOfScript += 1 00296 if numOfScript == 0: 00297 raise Exception("Could not find test %s" % testname) 00298 if( numOfScript > 1) : 00299 print 'More than 1 scripts found for name '+testname 00300 print 'Using the following one '+ theScript 00301 print "Found", theScript 00302 return theScript 00303 00304 def ispythonscript(self, thescript): 00305 if(string.find(thescript,'.py', len(thescript)-3) > 0): 00306 return True 00307 else: 00308 return False 00309 00310 def locatedata(self, datafile): 00311 for repository in self.dataBaseDirectory : 00312 00313 # Skip hidden directories 00314 filter_hidden = ' | grep -vE "^\\." | grep -vE "/\\."' 00315 00316 # See if find understands -L or -follow (depends on find version) 00317 (err, a) = commands.getstatusoutput('find -L ' + repository+'/ 1>/dev/null 2>&1') 00318 if not err: 00319 findstr='find -L '+repository+'/ -name '+datafile+' -print 2>/dev/null' + filter_hidden 00320 else: 00321 findstr='find '+repository+'/ -follow -name '+datafile+' -print 2>/dev/null' + filter_hidden 00322 # A '/' is appended to the directory name; otherwise sometimes find doesn't find. 00323 # Also, ignore error messages such as missing directory permissions 00324 00325 (find_errorcode, a)=commands.getstatusoutput(findstr) # stdout and stderr 00326 #if find_errorcode != 0: 00327 # print >> sys.stderr, "%s failed: %s" % (findstr, a) 00328 retval='' 00329 b=[''] 00330 if(a!=''): 00331 b=string.split(a, '\n') 00332 retval=b[len(b)-1] 00333 if(len(b) > 1): 00334 print 'more than 1 file found with name '+datafile 00335 print 'will use', retval 00336 return retval 00337 raise Exception, 'Could not find datafile %s in the repository directories %s' \ 00338 % (datafile, self.dataBaseDirectory)
1.8.0
