doc:user:tutorials:test_python
This is an old revision of the document!
#!/usr/bin/env python # -*- coding: utf-8 -*- # # lance une serie de tests dans des repertoires donnes from __future__ import print_function from __future__ import absolute_import from __future__ import division from future import standard_library standard_library.install_aliases() from builtins import zip from builtins import range from builtins import object from past.utils import old_div import fnmatch, sys, os, os.path, shutil, glob, platform, datetime import re, socket import threading, queue, time import traceback, subprocess from operator import itemgetter from .pyutils import * import toolbox.stp2py from .concolor import * thcolors = [ fg_green, fg_cyan, fg_yellow, fg_magenta, fg_blue, fg_red ] writelock = threading.Lock() # sortie stdout fslock = threading.Lock() # file system lock (mkdir/rmdir) popenlock = threading.Lock() class BatTest(object): def __init__(self, battery, fullfile, num=0): self.battery = battery self.fullfile = fullfile self.num = num self.badstatus=False #self.pid=None def write(self, string): writelock.acquire() setColor(thcolors[self.num%len(thcolors)]) sys.stdout.write( "[%d] %s" % (self.num, string) ) resetColor() writelock.release() # -- LANCE METAFOR -- def runModule(self): cleanneeded=False files=glob.glob(self.fullfile) files.sort() for pyfile in files: # utile si serie module = fileToModule(pyfile, verb=False) #self.write("running module: %s" % module) resfile = module.replace('.',os.sep)+'.res' # fichier resultat resdir = os.path.dirname(resfile) if resdir == '' : # laisser la version ci dessus sinon il fait une recursion infinie resdir = os.path.dirname(os.path.abspath(resfile)) # abspath necessaire si le test est dans "." startT = datetime.datetime.now() # determine method method='meta' for mods in self.battery.cplx_exec: if "*" not in mods: mods = mods + "*" if fnmatch.fnmatch(module, mods): method='exec' break for mods in self.battery.cplx_import: if "*" not in mods: mods = mods + "*" if fnmatch.fnmatch(module, mods): method='import' break for mods in self.battery.restart: if "*" not in mods: mods = mods + "*" if fnmatch.fnmatch(module, mods): method='restart' break # restart : on renomme le .res en .res1 if method == 'restart' : resfile = module.replace('.',os.sep)+'.restart.res' # fichier resultat #print "restart res file = ", resfile # test metafor if not os.path.isfile(resfile) \ or os.path.getmtime(resfile) < os.path.getmtime(pyfile): cleanneeded=True try: cwd=os.getcwd() # resDir fslock.acquire() if not os.path.isdir(resdir): os.makedirs(resdir) fslock.release() # resfile fileout = open(resfile,'w') #cmd=[self.battery.python] #if self.battery.nice: # cmd = [self.battery.nice]+cmd cmd = self.battery.startCmd + self.battery.nice + self.battery.affinity + [self.battery.python] # print ("cmd = ", cmd) # python 3 "print" stament is buffered by default (unlike python2) # this prevents us from seeing python print and std::cout is the correct order # => we set PYTHONUNBUFFERED to true (we could use 'python -u' too) os.environ['PYTHONUNBUFFERED']='TRUE' # subprocess global popenlock popenlock.acquire() if isUnix(): # sans close_fds, ca freeze p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fileout, stderr=fileout, env=os.environ, shell=False,close_fds=True) else: # si nice+Windows => "shell=True" (pour "start") subProcessFlag = self.battery.getSubprocessFlags() p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fileout, stderr=fileout, env=os.environ, shell=True, creationflags=subProcessFlag) popenlock.release() # commandes communes : pin = p.stdin pin.write(b"from __future__ import print_function\n") pin.write(b"from past.builtins import execfile\n") if method=='exec': pin.write( ("__file__=r'%s'\n" % pyfile).encode('utf-8')) # needed by .pythonrc.py pin.write(b"import sys\n") pin.write(b"print (sys.argv)\n") # avant execution du .pythonrc if self.battery.useFPE : pin.write(b"sys.argv.append('-fpe')\n") # avant execution du .pythonrc.py pin.write(b"sys.argv.append('-nogui')\n") # definition du nombre de threads pin.write(b"sys.argv.append('-k')\n") # avant execution du .pythonrc pin.write( ("sys.argv.append('%d')\n" % self.battery.numThreads).encode('utf-8')) # avant execution du .pythonrc # pin.write(b"print('sys.argv=', sys.argv)\n") # avant execution du .pythonrc # Execution .pythonrc.py pin.write( ("execfile(r'%s')\n"%os.path.join(self.battery.mtfdir, '.pythonrc.py') ).encode('utf-8') ) #pin.write("setNumTasks(1)\n") # de maniere a ce que les params ne demarrent pas nbTasks tests ... pin.write( ("setNumTasks(%d)\n"%self.battery.numTasks).encode('utf-8')) # WDirRoot (pour sauvegarde sur disques locaux sur le cluster) if self.battery.wDRoot != '' : self.write("setTheWDirRoot('%s')\n"%self.battery.wDRoot) pin.write( ("setTheWDirRoot('%s')\n"%self.battery.wDRoot).encode('utf-8')) # selon type de run if method=='exec': self.write("\texecfile(r'%s')\n" % pyfile) # note: past.builtins.execfile does not always work # (apps.remeshing2.fullAuto.taylor2dImpRemeshing fails?) pin.write( ("exec(open(r'%s').read())\n" % pyfile).encode('utf-8') ) elif method=='import': self.write("\timport %s\n" % module) pin.write( ("import %s\n" % module).encode('utf-8')) elif method=='restart': self.write("\trestart %s\n" % module) pin.write( ("load(r'%s')\n" % module).encode('utf-8')) pin.write( ("restart(%d)\n"%self.battery.restartStepNo).encode('utf-8')) else: self.write("\tmeta('%s')\n" % module) pin.write( ("load(r'%s')\n" % module).encode('utf-8')) pin.write(b"meta()\n") self.write("\t\tstarted on: %s\n" % startT) pin.close() retcode = p.wait() fileout.close() # 0 < retcode < 127 => USER # 128 < retcode < 255 => Unix Signals # 139 => Unix seg-fault (sigsegv = 11 = 139-128) # retcode = -1073741510 : sig-break windows if not isUnix() and retcode<0 and not self.battery.keepFacs: # win multi-thread Sig-Break self.cleanModule() raise Exception("bad retcode: %d" % retcode) #print "[%d]job done. %d" % (self.num, retcode) #self.pid=None except (Exception, KeyboardInterrupt) as e: # catch SigBreaks, bad retcodes, etc self.write("Exception in runModule! : %s\n" % e) try : # si l'exception arrive avant qu'on ai defini fileout fileout.close() except : pass self.cleanModule() raise stopT = datetime.datetime.now() self.write("\t\telapsed: %s\n" % (stopT-startT)) self.postInfos(pyfile, resfile) if self.badstatus: return if cleanneeded and not self.battery.keepFacs: self.postCleanModule() def postInfos(self, pyfile, resfile): if os.path.isfile(resfile): tscfound=False for line in open(resfile,'r'): if not tscfound and line.find("[TSC")!=-1: tscfound=True if line.find("Traceback")!=-1: #print "line = ", line self.dispWarningAndTouch(pyfile, resfile, "python exception detected!") if line.find(" raise")!=-1: self.dispWarningAndTouch(pyfile, resfile, line.replace('\n','')) if line.find("FATAL ERROR")!=-1: self.dispWarningAndTouch(pyfile, resfile, "Metafor Fatal Error detected!") if not tscfound: self.dispWarningAndTouch(pyfile, resfile, "no TSC in %s!" % os.path.basename(resfile) ) else: self.dispWarningAndTouch(pyfile, resfile, '%s file is missing!' % os.path.basename(resfile) ) def dispWarningAndTouch(self, pyfile, resfile, msg): self.badstatus=True self.write("\t\tWARNING: %s\n" % msg) # touch pyfile if os.path.isfile(pyfile): #os.utime(pyfile, None) os.utime(pyfile, (time.time()+10.0,time.time()+10.0)) # decallage du touch de 10 sec pour forcer le rerun (parfois ca marche pas) def cleanModule(self): # -- Nettoyage des crasses dans le dir du fichier python -- self.postCleanModule() # au cas ou... for pyfile in glob.glob(self.fullfile): # .pyc for ext in ['.pyo','.pyc']: pycfile = pyfile.replace('.py',ext) if os.path.isfile(pycfile): try: # sur les versions installees, les pyc appartiennent a l'os. les utilisateurs ne peuvent pas toujours y toucher... os.remove(pycfile) self.write("\t%s deleted\n" % os.path.basename(pycfile)) except: self.write("\tUnable to remove %s : file protected by the os.\n" % os.path.basename(pycfile)) # .res module = fileToModule(pyfile, verb=False) resfile = module.replace('.',os.sep)+'.res' # fichier resultat if os.path.isfile(resfile): os.remove(resfile) self.write("\t%s deleted\n" % resfile) # .tsc tscfile = module.replace('.',os.sep)+'.tsc' # fichier resultat if os.path.isfile(tscfile): os.remove(tscfile) self.write("\t%s deleted\n" % tscfile) # res dir resdir = os.path.dirname(os.path.abspath(resfile)) # abspath necessaire si le test est dans "." fslock.acquire() self.cleanDir(resdir) fslock.release() def cleanDir(self, dir): # recursive delete ld=glob.glob(dir) for fil in ld: if os.path.isdir(fil): try: #self.write("\ttrying to clean %s\n" % dir) os.rmdir(dir) # only works if empty self.write("\t%s deleted\n" % dir) except: pass def postCleanModule(self): # -- Suppression du workspace -- try: #print "postclean...", self.fullfile if self.battery.wDRoot != '' : workspaceBase = os.path.join(self.battery.wDRoot, 'workspace') else: workspaceBase ='workspace' for file in glob.glob(self.fullfile): # facs module = fileToModule(file, verb=False) workspace = os.path.join(workspaceBase,module.replace('.','_')) if os.path.isdir(workspace): shutil.rmtree(workspace) self.write("\t%s deleted\n" % workspace) # parametric workspace = os.path.join(workspaceBase,os.path.basename(file).replace('.py','')) if os.path.isdir(workspace): shutil.rmtree(workspace) self.write("\t%s deleted\n" % workspace) # teste si c'est une serie (tests par enchainement) reg1=r"(.+)_0*([1-9][0-9]*)" exp1= re.compile(reg1) m = exp1.match(module) if m: module = m.group(1) workspace = os.path.join(workspaceBase,module.replace('.','_')) if os.path.isdir(workspace): shutil.rmtree(workspace) self.write("\t%s deleted\n" % workspace) # crasses annexes # ... except: # WindowsError si fichiers en cours d'utilisation pass def verifModule(self): tsc=[] leaks=[] files=glob.glob(self.fullfile) files.sort() for pyfile in files: module = fileToModule(pyfile, verb=False) resfile = module.replace('.',os.sep)+'.res' # fichier resultat if os.path.isfile(resfile): for line in open(resfile,'r'): if line.find("[TSC-LKS]")!=-1: leaks.append(line) elif line.find("[TSC")!=-1: tsc.append(line) if(len(tsc)): return tsc + leaks # on retourne les leaks uniquement si le test retourne qq cghose d'autre else: return tsc def buildTSCFile(self): files = glob.glob(self.fullfile) files.sort() for pyfile in files: tscs = [] leaks = [] module = fileToModule(pyfile, verb=False) resfile = module.replace('.',os.sep)+'.res' # fichier resultat if os.path.isfile(resfile): for line in open(resfile,'r'): if line.find("[TSC-LKS]")!=-1: leaks.append(line) elif line.find("[TSC")!=-1: tscs.append(line) fname = module.replace('.',os.sep)+'.tsc' # fichier TSC tscFile = open(fname, 'w') if len(tscs) == 0: tscFile.write("[TSC-FAILED]\n") else: for tsc in tscs: tscFile.write("%s" % tsc) for leak in leaks: tscFile.write("%s" % leak) tscFile.close() def diffTSC(self): import re, difflib # diff de valeur : # regTsc : # groupe 1 : code du TSC # groupe 2 : nom de la valeur # groupe 3 : valeur self.regTsc = re.compile(r'\[TSC-(.+)\]\s*(.+)\s*:\s*([+\-0-9\.eE]*)') self.regTscTol = re.compile(r'\[TSC-(.+)\]\s*(.+)\s*:\s*([+\-0-9\.eE]*)\s*Tol\s*:\s*([+\-0-9\.eE])') self.regFailed = re.compile(r'\[TSC-FAILED\]') vals={} files = glob.glob(self.fullfile) files.sort() for pyfile in files: tscs = {} module = fileToModule(pyfile, verb=False) srcFile = pyfile.replace('.py','.tsc') # resultat commite self.fillEntries(srcFile, module, tscs, src=True) tscFile = os.path.abspath(module.replace('.',os.sep)+'.tsc') # resultat battery self.fillEntries(tscFile, module, tscs, src=False) resFile = os.path.abspath(module.replace('.',os.sep)+'.res') # res files #Check Values for code, txt in list(tscs.items()): if code not in vals: vals[code]=[] for name, diff in list(txt.items()) : if diff.old == diff.new: continue elif diff.old != Diff.MISSING and diff.new != Diff.MISSING: try: old = float(diff.old) new = float(diff.new) if new < (old-diff.tol) or (old+diff.tol) < new : color = "#FFFFFF" # classic diff => white ad = new-old if ad.is_integer(): absDiff = '%d'%ad else: absDiff = '%4.2e'%ad try: vd = old_div((new-old), (old+new))*2*100 vDiff = '%6.1f'%vd except: vDiff = '-' vals[code].append([color, name, pyfile.replace('\\','\\\\'), resFile.replace('\\','\\\\'), srcFile.replace('\\','\\\\'), tscFile.replace('\\','\\\\'), diff.old, diff.new, absDiff, vDiff]) except: absDiff = 'NaN' vDiff = 'NaN' vals[code].append([color, name, pyfile.replace('\\','\\\\'), resFile.replace('\\','\\\\'), srcFile.replace('\\','\\\\'), tscFile.replace('\\','\\\\'), diff.old, diff.new, absDiff, vDiff]) elif diff.old==Diff.MISSING and diff.new != Diff.MISSING: color="#CCFFCC" # new value => green absDiff = diff.new vDiff = '-' vals[code].append([color, name, pyfile.replace('\\','\\\\'), resFile.replace('\\','\\\\'), srcFile.replace('\\','\\\\'), tscFile.replace('\\','\\\\'), diff.old, diff.new, absDiff, vDiff]) elif diff.old!=Diff.MISSING and diff.new==Diff.MISSING: color="#FFCCCC" # old value - new missing => red absDiff = diff.old vDiff = '-' vals[code].append([color, name, pyfile.replace('\\','\\\\'), resFile.replace('\\','\\\\'), srcFile.replace('\\','\\\\'), tscFile.replace('\\','\\\\'), diff.old, diff.new, absDiff, vDiff]) return vals def fillEntries(self, file, module, diffs, src=True): if os.path.isfile(file): for line in open(file,'r'): n = self.regTsc.match(line) if src : m = self.regTscTol.match(line) else : m = None o = self.regFailed.match(line) if m : code = m.group(1) txt = module + " - " + m.group(2) val = m.group(3) if code not in diffs: diffs[code]={} if txt not in diffs[code]: diffs[code][txt]=Diff() diffs[code][txt].old=val diffs[code][txt].tol=float(m.group(4)) elif n: code = n.group(1) txt = module + " - " + n.group(2) val = n.group(3) if code not in diffs: diffs[code]={} if txt not in diffs[code]: diffs[code][txt]=Diff() if src: diffs[code][txt].old=val else: diffs[code][txt].new=val elif o : code = 'FAILED' if code not in diffs: diffs[code]={} if module not in diffs[code]: diffs[code][module]=Diff() if src: diffs[code][module].old='failed!' else: diffs[code][module].new='failed!' else : print("Error : no regExp math line : ") print("\t", line) else: code = 'FAILED' if code not in diffs: diffs[code]={} if module not in diffs[code]: diffs[code][module]=Diff() if src: diffs[code][module].old='failed!' else: diffs[code][module].new='failed!' # ---------------------------------------------------------------------------------------- # -- diffs -- head=r""" <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml"> <head> <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> <title>Diffs %s - %s</title> <style type="text/css"> <!-- body { font-family: Geneva, Arial, Helvetica, sans-serif; font-size: 12px; } p { font-family: "Courier New", Courier, monospace; font-size: 12px; } table { font-size: 12px; } h2 { font-size: 18px; } th { color: #FFFFFF; background-color: #000000; } .style1 {font-family: "Courier New", Courier, monospace} --> </style> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.3/jquery.min.js"></script> <script src="http://code.jquery.com/ui/1.9.2/jquery-ui.js"></script> <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/dt/dt-1.10.15/datatables.min.css"/> <script type="text/javascript" src="https://cdn.datatables.net/v/dt/dt-1.10.15/datatables.min.js"></script> <script langage="Javascript" type="text/javascript"> function initDataTable(tableID) { $("#"+tableID).DataTable({ "bSortClasses": false, "bJQueryUI": false, "bAutoWidth": true, "bDeferRender": true, "bPaginate":true, "sPaginationType": "full_numbers", "serverSide": false, "stateSave": true, "ordering" : true, "aaSorting": [[ 4, "desc" ]], "columns": [ { "type": "html" }, { "type": "html-num-fmt" }, { "type": "html-num-fmt" }, { "type": "html-num-fmt" }, { "type": "html-num-fmt" }] }); $("#"+tableID).show(); } function initDataTableWithTotal(tableID) { $("#"+tableID).DataTable({ "bSortClasses": false, "bJQueryUI": false, "bAutoWidth": true, "bDeferRender": true, "bPaginate":true, "sPaginationType": "full_numbers", "serverSide": false, "stateSave": true, "ordering" : true, "aaSorting": [[ 4, "desc" ]], "columns": [ { "type": "html" }, { "type": "html-num-fmt" }, { "type": "html-num-fmt" }, { "type": "html-num-fmt" }, { "type": "html-num-fmt" }], footerCallback: function ( row, data, start, end, display ) { var api = this.api(); // Remove the formatting to get float data for summation var floatVal = function ( i ) { if(typeof(i) == "string"){ var temp = Number(i.replace(/^<div[^>]*>|<\/div>$/g, "")); if(Number.isNaN(temp)) return 0; else return temp;} else{ return Number(i);} }; for (var i = 1; i < 5; ++i) { // Total over all pages var total = api.column(i).data().reduce( function (a, b) { return floatVal(a) + floatVal(b); }, 0 ); // Total over this page var pageTotal = api.column( i, { page: 'current'} ).data().reduce( function (a, b) { return floatVal(a) + floatVal(b); }, 0 ); var percent = pageTotal/total*100; // Update footer $( api.column( i ).footer() ).html( ""+pageTotal.toExponential(5)+" ( "+ percent.toFixed(5) +"%% of "+total.toExponential(5)+")" ); } } }); $("#"+tableID).show(); } function diffTSC(color,moduleCodeString, pyFile, resFile, oldValue, newValue, absDiffVal, diffValue ) { var msg1 = '<tr bgcolor='+color+'>' msg1 += '<td><span class="style1"> '+moduleCodeString+' </span>' if (!(pyFile == '')){msg1 += '[<A type="application/python" href="file:///'+pyFile+'"> .py</A>]'} if (!(resFile == '')){msg1 += '[<A href="file:///'+resFile+'"> .res</A>]'} msg1 +='</td>' var msg2 = '<td><div align="center">'+oldValue+'</div></td>' var msg3 = '<td><div align="center">'+newValue+'</div></td>' var msg4 = '<td><div align="center">'+absDiffVal+'</div></td>' var msg5 = '<td><div align="center">'+diffValue+'</div></td>' document.writeln(msg1+msg2+msg3+msg4+msg5) } function diffTSC2(color,moduleCodeString, pyFile, resFile, oldTscFile, newTscFile, oldValue, newValue, absDiffVal, diffValue ) { var msg1 = '<tr bgcolor='+color+'>' msg1 += '<td><span class="style1"> '+moduleCodeString+' </span>' if (!(pyFile == '')){msg1 += '[<A type="application/python" href="file:///'+pyFile+'"> .py</A>]'} if (!(resFile == '')){msg1 += '[<A href="file:///'+resFile+'"> .res</A>]'} if (!(oldTscFile == '')){msg1 += '[<A type="application/python" href="file:///'+oldTscFile+'"> old .tsc</A>]'} if (!(newTscFile == '')){msg1 += '[<A type="application/python" href="file:///'+newTscFile+'"> new .tsc</A>]'} msg1 +='</td>' var msg2 = '<td><div align="center">'+oldValue+'</div></td>' var msg3 = '<td><div align="center">'+newValue+'</div></td>' var msg4 = '<td><div align="center">'+absDiffVal+'</div></td>' var msg5 = '<td><div align="center">'+diffValue+'</div></td>' document.writeln(msg1+msg2+msg3+msg4+msg5) } function headTxt(ref, fileName) { var msg1 = '<a name='+ref+' />' msg1 += '<h2>'+fileName+'</h2>' var msg2 = '<a href="#top">[top]</a>' var msg3 = '<table width="100 %%" border="1" id='+ref+' style="display:none"> <thead> <tr> <th>Test</th> <th>Old</th> <th>New</th> <th>Abs Diff</th><th> %% Diff</th> </tr> </thead> \n <tbody>' document.writeln(msg1+msg2+msg3) } function headTxtWithTotal(ref, fileName) { var msg1 = '<a name='+ref+' />' msg1 += '<h2>'+fileName+'</h2>' var msg2 = '<a href="#top">[top]</a>' var msg3 = '<table width="100 %%" border="1" id='+ref+' style="display:none"> <thead> <tr> <th>Test</th> <th>Old</th> <th>New</th> <th>Abs Diff</th><th> %% Diff</th> </tr> </thead> \n' var msg4 = '<tfoot> <tr> <th style="text-align:right">Total:</th> <th> 0</th> <th> 0</th> <th> 0</th> <th> 0</th></tr> </tfoot> \n <tbody>' document.writeln(msg1+msg2+msg3+msg4) } function switchMode() { """ class Diff(object): """ One diff (old & new numeric values) """ MISSING="missing!" def __init__(self): self.new = self.MISSING self.old = self.MISSING self.tol = 0.0 class DiffGenerator(object): def __init__(self, battery, part=False, codes=None): self.battery = battery # for parameters self.diffpart = part if codes : self.codes = codes # for diff on certain codes only else : self.codes = battery.codes # for diff on certain codes only def htmlhead(self): global head return head % (machineid(), socket.gethostname()) def diffOneCode(self, fname, lines, code): """ diff one given result file - returns html code """ import sys, re diffs={} # match a "parametric/normal" diff regu=r"^([\+\-])([^\s+]+).+([\[].+[\]])\s+([^:]+)[:\s]*(\w+\s=\s)?([+\-0-9\.eE]*)" # ok pour "svn diff" et "difflib.unified_diff" # group(1) : + ou - # group(2) : nom du module # group(3) : [TSC-*] # group(4) : nom de la courbe OU exp_* (parametric) # group(5) : nom de la variable (parametric) # group(6) : valeur exp1 = re.compile( regu ) # match failed test regu=r"^([\+\-])([^\s+]+).+(TSC-FAILED)" # ok pour "svn diff" et "difflib.unified_diff" # group(1) : + ou - # group(2) : nom du module # group(3) : FAILED! exp2 = re.compile( regu ) #for l in lines: # (pour difflib) # l=l[:-1] # remove newline (pour difflib) #for l in lines.split('\n'): for l in lines.splitlines(): m = exp1.match(l) n = exp2.match(l) if m: # a diff bkey = m.group(2)+" - "+m.group(4) if m.group(5): bkey=bkey+" - "+m.group(5) key=bkey no=1 while True: #print "key=",key if key not in diffs: diffs[ key ] = Diff() d = diffs[ key ] if m.group(1)=='+' and d.new==Diff.MISSING: d.new = m.group(6) break if m.group(1)=='-' and d.old==Diff.MISSING: d.old = m.group(6) break no+=1 key = '%s (%d)' % (bkey, no) # manage multiple output for the same test elif n: # a failed test key=n.group(2) if key not in diffs: diffs[ key ] = Diff() d = diffs[ key ] if n.group(1)=='+': d.new = "failed!" else: d.old = "failed!" if self.diffpart: # on va virer tous les tests non specifies en ligne de commande # !! utile si on lance un repertoire particulier # !! ennuyeux si des tests ont ete renommes ou deplaces tests=set() for mods in self.battery.loopOn(self.battery.dirs): for mod in glob.glob(mods): tests.add(fileToModule(mod, verb=False)) sk = [x for x in list(diffs.keys()) if x.split()[0] in tests] # on garde que les tests lances #sk = diffs.keys() sk.sort() # sorted module names vk = [diffs[k] for k in sk] # sorted diffs else: # on garde toutes les diffs independamment de ce qu'on a lance sk = list(diffs.keys()) sk.sort() vk = [diffs[k] for k in sk] title = os.path.basename(fname) tex = '<script langage="Javascript" type="text/javascript">\n' tex += '<!--\n' if(code == "CPU" or code == "MEM"): tex += 'headTxtWithTotal("%s","%s")\n' % (code, title) else: tex += 'headTxt("%s","%s")\n' % (code, title) for k, v in zip(sk, vk): color="#FFFFFF" if v.old==v.new: color="#CCFFCC" elif (v.old==Diff.MISSING or v.new==Diff.MISSING): color="#FFCCCC" vdiff = None absDiff = None trueDiff = True try: vnew=float(v.new) vold=float(v.old) vdiff=(float(v.new)-float(v.old))/float(v.old)*100 absDiff = (float(v.new)-float(v.old)) #if (abs(vdiff)<0.1): # trueDiff = False vdiff = '%6.1f' % vdiff if absDiff.is_integer() : absDiff = '%d' % absDiff else: absDiff = '%4.2e' % absDiff except: vdiff = 'NaN' absDiff = 'NaN' #tex += '<tr bgcolor="%s">\n' % color module = k.split()[0] pyfile = moduleToFile(module) if not(os.path.isfile(pyfile)): pyfile='' resfile = os.path.abspath(module.replace('.',os.sep)+'.res') # fichier resultat if not(os.path.isfile(resfile)): resfile = '' if (trueDiff): tex += 'diffTSC("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")\n'% (color, k, pyfile.replace('\\','\\\\'), resfile.replace('\\','\\\\'), v.old, v.new, absDiff, vdiff) tex += '-->\n' tex += '</script>\n' tex += '</tbody>\n' tex += '</table>\n' return (tex, len(sk)) def generate(self): self.battery.setPaths() print("'diff' battery results") html1=self.htmlhead() html2 = '<body style="display:none"> \n' html2+='<a name="top" />\n' html2+='<h1>HTML report on %s (%s)' % (machineid(),compilerid()) html2+= '<div style="float: right"><input type="checkbox" id="cbox" onchange="switchMode()"><label for="cbox"> Advanced Mode </label></div></h1>' html2+='<h3>(%s, %s)</h3>\n' % (socket.gethostname(), os.getcwd()) html2+='<p><ul>' html3 = "" html4 = '<script type="text/javascript" class="init"> \n' html4 += '$(document).ready( function () {\n' html4 += ' $("body").show();\n' for c in self.codes : if c not in self.battery.excludeCodesDiffs: fname = '%s%s%s-%s-%s.txt' % (self.battery.verifsrc, os.sep,c, machineid(),compilerid()) print("\trunning 'git diff' on %s" % fname) # (using "difflib") #import difflib #srcname = '%s%s%s-%s-%s.txt' % (self.battery.verifsrc, os.sep,c, machineid(), compilerid()) #srclines = open(srcname, 'U').readlines() #flines = open(fname, 'U').readlines() #diff = difflib.unified_diff(srclines, flines) #cmd = 'svn diff %s' % fname fname_dir, fname_base = os.path.split(fname) cwd = os.getcwd() os.chdir(fname_dir) if isUnix(): cmd = 'git diff HEAD -- %s' % fname_base else: cmd = 'git diff --ignore-cr-at-eol HEAD -- %s' % fname_base child = os.popen(cmd) diff = child.read() err = child.close() if err: print('\t!! %r failed with exit code %d' % (cmd, err)) # nouveau code / fichier absent / svn pourri os.chdir(cwd) tex, nb = self.diffOneCode(fname, diff, c) if nb>0: html3 += tex html2 += '<li><a href="#%s">%s</a>: %d diffs</li>' % (c, fname, nb) html1 += ' if(document.getElementById("cbox").checked){\n' if( c == "CPU" or c == "MEM"): html1 += ' initDataTableWithTotal("%s");}\n' % (c) else: html1 += ' initDataTable("%s");}\n' % (c) html1 += ' else{\n' html1 += ' $("#%s").DataTable().destroy();} \n' % (c) html4 += ' $("#%s").show(); \n' % (c) else: html2 += '<li>%s: %d diffs</li>' % (fname, nb) html1 += "}\n</script>\n" html2 +='</ul></p>\n' html3 += "\n</body>\n</html>\n" html4 += '} ); \n</script>\n' html1 += html4 html1 += '</head>\n' if self.diffpart: fname = '%s/%s-diffsPart.html' % (self.battery.verifdir, machineid()) else : fname = '%s/%s-diffs.html' % (self.battery.verifdir, machineid()) print("\t=> %s created" % fname) file = open(fname,'w') file.write(html1+html2+html3) file.close() self.battery.resetPaths() # ---------------------------------------------------------------------------------------- class DiffGeneratorTsc(object): def __init__(self, battery, codes=None): self.battery = battery # for parameters if codes : self.codes = codes # for diff on certain codes only else : self.codes = battery.codes # for diff on certain codes only def htmlhead(self): global head return head % (machineid(), socket.gethostname()) def generate(self): self.battery.setPaths() diffs = {} for code in self.battery.codes : diffs[code]=[] for mod in self.battery.loopOn(self.battery.dirs): bt = BatTest(self.battery,mod) modDiffs = bt.diffTSC() for code in self.battery.codes : if code in modDiffs: diffs[code].extend(modDiffs[code]) print("'diff' battery results") html1=self.htmlhead() html2 = '<body style="display:none"> \n' html2+='<a name="top" />\n' html2+='<h1>HTML report on %s (%s)' % (machineid(),compilerid()) html2+= '<div style="float: right"><input type="checkbox" id="cbox" onchange="switchMode()"><label for="cbox"> Advanced Mode </label></div></h1>' html2+='<h3>(%s, %s)</h3>\n' % (socket.gethostname(), os.getcwd()) html2+='<p><ul>' html3 = "" html4 = '<script type="text/javascript" class="init"> \n' html4 += '$(document).ready( function () {\n' html4 += ' $("body").show();\n' for code in self.battery.codes : nb = len(diffs[code]) if nb > 0: html2 += '<li><a href="#%s">%s</a>: %d diffs</li>' % (code, code, nb) html3 += '<script langage="Javascript" type="text/javascript">\n' html3 += '<!--\n' if(code == "CPU" or code == "MEM"): html3 += 'headTxtWithTotal("%s","%s")\n' % (code,code) else: html3 += 'headTxt("%s","%s")\n' % (code,code) #diffs[code].sort() #comment trier sur base des noms => 1 er indice du tableau for d in sorted(diffs[code], key=itemgetter(1)): if len(d) > 0: html3 += 'diffTSC2("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")\n'%(d[0],d[1],d[2],d[3],d[4],d[5],d[6],d[7],d[8],d[9]) html3 += '-->\n' html3 += '</script>\n' html3 += '</tbody>\n' html3 += '</table>\n' html1 += ' if(document.getElementById("cbox").checked){\n' if( code == "CPU" or code == "MEM"): html1 += ' initDataTableWithTotal("%s");}\n' % (code) else: html1 += ' initDataTable("%s");}\n' % (code) html1 += ' else{\n' html1 += ' $("#%s").DataTable().destroy();} \n' % (code) html4 += ' $("#%s").show(); \n' % (code) else: html2 += '<li>%s: %d diffs</li>' % (code, nb) html1 += "}\n</script>\n" html2 +='</ul></p>\n' html3 += "\n</body>\n</html>\n" html4 += '} ); \n</script>\n' html1 += html4 html1 += '</head>\n' # ecriture du fichier fname = '%s/%s-diffsTsc.html' % (self.battery.verifdir, machineid()) print("\t=> %s created" % fname) file = open(fname,'w') file.write(html1+html2+html3) file.close() self.battery.resetPaths() # ---------------------------------------------------------------------------------------- class WorkerThread(threading.Thread): """ A worker thread reading and executing a task queue filled by the main thread """ def __init__(self, battery, num=0, queue=None): threading.Thread.__init__(self) self.battery = battery self.num = num self.queue = queue #self.setDaemon(True) self.test=None def write(self, string): writelock.acquire() setColor(thcolors[self.num%len(thcolors)]) sys.stdout.write("[%d] %s" % (self.num, string)) resetColor() writelock.release() def run(self): self.write("START\n") while True: cmd, file = self.queue.get() if cmd == 'STOP': self.queue.task_done() break try: self.runFile(cmd, file) except (Exception, KeyboardInterrupt) as e: self.write("%s: %s" % (sys.exc_info()[0], sys.exc_info()[1]) ) writelock.acquire() traceback.print_exc() writelock.release() self.battery.errorEvent.set() # signale au main qu'il y a eu une couille if not self.queue.empty(): cmd, file = self.queue.get() # libere le main thread self.queue.task_done() break self.queue.task_done() self.write("END\n") def runFile(self, cmd, files): self.test = BatTest(self.battery, files, self.num) if cmd=='clean': self.test.cleanModule() elif cmd=='run': self.test.runModule() if self.battery.buildTsc: self.test.buildTSCFile() elif cmd=='rerun': self.test.cleanModule() self.test.runModule() if self.battery.buildTsc: self.test.buildTSCFile() self.test=None # def kill(self): # print "kill......", self.test.pid # if self.test and self.test.pid: # print "killing pid=", self.test.pid # import ctypes # PROCESS_TERMINATE = 1 # handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, self.test.pid) # ctypes.windll.kernel32.TerminateProcess(handle, -1) # ctypes.windll.kernel32.CloseHandle(handle) # print "done." # ---------------------------------------------------------------------------------------- class Battery(object): """ Main class managing a test battery """ def __init__(self): """ feel free to modify public variables to modify the battery behavior """ # default dirs self.dirs = 'tests' # verif dirs self.verifsrc = None self.verifdir = 'verif' self.buildTsc = False # metafor dir self.mtfdir = '.' # Working Dir Root self.wDRoot = '' # modules to be run by "execfile" instead of "meta" self.cplx_exec = [ ] # modules to be run by "import" instead of "meta" self.cplx_import = [ ] # modules to be run by in parallel (TBB/Blas) (one at a time) self.parallel = [ ] # modules to be run by restart (for launch & launchGui) self.restart = [] self.restartStepNo = -1 # result codes self.codes = [ 'FAILED' ] self.excludeCodesDiffs = [ ] # platform specific self.skips = [ '__init__.py', '*Bacon.py', '*stp.py', '*Dat.py'] self.skipdirs = [ '.svn', 'toolbox', 'tools', 'matlab'] # global options self.useFPE = False self.withWER = False # if true, activate the Windows Error Report popup opening on failure self.keepFacs = True # parallel parameters self.numTasks = 1 self.numThreads = 1 # cmd self.nice = [] self.affinity = [] if isUnix(): self.startCmd = [] else: self.startCmd = ['start', '/B', '/WAIT'] # threads related parms self.queue = None self.threads = [] self.errorEvent = threading.Event() self.python = sys.executable #series management (chaining tests) self.lasts=None def useDebug(self): if not isUnix(): self.python = sys.executable.replace('.exe','_d.exe') def getSubprocessFlags(self): # from http://stackoverflow.com/questions/24130623/using-python-subprocess-popen-cant-prevent-exe-stopped-working-prompt if self.withWER : subProcessFlags = 0 else : import ctypes SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX); CREATE_NO_WINDOW = 0x08000000 # From Windows API subProcessFlags = CREATE_NO_WINDOW return subProcessFlags def write(self, string): writelock.acquire() sys.stdout.write("[0] %s" % string) writelock.release() def addCplxExecPath(self, path): mod = fileToModule(path) print("addCplxExecPath : ") print(" path = ", path) print(" cplxModule = ", mod) self.cplx_exec.append(mod) print(" self.cplx_exec = ", self.cplx_exec) def addCplxImportPath(self, path): mod = fileToModule(path) print("addCplxImportPath : ") print(" path = ", path) print(" cplxModule = ", mod) self.cplx_import.append(mod) print(" self.cplx_import = ", self.cplx_import) def addParallelPath(self, path): mod = fileToModule(path) print("addParallelMetaPath : ") print(" path = ", path) print(" parallelMetaModule = ", mod) self.parallel.append(mod) print(" self.parallel = ", self.parallel) def addRestartPath(self, path): mod = fileToModule(path) print("addRestartPath : ") print(" path = ", path) print(" restartModule = ", mod) self.restart.append(mod) print(" self.restart = ", self.restart) def setRestartStepNo(self, no): self.restartStepNo = no def setWDRoot(self, wd): self.wDRoot = wd def setNumThreads(self, num): self.numThreads = num def setNumTasks(self, num): self.numTasks = num def setNice(self, niceVal): if isUnix(): self.nice = ['nice', '-%d'%niceVal] else: if niceVal > 14 : prior = '/LOW' elif niceVal > 8 : prior = '/BELOWNORMAL' elif niceVal > 4 : prior = '/NORMAL' elif niceVal > 2 : prior = '/ABOVENORMAL' else: prior = '/HIGH' #self.nice = ['start', prior, '/B', '/WAIT'] self.nice = [prior] print("niceVal = ", niceVal) print("niceCmd = ", self.nice) def setAffinity(self, affinity): if isUnix(): if self.hasSysCmd('numactl'): self.affinity = [ "numactl", "--physcpubind", affinity] #, "--localalloc" ] elif self.hasSysCmd('taskset'): self.affinity = [ "taskset", "-c", affinity ] else: print("neither numactl nor taskset found on the system => affinity not fixed") else: self.affinity = [] # to do according to doc below ... print("setAffinity cmd under windows has to be impremented & tested ...") #affCmd=['/AFFINITY', traduire les numero de coeurs en hexadecimal mask : pour un 8 coeurs : 87654321 => 10101010 ...] #affCmd=['/NODE', si on vuet faire les choses bien ...] ''' NODE Specifies the preferred Non-Uniform Memory Architecture (NUMA) node as a decimal integer. AFFINITY Specifies the processor affinity mask as a hexadecimal number. The process is restricted to running on these processors. The affinity mask is interpreted differently when /AFFINITY and /NODE are combined. Specify the affinity mask as if the NUMA node's processor mask is right shifted to begin at bit zero. The process is restricted to running on those processors in common between the specified affinity mask and the NUMA node. If no processors are in common, the process is restricted to running on the specified NUMA node. Specifying /NODE allows processes to be created in a way that leverages memory locality on NUMA systems. For example, two processes that communicate with each other heavily through shared memory can be created to share the same preferred NUMA node in order to minimize memory latencies. They allocate memory from the same NUMA node when possible, and they are free to run on processors outside the specified node. start /NODE 1 application1.exe start /NODE 1 application2.exe These two processes can be further constrained to run on specific processors within the same NUMA node. In the following example, application1 runs on the low-order two processors of the node, while application2 runs on the next two processors of the node. This example assumes the specified node has at least four logical processors. Note that the node number can be changed to any valid node number for that computer without having to change the affinity mask. start /NODE 1 /AFFINITY 0x3 application1.exe start /NODE 1 /AFFINITY 0xc application2.exe ''' def runFileMT(self, cmd, file): if self.numTasks>1: #print '[0] %s %s => queue' % (cmd, file) self.queue.put( (cmd,file) ) if self.errorEvent.isSet(): raise Exception("error detected in some thread") else: # single-threaded run t = WorkerThread(self) t.runFile(cmd, file) def setPaths(self): self.old_sys_path = sys.path if 'PYTHONPATH' in os.environ: self.old_os_environ = os.environ['PYTHONPATH'] else: self.old_os_environ = '' # needed by python for this script to behave like metafor self.mtfdir = os.path.abspath(self.mtfdir) if os.path.isdir(self.mtfdir): sys.path.append(os.path.abspath(self.mtfdir)) # needed by spawned python (e.g. 'complex') to find 'wrap' os.environ['PYTHONPATH'] = self.mtfdir + pythonPathSep() + os.getcwd() + pythonPathSep() + self.old_os_environ else: self.write("metafor dir (\"%s\") not found!\n" % self.mtfdir) sys.exit() # adds nda modules ndadir = os.path.abspath(os.path.join(self.mtfdir, '..%soo_nda' % os.sep)) #print ndadir if os.path.isdir(ndadir): sys.path.append(ndadir) def resetPaths(self): sys.path = self.old_sys_path os.environ['PYTHONPATH'] = self.old_os_environ def isAParallelModule(self,file): mod = fileToModule(file) isParallel = False for mods in self.parallel : if mod.find(mods)!=-1: isParallel = True break return isParallel def start(self, cmd): """ multithreaded execution of the commands : clean, run, rerun """ self.setPaths() # generation of wDirRoot to avoid race between tasks in mt if self.wDRoot != '' : createAndCheckDir(self.wDRoot) # function of pyutils # runs // : try: #print "main loop tests //\n" # main loop tests // (run en nbtasks = 1) print("running parallel modules : ") numThreads = self.numThreads # backup nbThreads numTasks = self.numTasks # backup nbTasks #self.numTasks = 1 # 1 test at a time self.numThreads = numTasks # ran with "numTasks" threads for mod in self.loopOn(self.dirs): if self.isAParallelModule(mod) : #print ("running a parallel module : " , mod) #self.runFileMT(cmd, mod) t = WorkerThread(self) t.runFile(cmd, mod) self.numThreads = numThreads # recover nbThreads #self.numTasks = numTasks # recover nbTasks print("running parallel modules : done") except (Exception, KeyboardInterrupt) as e: # abnormal termination (sig-break, syntax error in this file) self.write("%s: %s\n" % (sys.exc_info()[0], sys.exc_info()[1]) ) writelock.acquire() traceback.print_exc() writelock.release() sys.exit() self.lasts = None # Il est necessaire de reinitialiser cet attribut suite a la premiere boucle sur les modules pour les tests paralleles. # Dans le cas contraire, les tests series ci-dessous sont skippes dans le cas du run d'un et seul module par enchainement (et a fortiori avec launch.py). #============= # runs serie : # starts the threads if self.numTasks>1: self.errorEvent.clear() self.queue = queue.Queue(self.numTasks) for t in range(self.numTasks): tr = WorkerThread(self, t+1, self.queue) self.threads.append(tr) tr.start() try: # main loop tests serie #print "main loop tests serie\n" #print "self.dirs : ",self.dirs print("running serial modules : ") for mod in self.loopOn(self.dirs): #print "mod : ",mod if not self.isAParallelModule(mod) : #print "running a serial module : " , mod self.runFileMT(cmd, mod) # normal termination... stops the threads if self.numTasks>1: self.write(" joining...\n") for t in self.threads: self.queue.put( ('STOP', '') ) # waits for the threads for t in self.threads: t.join() print("running serial modules : done") except (Exception, KeyboardInterrupt) as e: # abnormal termination (sig-break, syntax error in this file) self.write("%s: %s\n" % (sys.exc_info()[0], sys.exc_info()[1]) ) writelock.acquire() traceback.print_exc() writelock.release() if self.numTasks>1: self.write(" joining...\n") for t in self.threads: t.join() sys.exit() self.resetPaths() def loopOn(self, modules): """ loop on the modules and - convert modules to paths ( intelSig.tests => ..\oo_nda\intelSig\tests) - expand dos wildcards (apps\qs\cont* => [apps.qs.cont2, apps.qs.cont2ILU0, ...]) """ modules.sort() #print " modules : ",modules for dir in modules: #print " dir : ",dir dir = dir.replace('/',os.sep).replace('\\',os.sep) if dir.find(os.sep)!=-1 or fnmatch.fnmatch(dir,'*.py'): # c'est un fichier # case #1: "battery run apps\qs\cont2.py" filemod = os.path.abspath(dir) else: # case #2: "battery run apps.qs.cont2" filemod = moduleToFile(dir) # avoids "import" if filemod=='': self.write("ERROR: \"%s\" not found in PYTHONPATH\n" % dir) sys.exit() elif filemod.find('*')!=-1: # expand DOS wildcard # case #3: "battery run apps\qs\cont*" #print "Expanding DOS wildcard!" files=glob.glob(filemod) files.sort() for file in files: for mod in self.loopOn([file, ]): # recursive call yield mod elif os.path.isdir(filemod): # c'est un dir # case #4: "battery run apps\qs\" or "battery run apps.qs" good=True for sk in self.skipdirs: if fnmatch.fnmatch(os.path.basename(filemod), sk): good=False break if good: files = os.listdir(filemod) files.sort() for file in files: fullfile = os.path.join(filemod, file) for mod in self.loopOn([fullfile, ]): yield mod else: # c'est un simple fichier #print "+++", filemod good=False if fnmatch.fnmatch(filemod,'*.py'): good=True for sk in self.skips: if fnmatch.fnmatch(os.path.basename(filemod), sk): good=False break if good: # teste si c'est une serie (tests par enchainement) reg1=r"(.+)_0*([1-9][0-9]*)\.py" exp1= re.compile(reg1) m = exp1.match(os.path.basename(filemod)) if m: # c'est potentiellement une serie #print "serie: %s %d" % (m.group(1), int(m.group(2))) #print "self.lasts : ",self.lasts if not m.group(1)==self.lasts: # c'est pas la meme que la derniere => on skippe self.lasts = m.group(1) toyield = "%s_*.py" % os.path.join(os.path.dirname(filemod), m.group(1)) #print "toyield : ",toyield yield toyield else: # c'en est pas une #print "filemod : ",filemod yield filemod def verif(self): self.setPaths() ofiles = {} self.write("Verifying battery results...\n") # new: create local verifdir is needed # if self.verifsrc and os.path.isdir(self.verifsrc): # if os.path.isdir(self.verifdir): # shutil.rmtree(self.verifdir,ignore_errors=False, onerror=handleRemoveReadonly) # if os.path.isdir(self.verifdir): # garde-fou pour le .copytree # self.write("sleeping...") # sleep(1) # shutil.copytree(self.verifsrc, self.verifdir) if not os.path.isdir(self.verifdir): os.mkdir(self.verifdir) # open files for c in self.codes: fname = '%s/%s-%s-%s.txt' % (self.verifsrc, c, machineid(), compilerid()) ofiles[c] = open(fname, 'w') for files in self.loopOn(self.dirs): gfiles=glob.glob(files) gfiles.sort() for file in gfiles: # peut etre une serie module = fileToModule(file, verb=False) t = BatTest(self, file) tscs = t.verifModule() # chaque ligne contient des EOFs if len(tscs) == 0: ofiles['FAILED'].write( "%s : [TSC-FAILED]\n" % module ) else: for c in self.codes: for tsc in tscs: if tsc.find('[TSC-%s]' % c)!=-1: ofiles[c].write("%s : %s" % (module, tsc)) self.resetPaths() def buildTscFiles(self): self.write("Building TSC Files...\n") for files in self.loopOn(self.dirs): gfiles=glob.glob(files) gfiles.sort() for file in gfiles: # peut etre une serie module = fileToModule(file, verb=False) t = BatTest(self, file) t.buildTSCFile() # --- def handleRemoveReadonly(func, path, exc): import stat #http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied if not os.access(path, os.W_OK): # Is the error an access error ? os.chmod(path, stat.S_IWUSR) func(path) else: raise # --- def machineid(): uname = platform.uname() if uname[0] == 'Windows' or uname[2] == 'Windows' or uname[0].find("CYGWIN")!=-1: return "Windows" elif uname[0] == 'Linux': if uname[4] == 'x86_64': return "Linux64" elif uname[4] == 'alpha': return "AlphaLinux" else: return "Linux" else: return uname[0] def compilerid(): from wrap.mtGlobalw import getCompilerName name=getCompilerName() if name[:4]=='msvc': name='msvc' return name # --- gestion sig-break # lors d'un CTRL-BREAK: # windows: # single-thread: le signal est appele immediatement et l'exception est lancee # multi-threads: le main thread est bloque dans queue.put # l'object Popen.wait() renvoie un retcode negatif # il est necessaire de debloquer la queue pour que le main thread appelle le signal handler # linux: sig-break marche pas => Ctrl-C # single-thread: KeyboardInterrupt est envoye au main thread => arret propre # multi-threads: KeyboardInterrupt est envoye au main thread # les autres threads sortent de Popen via FATAL_ERROR de Metafor => continuent! class SigBreak(Exception): def __init__(self): self.msg="SIG-BREAK" def __str__(self): return self.msg def sigbreak(sig, arg): # seul le main thread recoit le signal et appelle cette fonction writelock.acquire() setColor(fg_red) print("SIG-BREAK!") setColor(fg_dark_white) writelock.release() raise SigBreak() # -- MAIN -- def moduleToFile(module): for p in sys.path: #print 'testing %s' % p dir = os.path.join(p, module.replace('.',os.sep)) dirini = os.path.join(dir, '__init__.py') file = '%s.py' % dir #print '\tdir=%s\n\tfile=%s' %(dir, file) if os.path.isfile(dirini): return dir elif os.path.isfile(file): return file return '' def fileToModule(file, verb=False): if verb: print('os.path.sep=',os.path.sep) file=os.path.abspath(file) if verb: print('file=',file) for dirname in sys.path: dirname = os.path.abspath(dirname).lower() if verb: print('module in', dirname, '?', end=' ') common = os.path.commonprefix( (file.lower(), dirname) ) if common == dirname: if verb: print('YES') strip = file[len(dirname):] if strip[0]==os.path.sep: strip=strip[1:] strip = os.path.splitext(strip)[0] strip = strip.replace(os.path.sep,'.') if verb: print('module=', strip) return strip break else: if verb: print('NO') return '' def main(battery=Battery()): # trap ctrl-break (doesn t work with GUI) # note: only main thread catchs the signals try: import signal signal.signal(signal.SIGBREAK, sigbreak) #signal.signal(signal.SIGINT, sigbreak) except: pass import argparse parser = argparse.ArgumentParser(description="Running Metafor's battery", formatter_class=argparse.RawTextHelpFormatter) # Commandes cmdHelp = [] cmdHelp.append('run : run some modules (files and/or directories)') cmdHelp.append('rerun : rerun some modules (files and/or directories)') cmdHelp.append('verif : gather results to verifdir/*.txt') cmdHelp.append('buildTsc : gather results to TSC individual files') cmdHelp.append('diff : create a diff HTML report in workspace') cmdHelp.append('diffPart : create a diff HTML report of defined module only ') cmdHelp.append('diffTsc : create a diff HTML report based on tsc files ') cmdHelp.append('clean : clean some modules') # cmd parser parser.add_argument('cmd', choices=['run','rerun','verif','buildTsc','diff','diffPart','diffTsc','clean'], nargs='?', help="battery command\n"+'\n'.join(cmdHelp)) parser.add_argument('modules', nargs='*', help='Module selection') # Options parser.add_argument('--wdroot', type=str, nargs=1, action='store',metavar='path', help='Working Dir Root') parser.add_argument('--keep', dest='keepFacs', action='store_true', help="Keep Results") parser.add_argument('--noKeep', dest='keepFacs', action='store_false', help="do Not Keep Results (default)") parser.add_argument('--fpe', action = 'store_true', help='Run in -fpe mode (NaN not allowed => failed)') parser.add_argument('--withWER', action = 'store_true', help='Enable the "Windows Error Report" popup on test failure') parser.add_argument('-d', '--debug', action = 'store_true', help='Run debug version of Metafor using python_d') parser.add_argument('-l', '--low', dest='niceVal', action='store_const', const=19, help='Low priority level run (equivalent --nice 19)') parser.add_argument('-n', '--nice', type=int, dest='niceVal', choices=list(range(1,20)), help='Define nice priority level run (1-19)') parser.add_argument('-j', '--nbTasks', type=int, metavar='N', help='Number of tasks running simultaneously') parser.add_argument('-k', '--nbThreads', type=int, metavar='N', help='Number of threads (tbb, blas,...) by tasks') parser.add_argument('-c','--codes', type=str, nargs='+', action='store', metavar='TSC', choices=['FAILED', 'STP', 'ITE', 'INW', 'EXW', 'CPU', 'MEM'], help='TSC codes list for diff among : FAILED STP ITE INW EXW CPU MEM') # difficulte a passer la liste => laisse a l'interface launch #parser.add_argument('--affinity', type=int, nargs='+', metavar='N', # help='affinity (core list)') # Parsing arguments print(sys.argv) args = parser.parse_args() #print "args", args #print "args.codes", args.codes # configure battery according to args if args.keepFacs : battery.keepFacs = args.keepFacs if args.fpe: battery.useFPE = True if args.withWER: battery.withWER = True if args.debug: battery.useDebug() if args.niceVal : battery.setNice(args.niceVal) if args.nbTasks: battery.setNumTasks(args.nbTasks) if args.nbThreads: battery.setNumThreads(args.nbThreads) # choosing defining modules to run if args.modules : battery.dirs = args.modules # WorkDir Root if args.wdroot: battery.setWDRoot(args.wdroot) # debug purpose print(args) # starting cmd if args.cmd in ['clean', 'run', 'rerun'] : battery.start(args.cmd) elif args.cmd == 'verif': battery.verif() elif args.cmd == 'buildTsc': battery.buildTscFiles() elif args.cmd == 'diff': if args.codes : diff = DiffGenerator(battery, False, args.codes) else : diff = DiffGenerator(battery, False, battery.codes) diff.generate() elif args.cmd == 'diffPart': diff = DiffGenerator(battery, True, battery.codes) diff.generate() elif args.cmd == 'diffTsc': diff = DiffGeneratorTsc(battery, battery.codes) diff.generate() else : # par defaut battery.start('run') battery.verif()
doc/user/tutorials/test_python.1599573968.txt.gz · Last modified: by boman