test.py 2.56 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
#!/usr/bin/env python

import sys
import os
import tempfile
import datetime
import subprocess
import glob
import re

sdir = os.path.dirname( os.path.abspath( sys.argv[0] ) )

def start( cmd ):
  tmp_path = tempfile.mkdtemp()
  os.chdir( tmp_path )
 
  print tmp_path
  print cmd
  # starts easywave and counts time differential between start and end
  t0 = datetime.datetime.now()
  ewrun = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
  ewrun.wait()
  delta_t = ( datetime.datetime.now() - t0 )
  print 'Runtime: %u.%u sec' % (delta_t.seconds, delta_t.microseconds)

  cmds[ cmd ] = tmp_path

def compareCmds( cmd ):

  path = []
  files = []

  maxDiff = 0.0

  for i in range( 0, 2 ):

    path.append( cmds[ cmd[i] ] )

    files.append( glob.glob( path[i] + "/*.ssh" ) )
    files[i] += glob.glob( path[i] + "/*.sshmax" )
    #files[i] += glob.glob( path[i] + "/*.time" )
  
  for f in files[0]:
    basef = os.path.basename( f )
    fnext = path[1] + '/' + basef
    
    if fnext in files[1]:
      #compare.py
      compare = subprocess.Popen( sdir + '/compare.py %s %s' % (f, fnext), shell=True, stdout=subprocess.PIPE )
      compare.wait()      
      diff = float( re.search( 'Max difference: (.*)\n', compare.stdout.read() ).group(1) )
      maxDiff = max( maxDiff, diff )
      
      if verbose:
	print basef + ": %f" % diff
      
      files[1].remove( fnext )
    else:
      print 'File "' + basef + '" is missing in 1.'
  
  for f in files[1]:
    print 'File "' + basef + '" is missing in 0.'
      
  if maxDiff > tolerance:
    print "\033[91mFailed:\033[0m",
  else:
    print "\033[92mSuccessful:\033[0m",
    
  print '%f' % maxDiff
    
cfg_path = sdir + "/config.cfg"
scs_path = sdir + "/scenarios.cfg"

runs = []
scenarios = []
cmds = {}

tolerance = 0.02
verbose = True

# read config file to get a list of programs to compare
f = open( cfg_path, 'r' )

for line in f:
  if line.lstrip()[0] != '#':
    progs = line.replace('\n','').replace('@',sdir+'/').split( ';' )
    if len( progs ) > 2:
      print 'Warning: Can compare only two programs at a time. Ignoring additional entries.'
      del progs[2:]
    runs.append( progs )

f.close()

# read scenario file to get a list of scenarios used to compare the given programs
f = open( scs_path, 'r' )

for line in f:
  if line.lstrip()[0] != '#':
    scenarios.append( line.replace('\n','').replace('@',sdir+'/') )

f.close()

# input was read successfully
for run in runs:
  
  for sc in scenarios:
        
    cmd = []
    for prog in run:
      cmd.append( prog + ' ' + sc )
      start( cmd[-1] )
      
    compareCmds( cmd )