Commit dc1b3e86 authored by Matthias Rüster's avatar Matthias Rüster
Browse files

Tests: Use expected results for comparing and split CPU and GPU tests

parent e356afe7
BengkuluSept2007
MentawaiOct2010
#!/usr/bin/env python
#!/usr/bin/env python3
import sys
import struct
if __name__ == '__main__':
fileA = open(sys.argv[1], 'r')
fileB = open(sys.argv[2], 'r')
# fileA: expected output
# fileB: computation result
# Note: computed result file could be bigger than expected output file
# computed files with GPU can be bigger than computed by CPU
try:
fileA = open(sys.argv[1], 'rb')
except OSError:
print('File ' + sys.argv[1] + ' could not be read.')
sys.exit(1)
try:
fileB = open(sys.argv[2], 'rb')
except OSError:
print('File ' + sys.argv[1] + ' could not be read.')
sys.exit(1)
numDiff = 0
maxDiff = 0.0
......@@ -16,15 +30,25 @@ if __name__ == '__main__':
while True:
strfA = fileA.read(bufSize)
if strfA == '':
if not strfA:
# EOF
break
strfB = fileB.read(bufSize)
if len(strfA) != len(strfB):
print('The files have different sizes!')
if len(strfA) > len(strfB):
print('The computed output file is smaller than expected.')
sys.exit(1)
count = len(strfA) / 4
# compare only matching data points
# the remaining bytes can differ in the very last read
# if the files have different sizes
cmp_count = min(len(strfA), len(strfB))
# make read byte array to fit with comparison byte array
strfB = strfB[:len(strfA)]
count = cmp_count / 4
bufA = struct.unpack('%uf' % count, strfA)
bufB = struct.unpack('%uf' % count, strfB)
......@@ -34,7 +58,7 @@ if __name__ == '__main__':
maxDiff = max(maxDiff, abs(valA - valB))
if fileB.read(1) != '':
print('File A is shorter than file B!')
print('Expected output is shorter than computed output!')
print('Differences: %u' % numDiff)
print('Max difference: %f' % maxDiff)
......
@../src/easywave; @../src/easywave -gpu
#!/usr/bin/env python
import sys
import os
import subprocess
import re
if __name__ == '__main__':
sdir = os.path.dirname(os.path.abspath(sys.argv[0]))
grids = sdir + '/grids'
faults = sdir + '/faults'
pois = sdir + '/pois'
if (
os.access(grids, os.R_OK) or
os.access(faults, os.R_OK) or
os.access(pois, os.R_OK)
):
print(
'The directories grids/, faults/ and pois/ are already present. ' +
'Abort.'
)
sys.exit(1)
p = subprocess.Popen('svn info --xml', shell=True, stdout=subprocess.PIPE)
ret = p.wait()
lines = ''
for line in p.stdout.readlines():
lines += line
matchBase = re.search('<root>(.*)</root>', lines)
matchPath = re.search('<wcroot-abspath>(.*)</wcroot-abspath>', lines)
if matchBase and matchPath:
svnbase = matchBase.group(1)
svnpath = matchPath.group(1)
if svnbase == 'http://svnext.gfz-potsdam.de/easywave':
# data directory should be present, so just create symbolic links
os.symlink(svnpath + '/data/grids', grids)
os.symlink(svnpath + '/data/faults', faults)
os.symlink(svnpath + '/data/pois', pois)
sys.exit(0)
# directories not present and no suitable svn root was found
# --> download contents
link = 'http://svnext.gfz-potsdam.de/easywave/data'
ret = 0
ret += subprocess.Popen(
'svn export ' + link + '/grids', shell=True
).wait()
ret += subprocess.Popen(
'svn export ' + link + '/faults', shell=True
).wait()
ret += subprocess.Popen(
'svn export ' + link + '/pois', shell=True
).wait()
if ret > 0:
print('Error: Could not download data.')
sys.exit(1)
sys.exit(0)
-grid @grids/e2Asean.grd -source @faults/BengkuluSept2007.flt -time 300
-grid @grids/e2Indian.grd -source @faults/BengkuluSept2007.flt -time 300
-grid @grids/g08Indonesia.grd -source @faults/BengkuluSept2007.flt -time 300
-grid @grids/g08Japan.grd -source @faults/uz.Tohoku11.grd -time 300
-grid @grids/g08Lisboa.grd -source @faults/faultPortugal.inp -time 300
-grid @grids/e2r4Pacific.grd -source @faults/uz.Tohoku11.grd -time 300
#!/bin/bash
PYTHON="python3"
ADDITIONAL_EASYWAVE_ARGS=""
cd "$(dirname "$0")"
echo "Extracting expected output directories"
for i in *.tar.xz; do
base=${i%.tar.xz}
if [ -d "${base}" ]; then
echo "${base} already exists, skipping ${i}"
continue
fi
tar xf "$i"
done
${PYTHON} test.py ${ADDITIONAL_EASYWAVE_ARGS}
#!/bin/bash
PYTHON="python3"
ADDITIONAL_EASYWAVE_ARGS="-gpu"
cd "$(dirname "$0")"
echo "Extracting expected output directories"
for i in *.tar.xz; do
base=${i%.tar.xz}
if [ -d "${base}" ]; then
echo "${base} already exists, skipping ${i}"
continue
fi
tar xf "$i"
done
${PYTHON} test.py ${ADDITIONAL_EASYWAVE_ARGS}
#!/usr/bin/env python
#!/usr/bin/env python3
import sys
import os
......@@ -10,13 +10,34 @@ import re
sdir = os.path.dirname(os.path.abspath(sys.argv[0]))
# how to call easywave
easywave = '@../src/easywave'
test_scenarios = [
# fault with slip, length and width:
{
'grid': '@../../data/grids/e2r4Asean.grd',
'fault': '@../../data/faults/BengkuluSept2007.flt',
# directory which contains the expected outputs
'expected': '@BengkuluSept2007',
},
# fault with magnitude:
{
'grid': '@../../data/grids/e2r4Asean.grd',
'fault': '@../../data/faults/MentawaiOct2010.flt',
'expected': '@MentawaiOct2010',
},
]
def start(cmd):
tmp_path = tempfile.mkdtemp()
os.chdir(tmp_path)
print(tmp_path)
print('Writing computation results to: ' + tmp_path)
print('Starting easyWave computation:')
print(cmd)
# starts easywave and counts time differential between start and end
t0 = datetime.datetime.now()
ewrun = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
......@@ -24,101 +45,104 @@ def start(cmd):
delta_t = (datetime.datetime.now() - t0)
print('Runtime: %u.%u sec' % (delta_t.seconds, delta_t.microseconds))
cmds[cmd] = tmp_path
return tmp_path
def compareCmds(cmd):
def compareResults(expected, computed):
path = []
files = []
maxDiff = 0.0
tolerance = 0.02
success = True
for i in range(0, 2):
path.append(cmds[cmd[i]])
path.append(expected)
path.append(computed)
for i in range(0, 2):
files.append(glob.glob(path[i] + '/*.ssh'))
files[i] += glob.glob(path[i] + '/*.sshmax')
# files[i] += glob.glob( path[i] + '/*.time' )
files[i] += glob.glob(path[i] + '/*.time')
for f in files[0]:
# go through expected files
basef = os.path.basename(f)
fnext = path[1] + '/' + basef
if fnext in files[1]:
# compare.py
# we found the expected file in the computed results
files[1].remove(fnext)
print('Comparing ' + '%s with %s' % (f, fnext))
compare = subprocess.Popen(
sdir + '/compare.py %s %s' % (f, fnext),
shell=True,
stdout=subprocess.PIPE
)
compare.wait()
# convert byte-like object to string
output = compare.stdout.read().decode('utf-8')
match = re.search(
'Max difference: (.*)\n',
output
)
if match is None:
print('Output of compare.py was not as expected.')
success = False
continue
diff = float(
re.search(
'Max difference: (.*)\n',
compare.stdout.read()
).group(1)
match.group(1)
)
maxDiff = max(maxDiff, diff)
if verbose:
print(basef + ': %f' % diff)
files[1].remove(fnext)
print(basef + ': %f' % diff)
else:
print('File "' + basef + '" is missing in 1.')
print('File "' + basef + '" is missing in computed results.')
success = False
for f in files[1]:
print('File "' + basef + '" is missing in 0.')
print('File "' + f + '" of the computation result is not expected.')
success = False
if maxDiff > tolerance:
print('\033[91mFailed:\033[0m')
success = False
if success:
print('\033[92mSuccessful:\033[0m %f\n' % maxDiff)
else:
print('\033[92mSuccessful:\033[0m')
print('\033[91mFailed:\033[0m %f\n' % maxDiff)
print('%f' % maxDiff)
return success
if __name__ == '__main__':
cfg_path = sdir + '/config.cfg'
scs_path = sdir + '/scenarios.cfg'
easywave = easywave.replace('@', sdir + '/')
ret = 0
runs = []
scenarios = []
cmds = {}
# add extra arguments to easyWave if needed
if len(sys.argv) > 1 and sys.argv[1]:
easywave = easywave + ' ' + sys.argv[1]
tolerance = 0.02
verbose = True
# read config file to get a list of programs to compare
f = open(cfg_path, 'r')
for line in f:
if line.lstrip()[0] != '#':
progs = line.replace('\n', '').replace('@', sdir+'/').split(';')
if len(progs) > 2:
print(
'Warning: Can compare only two programs at a time. ' +
'Ignoring additional entries.'
)
del progs[2:]
runs.append(progs)
f.close()
# read scenario file to get a list of scenarios used to compare the given
# programs
f = open(scs_path, 'r')
for line in f:
if line.lstrip()[0] != '#':
scenarios.append(line.replace('\n', '').replace('@', sdir+'/'))
f.close()
# input was read successfully
for run in runs:
for sc in scenarios:
cmd = []
for prog in run:
cmd.append(prog + ' ' + sc)
start(cmd[-1])
compareCmds(cmd)
for sc in test_scenarios:
grid = sc['grid'].replace('@', sdir + '/')
fault = sc['fault'].replace('@', sdir + '/')
args = '-grid ' + grid + ' -source ' + fault + ' -time 300'
cmd = []
cmd.append(easywave + ' ' + args)
tmpdir = start(cmd[-1])
expected = sc['expected'].replace('@', sdir + '/')
successful = compareResults(expected, tmpdir)
if not successful:
ret = 1
if ret == 1:
print('Some tests failed.')
else:
print('All tests passed successfully.')
sys.exit(ret)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment