Skip to content

Commit 82c85e2

Browse files
slefrancoisslefrancois
committed
Add JUnit writer for speed tests, remove hardcoded reference times
1 parent 500c214 commit 82c85e2

File tree

1 file changed

+29
-88
lines changed

1 file changed

+29
-88
lines changed

‎code/test.py

Lines changed: 29 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -98,43 +98,7 @@ def speed():
9898
do_gpu = True
9999

100100
algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]]
101-
#Timming expected are from the buildbot that have an i7-920 @
102-
# 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX
103-
# 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD
104-
# 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread.
105-
106-
expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4,
107-
346.9, 355.0, 268.2, 115.8, 16.8, 91.6])
108-
expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71,
109-
191.2, 199.0, 201.9, 107, 12.6, 61.3])
110-
111-
# Number with just 1 decimal are new value that are faster with
112-
# the Theano version 0.5rc2 Other number are older. They are not
113-
# updated, as we where faster in the past!
114-
# TODO: find why and fix this!
115-
116-
# Here is the value for the buildbot on February 3th 2012 with a GTX 285
117-
# sgd, cg mlp conv da
118-
# sda dbn rbm
119-
# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657,
120-
# 18.30802011, 53.38651466, 285.41386175]
121-
# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450,
122-
# 24.77524018, 92.66246653, 322.340329170]
123-
# sgd, cg mlp conv da
124-
# sda dbn rbm
125-
#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138
126-
# 1.35324519 1.7356905 1.12937868]
127-
128-
expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785,
129-
5.8, 19.2,
130-
11.2, 7.3, 122, 112.5, 31.1, 8.3])
131-
expected_times_64 = [s for idx, s in enumerate(expected_times_64)
132-
if to_exec[idx]]
133-
expected_times_32 = [s for idx, s in enumerate(expected_times_32)
134-
if to_exec[idx]]
135-
expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu)
136-
if to_exec[idx]]
137-
101+
138102
def time_test(m, l, idx, f, **kwargs):
139103
if not to_exec[idx]:
140104
return
@@ -196,19 +160,13 @@ def do_tests():
196160
float64_times = do_tests()
197161
print(algo_executed, file=sys.stderr)
198162
print('float64 times', float64_times, file=sys.stderr)
199-
print('float64 expected', expected_times_64, file=sys.stderr)
200-
print('float64 % expected/get', (
201-
expected_times_64 / float64_times), file=sys.stderr)
202163

203164
#test in float32 in FAST_RUN mode on the cpu
204165
theano.config.floatX = 'float32'
205166
if do_float32:
206167
float32_times = do_tests()
207168
print(algo_executed, file=sys.stderr)
208169
print('float32 times', float32_times, file=sys.stderr)
209-
print('float32 expected', expected_times_32, file=sys.stderr)
210-
print('float32 % expected/get', (
211-
expected_times_32 / float32_times), file=sys.stderr)
212170

213171
if do_float64:
214172
print('float64/float32', (
@@ -218,18 +176,10 @@ def do_tests():
218176
'in one place'), file=sys.stderr)
219177
print(algo_executed, file=sys.stderr)
220178
print('float64 times', float64_times, file=sys.stderr)
221-
print('float64 expected', expected_times_64, file=sys.stderr)
222-
print('float64 % expected/get', (
223-
expected_times_64 / float64_times), file=sys.stderr)
224179
print('float32 times', float32_times, file=sys.stderr)
225-
print('float32 expected', expected_times_32, file=sys.stderr)
226-
print('float32 % expected/get', (
227-
expected_times_32 / float32_times), file=sys.stderr)
228180

229181
print('float64/float32', (
230182
float64_times / float32_times), file=sys.stderr)
231-
print('expected float64/float32', (
232-
expected_times_64 / float32_times), file=sys.stderr)
233183

234184
#test in float32 in FAST_RUN mode on the gpu
235185
import theano.sandbox.cuda
@@ -238,9 +188,6 @@ def do_tests():
238188
gpu_times = do_tests()
239189
print(algo_executed, file=sys.stderr)
240190
print('gpu times', gpu_times, file=sys.stderr)
241-
print('gpu expected', expected_times_gpu, file=sys.stderr)
242-
print('gpu % expected/get', (
243-
expected_times_gpu / gpu_times), file=sys.stderr)
244191

245192
if do_float64:
246193
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
@@ -252,50 +199,44 @@ def do_tests():
252199
print(algo_executed, file=sys.stderr)
253200
if do_float64:
254201
print('float64 times', float64_times, file=sys.stderr)
255-
print('float64 expected', expected_times_64, file=sys.stderr)
256-
print('float64 % expected/get', (
257-
expected_times_64 / float64_times), file=sys.stderr)
258202
if do_float32:
259203
print('float32 times', float32_times, file=sys.stderr)
260-
print('float32 expected', expected_times_32, file=sys.stderr)
261-
print('float32 % expected/get', (
262-
expected_times_32 / float32_times), file=sys.stderr)
263204
if do_gpu:
264205
print('gpu times', gpu_times, file=sys.stderr)
265-
print('gpu expected', expected_times_gpu, file=sys.stderr)
266-
print('gpu % expected/get', (
267-
expected_times_gpu / gpu_times), file=sys.stderr)
268206

269207
print()
270208
if do_float64 and do_float32:
271209
print('float64/float32', (
272210
float64_times / float32_times), file=sys.stderr)
273-
print('expected float64/float32', (
274-
expected_times_64 / float32_times), file=sys.stderr)
275211
if do_float64 and do_gpu:
276212
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
277-
print('expected float64/gpu', (
278-
expected_times_64 / gpu_times), file=sys.stderr)
279213
if do_float32 and do_gpu:
280214
print('float32/gpu', float32_times / gpu_times, file=sys.stderr)
281-
print('expected float32/gpu', (
282-
expected_times_32 / gpu_times), file=sys.stderr)
283-
284-
def compare(x, y):
285-
ratio = x / y
286-
# If there is more then 5% difference between the expected
287-
# time and the real time, we consider this an error.
288-
return sum((ratio < 0.95) + (ratio > 1.05))
289-
290-
print(file=sys.stderr)
291-
if do_float64:
292-
err = compare(expected_times_64, float64_times)
293-
print('speed_failure_float64=' + str(err), file=sys.stderr)
294-
if do_float32:
295-
err = compare(expected_times_32, float32_times)
296-
print('speed_failure_float32=' + str(err), file=sys.stderr)
297-
if do_gpu:
298-
err = compare(expected_times_gpu, gpu_times)
299-
print('speed_failure_gpu=' + str(err), file=sys.stderr)
300-
301-
assert not numpy.isnan(gpu_times).any()
215+
216+
# Write JUnit xml for speed test performance report
217+
218+
speed_file = 'speedtests_time.xml'
219+
220+
# Define speed test file write method
221+
def write_junit(filename, algos, times, label):
222+
with open(filename, 'a') as f:
223+
for algo, time in zip(algos, times):
224+
f.write(' <testcase classname="{label}" name="{algo}" time="{time}">'
225+
.format(label=label, algo=algo, time=time))
226+
f.write(' </testcase>\n')
227+
228+
test_total = numpy.size(float64_times) \
229+
+ numpy.size(float32_times) \
230+
+ numpy.size(gpu_times)
231+
232+
with open(speed_file, 'w') as f:
233+
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
234+
f.write('<testsuite name="theano_speedtests" tests="{ntests}">\n'
235+
.format(ntests=numpy.size(test_total)))
236+
237+
write_junit(speed_file, algo_executed, float64_times, label='float64')
238+
write_junit(speed_file, algo_executed, float32_times, label='float32')
239+
write_junit(speed_file, algo_executed, gpu_times, label='gpu')
240+
241+
with open(speed_file, 'a') as f:
242+
f.write('</testsuite>\n')

0 commit comments

Comments
 (0)