Skip to content

Commit b8e9b29

Browse files
maleadtmaleadt
committed
Fix test runner.
1 parent cade284 commit b8e9b29

File tree

2 files changed

+89
-63
lines changed

2 files changed

+89
-63
lines changed

‎test/runtests.jl‎

Lines changed: 69 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -330,73 +330,85 @@ println("Testing finished in $elapsed")
330330

331331
# construct a testset to render the test results
332332
o_ts = Test.DefaultTestSet("Overall")
333-
Test.push_testset(o_ts)
334-
completed_tests = Set{String}()
335-
for (testname, (resp,)) in results
336-
push!(completed_tests, testname)
337-
if isa(resp, Test.DefaultTestSet)
338-
Test.push_testset(resp)
339-
Test.record(o_ts, resp)
340-
Test.pop_testset()
341-
elseif isa(resp, Tuple{Int,Int})
342-
fake = Test.DefaultTestSet(testname)
343-
for i in 1:resp[1]
344-
Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
345-
end
346-
for i in 1:resp[2]
347-
Test.record(fake, Test.Broken(:test, nothing))
348-
end
349-
Test.push_testset(fake)
350-
Test.record(o_ts, fake)
351-
Test.pop_testset()
352-
elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException)
353-
println("Worker $(resp.pid) failed running test $(testname):")
354-
Base.showerror(stdout, resp.captured)
355-
println()
356-
fake = Test.DefaultTestSet(testname)
357-
for i in 1:resp.captured.ex.pass
358-
Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
359-
end
360-
for i in 1:resp.captured.ex.broken
361-
Test.record(fake, Test.Broken(:test, nothing))
362-
end
363-
for t in resp.captured.ex.errors_and_fails
364-
Test.record(fake, t)
365-
end
366-
Test.push_testset(fake)
367-
Test.record(o_ts, fake)
368-
Test.pop_testset()
333+
function with_testset(f, testset)
334+
@static if VERSION >= v"1.13.0-DEV.1044"
335+
Test.@with_testset testset f()
369336
else
370-
if !isa(resp, Exception)
371-
resp = ErrorException(string("Unknown result type : ", typeof(resp)))
337+
Test.push_testset(testset)
338+
try
339+
f()
340+
finally
341+
Test.pop_testset()
372342
end
373-
# If this test raised an exception that is not a remote testset exception,
374-
# i.e. not a RemoteException capturing a TestSetException that means
375-
# the test runner itself had some problem, so we may have hit a segfault,
376-
# deserialization errors or something similar. Record this testset as Errored.
377-
fake = Test.DefaultTestSet(testname)
378-
Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1)))
379-
Test.push_testset(fake)
380-
Test.record(o_ts, fake)
381-
Test.pop_testset()
382343
end
383344
end
384-
for test in tests
385-
(test in completed_tests) && continue
386-
fake = Test.DefaultTestSet(test)
387-
Test.record(fake, Test.Error(:test_interrupted, test, nothing,
388-
[("skipped", [])], LineNumberNode(1)))
389-
Test.push_testset(fake)
390-
Test.record(o_ts, fake)
391-
Test.pop_testset()
345+
with_testset(o_ts) do
346+
completed_tests = Set{String}()
347+
for (testname, (resp,)) in results
348+
push!(completed_tests, testname)
349+
if isa(resp, Test.DefaultTestSet)
350+
with_testset(resp) do
351+
Test.record(o_ts, resp)
352+
end
353+
elseif isa(resp, Tuple{Int,Int})
354+
fake = Test.DefaultTestSet(testname)
355+
for i in 1:resp[1]
356+
Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
357+
end
358+
for i in 1:resp[2]
359+
Test.record(fake, Test.Broken(:test, nothing))
360+
end
361+
with_testset(fake) do
362+
Test.record(o_ts, fake)
363+
end
364+
elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException)
365+
println("Worker $(resp.pid) failed running test $(testname):")
366+
Base.showerror(stdout, resp.captured)
367+
println()
368+
fake = Test.DefaultTestSet(testname)
369+
for i in 1:resp.captured.ex.pass
370+
Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing))
371+
end
372+
for i in 1:resp.captured.ex.broken
373+
Test.record(fake, Test.Broken(:test, nothing))
374+
end
375+
for t in resp.captured.ex.errors_and_fails
376+
Test.record(fake, t)
377+
end
378+
with_testset(fake) do
379+
Test.record(o_ts, fake)
380+
end
381+
else
382+
if !isa(resp, Exception)
383+
resp = ErrorException(string("Unknown result type : ", typeof(resp)))
384+
end
385+
# If this test raised an exception that is not a remote testset exception,
386+
# i.e. not a RemoteException capturing a TestSetException that means
387+
# the test runner itself had some problem, so we may have hit a segfault,
388+
# deserialization errors or something similar. Record this testset as Errored.
389+
fake = Test.DefaultTestSet(testname)
390+
Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack([(exception=resp,backtrace=[])]), LineNumberNode(1)))
391+
with_testset(fake) do
392+
Test.record(o_ts, fake)
393+
end
394+
end
395+
end
396+
for test in tests
397+
(test in completed_tests) && continue
398+
fake = Test.DefaultTestSet(test)
399+
Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack([(exception="skipped",backtrace=[])]), LineNumberNode(1)))
400+
with_testset(fake) do
401+
Test.record(o_ts, fake)
402+
end
403+
end
392404
end
393405
println()
394406
Test.print_test_results(o_ts, 1)
395-
if !o_ts.anynonpass
407+
if (VERSION >= v"1.13.0-DEV.1037" && !Test.anynonpass(o_ts)) ||
408+
(VERSION < v"1.13.0-DEV.1037" && !o_ts.anynonpass)
396409
println(" \033[32;1mSUCCESS\033[0m")
397410
else
398411
println(" \033[31;1mFAILURE\033[0m\n")
399412
Test.print_test_errors(o_ts)
400413
throw(Test.FallbackTestSetException("Test run finished with errors"))
401414
end
402-

‎test/setup.jl‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,15 @@ for file in readdir(joinpath(@__DIR__, "helpers"))
1111
end
1212
using .FileCheck
1313

14+
if VERSION >= v"1.13.0-DEV.1044"
15+
using Base.ScopedValues
16+
end
17+
1418

1519
## entry point
1620

1721
function runtests(f, name)
18-
old_print_setting = Test.TESTSET_PRINT_ENABLE[]
19-
Test.TESTSET_PRINT_ENABLE[] = false
20-
21-
try
22+
function inner()
2223
# generate a temporary module to execute the tests in
2324
mod_name = Symbol("Test", rand(1:100), "Main_", replace(name, '/' => '_'))
2425
mod = @eval(Main, module $mod_name end)
@@ -61,8 +62,21 @@ function runtests(f, name)
6162

6263
GC.gc(true)
6364
res
64-
finally
65-
Test.TESTSET_PRINT_ENABLE[] = old_print_setting
65+
end
66+
67+
old_print_setting = Test.TESTSET_PRINT_ENABLE[]
68+
@static if VERSION >= v"1.13.0-DEV.1044"
69+
@with Test.TESTSET_PRINT_ENABLE=>false begin
70+
inner()
71+
end
72+
else
73+
old_print_setting = Test.TESTSET_PRINT_ENABLE[]
74+
Test.TESTSET_PRINT_ENABLE[] = false
75+
try
76+
inner()
77+
finally
78+
Test.TESTSET_PRINT_ENABLE[] = old_print_setting
79+
end
6680
end
6781
end
6882

0 commit comments

Comments
 (0)