Add WorkerTestSet to avoid loss of context due to TestSetException

src/ParallelTestRunner.jl

@@ -13,6 +13,16 @@ import Test
 import Random
 import IOCapture
 
+function anynonpass(ts::Test.AbstractTestSet)
+    @static if VERSION >= v"1.13.0-DEV.1037"
+        return Test.anynonpass(ts)
+    else
+        Test.get_test_counts(ts)
+        return ts.anynonpass
+    end
+end
+
+
 #Always set the max rss so that if tests add large global variables (which they do) we don't make the GC's life too hard
 if Sys.WORD_SIZE == 64
     const JULIA_TEST_MAXRSS_MB = 3800
@@ -214,12 +224,39 @@ end
 # entry point
 #
 
+mutable struct WorkerTestSet <: Test.AbstractTestSet
+    const name::String
+    wrapped_ts::Test.DefaultTestSet
+    function WorkerTestSet(name::AbstractString)
+        new(name)
+    end
+end
+
+function Test.record(ts::WorkerTestSet, res)
+    @assert res isa Test.DefaultTestSet
+    @assert !isdefined(ts, :wrapped_ts)
+    ts.wrapped_ts = res
+    return nothing
+end
+
+function Test.finish(ts::WorkerTestSet)
+    # This testset is just a placeholder,
+    # so it must be the top-most
+    @assert Test.get_testset_depth() == 0
+    @assert isdefined(ts, :wrapped_ts)
+    return ts
+end
+
+anynonpass(ts::WorkerTestSet) = anynonpass(ts.wrapped_ts)
+
 function runtest(::Type{TestRecord}, f, name, init_code, color)
     function inner()
         # generate a temporary module to execute the tests in
         mod = @eval(Main, module $(gensym(name)) end)
         @eval(mod, import ParallelTestRunner: Test, Random)
         @eval(mod, using .Test, .Random)
+        @eval(mod, import ParallelTestRunner: WorkerTestSet)
+        @eval(mod, import Test: DefaultTestSet)
 
         Core.eval(mod, init_code)
 
@@ -230,10 +267,13 @@ function runtest(::Type{TestRecord}, f, name, init_code, color)
             mktemp() do path, io
                 stats = redirect_stdio(stdout=io, stderr=io) do
                     @timed try
-                        @testset $name begin
-                            $f
+                        @testset WorkerTestSet "placeholder" begin
+                            @testset DefaultTestSet $name begin
+                                $f
+                            end
                         end
                     catch err
+                        # TODO: Should never receive a TestSetException here
                         isa(err, Test.TestSetException) || rethrow()
 
                         # return the error to package it into a TestRecord
@@ -722,7 +762,12 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
                         if record.value isa Exception
                             print_test_failed(record, wrkr, test_name, io_ctx)
                         else
-                            print_test_finished(record, wrkr, test_name, io_ctx)
+                            ts = record.value::WorkerTestSet
+                            if anynonpass(ts)
+                                print_test_failed(record, wrkr, test_name, io_ctx)
+                            else
+                                print_test_finished(record, wrkr, test_name, io_ctx)
+                            end
                         end
 
                     elseif msg_type == :crashed
@@ -932,33 +977,19 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
             for (testname, result, start, stop) in results
                 push!(completed_tests, testname)
 
-                # decode or fake a testset
+                # decode testset
                 if result isa AbstractTestRecord
-                    if result.value isa Test.AbstractTestSet
-                        testset = result.value
-                        historical_durations[testname] = stop - start
-                    else
-                        # TODO: improve the Test stdlib to keep track of the exact failure
-                        #       instead of flattening into an exception without provenance
-                        @assert result.value isa Test.TestSetException
-                        testset = create_testset(testname; start, stop)
-                        for i in 1:result.value.pass
-                            Test.record(testset, Test.Pass(:test, nothing, nothing, nothing, LineNumberNode(@__LINE__, @__FILE__)))
-                        end
-                        for i in 1:result.value.broken
-                            Test.record(testset, Test.Broken(:test, nothing))
-                        end
-                        for t in result.value.errors_and_fails
-                            Test.record(testset, t)
-                        end
-                    end
+                    @assert result.value isa WorkerTestSet
+                    testset = result.value.wrapped_ts
+                    historical_durations[testname] = stop - start
                 else
                     # If this test raised an exception that is not a remote testset
                     # exception, that means the test runner itself had some problem, so we
                     # may have hit a segfault, deserialization errors or something similar.
                     # Record this testset as Errored.
                     @assert result isa Exception
                     testset = create_testset(testname; start, stop)
+                    # TODO: Send backtrace with Exception
                     Test.record(testset, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(NamedTuple[(;exception = result, backtrace = [])]), LineNumberNode(1)))
                 end
 
@@ -1003,8 +1034,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
         end
         print(io_ctx.stdout, c.output)
     end
-    if (VERSION >= v"1.13.0-DEV.1037" && !Test.anynonpass(o_ts)) ||
-            (VERSION < v"1.13.0-DEV.1037" && !o_ts.anynonpass)
+    if !anynonpass(o_ts)
         println(io_ctx.stdout, "    \033[32;1mSUCCESS\033[0m")
     else
         println(io_ctx.stderr, "    \033[31;1mFAILURE\033[0m\n")

test/runtests.jl

@@ -97,6 +97,36 @@ end
     @test contains(str, "1 == 2")
 end
 
+@testset "nested failure" begin
+    custom_tests = Dict(
+        "nested" => quote
+            @test true
+            @testset "foo" begin
+                @test true
+                @testset "bar" begin
+                    @test false
+                end
+            end
+        end
+    )
+    error_line = @__LINE__() - 5
+
+    io = IOBuffer()
+    @test_throws Test.FallbackTestSetException("Test run finished with errors") begin
+        runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io)
+    end
+
+    str = String(take!(io))
+    @test contains(str, r"nested .+ started at")
+    @test contains(str, r"nested .+ failed at")
+    @test contains(str, r"nested .+ \| .+ 2 .+ 1 .+ 3")
+    @test contains(str, r"foo .+ \| .+ 1 .+ 1 .+ 2")
+    @test contains(str, r"bar .+ \| .+ 1 .+ 1")
+    @test contains(str, "FAILURE")
+    @test contains(str, "Error in testset bar")
+    @test contains(str, "$(basename(@__FILE__)):$error_line")
+end
+
 @testset "throwing test" begin
     custom_tests = Dict(
         "throwing test" => quote