RemoteTestSet to preserve testset metadata across processes

src/ParallelTestRunner.jl

@@ -13,6 +13,8 @@ import Test
 import Random
 import IOCapture
 
+include("remotetestset.jl")
+
 #Always set the max rss so that if tests add large global variables (which they do) we don't make the GC's life too hard
 if Sys.WORD_SIZE == 64
     const JULIA_TEST_MAXRSS_MB = 3800
@@ -218,8 +220,9 @@ function runtest(::Type{TestRecord}, f, name, init_code, color)
     function inner()
         # generate a temporary module to execute the tests in
         mod = @eval(Main, module $(gensym(name)) end)
-        @eval(mod, import ParallelTestRunner: Test, Random)
-        @eval(mod, using .Test, .Random)
+        @eval(mod, import ParallelTestRunner: Test, Random, RemoteTestSets)
+        @eval(mod, using .Test, .Random, .RemoteTestSets)
+        @eval(mod, using .Test: DefaultTestSet) # Necessary because VERSION <= v"1.10.0-" does not support unexported TestSets the @testset
 
         Core.eval(mod, init_code)
 
@@ -230,7 +233,7 @@ function runtest(::Type{TestRecord}, f, name, init_code, color)
             mktemp() do path, io
                 stats = redirect_stdio(stdout=io, stderr=io) do
                     @timed try
-                        @testset $name begin
+                        @remote_testset $name begin
                             $f
                         end
                     catch err
@@ -399,6 +402,15 @@ function addworker(; env=Vector{Pair{String, String}}())
     return wrkr
 end
 
+@static if VERSION >= v"1.13.0-DEV.1037"
+    compat_anynonpass(ts::Test.AbstractTestSet) = Test.anynonpass(ts)
+else
+    function compat_anynonpass(ts::Test.AbstractTestSet)
+        Test.get_test_counts(ts)
+        return ts.anynonpass
+    end
+end
+
 """
     runtests(mod::Module, ARGS; RecordType = TestRecord,
                                 test_filter = Returns(true),
@@ -719,7 +731,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
                         test_name, wrkr, record = msg[2], msg[3], msg[4]
 
                         clear_status()
-                        if record.value isa Exception
+                        if compat_anynonpass(record.value)
                             print_test_failed(record, wrkr, test_name, io_ctx)
                         else
                             print_test_finished(record, wrkr, test_name, io_ctx)
@@ -766,7 +778,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
 
     worker_tasks = Task[]
     for p in workers
-        push!(worker_tasks, @async begin
+    push!(worker_tasks, @async begin
             while !done
                 # if a worker failed, spawn a new one
                 if !Malt.isrunning(p)
@@ -1003,8 +1015,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
         end
         print(io_ctx.stdout, c.output)
     end
-    if (VERSION >= v"1.13.0-DEV.1037" && !Test.anynonpass(o_ts)) ||
-            (VERSION < v"1.13.0-DEV.1037" && !o_ts.anynonpass)
+    if !compat_anynonpass(o_ts)
         println(io_ctx.stdout, "    \033[32;1mSUCCESS\033[0m")
     else
         println(io_ctx.stderr, "    \033[31;1mFAILURE\033[0m\n")

src/remotetestset.jl

@@ -0,0 +1,53 @@
+module RemoteTestSets
+
+export RemoteTestSet, @remote_testset
+
+import Test
+import Test: AbstractTestSet, DefaultTestSet, Broken, Pass, Fail, Error, @testset
+
+struct RemoteTestSet <: AbstractTestSet
+    ts::DefaultTestSet
+
+    RemoteTestSet(dts::DefaultTestSet) = new(dts)
+end
+
+RemoteTestSet(args...; kwargs...) = RemoteTestSet(DefaultTestSet(args...; kwargs...))
+
+# Record testsets as usual
+Test.record(ts::RemoteTestSet, t::Union{Broken, Pass, Fail, Error}; kwargs...) = Test.record(ts.ts, t; kwargs...)
+Test.record(ts::RemoteTestSet, t::AbstractTestSet) = Test.record(ts.ts, t)
+
+# This is the single method that needs changing
+function Test.finish(ts::RemoteTestSet; print_results::Bool=Test.TESTSET_PRINT_ENABLE[])
+    # This testset is just a placeholder,
+    #   so it must be the top-most
+    @assert Test.get_testset_depth() == 0
+
+    # There should only ever be one child testset
+    return only(ts.ts.results)
+end
+
+
+macro remote_testset(args...)
+    testsettype = nothing
+    otherargs = []
+
+    for arg in args[1:end-1]
+        if isa(arg, Symbol) || Base.isexpr(arg, :.)
+            testsettype = arg
+        else
+            push!(otherargs, arg)
+        end
+    end
+
+    source = args[end]
+    testsettype = isnothing(testsettype) ? :(DefaultTestSet) : testsettype
+
+    return esc(quote
+        @testset RemoteTestSet "wrapper" begin
+            @testset $testsettype $(otherargs...) $source
+        end
+    end)
+end
+
+end

test/basicts.jl

@@ -0,0 +1,3 @@
+@testset "basic in testset" begin
+    @test true
+end