From 1d32fd7cec8361c9ca5d641a74aafcee63a964b0 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Thu, 16 Oct 2025 15:53:48 -0300 Subject: [PATCH 1/6] RemoteTestSet to keep testset data across processes --- src/ParallelTestRunner.jl | 27 ++++++++++++----- src/remotetestset.jl | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 src/remotetestset.jl diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index a5409e0..bc5fa93 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -13,6 +13,8 @@ import Test import Random import IOCapture +include("remotetestset.jl") + #Always set the max rss so that if tests add large global variables (which they do) we don't make the GC's life too hard if Sys.WORD_SIZE == 64 const JULIA_TEST_MAXRSS_MB = 3800 @@ -218,8 +220,9 @@ function runtest(::Type{TestRecord}, f, name, init_code, color) function inner() # generate a temporary module to execute the tests in mod = @eval(Main, module $(gensym(name)) end) - @eval(mod, import ParallelTestRunner: Test, Random) + @eval(mod, import ParallelTestRunner: Test, Random, RemoteTestSet) @eval(mod, using .Test, .Random) + @eval(mod, using .Test: DefaultTestSet) # Necessary because VERSION <= v"1.10.0-" does not support unexported TestSets the @testset Core.eval(mod, init_code) @@ -230,8 +233,10 @@ function runtest(::Type{TestRecord}, f, name, init_code, color) mktemp() do path, io stats = redirect_stdio(stdout=io, stderr=io) do @timed try - @testset $name begin - $f + @testset RemoteTestSet "wrapper" begin + @testset DefaultTestSet $name begin + $f + end end catch err isa(err, Test.TestSetException) || rethrow() @@ -399,6 +404,15 @@ function addworker(; env=Vector{Pair{String, String}}()) return wrkr end +@static if VERSION >= v"1.13.0-DEV.1037" + compat_anynonpass(ts::Test.AbstractTestSet) = Test.anynonpass(ts) +else + function compat_anynonpass(ts::Test.AbstractTestSet) + Test.get_test_counts(ts) + return ts.anynonpass + end +end + """ runtests(mod::Module, ARGS; RecordType = TestRecord, test_filter = Returns(true), @@ -719,7 +733,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T test_name, wrkr, record = msg[2], msg[3], msg[4] clear_status() - if record.value isa Exception + if compat_anynonpass(record.value) print_test_failed(record, wrkr, test_name, io_ctx) else print_test_finished(record, wrkr, test_name, io_ctx) @@ -766,7 +780,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T worker_tasks = Task[] for p in workers - push!(worker_tasks, @async begin + push!(worker_tasks, @async begin while !done # if a worker failed, spawn a new one if !Malt.isrunning(p) @@ -1003,8 +1017,7 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T end print(io_ctx.stdout, c.output) end - if (VERSION >= v"1.13.0-DEV.1037" && !Test.anynonpass(o_ts)) || - (VERSION < v"1.13.0-DEV.1037" && !o_ts.anynonpass) + if !compat_anynonpass(o_ts) println(io_ctx.stdout, " \033[32;1mSUCCESS\033[0m") else println(io_ctx.stderr, " \033[31;1mFAILURE\033[0m\n") diff --git a/src/remotetestset.jl b/src/remotetestset.jl new file mode 100644 index 0000000..e305482 --- /dev/null +++ b/src/remotetestset.jl @@ -0,0 +1,63 @@ +module RemoteTestSets + +export RemoteTestSet, @remote_testset + +import Test +import Test: AbstractTestSet, DefaultTestSet, Broken, Pass, Fail, Error + +struct RemoteTestSet <: AbstractTestSet + ts::DefaultTestSet + + RemoteTestSet(dts::DefaultTestSet) = new(dts) +end + +RemoteTestSet(args...; kwargs...) = RemoteTestSet(DefaultTestSet(args...; kwargs...)) + +function Base.propertynames(x::RemoteTestSet) + (:ts, Base.propertynames(x.ts)...) +end +function Base.getproperty(ts::RemoteTestSet, sym::Symbol) + if sym === :ts + return Base.getfield(ts, :ts) + end + return Base.getfield(Base.getfield(ts, :ts), sym) +end +function Base.setproperty!(ts::RemoteTestSet, sym::Symbol, v) + return Base.setproperty!(ts.ts, sym, v) +end + +# Record testsets as usual +Test.record(ts::RemoteTestSet, t::Union{Broken, Pass, Fail, Error}; kwargs...) = Test.record(ts.ts, t; kwargs...) +Test.record(ts::RemoteTestSet, t::AbstractTestSet) = Test.record(ts.ts, t) + +# This is the single method that needs changing +function Test.finish(ts::RemoteTestSet; print_results::Bool=Test.TESTSET_PRINT_ENABLE[]) + if Test.get_testset_depth() != 0 + throw(ErrorException("RemoteTestSet should only ever be a top-level TestSet")) + end + + # Otherwise, just return the testset so it is returned from the @testset macro + return only(ts.results) +end + +Test.filter_errors(ts::RemoteTestSet) = Test.filter_errors(ts.ts) +Test.get_test_counts(ts::RemoteTestSet) = Test.get_test_counts(ts.ts) +Test.get_alignment(ts::RemoteTestSet, depth::Int) = Test.get_alignment(ts.ts, depth) + +@static if isdefined(Test, :results) #VERSION > v"1.11.0-??" + Test.results(ts::RemoteTestSet) = Test.results(ts.ts) +end +@static if isdefined(Test, :print_verbose) #VERSION > v"1.11.0-??" + Test.print_verbose(ts::RemoteTestSet) = Test.print_verbose(ts.ts) +end +@static if isdefined(Test, :format_duration) #VERSION > v"1.?.0-" + Test.format_duration(ts::RemoteTestSet) = Test.format_duration(ts.ts) +end +@static if isdefined(Test, :get_rng) #VERSION > v"1.12.0-" + Test.get_rng(ts::RemoteTestSet) = Test.get_rng(ts.ts) +end +@static if isdefined(Test, :anynonpass) #VERSION > v"1.13.0-" + Test.anynonpass(ts::RemoteTestSet) = Test.anynonpass(ts.ts) +end + +end From 2e821a8c3cd6fc8b6d21731222e7b140b057db77 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Thu, 16 Oct 2025 18:05:03 -0300 Subject: [PATCH 2/6] `@remote_testset` macro --- src/ParallelTestRunner.jl | 11 ++++------- src/remotetestset.jl | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index bc5fa93..808b299 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -220,9 +220,8 @@ function runtest(::Type{TestRecord}, f, name, init_code, color) function inner() # generate a temporary module to execute the tests in mod = @eval(Main, module $(gensym(name)) end) - @eval(mod, import ParallelTestRunner: Test, Random, RemoteTestSet) - @eval(mod, using .Test, .Random) - @eval(mod, using .Test: DefaultTestSet) # Necessary because VERSION <= v"1.10.0-" does not support unexported TestSets the @testset + @eval(mod, import ParallelTestRunner: Test, Random, RemoteTestSets) + @eval(mod, using .Test, .Random, .RemoteTestSets) Core.eval(mod, init_code) @@ -233,10 +232,8 @@ function runtest(::Type{TestRecord}, f, name, init_code, color) mktemp() do path, io stats = redirect_stdio(stdout=io, stderr=io) do @timed try - @testset RemoteTestSet "wrapper" begin - @testset DefaultTestSet $name begin - $f - end + @remote_testset $name begin + $f end catch err isa(err, Test.TestSetException) || rethrow() diff --git a/src/remotetestset.jl b/src/remotetestset.jl index e305482..f2cfc6d 100644 --- a/src/remotetestset.jl +++ b/src/remotetestset.jl @@ -60,4 +60,40 @@ end Test.anynonpass(ts::RemoteTestSet) = Test.anynonpass(ts.ts) end +macro remote_testset(args...) + testsettype = nothing + otherargs = [] + + for arg in args[1:end-1] + if isa(arg, Symbol) || Base.isexpr(arg, :.) + testsettype = arg + else + push!(otherargs, arg) + end + end + + source = args[end] + if isnothing(testsettype) + testsettype = :(Test.DefaultTestSet) + end + + # Build the inner @testset call + inner_testset = Expr(:macrocall, + :(Test.var"@testset"), + LineNumberNode(@__LINE__, @__FILE__), + testsettype, + otherargs..., + source) + + # Build the outer @testset call with RemoteTestSet + outer_testset = Expr(:macrocall, + :(Test.var"@testset"), + LineNumberNode(@__LINE__, @__FILE__), + :RemoteTestSet, + "wrapper", + Expr(:block, inner_testset)) + + return esc(outer_testset) +end + end From ec43ff27bfa390d11e569b3179866a6fb06125e2 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Thu, 16 Oct 2025 18:30:35 -0300 Subject: [PATCH 3/6] Fix --- src/ParallelTestRunner.jl | 1 + src/remotetestset.jl | 28 +++++++--------------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 808b299..d123a2d 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -222,6 +222,7 @@ function runtest(::Type{TestRecord}, f, name, init_code, color) mod = @eval(Main, module $(gensym(name)) end) @eval(mod, import ParallelTestRunner: Test, Random, RemoteTestSets) @eval(mod, using .Test, .Random, .RemoteTestSets) + @eval(mod, using .Test: DefaultTestSet) # Necessary because VERSION <= v"1.10.0-" does not support unexported TestSets the @testset Core.eval(mod, init_code) diff --git a/src/remotetestset.jl b/src/remotetestset.jl index f2cfc6d..aee2f73 100644 --- a/src/remotetestset.jl +++ b/src/remotetestset.jl @@ -3,7 +3,7 @@ module RemoteTestSets export RemoteTestSet, @remote_testset import Test -import Test: AbstractTestSet, DefaultTestSet, Broken, Pass, Fail, Error +import Test: AbstractTestSet, DefaultTestSet, Broken, Pass, Fail, Error, @testset struct RemoteTestSet <: AbstractTestSet ts::DefaultTestSet @@ -73,27 +73,13 @@ macro remote_testset(args...) end source = args[end] - if isnothing(testsettype) - testsettype = :(Test.DefaultTestSet) - end + testsettype = isnothing(testsettype) ? :(DefaultTestSet) : testsettype - # Build the inner @testset call - inner_testset = Expr(:macrocall, - :(Test.var"@testset"), - LineNumberNode(@__LINE__, @__FILE__), - testsettype, - otherargs..., - source) - - # Build the outer @testset call with RemoteTestSet - outer_testset = Expr(:macrocall, - :(Test.var"@testset"), - LineNumberNode(@__LINE__, @__FILE__), - :RemoteTestSet, - "wrapper", - Expr(:block, inner_testset)) - - return esc(outer_testset) + return esc(quote + @testset RemoteTestSet "wrapper" begin + @testset $testsettype $(otherargs...) $source + end + end) end end From d8d83d40a8343ee9158e2ae03d7151b55544c635 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Thu, 16 Oct 2025 18:30:58 -0300 Subject: [PATCH 4/6] Remove stuff --- src/remotetestset.jl | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/remotetestset.jl b/src/remotetestset.jl index aee2f73..c0877f4 100644 --- a/src/remotetestset.jl +++ b/src/remotetestset.jl @@ -13,18 +13,18 @@ end RemoteTestSet(args...; kwargs...) = RemoteTestSet(DefaultTestSet(args...; kwargs...)) -function Base.propertynames(x::RemoteTestSet) - (:ts, Base.propertynames(x.ts)...) -end -function Base.getproperty(ts::RemoteTestSet, sym::Symbol) - if sym === :ts - return Base.getfield(ts, :ts) - end - return Base.getfield(Base.getfield(ts, :ts), sym) -end -function Base.setproperty!(ts::RemoteTestSet, sym::Symbol, v) - return Base.setproperty!(ts.ts, sym, v) -end +# function Base.propertynames(x::RemoteTestSet) +# (:ts, Base.propertynames(x.ts)...) +# end +# function Base.getproperty(ts::RemoteTestSet, sym::Symbol) +# if sym === :ts +# return Base.getfield(ts, :ts) +# end +# return Base.getfield(Base.getfield(ts, :ts), sym) +# end +# function Base.setproperty!(ts::RemoteTestSet, sym::Symbol, v) +# return Base.setproperty!(ts.ts, sym, v) +# end # Record testsets as usual Test.record(ts::RemoteTestSet, t::Union{Broken, Pass, Fail, Error}; kwargs...) = Test.record(ts.ts, t; kwargs...) @@ -37,28 +37,28 @@ function Test.finish(ts::RemoteTestSet; print_results::Bool=Test.TESTSET_PRINT_E end # Otherwise, just return the testset so it is returned from the @testset macro - return only(ts.results) + return only(ts.ts.results) end -Test.filter_errors(ts::RemoteTestSet) = Test.filter_errors(ts.ts) -Test.get_test_counts(ts::RemoteTestSet) = Test.get_test_counts(ts.ts) -Test.get_alignment(ts::RemoteTestSet, depth::Int) = Test.get_alignment(ts.ts, depth) - -@static if isdefined(Test, :results) #VERSION > v"1.11.0-??" - Test.results(ts::RemoteTestSet) = Test.results(ts.ts) -end -@static if isdefined(Test, :print_verbose) #VERSION > v"1.11.0-??" - Test.print_verbose(ts::RemoteTestSet) = Test.print_verbose(ts.ts) -end -@static if isdefined(Test, :format_duration) #VERSION > v"1.?.0-" - Test.format_duration(ts::RemoteTestSet) = Test.format_duration(ts.ts) -end -@static if isdefined(Test, :get_rng) #VERSION > v"1.12.0-" - Test.get_rng(ts::RemoteTestSet) = Test.get_rng(ts.ts) -end -@static if isdefined(Test, :anynonpass) #VERSION > v"1.13.0-" - Test.anynonpass(ts::RemoteTestSet) = Test.anynonpass(ts.ts) -end +# Test.filter_errors(ts::RemoteTestSet) = Test.filter_errors(ts.ts) +# Test.get_test_counts(ts::RemoteTestSet) = Test.get_test_counts(ts.ts) +# Test.get_alignment(ts::RemoteTestSet, depth::Int) = Test.get_alignment(ts.ts, depth) + +# @static if isdefined(Test, :results) #VERSION > v"1.11.0-??" +# Test.results(ts::RemoteTestSet) = Test.results(ts.ts) +# end +# @static if isdefined(Test, :print_verbose) #VERSION > v"1.11.0-??" +# Test.print_verbose(ts::RemoteTestSet) = Test.print_verbose(ts.ts) +# end +# @static if isdefined(Test, :format_duration) #VERSION > v"1.?.0-" +# Test.format_duration(ts::RemoteTestSet) = Test.format_duration(ts.ts) +# end +# @static if isdefined(Test, :get_rng) #VERSION > v"1.12.0-" +# Test.get_rng(ts::RemoteTestSet) = Test.get_rng(ts.ts) +# end +# @static if isdefined(Test, :anynonpass) #VERSION > v"1.13.0-" +# Test.anynonpass(ts::RemoteTestSet) = Test.anynonpass(ts.ts) +# end macro remote_testset(args...) testsettype = nothing From 5ce7945d17ed022adfcc46dfd1c842af87efe73f Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Thu, 16 Oct 2025 23:00:19 -0300 Subject: [PATCH 5/6] Changes from #55 and new test --- src/remotetestset.jl | 8 ++++---- test/basicts.jl | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 test/basicts.jl diff --git a/src/remotetestset.jl b/src/remotetestset.jl index c0877f4..93d82e3 100644 --- a/src/remotetestset.jl +++ b/src/remotetestset.jl @@ -32,11 +32,11 @@ Test.record(ts::RemoteTestSet, t::AbstractTestSet) = Test.record(ts.ts, t) # This is the single method that needs changing function Test.finish(ts::RemoteTestSet; print_results::Bool=Test.TESTSET_PRINT_ENABLE[]) - if Test.get_testset_depth() != 0 - throw(ErrorException("RemoteTestSet should only ever be a top-level TestSet")) - end + # This testset is just a placeholder, + # so it must be the top-most + @assert Test.get_testset_depth() == 0 - # Otherwise, just return the testset so it is returned from the @testset macro + # There should only ever be one child testset return only(ts.ts.results) end diff --git a/test/basicts.jl b/test/basicts.jl new file mode 100644 index 0000000..7ba4e48 --- /dev/null +++ b/test/basicts.jl @@ -0,0 +1,3 @@ +@testset "basic in testset" begin + @test true +end From 2323688a2305a19a26e02dec06c8ab2d5f4eb674 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Thu, 16 Oct 2025 23:01:29 -0300 Subject: [PATCH 6/6] Cleanup --- src/remotetestset.jl | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/src/remotetestset.jl b/src/remotetestset.jl index 93d82e3..c80f861 100644 --- a/src/remotetestset.jl +++ b/src/remotetestset.jl @@ -13,19 +13,6 @@ end RemoteTestSet(args...; kwargs...) = RemoteTestSet(DefaultTestSet(args...; kwargs...)) -# function Base.propertynames(x::RemoteTestSet) -# (:ts, Base.propertynames(x.ts)...) -# end -# function Base.getproperty(ts::RemoteTestSet, sym::Symbol) -# if sym === :ts -# return Base.getfield(ts, :ts) -# end -# return Base.getfield(Base.getfield(ts, :ts), sym) -# end -# function Base.setproperty!(ts::RemoteTestSet, sym::Symbol, v) -# return Base.setproperty!(ts.ts, sym, v) -# end - # Record testsets as usual Test.record(ts::RemoteTestSet, t::Union{Broken, Pass, Fail, Error}; kwargs...) = Test.record(ts.ts, t; kwargs...) Test.record(ts::RemoteTestSet, t::AbstractTestSet) = Test.record(ts.ts, t) @@ -40,25 +27,6 @@ function Test.finish(ts::RemoteTestSet; print_results::Bool=Test.TESTSET_PRINT_E return only(ts.ts.results) end -# Test.filter_errors(ts::RemoteTestSet) = Test.filter_errors(ts.ts) -# Test.get_test_counts(ts::RemoteTestSet) = Test.get_test_counts(ts.ts) -# Test.get_alignment(ts::RemoteTestSet, depth::Int) = Test.get_alignment(ts.ts, depth) - -# @static if isdefined(Test, :results) #VERSION > v"1.11.0-??" -# Test.results(ts::RemoteTestSet) = Test.results(ts.ts) -# end -# @static if isdefined(Test, :print_verbose) #VERSION > v"1.11.0-??" -# Test.print_verbose(ts::RemoteTestSet) = Test.print_verbose(ts.ts) -# end -# @static if isdefined(Test, :format_duration) #VERSION > v"1.?.0-" -# Test.format_duration(ts::RemoteTestSet) = Test.format_duration(ts.ts) -# end -# @static if isdefined(Test, :get_rng) #VERSION > v"1.12.0-" -# Test.get_rng(ts::RemoteTestSet) = Test.get_rng(ts.ts) -# end -# @static if isdefined(Test, :anynonpass) #VERSION > v"1.13.0-" -# Test.anynonpass(ts::RemoteTestSet) = Test.anynonpass(ts.ts) -# end macro remote_testset(args...) testsettype = nothing