From 874a40fad158ec881ad02447c570a96534b04488 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Oct 2025 17:43:44 +0200 Subject: [PATCH 1/7] Replace test_filter/custom_tests by a single testsuite argument. --- README.md | 31 +++++++-- src/ParallelTestRunner.jl | 134 ++++++++++++++++++++------------------ test/runtests.jl | 56 +++++++++------- 3 files changed, 128 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 30b6137..27995a5 100644 --- a/README.md +++ b/README.md @@ -40,19 +40,36 @@ using ParallelTestRunner runtests(MyModule, ARGS) ``` -### Filtering +### Customizing the test suite -`runtests` takes a keyword argument that acts as a filter function +By default, `runtests` automatically discovers all `.jl` files in your `test/` directory (excluding `runtests.jl` itself) using the `find_tests` function. You can customize which tests to run by providing a custom `testsuite` dictionary: ```julia -function test_filter(test) - if Sys.iswindows() && test == "ext/specialfunctions" - return false +# Manually define your test suite +testsuite = Dict( + "basic" => quote + include("basic.jl") + end, + "advanced" => quote + include("advanced.jl") end - return true +) + +runtests(MyModule, ARGS; testsuite) +``` + +You can also use `find_tests` to automatically discover tests and then filter or modify them: + +```julia +# Start with autodiscovered tests +testsuite = find_tests(pwd()) + +# Remove tests that shouldn't run on Windows +if Sys.iswindows() + delete!(testsuite, "ext/specialfunctions") end -runtests(MyModule, ARGS; test_filter) +runtests(MyModule, ARGS; testsuite) ``` ### Provide defaults diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index dc0d4a2..adfc466 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -1,6 +1,6 @@ module ParallelTestRunner -export runtests, addworkers, addworker +export runtests, addworkers, addworker, find_tests using Malt using Dates @@ -445,9 +445,51 @@ function addworker(; env=Vector{Pair{String, String}}()) end """ - runtests(mod::Module, ARGS; RecordType = TestRecord, - test_filter = Returns(true), - custom_tests = Dict(), + find_tests(dir::String) -> Dict{String, Expr} + +Discover test files in a directory and return a test suite dictionary. + +Walks through `dir` and finds all `.jl` files (excluding `runtests.jl`), returning a +dictionary mapping test names to expression that include each test file. +""" +function find_tests(dir::String) + tests = Dict{String, Expr}() + for (rootpath, dirs, files) in walkdir(dir) + # find Julia files + filter!(files) do file + endswith(file, ".jl") && file !== "runtests.jl" + end + isempty(files) && continue + + # strip extension + files = map(files) do file + file[1:(end - 3)] + end + + # prepend subdir + subdir = relpath(rootpath, dir) + if subdir != "." + files = map(files) do file + joinpath(subdir, file) + end + end + + # unify path separators + files = map(files) do file + replace(file, path_separator => '/') + end + + for file in files + path = joinpath(rootpath, file * ".jl") + tests[file] = :(include($path)) + end + end + return tests +end + +""" + runtests(mod::Module, ARGS; testsuite::Dict{String,Expr}=find_tests(pwd()), + RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing), stdout = Base.stdout, @@ -463,9 +505,9 @@ Run Julia tests in parallel across multiple worker processes. Several keyword arguments are also supported: +- `testsuite`: Dictionary mapping test names to expressions to execute (default: `find_tests(pwd())`). + By default, automatically discovers all `.jl` files in the test directory. - `RecordType`: Type of test record to use for tracking test results (default: `TestRecord`) -- `test_filter`: Optional function to filter which tests to run (default: run all tests) -- `custom_tests`: Optional dictionary of custom tests, mapping test names to expressions. - `init_code`: Code use to initialize each test's sandbox module (e.g., import auxiliary packages, define constants, etc). - `test_worker`: Optional function that takes a test name and returns a specific worker. @@ -494,14 +536,24 @@ Several keyword arguments are also supported: ## Examples ```julia -# Run all tests with default settings +# Run all tests with default settings (auto-discovers .jl files) runtests(MyModule, ARGS) # Run only tests matching "integration" runtests(MyModule, ["integration"]) -# Run with custom filter function -runtests(MyModule, ARGS; test_filter = test -> occursin("unit", test)) +# Customize the test suite +testsuite = find_tests(pwd()) +delete!(testsuite, "slow_test") # Remove a specific test +runtests(MyModule, ARGS; testsuite) + +# Define a custom test suite manually +testsuite = Dict( + "custom" => quote + @test 1 + 1 == 2 + end +) +runtests(MyModule, ARGS; testsuite) # Use custom test record type runtests(MyModule, ARGS; RecordType = MyCustomTestRecord) @@ -512,9 +564,9 @@ runtests(MyModule, ARGS; RecordType = MyCustomTestRecord) Workers are automatically recycled when they exceed memory limits to prevent out-of-memory issues during long test runs. The memory limit is set based on system architecture. """ -function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = TestRecord, - custom_tests::Dict{String, Expr}=Dict{String, Expr}(), init_code = :(), - test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr) +function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(pwd()), + RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing), + stdout = Base.stdout, stderr = Base.stderr) # # set-up # @@ -545,51 +597,8 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)") end - WORKDIR = pwd() - - # choose tests - tests = [] - test_runners = Dict() - ## custom tests by the user - for (name, runner) in custom_tests - push!(tests, name) - test_runners[name] = runner - end - ## files in the test folder - for (rootpath, dirs, files) in walkdir(WORKDIR) - # find Julia files - filter!(files) do file - endswith(file, ".jl") && file !== "runtests.jl" - end - isempty(files) && continue - - # strip extension - files = map(files) do file - file[1:(end - 3)] - end - - # prepend subdir - subdir = relpath(rootpath, WORKDIR) - if subdir != "." - files = map(files) do file - joinpath(subdir, file) - end - end - - # unify path separators - files = map(files) do file - replace(file, path_separator => '/') - end - - append!(tests, files) - for file in files - test_runners[file] = quote - include($(joinpath(WORKDIR, file * ".jl"))) - end - end - end - ## finalize - unique!(tests) + # determine test order + tests = collect(keys(testsuite)) Random.shuffle!(tests) historical_durations = load_test_history(mod) sort!(tests, by = x -> -get(historical_durations, x, Inf)) @@ -603,11 +612,8 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T exit(0) end - # filter tests - if isempty(ARGS) - filter!(test_filter, tests) - else - # let the user filter + # filter tests based on command-line arguments + if !isempty(ARGS) filter!(tests) do test any(arg -> startswith(test, arg), ARGS) end @@ -834,8 +840,8 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T put!(printer_channel, (:started, test, worker_id(wrkr))) result = try Malt.remote_eval_wait(Main, wrkr, :(import ParallelTestRunner)) - Malt.remote_call_fetch(invokelatest, wrkr, runtest, RecordType, test_runners[test], test, - init_code, io_ctx.color) + Malt.remote_call_fetch(invokelatest, wrkr, runtest, RecordType, + testsuite[test], test, init_code, io_ctx.color) catch ex if isa(ex, InterruptException) # the worker got interrupted, signal other tasks to stop diff --git a/test/runtests.jl b/test/runtests.jl index d5cabd8..5971bfe 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,7 +5,7 @@ cd(@__DIR__) @testset "ParallelTestRunner" verbose=true begin -@testset "basic test" begin +@testset "basic use" begin io = IOBuffer() io_color = IOContext(io, :color => true) runtests(ParallelTestRunner, ["--verbose"]; stdout=io_color, stderr=io_color) @@ -24,7 +24,23 @@ cd(@__DIR__) @test isfile(ParallelTestRunner.get_history_file(ParallelTestRunner)) end -@testset "custom tests and init code" begin +@testset "custom tests" begin + testsuite = Dict( + "custom" => quote + @test true + end + ) + + io = IOBuffer() + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) + + str = String(take!(io)) + @test !contains(str, r"basic .+ started at") + @test contains(str, r"custom .+ started at") + @test contains(str, "SUCCESS") +end + +@testset "init code" begin init_code = quote using Test should_be_defined() = true @@ -33,7 +49,7 @@ end return :(true) end end - custom_tests = Dict( + testsuite = Dict( "custom" => quote @test should_be_defined() @test @should_also_be_defined() @@ -41,10 +57,9 @@ end ) io = IOBuffer() - runtests(ParallelTestRunner, ["--verbose"]; init_code, custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; init_code, testsuite, stdout=io, stderr=io) str = String(take!(io)) - @test contains(str, r"basic .+ started at") @test contains(str, r"custom .+ started at") @test contains(str, "SUCCESS") end @@ -56,7 +71,7 @@ end end return nothing end - custom_tests = Dict( + testsuite = Dict( "needs env var" => quote @test ENV["SPECIAL_ENV_VAR"] == "42" end, @@ -66,17 +81,16 @@ end ) io = IOBuffer() - runtests(ParallelTestRunner, ["--verbose"]; test_worker, custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; test_worker, testsuite, stdout=io, stderr=io) str = String(take!(io)) - @test contains(str, r"basic .+ started at") @test contains(str, r"needs env var .+ started at") @test contains(str, r"doesn't need env var .+ started at") @test contains(str, "SUCCESS") end @testset "failing test" begin - custom_tests = Dict( + testsuite = Dict( "failing test" => quote @test 1 == 2 end @@ -85,11 +99,10 @@ end io = IOBuffer() @test_throws Test.FallbackTestSetException("Test run finished with errors") begin - runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) end str = String(take!(io)) - @test contains(str, r"basic .+ started at") @test contains(str, r"failing test .+ failed at") @test contains(str, "$(basename(@__FILE__)):$error_line") @test contains(str, "FAILURE") @@ -98,7 +111,7 @@ end end @testset "nested failure" begin - custom_tests = Dict( + testsuite = Dict( "nested" => quote @test true @testset "foo" begin @@ -113,7 +126,7 @@ end io = IOBuffer() @test_throws Test.FallbackTestSetException("Test run finished with errors") begin - runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) end str = String(take!(io)) @@ -128,7 +141,7 @@ end end @testset "throwing test" begin - custom_tests = Dict( + testsuite = Dict( "throwing test" => quote error("This test throws an error") end @@ -137,11 +150,10 @@ end io = IOBuffer() @test_throws Test.FallbackTestSetException("Test run finished with errors") begin - runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) end str = String(take!(io)) - @test contains(str, r"basic .+ started at") @test contains(str, r"throwing test .+ failed at") @test contains(str, "$(basename(@__FILE__)):$error_line") @test contains(str, "FAILURE") @@ -150,7 +162,7 @@ end end @testset "crashing test" begin - custom_tests = Dict( + testsuite = Dict( "abort" => quote abort() = ccall(:abort, Nothing, ()) abort() @@ -159,7 +171,7 @@ end io = IOBuffer() @test_throws Test.FallbackTestSetException("Test run finished with errors") begin - runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) end str = String(take!(io)) @@ -171,14 +183,14 @@ end end @testset "test output" begin - custom_tests = Dict( + testsuite = Dict( "output" => quote println("This is some output from the test") end ) io = IOBuffer() - runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) str = String(take!(io)) @test contains(str, r"output .+ started at") @@ -187,14 +199,14 @@ end end @testset "warnings" begin - custom_tests = Dict( + testsuite = Dict( "warning" => quote @test_warn "3.0" @warn "3.0" end ) io = IOBuffer() - runtests(ParallelTestRunner, ["--verbose"]; custom_tests, stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; testsuite, stdout=io, stderr=io) str = String(take!(io)) @test contains(str, r"warning .+ started at") From 605dd0b4934818db2bdd3f01fba46c92039977a9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 20 Oct 2025 08:58:27 +0200 Subject: [PATCH 2/7] Rework args to support running filtered tests. --- README.md | 15 ++- src/ParallelTestRunner.jl | 239 ++++++++++++++++++++++++-------------- 2 files changed, 163 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 27995a5..80cce59 100644 --- a/README.md +++ b/README.md @@ -58,18 +58,23 @@ testsuite = Dict( runtests(MyModule, ARGS; testsuite) ``` -You can also use `find_tests` to automatically discover tests and then filter or modify them: +You can also use `find_tests` to automatically discover tests and then filter or modify them. This requires manually parsing arguments so that filtering is only applied when the user did not request specific tests to run: ```julia # Start with autodiscovered tests testsuite = find_tests(pwd()) -# Remove tests that shouldn't run on Windows -if Sys.iswindows() - delete!(testsuite, "ext/specialfunctions") +# Parse arguments +args = parse_args(ARGS) + +if filter_tests!(testsuite, args) + # Remove tests that shouldn't run on Windows + if Sys.iswindows() + delete!(testsuite, "ext/specialfunctions") + end end -runtests(MyModule, ARGS; testsuite) +runtests(MyModule, args; testsuite) ``` ### Provide defaults diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index adfc466..a2f69aa 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -1,6 +1,6 @@ module ParallelTestRunner -export runtests, addworkers, addworker, find_tests +export runtests, addworkers, addworker, find_tests, parse_args, filter_tests! using Malt using Dates @@ -35,28 +35,6 @@ end const max_worker_rss = JULIA_TEST_MAXRSS_MB * 2^20 -# parse some command-line arguments -function extract_flag!(args, flag, default = nothing; typ = typeof(default)) - for f in args - if startswith(f, flag) - # Check if it's just `--flag` or if it's `--flag=foo` - if f != flag - val = split(f, '=')[2] - if !(typ === Nothing || typ <: AbstractString) - val = parse(typ, val) - end - else - val = default - end - - # Drop this value from our args - filter!(x -> x != f, args) - return (true, val) - end - end - return (false, default) -end - function with_testset(f, testset) @static if VERSION >= v"1.13.0-DEV.1044" Test.@with_testset testset f() @@ -487,13 +465,130 @@ function find_tests(dir::String) return tests end +struct ParsedArgs + jobs::Union{Some{Int}, Nothing} + verbose::Union{Some{Nothing}, Nothing} + quickfail::Union{Some{Nothing}, Nothing} + list::Union{Some{Nothing}, Nothing} + + custom::Dict{String,Any} + + positionals::Vector{String} +end + +# parse some command-line arguments +function extract_flag!(args, flag; typ = Nothing) + for f in args + if startswith(f, flag) + # Check if it's just `--flag` or if it's `--flag=foo` + val = if f == flag + nothing + else + parts = split(f, '=') + if typ === Nothing || typ <: AbstractString + parts[2] + else + parse(typ, parts[2]) + end + end + + # Drop this value from our args + filter!(x -> x != f, args) + return Some(val) + end + end + return nothing +end + +""" + parse_args(args; [custom::Array{String}]) -> ParsedArgs + +Parse command-line arguments for `runtests`. Typically invoked by passing `Base.ARGS`. + +Fields of this structure represent command-line options, containing `nothing` when the +option was not specified, or `Some(optional_value=nothing)` when it was. + +Custom arguments can be specified via the `custom` keyword argument, which should be +an array of strings representing custom flag names (without the `--` prefix). Presence +of these flags will be recorded in the `custom` field of the returned `ParsedArgs` object. +""" +function parse_args(args; custom::Array{String} = String[]) + args = copy(args) + + help = extract_flag!(args, "--help") + if help !== nothing + usage = + """ + Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...] + + --help Show this text. + --list List all available tests. + --verbose Print more information during testing. + --quickfail Fail the entire run as soon as a single test errored. + --jobs=N Launch `N` processes to perform tests.""" + + if !isempty(custom) + usage *= "\n\nCustom arguments:" + for flag in custom + usage *= "\n --$flag" + end + end + usage *= "\n\nRemaining arguments filter the tests that will be executed." + println(usage) + exit(0) + end + + jobs = extract_flag!(args, "--jobs"; typ = Int) + verbose = extract_flag!(args, "--verbose") + quickfail = extract_flag!(args, "--quickfail") + list = extract_flag!(args, "--list") + + custom_args = Dict{String,Any}() + for flag in custom + custom_args[flag] = extract_flag!(args, "--$flag") + end + + ## no options should remain + optlike_args = filter(startswith("-"), args) + if !isempty(optlike_args) + error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)") + end + + return ParsedArgs(jobs, verbose, quickfail, list, custom_args, args) +end + +""" + filter_tests!(testsuite, args::ParsedArgs) -> Bool + +Filter tests in `testsuite` based on command-line arguments in `args`. + +Returns `true` if additional filtering may be done by the caller, `false` otherwise. +""" +function filter_tests!(testsuite, args::ParsedArgs) + # the user did not request specific tests, so let the caller do its own filtering + isempty(args.positionals) && return true + + # only select tests matching positional arguments + tests = collect(keys(testsuite)) + for test in tests + if !any(arg -> startswith(test, arg), args.positionals) + delete!(testsuite, test) + end + end + + # the user requested specific tests, so don't allow further filtering + return false +end + """ - runtests(mod::Module, ARGS; testsuite::Dict{String,Expr}=find_tests(pwd()), - RecordType = TestRecord, - init_code = :(), - test_worker = Returns(nothing), - stdout = Base.stdout, - stderr = Base.stderr) + runtests(mod::Module, args::ParsedArgs; + testsuite::Dict{String,Expr}=find_tests(pwd()), + RecordType = TestRecord, + init_code = :(), + test_worker = Returns(nothing), + stdout = Base.stdout, + stderr = Base.stderr) + runtests(mod::Module, ARGS; ...) Run Julia tests in parallel across multiple worker processes. @@ -501,7 +596,8 @@ Run Julia tests in parallel across multiple worker processes. - `mod`: The module calling runtests - `ARGS`: Command line arguments array, typically from `Base.ARGS`. When you run the tests - with `Pkg.test`, this can be changed with the `test_args` keyword argument. + with `Pkg.test`, this can be changed with the `test_args` keyword argument. If the caller + needs to accept args too, consider using `parse_args` to parse the arguments first. Several keyword arguments are also supported: @@ -542,12 +638,7 @@ runtests(MyModule, ARGS) # Run only tests matching "integration" runtests(MyModule, ["integration"]) -# Customize the test suite -testsuite = find_tests(pwd()) -delete!(testsuite, "slow_test") # Remove a specific test -runtests(MyModule, ARGS; testsuite) - -# Define a custom test suite manually +# Define a custom test suite testsuite = Dict( "custom" => quote @test 1 + 1 == 2 @@ -555,8 +646,14 @@ testsuite = Dict( ) runtests(MyModule, ARGS; testsuite) -# Use custom test record type -runtests(MyModule, ARGS; RecordType = MyCustomTestRecord) +# Customize the test suite +testsuite = find_tests(pwd()) +args = parse_args(ARGS) +if filter_tests!(testsuite, args) + # Remove a specific test + delete!(testsuite, "slow_test") +end +runtests(MyModule, args; testsuite) ``` ## Memory Management @@ -564,38 +661,25 @@ runtests(MyModule, ARGS; RecordType = MyCustomTestRecord) Workers are automatically recycled when they exceed memory limits to prevent out-of-memory issues during long test runs. The memory limit is set based on system architecture. """ -function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(pwd()), +function runtests(mod::Module, args::ParsedArgs; + testsuite::Dict{String,Expr} = find_tests(pwd()), RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr) # # set-up # - do_help, _ = extract_flag!(ARGS, "--help") - if do_help - println( - """ - Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...] - - --help Show this text. - --list List all available tests. - --verbose Print more information during testing. - --quickfail Fail the entire run as soon as a single test errored. - --jobs=N Launch `N` processes to perform tests. - - Remaining arguments filter the tests that will be executed.""" - ) + # list tests, if requested + if args.list !== nothing + println(stdout, "Available tests:") + for test in keys(testsuite) + println(stdout, " - $test") + end exit(0) end - set_jobs, jobs = extract_flag!(ARGS, "--jobs"; typ = Int) - do_verbose, _ = extract_flag!(ARGS, "--verbose") - do_quickfail, _ = extract_flag!(ARGS, "--quickfail") - do_list, _ = extract_flag!(ARGS, "--list") - ## no options should remain - optlike_args = filter(startswith("-"), ARGS) - if !isempty(optlike_args) - error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)") - end + + # filter tests + filter_tests!(testsuite, args) # determine test order tests = collect(keys(testsuite)) @@ -603,26 +687,8 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p historical_durations = load_test_history(mod) sort!(tests, by = x -> -get(historical_durations, x, Inf)) - # list tests, if requested - if do_list - println(stdout, "Available tests:") - for test in sort(tests) - println(stdout, " - $test") - end - exit(0) - end - - # filter tests based on command-line arguments - if !isempty(ARGS) - filter!(tests) do test - any(arg -> startswith(test, arg), ARGS) - end - end - # determine parallelism - if !set_jobs - jobs = default_njobs() - end + jobs = something(args.jobs, default_njobs()) jobs = clamp(jobs, 1, length(tests)) println(stdout, "Running $jobs tests in parallel. If this is too many, specify the `--jobs=N` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable.") workers = addworkers(min(jobs, length(tests))) @@ -761,7 +827,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p test_name, wrkr = msg[2], msg[3] # Optionally print verbose started message - if do_verbose + if args.verbose !== nothing clear_status() print_test_started(RecordType, wrkr, test_name, io_ctx) end @@ -868,7 +934,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException @assert result isa Exception put!(printer_channel, (:crashed, test, worker_id(wrkr))) - if do_quickfail + if args.quickfail !== nothing stop_work() end @@ -977,7 +1043,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p return testset end t1 = time() - o_ts = create_testset("Overall"; start=t0, stop=t1, verbose=do_verbose) + o_ts = create_testset("Overall"; start=t0, stop=t1, verbose=!isnothing(args.verbose)) function collect_results() with_testset(o_ts) do completed_tests = Set{String}() @@ -1054,6 +1120,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p end return -end # runtests +end +runtests(mod::Module, ARGS; kwargs...) = runtests(mod, parse_args(ARGS); kwargs...) -end # module ParallelTestRunner +end From 78e51fee4deff5d7789b172fb271859181329ca4 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 20 Oct 2025 09:04:21 +0200 Subject: [PATCH 3/7] Remove RecordType from the public API for now. --- src/ParallelTestRunner.jl | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index a2f69aa..2023710 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -92,7 +92,7 @@ struct TestIOContext rss_align::Int end -function test_IOContext(::Type{TestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int) +function test_IOContext(stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int) elapsed_align = textwidth("Time (s)") gc_align = textwidth("GC (s)") percent_align = textwidth("GC %") @@ -107,7 +107,7 @@ function test_IOContext(::Type{TestRecord}, stdout::IO, stderr::IO, lock::Reentr ) end -function print_header(::Type{TestRecord}, ctx::TestIOContext, testgroupheader, workerheader) +function print_header(ctx::TestIOContext, testgroupheader, workerheader) lock(ctx.lock) try printstyled(ctx.stdout, " "^(ctx.name_align + textwidth(testgroupheader) - 3), " │ ") @@ -121,7 +121,7 @@ function print_header(::Type{TestRecord}, ctx::TestIOContext, testgroupheader, w end end -function print_test_started(::Type{TestRecord}, wrkr, test, ctx::TestIOContext) +function print_test_started(wrkr, test, ctx::TestIOContext) lock(ctx.lock) try printstyled(ctx.stdout, test, lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │", color = :white) @@ -185,7 +185,7 @@ function print_test_failed(record::TestRecord, wrkr, test, ctx::TestIOContext) end end -function print_test_crashed(::Type{TestRecord}, wrkr, test, ctx::TestIOContext) +function print_test_crashed(wrkr, test, ctx::TestIOContext) lock(ctx.lock) try printstyled(ctx.stderr, test, color = :red) @@ -236,7 +236,7 @@ function Test.finish(ts::WorkerTestSet) return ts.wrapped_ts end -function runtest(::Type{TestRecord}, f, name, init_code, color) +function runtest(f, name, init_code, color) function inner() # generate a temporary module to execute the tests in mod = @eval(Main, module $(gensym(name)) end) @@ -583,7 +583,6 @@ end """ runtests(mod::Module, args::ParsedArgs; testsuite::Dict{String,Expr}=find_tests(pwd()), - RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing), stdout = Base.stdout, @@ -603,7 +602,6 @@ Several keyword arguments are also supported: - `testsuite`: Dictionary mapping test names to expressions to execute (default: `find_tests(pwd())`). By default, automatically discovers all `.jl` files in the test directory. -- `RecordType`: Type of test record to use for tracking test results (default: `TestRecord`) - `init_code`: Code use to initialize each test's sandbox module (e.g., import auxiliary packages, define constants, etc). - `test_worker`: Optional function that takes a test name and returns a specific worker. @@ -663,7 +661,7 @@ issues during long test runs. The memory limit is set based on system architectu """ function runtests(mod::Module, args::ParsedArgs; testsuite::Dict{String,Expr} = find_tests(pwd()), - RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing), + init_code = :(), test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr) # # set-up @@ -731,8 +729,8 @@ function runtests(mod::Module, args::ParsedArgs; stderr.lock = print_lock end - io_ctx = test_IOContext(RecordType, stdout, stderr, print_lock, name_align) - print_header(RecordType, io_ctx, testgroupheader, workerheader) + io_ctx = test_IOContext(stdout, stderr, print_lock, name_align) + print_header(io_ctx, testgroupheader, workerheader) status_lines_visible = Ref(0) @@ -829,7 +827,7 @@ function runtests(mod::Module, args::ParsedArgs; # Optionally print verbose started message if args.verbose !== nothing clear_status() - print_test_started(RecordType, wrkr, test_name, io_ctx) + print_test_started(wrkr, test_name, io_ctx) end elseif msg_type == :finished @@ -846,7 +844,7 @@ function runtests(mod::Module, args::ParsedArgs; test_name, wrkr = msg[2], msg[3] clear_status() - print_test_crashed(RecordType, wrkr, test_name, io_ctx) + print_test_crashed(wrkr, test_name, io_ctx) end end @@ -906,7 +904,7 @@ function runtests(mod::Module, args::ParsedArgs; put!(printer_channel, (:started, test, worker_id(wrkr))) result = try Malt.remote_eval_wait(Main, wrkr, :(import ParallelTestRunner)) - Malt.remote_call_fetch(invokelatest, wrkr, runtest, RecordType, + Malt.remote_call_fetch(invokelatest, wrkr, runtest, testsuite[test], test, init_code, io_ctx.color) catch ex if isa(ex, InterruptException) @@ -922,7 +920,6 @@ function runtests(mod::Module, args::ParsedArgs; # act on the results if result isa AbstractTestRecord - @assert result isa RecordType put!(printer_channel, (:finished, test, worker_id(wrkr), result)) if memory_usage(result) > max_worker_rss From 66a5875f0f1adcacaf197ffceac82890487ece00 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 22 Oct 2025 11:33:14 +0200 Subject: [PATCH 4/7] Make sure we don't spawn workers if all we need are custom ones. --- src/ParallelTestRunner.jl | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 2023710..59dece1 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -689,12 +689,12 @@ function runtests(mod::Module, args::ParsedArgs; jobs = something(args.jobs, default_njobs()) jobs = clamp(jobs, 1, length(tests)) println(stdout, "Running $jobs tests in parallel. If this is too many, specify the `--jobs=N` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable.") - workers = addworkers(min(jobs, length(tests))) - nworkers = length(workers) + nworkers = min(jobs, length(tests)) + workers = fill(nothing, nworkers) t0 = time() results = [] - running_tests = Dict{String, Tuple{Int, Float64}}() # test => (worker, start_time) + running_tests = Dict{String, Float64}() # test => start_time test_lock = ReentrantLock() # to protect crucial access to tests and running_tests done = false @@ -755,9 +755,9 @@ function runtests(mod::Module, args::ParsedArgs; line1 = "" # line 2: running tests - test_list = sort(collect(running_tests), by = x -> x[2][2]) - status_parts = map(test_list) do (test, (wrkr, _)) - "$test ($wrkr)" + test_list = sort(collect(keys(running_tests)), by = x -> running_tests[x]) + status_parts = map(test_list) do test + "$test" end line2 = "Running: " * join(status_parts, ", ") ## truncate @@ -777,7 +777,7 @@ function runtests(mod::Module, args::ParsedArgs; est_remaining = 0.0 ## currently-running - for (test, (_, start_time)) in running_tests + for (test, start_time) in running_tests elapsed = time() - start_time duration = get(historical_durations, test, est_per_test) est_remaining += max(0.0, duration - elapsed) @@ -883,21 +883,24 @@ function runtests(mod::Module, args::ParsedArgs; for p in workers push!(worker_tasks, @async begin while !done - # if a worker failed, spawn a new one - if !Malt.isrunning(p) - p = addworker() - end - # get a test to run - test, wrkr, test_t0 = Base.@lock test_lock begin + test, test_t0 = Base.@lock test_lock begin isempty(tests) && break test = popfirst!(tests) - wrkr = something(test_worker(test), p) test_t0 = time() - running_tests[test] = (worker_id(wrkr), test_t0) + running_tests[test] = test_t0 - test, wrkr, test_t0 + test, test_t0 + end + + # if a worker failed, spawn a new one + wrkr = test_worker(test) + if wrkr === nothing + wrkr = p + end + if wrkr === nothing || !Malt.isrunning(wrkr) + wrkr = p = addworker() end # run the test From 71c610ebbed6eaed3a03b39044491556105d632c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 22 Oct 2025 12:00:27 +0200 Subject: [PATCH 5/7] Use pipes for redirecting I/O. --- src/ParallelTestRunner.jl | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 59dece1..6209822 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -252,22 +252,27 @@ function runtest(f, name, init_code, color) GC.gc(true) Random.seed!(1) - mktemp() do path, io - stats = redirect_stdio(stdout=io, stderr=io) do - # @testset CustomTestRecord switches the all lower-level testset to our custom testset, - # so we need to have two layers here such that the user-defined testsets are using `DefaultTestSet`. - # This also guarantees our invariant about `WorkerTestSet` containing a single `DefaultTestSet`. - @timed @testset WorkerTestSet "placeholder" begin - @testset DefaultTestSet $name begin - $f - end + pipe = Pipe() + pipe_initialized = Channel{Nothing}(1) + reader = @async begin + take!(pipe_initialized) + read(pipe, String) + end + stats = redirect_stdio(stdout=pipe, stderr=pipe) do + put!(pipe_initialized, nothing) + + # @testset CustomTestRecord switches the all lower-level testset to our custom testset, + # so we need to have two layers here such that the user-defined testsets are using `DefaultTestSet`. + # This also guarantees our invariant about `WorkerTestSet` containing a single `DefaultTestSet`. + @timed @testset WorkerTestSet "placeholder" begin + @testset DefaultTestSet $name begin + $f end end - close(io) - output = read(path, String) - (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) - end + close(pipe.in) + output = fetch(reader) + (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) end # process results From 05287d791b23a10702029ed21629f3c1903dc0f9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 22 Oct 2025 12:22:24 +0200 Subject: [PATCH 6/7] Make quickfail work again. --- src/ParallelTestRunner.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 6209822..9d5e078 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -929,6 +929,10 @@ function runtests(mod::Module, args::ParsedArgs; # act on the results if result isa AbstractTestRecord put!(printer_channel, (:finished, test, worker_id(wrkr), result)) + if anynonpass(result[]) && args.quickfail !== nothing + stop_work() + break + end if memory_usage(result) > max_worker_rss # the worker has reached the max-rss limit, recycle it @@ -941,6 +945,7 @@ function runtests(mod::Module, args::ParsedArgs; put!(printer_channel, (:crashed, test, worker_id(wrkr))) if args.quickfail !== nothing stop_work() + break end # the worker encountered some serious failure, recycle it From d3235a64c13ec642b99364fc75aa12a29ea231a2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 22 Oct 2025 13:07:34 +0200 Subject: [PATCH 7/7] Bump version. --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 457b1e0..30ff4d5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ParallelTestRunner" uuid = "d3525ed8-44d0-4b2c-a655-542cee43accc" authors = ["Valentin Churavy "] -version = "1.0.3" +version = "2.0.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"