Replace test_filter/custom_tests by a single testsuite argument.

Author: maleadt

README.md

@@ -40,19 +40,36 @@ using ParallelTestRunner
 runtests(MyModule, ARGS)
 ```
 
-### Filtering
+### Customizing the test suite
 
-`runtests` takes a keyword argument that acts as a filter function
+By default, `runtests` automatically discovers all `.jl` files in your `test/` directory (excluding `runtests.jl` itself) using the `find_tests` function. You can customize which tests to run by providing a custom `testsuite` dictionary:
 
 ```julia
-function test_filter(test)
-    if Sys.iswindows() && test == "ext/specialfunctions"
-        return false
+# Manually define your test suite
+testsuite = Dict(
+    "basic" => quote
+        include("basic.jl")
+    end,
+    "advanced" => quote
+        include("advanced.jl")
     end
-    return true
+)
+
+runtests(MyModule, ARGS; testsuite)
+```
+
+You can also use `find_tests` to automatically discover tests and then filter or modify them:
+
+```julia
+# Start with autodiscovered tests
+testsuite = find_tests(pwd())
+
+# Remove tests that shouldn't run on Windows
+if Sys.iswindows()
+    delete!(testsuite, "ext/specialfunctions")
 end
 
-runtests(MyModule, ARGS; test_filter)
+runtests(MyModule, ARGS; testsuite)
 ```
 
 ### Provide defaults

src/ParallelTestRunner.jl

@@ -1,6 +1,6 @@
 module ParallelTestRunner
 
-export runtests, addworkers, addworker
+export runtests, addworkers, addworker, find_tests
 
 using Malt
 using Dates
@@ -445,9 +445,81 @@ function addworker(; env=Vector{Pair{String, String}}())
 end
 
 """
-    runtests(mod::Module, ARGS; RecordType = TestRecord,
-                                test_filter = Returns(true),
-                                custom_tests = Dict(),
+    find_tests([f], dir::String) -> Dict{String, Expr}
+
+Discover test files in a directory and return a test suite dictionary.
+
+Walks through `dir` and finds all `.jl` files (excluding `runtests.jl`), returning a
+dictionary mapping test names to expressions that run those tests.
+
+## Arguments
+
+- `f`: Optional function that takes a file path and returns an expression to execute
+  (default: `path -> :(include(\$path))`)
+- `dir`: Directory to search for test files
+
+## Returns
+
+A `Dict{String, Expr}` where keys are test names (file paths relative to `dir` with
+`.jl` extension removed and path separators normalized to `/`) and values are expressions
+to execute for each test.
+
+## Examples
+
+```julia
+# Auto-discover tests with default include behavior
+testsuite = find_tests(pwd())
+
+# Custom expression for each test file
+testsuite = find_tests(pwd()) do path
+    quote
+        @info "Running test: \$path"
+        include(\$path)
+    end
+end
+```
+"""
+function find_tests(f, dir::String)
+    tests = Dict{String, Expr}()
+    for (rootpath, dirs, files) in walkdir(dir)
+        # find Julia files
+        filter!(files) do file
+            endswith(file, ".jl") && file !== "runtests.jl"
+        end
+        isempty(files) && continue
+
+        # strip extension
+        files = map(files) do file
+            file[1:(end - 3)]
+        end
+
+        # prepend subdir
+        subdir = relpath(rootpath, dir)
+        if subdir != "."
+            files = map(files) do file
+                joinpath(subdir, file)
+            end
+        end
+
+        # unify path separators
+        files = map(files) do file
+            replace(file, path_separator => '/')
+        end
+
+        for file in files
+            path = joinpath(rootpath, file * ".jl")
+            tests[file] = f(path)
+        end
+    end
+    return tests
+end
+find_tests(dir::String) = find_tests(dir) do path
+    :(include($path))
+end
+
+"""
+    runtests(mod::Module, ARGS; testsuite::Dict{String,Expr}=find_tests(pwd()),
+                                RecordType = TestRecord,
                                 init_code = :(),
                                 test_worker = Returns(nothing),
                                 stdout = Base.stdout,
@@ -463,9 +535,9 @@ Run Julia tests in parallel across multiple worker processes.
 
 Several keyword arguments are also supported:
 
+- `testsuite`: Dictionary mapping test names to expressions to execute (default: `find_tests(pwd())`).
+  By default, automatically discovers all `.jl` files in the test directory.
 - `RecordType`: Type of test record to use for tracking test results (default: `TestRecord`)
-- `test_filter`: Optional function to filter which tests to run (default: run all tests)
-- `custom_tests`: Optional dictionary of custom tests, mapping test names to expressions.
 - `init_code`: Code use to initialize each test's sandbox module (e.g., import auxiliary
   packages, define constants, etc).
 - `test_worker`: Optional function that takes a test name and returns a specific worker.
@@ -494,14 +566,24 @@ Several keyword arguments are also supported:
 ## Examples
 
 ```julia
-# Run all tests with default settings
+# Run all tests with default settings (auto-discovers .jl files)
 runtests(MyModule, ARGS)
 
 # Run only tests matching "integration"
 runtests(MyModule, ["integration"])
 
-# Run with custom filter function
-runtests(MyModule, ARGS; test_filter = test -> occursin("unit", test))
+# Customize the test suite
+testsuite = find_tests(pwd())
+delete!(testsuite, "slow_test")  # Remove a specific test
+runtests(MyModule, ARGS; testsuite)
+
+# Define a custom test suite manually
+testsuite = Dict(
+    "custom" => quote
+        @test 1 + 1 == 2
+    end
+)
+runtests(MyModule, ARGS; testsuite)
 
 # Use custom test record type
 runtests(MyModule, ARGS; RecordType = MyCustomTestRecord)
@@ -512,9 +594,9 @@ runtests(MyModule, ARGS; RecordType = MyCustomTestRecord)
 Workers are automatically recycled when they exceed memory limits to prevent out-of-memory
 issues during long test runs. The memory limit is set based on system architecture.
 """
-function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = TestRecord,
-                  custom_tests::Dict{String, Expr}=Dict{String, Expr}(), init_code = :(),
-                  test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr)
+function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(pwd()),
+                  RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing),
+                  stdout = Base.stdout, stderr = Base.stderr)
     #
     # set-up
     #
@@ -545,51 +627,8 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
         error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)")
     end
 
-    WORKDIR = pwd()
-
-    # choose tests
-    tests = []
-    test_runners = Dict()
-    ## custom tests by the user
-    for (name, runner) in custom_tests
-        push!(tests, name)
-        test_runners[name] = runner
-    end
-    ## files in the test folder
-    for (rootpath, dirs, files) in walkdir(WORKDIR)
-        # find Julia files
-        filter!(files) do file
-            endswith(file, ".jl") && file !== "runtests.jl"
-        end
-        isempty(files) && continue
-
-        # strip extension
-        files = map(files) do file
-            file[1:(end - 3)]
-        end
-
-        # prepend subdir
-        subdir = relpath(rootpath, WORKDIR)
-        if subdir != "."
-            files = map(files) do file
-                joinpath(subdir, file)
-            end
-        end
-
-        # unify path separators
-        files = map(files) do file
-            replace(file, path_separator => '/')
-        end
-
-        append!(tests, files)
-        for file in files
-            test_runners[file] = quote
-                include($(joinpath(WORKDIR, file * ".jl")))
-            end
-        end
-    end
-    ## finalize
-    unique!(tests)
+    # determine test order
+    tests = collect(keys(testsuite))
     Random.shuffle!(tests)
     historical_durations = load_test_history(mod)
     sort!(tests, by = x -> -get(historical_durations, x, Inf))
@@ -603,11 +642,8 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
         exit(0)
     end
 
-    # filter tests
-    if isempty(ARGS)
-        filter!(test_filter, tests)
-    else
-        # let the user filter
+    # filter tests based on command-line arguments
+    if !isempty(ARGS)
         filter!(tests) do test
             any(arg -> startswith(test, arg), ARGS)
         end
@@ -834,8 +870,8 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T
                 put!(printer_channel, (:started, test, worker_id(wrkr)))
                 result = try
                     Malt.remote_eval_wait(Main, wrkr, :(import ParallelTestRunner))
-                    Malt.remote_call_fetch(invokelatest, wrkr, runtest, RecordType, test_runners[test], test,
-                                              init_code, io_ctx.color)
+                    Malt.remote_call_fetch(invokelatest, wrkr, runtest, RecordType,
+                                           testsuite[test], test, init_code, io_ctx.color)
                 catch ex
                     if isa(ex, InterruptException)
                         # the worker got interrupted, signal other tasks to stop