Test: Add @include_files

Author: IanButterworth

NEWS.md

@@ -108,6 +108,7 @@ Standard library changes
 * Transparent test sets (`@testset let`) now show context when tests error ([#58727]).
 * `@test_throws` now supports a three-argument form `@test_throws ExceptionType pattern expr` to test both exception type and message pattern in one call ([#59117]).
 * The testset stack was changed to use `ScopedValue` rather than task local storage ([#53462]).
+* New `@include_files` macro for organizing test files with optional filtering based on command-line arguments. Supports `Pkg.test(test_args=["--files=pattern"])` for substring matching and `Pkg.test(test_args=["--files-regex=pattern"])` for regex matching to run only matching test files. ([#59951])
 
 #### InteractiveUtils
 

stdlib/Test/docs/src/index.md

@@ -296,6 +296,51 @@ in the test set reporting. The test will not run but gives a `Broken` `Result`.
 Test.@test_skip
 ```
 
+## Organizing Test Files
+
+When working with larger test suites, it's common to split tests across multiple files. The
+`@include_files` macro provides a convenient way to include test files with optional filtering
+based on command-line arguments.
+
+```@docs
+Test.@include_files
+```
+
+For example, a package's `test/runtests.jl` might look like:
+
+```julia
+using Test
+
+# Include utility functions
+include("test_utils.jl")
+
+# Include test files, optionally filtered by command-line args
+@include_files [
+    "basics.jl",
+    "advanced.jl",
+    "performance.jl",
+]
+```
+
+This allows running specific test files:
+
+```julia
+# Run all tests
+Pkg.test("MyPackage")
+
+# Run only files containing "basics"
+Pkg.test("MyPackage", test_args=["--files=basics"])
+
+# Run files containing "basics" or "advanced"
+Pkg.test("MyPackage", test_args=["--files=basics,advanced"])
+
+# Use regex patterns for more complex filtering
+Pkg.test("MyPackage", test_args=["--files-regex=^test_.*_integration\\.jl\$"])
+
+# Match files with "basic" or "advanced" using regex
+Pkg.test("MyPackage", test_args=["--files-regex=basic|advanced"])
+```
+
 ## Test result types
 
 ```@docs

stdlib/Test/src/Test.jl

@@ -26,6 +26,7 @@ export @test, @test_throws, @test_broken, @test_skip,
 
 export @testset
 export @inferred
+export @include_files
 export detect_ambiguities, detect_unbound_args
 export GenericString, GenericSet, GenericDict, GenericArray, GenericOrder
 export TestSetException
@@ -2535,6 +2536,90 @@ function _check_bitarray_consistency(B::BitArray{N}) where N
     return true
 end
 
+function filter_test_files(files::Vector, args::Vector=ARGS)
+    # Parse args to extract file filters
+    filter_args = String[]
+    use_regex = false
+
+    # Check for --files-regex= argument (takes precedence)
+    regex_idx = findfirst(arg -> startswith(arg, "--files-regex="), args)
+    if regex_idx !== nothing
+        # Extract comma-separated list from --files-regex=
+        pattern_str = args[regex_idx][15:end]  # Skip "--files-regex="
+        append!(filter_args, split(pattern_str, ','))
+        use_regex = true
+    else
+        # Check for --files= argument
+        files_idx = findfirst(arg -> startswith(arg, "--files="), args)
+        if files_idx !== nothing
+            # Extract comma-separated list from --files=
+            files_str = args[files_idx][9:end]  # Skip "--files="
+            append!(filter_args, split(files_str, ','))
+        end
+    end
+
+    # Return all files if no filters
+    isempty(filter_args) && return files
+
+    # Filter files based on basename matching
+    return filter(files) do file
+        name = basename(file)
+        if use_regex
+            any(arg -> occursin(Regex(arg), name), filter_args)
+        else
+            any(arg -> occursin(arg, name), filter_args)
+        end
+    end
+end
+
+"""
+    @include_files(files)
+
+Include test files from a list, optionally filtered by command-line test arguments.
+
+When running tests via `Pkg.test()`, files can be filtered by passing `test_args`
+with the `--files=` or `--files-regex=` flag. If no test args are provided, all files are included.
+
+!!! compat "Julia 1.13"
+    `@include_files` was added in Julia 1.13, but is available and exported in Compat.jl
+    in earlier Julia versions.
+
+# Example
+```julia
+using Test
+
+# include common utilities
+include("utils.jl")
+
+@include_files [
+    "foo.jl",
+    "bar.jl",
+    "baz.jl",
+]
+```
+
+## Usage patterns:
+
+- `Pkg.test()` → includes all files
+- `Pkg.test(test_args=["--files=foo"])` → includes only "foo.jl"
+- `Pkg.test(test_args=["--files=foo,bar"])` → includes "foo.jl" and "bar.jl"
+- `Pkg.test(test_args=["--files-regex=^test_.*\\.jl\$"])` → uses regex pattern matching
+- `Pkg.test(test_args=["--files-regex=foo|bar"])` → includes files matching "foo" or "bar"
+"""
+macro include_files(files)
+    quote
+        let test_files = $(esc(files))
+            # Filter files based on ARGS
+            filtered_files = filter_test_files(test_files, ARGS)
+
+            # Include filtered files
+            for file in filtered_files
+                include(file)
+            end
+        end
+    end
+end
+
 include("logging.jl")
 include("precompile.jl")
 

stdlib/Test/test/include_test/runtests.jl

@@ -0,0 +1,3 @@
+using Test
+
+@include_files ["test_foo.jl", "test_bar.jl", "test_baz.jl"]

stdlib/Test/test/include_test/test_bar.jl

@@ -0,0 +1 @@
+println("BAR_RAN")

stdlib/Test/test/include_test/test_baz.jl

@@ -0,0 +1 @@
+println("BAZ_RAN")

stdlib/Test/test/include_test/test_foo.jl

@@ -0,0 +1 @@
+println("FOO_RAN")

stdlib/Test/test/runtests.jl

@@ -2096,3 +2096,71 @@ end
         @test !occursin("Finished testset:", output)
     end
 end
+
+@testset "filter_test_files" begin
+    files = ["test_foo.jl", "test_bar.jl", "test_baz.jl"]
+
+    @test Test.filter_test_files(files, String[]) == files
+    @test Test.filter_test_files(files, ["--files=foo"]) == ["test_foo.jl"]
+    @test Test.filter_test_files(files, ["--files=foo,bar"]) == ["test_foo.jl", "test_bar.jl"]
+    @test Test.filter_test_files(files, ["--files=ba"]) == ["test_bar.jl", "test_baz.jl"]
+    @test Test.filter_test_files(files, ["--files=nomatch"]) == String[]
+    @test Test.filter_test_files(files, ["--files=foo", "bar"]) == ["test_foo.jl"]  # Other args ignored
+    @test Test.filter_test_files(files, ["foo"]) == files  # No --files= means no filtering
+    @test Test.filter_test_files(files, ["bar", "baz"]) == files  # No --files= means no filtering
+
+    # Test regex patterns
+    @test Test.filter_test_files(files, ["--files-regex=^test_foo"]) == ["test_foo.jl"]
+    @test Test.filter_test_files(files, ["--files-regex=foo|bar"]) == ["test_foo.jl", "test_bar.jl"]
+    @test Test.filter_test_files(files, ["--files-regex=ba[rz]"]) == ["test_bar.jl", "test_baz.jl"]
+    @test Test.filter_test_files(files, ["--files-regex=^test_.*\\.jl\$"]) == files
+    @test Test.filter_test_files(files, ["--files-regex=nomatch"]) == String[]
+    @test Test.filter_test_files(files, ["--files-regex=foo", "--files=bar"]) == ["test_foo.jl"]  # --files-regex takes precedence
+end
+
+@testset "@include_files" begin
+    # Test in subprocess to avoid polluting ARGS and Main namespace
+    test_file = joinpath(@__DIR__, "include_test", "runtests.jl")
+
+    # Test 1: No args - all files included
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file`, String)
+    @test occursin("FOO_RAN", output)
+    @test occursin("BAR_RAN", output)
+    @test occursin("BAZ_RAN", output)
+
+    # Test 2: --files= with single filter
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file --files=foo`, String)
+    @test occursin("FOO_RAN", output)
+    @test !occursin("BAR_RAN", output)
+    @test !occursin("BAZ_RAN", output)
+
+    # Test 3: --files= with multiple filters
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file --files=bar,baz`, String)
+    @test !occursin("FOO_RAN", output)
+    @test occursin("BAR_RAN", output)
+    @test occursin("BAZ_RAN", output)
+
+    # Test 4: --files= with comma-separated list
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file --files=foo,bar`, String)
+    @test occursin("FOO_RAN", output)
+    @test occursin("BAR_RAN", output)
+    @test !occursin("BAZ_RAN", output)
+
+    # Test 5: Bare args without --files= should include all files
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file foo bar`, String)
+    @test occursin("FOO_RAN", output)
+    @test occursin("BAR_RAN", output)
+    @test occursin("BAZ_RAN", output)
+
+    # Test 6: --files-regex= with pattern
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file --files-regex=foo\|bar`, String)
+    @test occursin("FOO_RAN", output)
+    @test occursin("BAR_RAN", output)
+    @test !occursin("BAZ_RAN", output)
+
+    # Test 7: --files-regex= with anchored pattern
+    output = read(`$(Base.julia_cmd()) --startup-file=no $test_file --files-regex=^test_foo`, String)
+    @test occursin("FOO_RAN", output)
+    @test !occursin("BAR_RAN", output)
+    @test !occursin("BAZ_RAN", output)
+end

test/reflection.jl

@@ -193,7 +193,7 @@ let
         :TestModSub9475, :d7648, :eval, :f7648, :include])
     imported = Set(Symbol[:convert, :curmod_name, :curmod])
     usings_from_Test = Set(Symbol[
-        Symbol("@inferred"), Symbol("@test"), Symbol("@test_broken"), Symbol("@test_deprecated"),
+        Symbol("@inferred"), Symbol("@include_files"), Symbol("@test"), Symbol("@test_broken"), Symbol("@test_deprecated"),
         Symbol("@test_logs"), Symbol("@test_nowarn"), Symbol("@test_skip"), Symbol("@test_throws"),
         Symbol("@test_warn"), Symbol("@testset"), :GenericArray, :GenericDict, :GenericOrder,
         :GenericSet, :GenericString, :LogRecord, :Test, :TestLogger, :TestSetException,