Update to ParallelTestRunner v2 (#2690)

* Update to ParallelTestRunner v2

* [CI] Run macOS tests with 2 jobs instead of only one

Author: giordano

.github/workflows/CI.yml

@@ -127,13 +127,25 @@ jobs:
         id: buildpkg
         env:
           JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+      - name: "Set test arguments"
+        shell: bash
+        run: |
+          # Run tests in verbose mode
+          TEST_ARGS=(--verbose)
+          if [[ ${{ runner.os }} == "macOS" ]]; then
+              # Encourage ParallelTestRunner to use two jobs on macOS, rather
+              # than the single one it'd use by default.
+              TEST_ARGS+=(--jobs=2)
+          fi
+          echo "runtest_test_args=${TEST_ARGS[@]}" >> "${GITHUB_ENV}"
       - uses: julia-actions/julia-runtest@v1
         if: matrix.version != 'nightly' || steps.buildpkg.outcome == 'success'
         continue-on-error: ${{ matrix.version == 'nightly' }}
         id: run_tests
         # TODO restore coverage post https://github.com/JuliaGPU/GPUCompiler.jl/pull/711
         with:
           coverage: ${{ matrix.version != '1.11' || !matrix.assertions }}
+          test_args: ${{ env.runtest_test_args }}
         env:
           JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
       - uses: julia-actions/julia-processcoverage@v1
@@ -144,6 +156,7 @@ jobs:
           files: lcov.info
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false  # or true if you want CI to fail when Codecov fails
+
   enzymetestutils:
     timeout-minutes: 60
     name: EnzymeTestUtils - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.libEnzyme }} libEnzyme - ${{ github.event_name }}
@@ -198,6 +211,7 @@ jobs:
           files: lcov.info
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false  # or true if you want CI to fail when Codecov fails
+
   enzymecore:
     timeout-minutes: 20
     name: EnzymeCore - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.libEnzyme }} libEnzyme - ${{ github.event_name }}

test/Project.toml

@@ -30,4 +30,4 @@ EnzymeTestUtils = {path = "../lib/EnzymeTestUtils"}
 
 [compat]
 EnzymeTestUtils = "0.2.1"
-ParallelTestRunner = "1.0.1"
+ParallelTestRunner = "2.0.1"

test/multi-threads.jl

@@ -1,19 +1,33 @@
 import Enzyme
 import Enzyme_jll
-using ParallelTestRunner: runtests
+using ParallelTestRunner: addworker, filter_tests!, find_tests, parse_args, runtests
 
-function test_filter(test)
-    if test ∈ ("metal", "cuda", "amdgpu")
-        return false
-    end
-    if Sys.iswindows() && test == "ext/specialfunctions"
-        return false
+# Start with autodiscovered tests
+testsuite = find_tests(@__DIR__)
+# Add threads tests to be run with multiple Julia threads (will be configured in
+# `test_worker`).
+testsuite["threads/2"] = :(include($(joinpath(@__DIR__, "threads.jl"))))
+
+# Parse arguments
+args = parse_args(ARGS)
+
+if filter_tests!(testsuite, args)
+    # Skip GPU-specific tests by default.
+    delete!(testsuite, "metal")
+    delete!(testsuite, "cuda")
+    delete!(testsuite, "amdgpu")
+
+    # Skipped until https://github.com/EnzymeAD/Enzyme.jl/issues/2620 is fixed.
+    if Sys.iswindows()
+        delete!(testsuite, "ext/specialfunctions")
     end
-    if test == "threads"
-        # We run the "threads" tests via the "multi-threads" driver.
-        return false
+end
+
+function test_worker(name)
+    if name == "threads/2"
+        # Run the `threads/2` testset, with multiple threads.
+        return addworker(; exeflags = ["--threads=2"])
     end
-    return true
 end
 
 const init_code = quote
@@ -55,4 +69,4 @@ const init_code = quote
 end
 
 @info "Testing against" Enzyme_jll.libEnzyme
-runtests(Enzyme, ARGS; test_filter, init_code)
+runtests(Enzyme, args; testsuite, init_code, test_worker)