test: standardize testing

Author: avik-pal

.buildkite/pipeline.yml

@@ -30,6 +30,4 @@ steps:
 env:
   GROUP: CUDA
   JULIA_PKG_SERVER: "" # it often struggles with our large artifacts
-  RETESTITEMS_NWORKERS: 4
-  RETESTITEMS_NWORKER_THREADS: 2
   SECRET_CODECOV_TOKEN: "HC7K/ymhi62KUQ5OLU4DOl+11gaQt4JhXX/2nfTGlTsBB8mEMxQ8R+sHIp/2HjEup5eSXAN2IWQDQ7RDBuQvVp0T1UVtr2e4YNZFztKnsJXrFO15hXxYShJodI//X/8DzhlQd/lyTDOAOJu3eznsc3sC2CUgJzXZxLUtQN9YaZ1i3a+NoN1mO5UpkkHVhXigwF5gjy+0tei8fCdcP+SIhG0EanS5yd9q/SurtCpMHsHyUG97+ZVPglSKgdaqr31+PdmiPJ+ynp4+Hnc/esosxUSHSIL+ryRTO+28RNwPTiNf99J51RJLQmz1knWTR1ky6tiYIZ5218O6wvNil0SqNw==;U2FsdGVkX18nBY3t4LZYlEIz3EVKjpqCd994JNeJGt006up+sAjXEssI0tgCVXnfXsenVsP3NCCEoOS1GXc44g=="

.github/workflows/CI_NonlinearSolve.yml

@@ -35,7 +35,6 @@ jobs:
         version:
           - "min"
           - "1"
-          - "pre"
         os:
           - ubuntu-latest
           - macos-latest
@@ -70,8 +69,6 @@ jobs:
         shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
         env:
           GROUP: ${{ matrix.group }}
-          RETESTITEMS_NWORKERS: 4
-          RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
         with:
           directories: src,ext

.github/workflows/CI_SciMLJacobianOperators.yml

@@ -26,7 +26,6 @@ jobs:
         version:
           - "min"
           - "1"
-          - "pre"
         os:
           - ubuntu-latest
           - macos-latest

.github/workflows/Downgrade.yml

@@ -26,7 +26,3 @@ jobs:
           skip: Pkg,TOML
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
-        env:
-          JULIA_NUM_THREADS: 11
-          RETESTITEMS_NWORKERS: 4
-          RETESTITEMS_NWORKER_THREADS: 2

.github/workflows/Downstream.yml

@@ -55,9 +55,6 @@ jobs:
             @info "Not compatible with this release. No problem." exception=err
             exit(0)  # Exit immediately, as a success
           end
-        env:
-          RETESTITEMS_NWORKERS: 4
-          RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
         with:
           directories: src,ext

Project.toml

@@ -74,6 +74,7 @@ FastLevenbergMarquardt = "0.1"
 FiniteDiff = "2.22"
 FixedPointAcceleration = "0.3"
 ForwardDiff = "0.10.36"
+Hwloc = "3"
 LazyArrays = "1.8.2, 2"
 LeastSquaresOptim = "0.8.5"
 LineSearches = "7.2"
@@ -121,6 +122,8 @@ Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
 FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce"
 FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176"
+Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891"
 MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9"
 ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
@@ -142,4 +145,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "LeastSquaresOptim", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEq", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Symbolics", "Test", "Zygote"]
+test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEq", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Symbolics", "Test", "Zygote"]

test/core/23_test_problems_tests.jl

@@ -122,7 +122,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests, Sys.isapple() ? 1e-3 : 1e-4)
 end
 
-@testitem "Klement" retries=5 setup=[RobustnessTesting] tags=[:core] begin
+@testitem "Klement" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (Klement(), Klement(; init_jacobian = Val(:true_jacobian_diagonal)))
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)

test/core/rootfind_tests.jl

@@ -52,7 +52,7 @@ end
 
 # --- NewtonRaphson tests ---
 
-@testitem "NewtonRaphson" setup=[CoreRootfindTesting] tags=[:core] timeout=3600 begin
+@testitem "NewtonRaphson" setup=[CoreRootfindTesting] tags=[:core] begin
     @testset "LineSearch: $(_nameof(lsmethod)) LineSearch AD: $(_nameof(ad))" for lsmethod in (
             Static(), StrongWolfe(), BackTracking(), HagerZhang(), MoreThuente()),
         ad in (AutoForwardDiff(), AutoZygote(), AutoFiniteDiff())
@@ -118,7 +118,7 @@ end
 
 # --- TrustRegion tests ---
 
-@testitem "TrustRegion" setup=[CoreRootfindTesting] tags=[:core] skip=:(Sys.isapple()) timeout=3600 begin
+@testitem "TrustRegion" setup=[CoreRootfindTesting] tags=[:core] begin
     radius_update_schemes = [RadiusUpdateSchemes.Simple, RadiusUpdateSchemes.NocedalWright,
         RadiusUpdateSchemes.NLsolve, RadiusUpdateSchemes.Hei,
         RadiusUpdateSchemes.Yuan, RadiusUpdateSchemes.Fan, RadiusUpdateSchemes.Bastin]
@@ -236,7 +236,7 @@ end
 
 # --- LevenbergMarquardt tests ---
 
-@testitem "LevenbergMarquardt" setup=[CoreRootfindTesting] tags=[:core] timeout=3600 begin
+@testitem "LevenbergMarquardt" setup=[CoreRootfindTesting] tags=[:core] begin
     u0s = ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
     @testset "[OOP] u0: $(typeof(u0))" for u0 in u0s
         sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = LevenbergMarquardt())
@@ -322,7 +322,7 @@ end
 
 # --- DFSane tests ---
 
-@testitem "DFSane" setup=[CoreRootfindTesting] tags=[:core] timeout=3600 begin
+@testitem "DFSane" setup=[CoreRootfindTesting] tags=[:core] begin
     u0s = ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
 
     @testset "[OOP] u0: $(typeof(u0))" for u0 in u0s
@@ -393,7 +393,7 @@ end
 
 # --- PseudoTransient tests ---
 
-@testitem "PseudoTransient" setup=[CoreRootfindTesting] tags=[:core] timeout=3600 begin
+@testitem "PseudoTransient" setup=[CoreRootfindTesting] tags=[:core] begin
     # These are tests for NewtonRaphson so we should set alpha_initial to be high so that we
     # converge quickly
     @testset "PT: alpha_initial = 10.0 PT AD: $(ad)" for ad in (
@@ -462,7 +462,7 @@ end
 
 # --- Broyden tests ---
 
-@testitem "Broyden" setup=[CoreRootfindTesting] tags=[:core] skip=:(Sys.isapple()) timeout=3600 begin
+@testitem "Broyden" setup=[CoreRootfindTesting] tags=[:core] begin
     @testset "LineSearch: $(_nameof(lsmethod)) LineSearch AD: $(_nameof(ad)) Init Jacobian: $(init_jacobian) Update Rule: $(update_rule)" for lsmethod in (
             Static(), StrongWolfe(), BackTracking(),
             HagerZhang(), MoreThuente(), LiFukushimaLineSearch()),
@@ -512,7 +512,7 @@ end
 
 # --- Klement tests ---
 
-@testitem "Klement" setup=[CoreRootfindTesting] tags=[:core] skip=:(Sys.isapple()) timeout=3600 begin
+@testitem "Klement" setup=[CoreRootfindTesting] tags=[:core] begin
     @testset "LineSearch: $(_nameof(lsmethod)) LineSearch AD: $(_nameof(ad)) Init Jacobian: $(init_jacobian)" for lsmethod in (
             Static(), StrongWolfe(), BackTracking(), HagerZhang(), MoreThuente()),
         ad in (AutoForwardDiff(), AutoZygote(), AutoFiniteDiff()),
@@ -561,7 +561,7 @@ end
 
 # --- LimitedMemoryBroyden tests ---
 
-@testitem "LimitedMemoryBroyden" setup=[CoreRootfindTesting] tags=[:core] skip=:(Sys.isapple()) timeout=3600 begin
+@testitem "LimitedMemoryBroyden" setup=[CoreRootfindTesting] tags=[:core] begin
     @testset "LineSearch: $(_nameof(lsmethod)) LineSearch AD: $(_nameof(ad))" for lsmethod in (
             Static(), StrongWolfe(), BackTracking(),
             HagerZhang(), MoreThuente(), LiFukushimaLineSearch()),
@@ -611,7 +611,7 @@ end
 end
 
 # Miscellaneous Tests
-@testitem "Custom JVP" setup=[CoreRootfindTesting] tags=[:core] timeout=3600 begin
+@testitem "Custom JVP" setup=[CoreRootfindTesting] tags=[:core] begin
     function F(u::Vector{Float64}, p::Vector{Float64})
         Δ = Tridiagonal(-ones(99), 2 * ones(100), -ones(99))
         return u + 0.1 * u .* Δ * u - p

test/gpu/core_tests.jl

@@ -1,37 +1,39 @@
-@testitem "CUDA Tests" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) begin
+@testitem "CUDA Tests" tags=[:cuda] begin
     using CUDA, NonlinearSolve, LinearSolve, StableRNGs
 
-    CUDA.allowscalar(false)
+    if CUDA.functional()
+        CUDA.allowscalar(false)
 
-    A = cu(rand(StableRNG(0), 4, 4))
-    u0 = cu(rand(StableRNG(0), 4))
-    b = cu(rand(StableRNG(0), 4))
+        A = cu(rand(StableRNG(0), 4, 4))
+        u0 = cu(rand(StableRNG(0), 4))
+        b = cu(rand(StableRNG(0), 4))
 
-    linear_f(du, u, p) = (du .= A * u .+ b)
+        linear_f(du, u, p) = (du .= A * u .+ b)
 
-    prob = NonlinearProblem(linear_f, u0)
+        prob = NonlinearProblem(linear_f, u0)
 
-    SOLVERS = (NewtonRaphson(), LevenbergMarquardt(; linsolve = QRFactorization()),
-        LevenbergMarquardt(; linsolve = KrylovJL_GMRES()), PseudoTransient(),
-        Klement(), Broyden(; linesearch = LiFukushimaLineSearch()),
-        LimitedMemoryBroyden(; threshold = 2, linesearch = LiFukushimaLineSearch()),
-        DFSane(), TrustRegion(; linsolve = QRFactorization()),
-        TrustRegion(; linsolve = KrylovJL_GMRES(), concrete_jac = true),  # Needed if Zygote not loaded
-        nothing)
+        SOLVERS = (NewtonRaphson(), LevenbergMarquardt(; linsolve = QRFactorization()),
+            LevenbergMarquardt(; linsolve = KrylovJL_GMRES()), PseudoTransient(),
+            Klement(), Broyden(; linesearch = LiFukushimaLineSearch()),
+            LimitedMemoryBroyden(; threshold = 2, linesearch = LiFukushimaLineSearch()),
+            DFSane(), TrustRegion(; linsolve = QRFactorization()),
+            TrustRegion(; linsolve = KrylovJL_GMRES(), concrete_jac = true),  # Needed if Zygote not loaded
+            nothing)
 
-    @testset "[IIP] GPU Solvers" begin
-        for alg in SOLVERS
-            @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5)
+        @testset "[IIP] GPU Solvers" begin
+            for alg in SOLVERS
+                @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5)
+            end
         end
-    end
 
-    linear_f(u, p) = A * u .+ b
+        linear_f(u, p) = A * u .+ b
 
-    prob = NonlinearProblem{false}(linear_f, u0)
+        prob = NonlinearProblem{false}(linear_f, u0)
 
-    @testset "[OOP] GPU Solvers" begin
-        for alg in SOLVERS
-            @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5)
+        @testset "[OOP] GPU Solvers" begin
+            for alg in SOLVERS
+                @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5)
+            end
         end
     end
 end

test/runtests.jl

@@ -1,10 +1,17 @@
-using ReTestItems, CUDA
+using ReTestItems, NonlinearSolve, Hwloc, InteractiveUtils
 
-const GROUP = get(ENV, "GROUP", CUDA.functional() ? "All" : "All")
+@info sprint(InteractiveUtils.versioninfo)
 
-if GROUP == "All"
-    ReTestItems.runtests(@__DIR__)
-else
-    tags = [Symbol(lowercase(GROUP))]
-    ReTestItems.runtests(@__DIR__; tags)
-end
+const GROUP = lowercase(get(ENV, "GROUP", "All"))
+
+const RETESTITEMS_NWORKERS = parse(
+    Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 4))))
+const RETESTITEMS_NWORKER_THREADS = parse(Int,
+    get(ENV, "RETESTITEMS_NWORKER_THREADS",
+        string(max(Hwloc.num_virtual_cores() ÷ RETESTITEMS_NWORKERS, 1))))
+
+@info "Running tests for group: $LUXLIB_TEST_GROUP with $RETESTITEMS_NWORKERS workers"
+
+ReTestItems.runtests(NonlinearSolve; tags = (GROUP == "all" ? nothing : [Symbol(GROUP)]),
+    nworkers = RETESTITEMS_NWORKERS,
+    nworker_threads = RETESTITEMS_NWORKER_THREADS, testitem_timeout = 3600, retries=4)