Split backend tests into different workflows (#269)

* Split backend tests into different workflows

* Name groups

* Shorter names

* Remove process coverage

* Coverage dirs

* Fix group

* Typo

* Clean up

* Fix exclusions

* Remove problematic double

* Don't test folders

* Verbosity

Author: gdalle

.github/workflows/Documentation.yml

@@ -17,7 +17,7 @@ concurrency:
 jobs:
 
   docs:
-    name: ${{ matrix.pkg.name }}
+    name: ${{ matrix.pkg.shortcut }}
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -27,8 +27,10 @@ jobs:
       matrix:
         pkg:
           - name: DifferentiationInterface
+            shortcut: DI
             dir: './DifferentiationInterface'
           - name: DifferentiationInterfaceTest
+            shortcut: DIT
             dir: './DifferentiationInterfaceTest'
 
     steps:

.github/workflows/Test.yml

@@ -16,51 +16,115 @@ concurrency:
 
 jobs:
 
-  test:
-    name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }}
-    runs-on: ${{ matrix.os }}
+  test-DI:
+    name: DI (${{ matrix.group }}) - Julia ${{ matrix.version }}
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created
       actions: write
       contents: read
     strategy:
       fail-fast: false
       matrix:
-        pkg:
-          - name: DifferentiationInterface
-            dir: './DifferentiationInterface'
-          - name: DifferentiationInterfaceTest
-            dir: './DifferentiationInterfaceTest'
         version:
           - '1'
           - '1.6'
-        os:
-          - ubuntu-latest
-        arch:
-          - x64
+        group:
+          - Formalities
+          - Internals
+          - Single/Diffractor
+          - Single/Enzyme
+          - Single/FastDifferentiation
+          - Single/FiniteDiff
+          - Single/FiniteDifferences
+          - Single/ForwardDiff
+          - Single/PolyesterForwardDiff
+          - Single/ReverseDiff
+          - Single/Symbolics
+          - Single/Tapir
+          - Single/Tracker
+          - Single/Zygote
+          - Double/Enzyme-ForwardDiff
+          - Double/ForwardDiff-Zygote
+          - Double/ReverseDiff-Zygote
+        exclude:
+          - version: '1.6'
+            group: Formalities
+          - version: '1.6'
+            group: Single/Diffractor
+          - version: '1.6'
+            group: Single/Enzyme
+          - version: '1.6'
+            group: Single/FiniteDiff
+          - version: '1.6'
+            group: Single/FastDifferentiation
+          - version: '1.6'
+            group: Single/PolyesterForwardDiff
+          - version: '1.6'
+            group: Single/Symbolics
+          - version: '1.6'
+            group: Single/Tapir
+          - version: '1.6'
+            group: Double/Enzyme-ForwardDiff
+          - version: '1.6'
+            group: Double/ReverseDiff-Zygote
 
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
+          arch: x64
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          project: ./DifferentiationInterface
+        env:
+          JULIA_DI_TEST_GROUP: ${{ matrix.group }}
+      - uses: julia-actions/julia-processcoverage@v1
+        with:
+          directories: ./DifferentiationInterface/src,./DifferentiationInterface/ext
+      - uses: codecov/codecov-action@v4
+        with:
+          files: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+
+  test-DIT:
+    name: DIT - Julia ${{ matrix.version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created
+      actions: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1'
+          - '1.6'
+    
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: x64
       - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
       - name: Dev dependencies (temporary)
-        run: julia --project=${{ matrix.pkg.dir}}/ -e '
+        run: julia --project='./DifferentiationInterfaceTest' -e '
           using Pkg;
           Pkg.Registry.update();
-          if "${{ matrix.pkg.name}}" == "DifferentiationInterfaceTest";
-              Pkg.develop(PackageSpec(path="./DifferentiationInterface"));
-          end;
+          Pkg.develop(PackageSpec(path="./DifferentiationInterface"));
           Pkg.instantiate();'
       - uses: julia-actions/julia-runtest@v1
         with:
-          project: ${{ matrix.pkg.dir }}
+          project: ./DifferentiationInterfaceTest
       - uses: julia-actions/julia-processcoverage@v1
         with:
-          directories: ${{ matrix.pkg.dir }}/src,${{ matrix.pkg.dir }}/ext
+          directories: ./DifferentiationInterfaceTest/src,./DifferentiationInterfaceTest/ext
       - uses: codecov/codecov-action@v4
         with:
           files: lcov.info

DifferentiationInterface/Project.toml

@@ -70,9 +70,8 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
-# DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
 Diffractor = "9f5e2b26-1114-432f-b630-d3fe2085c51c"
-Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
 FastDifferentiation = "eb9bf01b-bf85-4b60-bf87-ee5de06c00be"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
@@ -96,8 +95,7 @@ test = [
     "ADTypes",
     "Aqua",
     "DataFrames",
-    # "DifferentiationInterfaceTest",
-    "Documenter",
+    "DifferentiationInterfaceTest",
     "JET",
     "JuliaFormatter",
     "Pkg",

DifferentiationInterface/test/Double/Enzyme-ForwardDiff.jl

@@ -2,7 +2,6 @@ using DifferentiationInterface, DifferentiationInterfaceTest
 import DifferentiationInterface as DI
 using Enzyme: Enzyme
 using ForwardDiff: ForwardDiff
-using Zygote: Zygote
 
 backends = [
     SecondOrder(AutoForwardDiff(), AutoEnzyme(Enzyme.Forward)),

DifferentiationInterface/test/Formalities/aqua.jl

@@ -0,0 +1,7 @@
+using Aqua: Aqua
+using DifferentiationInterface
+using Test
+
+Aqua.test_all(
+    DifferentiationInterface; ambiguities=false, deps_compat=(check_extras = false)
+)

DifferentiationInterface/test/Formalities/jet.jl

@@ -0,0 +1,5 @@
+using DifferentiationInterface
+using JET: JET
+using Test
+
+JET.test_package(DifferentiationInterface; target_defined_modules=true)

DifferentiationInterface/test/Formalities/juliaformatter.jl

@@ -0,0 +1,5 @@
+using DifferentiationInterface
+using JuliaFormatter: JuliaFormatter
+using Test
+
+@test JuliaFormatter.format(DifferentiationInterface; verbose=false, overwrite=false)

DifferentiationInterface/test/formal.jl

@@ -4,42 +4,15 @@ using Pkg
 using SparseConnectivityTracer
 using Test
 
-push!(Base.LOAD_PATH, Base.active_project())
-Pkg.activate(; temp=true)
-
-DI_PATH = joinpath(@__DIR__, "..", "..", "DifferentiationInterface")
+# DI_PATH = joinpath(@__DIR__, "..", "..", "DifferentiationInterface")
+# if isdir(DI_PATH)
+#     Pkg.develop(; path=DI_PATH)
+# end
 DIT_PATH = joinpath(@__DIR__, "..", "..", "DifferentiationInterfaceTest")
-if isdir(DI_PATH)
-    Pkg.develop(; path=DI_PATH)
-end
 if isdir(DIT_PATH)
     Pkg.develop(; path=DIT_PATH)
 end
 
-BACKENDS_1_6 = [
-    "FiniteDifferences",  #
-    "ForwardDiff",
-    "ReverseDiff",
-    "Tracker",
-    "Zygote",
-]
-
-BACKENDS_1_10 = [
-    "Diffractor",  # 
-    "Enzyme",
-    "FiniteDiff",
-    "FastDifferentiation",
-    "PolyesterForwardDiff",
-    "Symbolics",
-    "Tapir",
-]
-
-@static if VERSION >= v"1.10"
-    Pkg.add(vcat(BACKENDS_1_6, BACKENDS_1_10))
-else
-    Pkg.add(vcat(BACKENDS_1_6))
-end
-
 function MyAutoSparse(backend::AbstractADType)
     coloring_algorithm = GreedyColoringAlgorithm()
     sparsity_detector = TracerSparsityDetector()
@@ -48,31 +21,60 @@ end
 
 LOGGING = get(ENV, "CI", "false") == "false"
 
+GROUP = get(ENV, "JULIA_DI_TEST_GROUP", "All")
+
 ## Main tests
 
 @testset verbose = true "DifferentiationInterface.jl" begin
-    @static if VERSION >= v"1.10"
-        @info "Testing formalities"
-        @testset verbose = true "Formal tests" begin
-            include("formal.jl")
+    if GROUP == "Formalities" || GROUP == "All"
+        @testset "Formalities/$file" for file in readdir(joinpath(@__DIR__, "Formalities"))
+            @info "Testing Formalities/$file)"
+            include(joinpath(@__DIR__, "Formalities", file))
+        end
+    end
+
+    if GROUP == "Internals" || GROUP == "All"
+        @testset "Internals/$file" for file in readdir(joinpath(@__DIR__, "Internals"))
+            @info "Testing Internals/$file"
+            include(joinpath(@__DIR__, "Internals", file))
         end
     end
 
-    @testset verbose = true "$folder" for folder in ["Single", "Double", "Internals"]
-        folder_path = joinpath(@__DIR__, folder)
-        @testset verbose = true "$(file[1:end-3])" for file in readdir(folder_path)
-            endswith(file, ".jl") || continue
-            if (
-                VERSION < v"1.10" && any(
-                    part == backend for part in split(file[1:(end - 3)], '-') for
-                    backend in BACKENDS_1_10
-                )
-            )
-                @info "Skipping $folder - $(file[1:end-3])"
-            else
-                @info "Testing $folder - $(file[1:end-3])"
-                include(joinpath(folder_path, file))
+    if GROUP == "All"
+        Pkg.add([
+            "Diffractor",
+            "Enzyme",
+            "FiniteDiff",
+            "FiniteDifferences",
+            "FastDifferentiation",
+            "ForwardDiff",
+            "PolyesterForwardDiff",
+            "ReverseDiff",
+            "Symbolics",
+            "Tapir",
+            "Tracker",
+            "Zygote",
+        ])
+        @testset verbose = true "$folder" for folder in ("Single", "Double")
+            files = filter(f -> endswith(f, ".jl"), readdir(joinpath(@__DIR__, folder)))
+            @testset "$file" for file in files
+                @info "Testing $folder/$file"
+                include(joinpath(@__DIR__, folder, file))
             end
         end
+    elseif startswith(GROUP, "Single")
+        b1 = split(GROUP, '/')[2]
+        @testset "Single/$b1" begin
+            @info "Testing Single/$b1"
+            Pkg.add(b1)
+            include(joinpath(@__DIR__, "Single", "$b1.jl"))
+        end
+    elseif startswith(GROUP, "Double")
+        b1, b2 = split(split(GROUP, '/')[2], '-')
+        @testset "Single/$b1-$b2" begin
+            @info "Testing Double/$b1-$b2"
+            Pkg.add([b1, b2])
+            include(joinpath(@__DIR__, "Double", "$b1-$b2.jl"))
+        end
     end
 end;