Split up the test suite into three test groups

Author: DilumAluthge

.buildkite/pipeline.yml

@@ -1,5 +1,6 @@
-# env:
-#   SECRET_CODECOV_TOKEN: "..."
+env:
+  JULIA_NUM_THREADS: "1"
+  # SECRET_CODECOV_TOKEN: "..."
 
 steps:
   - label: "Julia 1.5"

.github/workflow/CompatHelper.yml renamed to .github/workflows/CompatHelper.yml

@@ -1,7 +1,7 @@
 name: CompatHelper
 on:
   schedule:
-    - cron: '00 00 * * *'
+    - cron: 0 0 * * *
   workflow_dispatch:
 jobs:
   CompatHelper:
@@ -12,5 +12,5 @@ jobs:
       - name: CompatHelper.main()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }}  # optional
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
         run: julia -e 'using CompatHelper; CompatHelper.main()'

.github/workflows/ci-julia-nightly.yml

@@ -9,20 +9,24 @@ on:
     tags: '*'
 jobs:
   test-julia-nightly:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    name: Julia ${{ matrix.version }}/${{ matrix.os }}/${{ matrix.arch }}/group ${{ matrix.group }}/${{ matrix.threads }} threads/${{ github.event_name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        JULIA_NUM_THREADS:
+        arch:
+          - x64
+        group:
           - '1'
-          - '6'
-        version:
-          - 'nightly'
+          - '2'
+          - '3'
         os:
           - ubuntu-latest
-        arch:
-          - x64
+        threads:
+          - '1'
+          - '2'
+        version:
+          - 'nightly'
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1
@@ -41,6 +45,8 @@ jobs:
             ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+        env:
+          JULIA_NUM_THREADS: ${{ matrix.threads }}
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:

.github/workflows/ci.yml

@@ -9,21 +9,25 @@ on:
     tags: '*'
 jobs:
   test:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    name: Julia ${{ matrix.version }}/${{ matrix.os }}/${{ matrix.arch }}/group ${{ matrix.group }}/${{ matrix.threads }} threads/${{ github.event_name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        JULIA_NUM_THREADS:
+        arch:
+          - x64
+        group:
+          - '1'
+          - '2'
+          - '3'
+        os:
+          - ubuntu-latest
+        threads:
           - '1'
-          - '6'
+          - '2'
         version:
           - '1.4'
           - '1' # automatically expands to the latest stable 1.x release of Julia
-        os:
-          - ubuntu-latest
-        arch:
-          - x64
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1
@@ -43,7 +47,7 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         env:
-          JULIA_NUM_THREADS: ${{ matrix.JULIA_NUM_THREADS }}
+          JULIA_NUM_THREADS: ${{ matrix.threads }}
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:

test/group-1.jl

@@ -0,0 +1,50 @@
+#===== stuff =====#
+
+t2 = time()
+
+@testset "parsing all the things" begin include("parsing.jl") end
+
+@testset "tests from Einsum.jl" begin include("einsum.jl") end
+
+@info @sprintf("Basic tests took %.1f seconds", time()-t2)
+
+@testset "internal pieces" begin include("utils.jl") end
+
+@testset "matrix multiplication" begin
+    # size 200 is big enough to test block_halves even with MINIBLOCK = 64^3
+    @testset "size $N, elements $T" for N in [2, 20, 200], T in [1:99, Float32, Float64, ComplexF64]
+        for f in [identity, adjoint]
+            A = f(rand(T, N,N));
+            B = f(rand(T, N,N));
+            @test A * B ≈ @tullio C[i,k] := A[i,j] * B[j,k]
+        end
+        if N < 200
+            X = rand(T, N,N+1);
+            Y = rand(T, N+1,N+2);
+            Z = rand(T, N+2,N+1);
+            @test X * Y * Z ≈ @tullio C[a,d] := X[a,b] * Y[b,c] * Z[c,d]
+        end
+    end
+    @testset "@allocated" begin
+        m!(C,A,B) = @tullio C[i,k] = A[i,j] * B[j,k] threads=false
+        C1, A1, B1 = rand(4,4), rand(4,4), rand(4,4)
+        @allocated m!(C1, A1, B1)
+        @test 0 == @allocated m!(C1, A1, B1)
+    end
+end
+
+#===== Tracker =====#
+
+t3 = time()
+using Tracker
+
+GRAD = :Tracker
+_gradient(x...) = Tracker.gradient(x...)
+
+@tullio grad=Base
+@testset "gradients: Tracker + DiffRules" begin include("gradients.jl") end
+
+@tullio grad=Dual
+@testset "gradients: Tracker + ForwardDiff" begin include("gradients.jl") end
+
+@info @sprintf("Tracker tests took %.1f seconds", time()-t3)

test/group-2.jl

@@ -0,0 +1,42 @@
+#===== KernelAbstractions =====#
+
+t4 = time()
+using KernelAbstractions
+
+using Tracker
+
+GRAD = :Tracker
+_gradient(x...) = Tracker.gradient(x...)
+
+@testset "KernelAbstractions + gradients" begin
+    A = (rand(3,4));
+    B = (rand(4,5));
+    @tullio C[i,k] := A[i,j] * B[j,k]  threads=false  # verbose=2
+    @test C ≈ A * B
+
+    @tullio threads=false # else KernelAbstractions CPU kernels not used
+    include("gradients.jl")
+    @tullio threads=true
+
+    for sy in Tullio.SYMBOLS
+        @test !isdefined(@__MODULE__, sy)
+    end
+end
+
+using CUDA
+
+if is_buildkite
+    # If we are on Buildkite, we should assert that we have a CUDA GPU available
+    @test CUDA.has_cuda_gpu()
+end
+
+if CUDA.has_cuda_gpu()
+    @info "===== found a GPU, starting CUDA tests ====="
+    @testset "===== CUDA tests on GPU =====" begin
+        include("cuda.jl")
+    end
+end
+
+@info @sprintf("KernelAbstractions tests took %.1f seconds", time()-t4)
+
+@tullio cuda=false