Split tests into groups + SafeTestsets (#30)

Author: lkdvos

Project.toml

@@ -20,13 +20,15 @@ Git_jll = "2.46.2"
 LibGit2 = "1.10"
 PkgSkeleton = "1.3.1"
 Preferences = "1.4.3"
+SafeTestsets = "0.1"
 Suppressor = "0.2.8"
 Test = "1.10"
 julia = "1.10"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "Test"]
+test = ["Aqua", "Test", "SafeTestsets"]

templates/project/Project.toml

@@ -5,12 +5,16 @@ version = "0.1.0"
 
 [compat]
 Aqua = "0.8.9"
+SafeTestsets = "0.1"
+Suppressor = "0.2"
 Test = "1.10"
 julia = "1.10"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 
 [targets]
-test = ["Aqua", "Test"]
+test = ["Aqua", "Test", "Suppressor", "SafeTestsets"]

templates/test/test/Project.toml

@@ -1,8 +1,6 @@
-@eval module $(gensym())
 using {PKGNAME}: {PKGNAME}
 using Test: @test, @testset
 
 @testset "{PKGNAME}" begin
   # Tests go here.
 end
-end

templates/test/test/test_basics.jl renamed to templates/test/test/basics/test_basics.jl

@@ -1,12 +1,49 @@
-@eval module $(gensym())
-using Test: @testset
+using SafeTestsets: @safetestset
+using Suppressor: @suppress
 
-@testset "{PKGNAME}.jl" begin
-  filenames = filter(readdir(@__DIR__)) do f
-    startswith("test_")(f) && endswith(".jl")(f)
+# check for filtered groups
+# either via `--group=ALL` or through ENV["GROUP"]
+const pat = r"(?:--group=)(\w+)"
+arg_id = findfirst(contains(pat), ARGS)
+const GROUP = uppercase(
+  if isnothing(arg_id)
+    get(ENV, "GROUP", "ALL")
+  else
+    only(match(pat, ARGS[arg_id]).captures)
+  end,
+)
+
+function istestfile(filename)
+  return isfile(filename) &&
+         endswith(filename, ".jl") &&
+         startswith(basename(filename), "test")
+end
+
+@time begin
+  # tests in groups based on folder structure
+  for testgroup in filter(isdir, readdir(@__DIR__))
+    if GROUP == "ALL" || GROUP == uppercase(testgroup)
+      for file in filter(istestfile, readdir(joinpath(@__DIR__, testgroup); join=true))
+        @eval @safetestset $file begin
+          include($file)
+        end
+      end
+    end
   end
-  @testset "Test $filename" for filename in filenames
-    include(filename)
+
+  # single files in top folder
+  for file in filter(istestfile, readdir(@__DIR__))
+    (file == basename(@__FILE__)) && continue
+    @eval @safetestset $file begin
+      include($file)
+    end
+  end
+
+  # test examples
+  examplepath = joinpath(@__DIR__, "..", "examples")
+  for file in filter(endswith(".jl"), readdir(examplepath; join=true))
+    @suppress @eval @safetestset $file begin
+      include($file)
+    end
   end
-end
 end

templates/test/test/runtests.jl

@@ -1,9 +1,7 @@
-@eval module $(gensym())
 using {PKGNAME}: {PKGNAME}
 using Aqua: Aqua
 using Test: @testset
 
 @testset "Code quality (Aqua.jl)" begin
   Aqua.test_all({PKGNAME})
 end
-end

templates/test/test/test_aqua.jl

@@ -1,4 +1,3 @@
-@eval module $(gensym())
 using ITensorPkgSkeleton: ITensorPkgSkeleton
 using Test: @test, @testset
 
@@ -43,4 +42,3 @@ using Test: @test, @testset
     end
   end
 end
-end

templates/test/test/test_examples.jl

@@ -1,12 +1,49 @@
-@eval module $(gensym())
-using Test: @testset
+using SafeTestsets: @safetestset
+using Suppressor: @suppress
 
-@testset "ITensorPkgSkeleton.jl" begin
-  filenames = filter(readdir(@__DIR__)) do f
-    startswith("test_")(f) && endswith(".jl")(f)
+# check for filtered groups
+# either via `--group=ALL` or through ENV["GROUP"]
+const pat = r"(?:--group=)(\w+)"
+arg_id = findfirst(contains(pat), ARGS)
+const GROUP = uppercase(
+  if isnothing(arg_id)
+    get(ENV, "GROUP", "ALL")
+  else
+    only(match(pat, ARGS[arg_id]).captures)
+  end,
+)
+
+function istestfile(filename)
+  return isfile(filename) &&
+         endswith(filename, ".jl") &&
+         startswith(basename(filename), "test")
+end
+
+@time begin
+  # tests in groups based on folder structure
+  for testgroup in filter(isdir, readdir(@__DIR__))
+    if GROUP == "ALL" || GROUP == uppercase(testgroup)
+      for file in filter(istestfile, readdir(joinpath(@__DIR__, testgroup); join=true))
+        @eval @safetestset $file begin
+          include($file)
+        end
+      end
+    end
   end
-  @testset "Test $filename" for filename in filenames
-    include(filename)
+
+  # single files in top folder
+  for file in filter(istestfile, readdir(@__DIR__))
+    (file == basename(@__FILE__)) && continue
+    @eval @safetestset $file begin
+      include($file)
+    end
+  end
+
+  # test examples
+  examplepath = joinpath(@__DIR__, "..", "examples")
+  for file in filter(endswith(".jl"), readdir(examplepath; join=true))
+    @suppress @eval @safetestset $file begin
+      include($file)
+    end
   end
-end
 end