tests: Allow running user-specified tests

Author: jpsamaroo

test/Project.toml

@@ -1,4 +1,5 @@
 [deps]
+ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"

test/runtests.jl

@@ -1,52 +1,101 @@
+tests = [
+    ("Thunk", "thunk.jl"),
+    ("Scheduler", "scheduler.jl"),
+    ("Processors", "processors.jl"),
+    ("Memory Spaces", "memory-spaces.jl"),
+    ("Logging", "logging.jl"),
+    ("Checkpointing", "checkpoint.jl"),
+    ("Scopes", "scopes.jl"),
+    ("Options", "options.jl"),
+    ("Mutation", "mutation.jl"),
+    ("Task Queues", "task-queues.jl"),
+    ("Datadeps", "datadeps.jl"),
+    ("Domain Utilities", "domain.jl"),
+    ("Array", "array.jl"),
+    ("Linear Algebra", "linalg.jl"),
+    ("Caching", "cache.jl"),
+    ("Disk Caching", "diskcaching.jl"),
+    ("File IO", "file-io.jl"),
+    #("Fault Tolerance", "fault-tolerance.jl"),
+]
+all_test_names = map(test -> replace(last(test), ".jl"=>""), tests)
+if PROGRAM_FILE != "" && realpath(PROGRAM_FILE) == @__FILE__
+    push!(LOAD_PATH, joinpath(@__DIR__, ".."))
+    push!(LOAD_PATH, @__DIR__)
+    using Pkg
+    Pkg.activate(@__DIR__)
+
+    using ArgParse
+    s = ArgParseSettings(description = "Dagger Testsuite")
+    @eval begin
+        @add_arg_table! s begin
+            "--test"
+                nargs = '*'
+                default = all_test_names
+                help = "Enables the specified test to run in the testsuite"
+            "-s", "--simulate"
+                action = :store_true
+                help = "Don't actually run the tests"
+        end
+    end
+    parsed_args = parse_args(s)
+    to_test = String[]
+    for test in parsed_args["test"]
+        if isdir(joinpath(@__DIR__, test))
+            for (_, other_test) in tests
+                if startswith(other_test, test)
+                    push!(to_test, other_test)
+                    continue
+                end
+            end
+        elseif test in all_test_names
+            push!(to_test, test)
+        else
+            println(stderr, "Unknown test: $test")
+            println(stderr, "Available tests:")
+            for ((test_title, _), test_name) in zip(tests, all_test_names)
+                println(stderr, "  $test_name: $test_title")
+            end
+            exit(1)
+        end
+    end
+    @info "Running tests: $(join(to_test, ", "))"
+    parsed_args["simulate"] && exit(0)
+else
+    to_test = all_test_names
+    @info "Running all tests"
+end
+
+
 using Distributed
 addprocs(3)
 
+include("util.jl")
+include("fakeproc.jl")
+
 using Test
 using Dagger
 using UUIDs
 import MemPool
 
-include("util.jl")
-include("fakeproc.jl")
-
-include("thunk.jl")
-
-#= FIXME: Unreliable, and some thunks still get retained
-# N.B. We need a few of these probably because of incremental WeakRef GC
-@everywhere GC.gc()
-@everywhere GC.gc()
-@everywhere GC.gc()
-@everywhere GC.gc()
-sleep(1)
-@test isempty(Dagger.Sch.EAGER_ID_MAP)
-state = Dagger.Sch.EAGER_STATE[]
-@test isempty(state.waiting)
-@test_broken length(keys(state.waiting_data)) == 1
-# Ensure that all cache entries have expired
-@test_broken isempty(state.cache)
-=#
-
-include("scheduler.jl")
-include("processors.jl")
-include("memory-spaces.jl")
-include("logging.jl")
-include("checkpoint.jl")
-include("scopes.jl")
-include("options.jl")
-include("mutation.jl")
-include("task-queues.jl")
-include("datadeps.jl")
-include("domain.jl")
-include("array.jl")
-include("linalg.jl")
-include("cache.jl")
-include("diskcaching.jl")
-include("file-io.jl")
-
-try # TODO: Fault tolerance is sometimes unreliable
-#include("fault-tolerance.jl")
+try
+    for test in to_test
+        test_title = tests[findfirst(x->x[2]==test * ".jl", tests)][1]
+        test_name = all_test_names[findfirst(x->x==test, all_test_names)]
+        println()
+        @info "Testing $test_title ($test_name)"
+        @testset "$test_title" include(test * ".jl")
+    end
 catch
+    printstyled(stderr, "Tests Failed!\n"; color=:red)
+    rethrow()
+finally
+    state = Dagger.Sch.EAGER_STATE[]
+    if state !== nothing
+        notify(state.halt)
+    end
+    sleep(1)
+    rmprocs(workers())
 end
-println(stderr, "tests done. cleaning up...")
-Dagger.cleanup()
-println(stderr, "all done.")
+
+printstyled(stderr, "Tests Completed!\n"; color=:green)