filtering: add pass and fail patterns

Author: rfourquet

src/ReTest.jl

@@ -1,6 +1,6 @@
 module ReTest
 
-export retest, @testset, @testset_macro, not, interpolated, reachable, depth
+export retest, @testset, @testset_macro, not, interpolated, reachable, depth, pass, fail
 
 using Distributed
 using Base.Threads: nthreads

src/patterns.jl

@@ -56,6 +56,15 @@ end
 struct Interpolated <: Pattern end
 
 
+### pass/fail
+
+struct Pass <: Pattern end
+const pass = Pass()
+
+struct Fail <: Pattern end
+const fail = Fail()
+
+
 ## alwaysmatches
 
 alwaysmatches(pat::And, d) = all(p -> alwaysmatches(p, d), pat.xs)
@@ -75,6 +84,8 @@ alwaysmatches(rx::Regex, _) = isempty(rx.pattern)
 alwaysmatches(id::Integer, _) = false
 alwaysmatches(pat::Reachable, d) = alwaysmatches(pat.x, d)
 alwaysmatches(dep::Depth, d) = dep.d == d
+alwaysmatches(::Pass, _) = false
+alwaysmatches(::Fail, _) = false
 
 
 ## matches
@@ -116,6 +127,12 @@ end
 
 matches(d::Depth, _, ts) = d.d == tsdepth(ts)
 
+matches(::Pass, subj::AbstractString, ts) = get(ts.pastresults, subj, false)
+matches(::Fail, subj::AbstractString, ts) = !get(ts.pastresults, subj, true)
+# TODO: test method below
+matches(::Union{Pass,Fail}, ::Missing, ts) =
+    isempty(ts.pastresults) ? false : missing
+
 
 ## make_pattern
 
@@ -156,6 +173,8 @@ hasinteger(pat::Not) = hasinteger(pat.x)
 hasinteger(::Interpolated) = false
 hasinteger(pat::Reachable) = hasinteger(pat.x)
 hasinteger(::Depth) = false
+hasinteger(::Pass) = false
+hasinteger(::Fail) = false
 
 
 ## exported pattern functions & singletons
@@ -477,3 +496,14 @@ julia> Depth.runtests(dry=true, verbose=3, depth.(2:3))
 ```
 """
 depth(x::Integer) = Depth(Int(x))
+
+"""
+    pass
+    fail
+
+Filtering patterns which match any testset which already ran,
+succesfully for `pass` or with at least one error for `fail`.
+The pattern `[pass, fail]` therefore matches any testset
+which already ran.
+"""
+pass, fail

src/testset.jl

@@ -65,6 +65,7 @@ mutable struct ReTestSet <: AbstractTestSet
     subject::String
     id::Int64
     overall::Bool # TODO: could be conveyed by having self.mod == ""
+    pastresults::Dict{String,Bool}
     results::Vector
     n_passed::Int
     anynonpass::Bool
@@ -74,11 +75,12 @@ mutable struct ReTestSet <: AbstractTestSet
 end
 
 function ReTestSet(mod, desc::String, id::Integer=0;
-                   overall=false, verbose=true, parent=nothing)
+                   overall=false, verbose=true, parent=nothing,
+                   pastresults=Dict{String,Bool}())
     parentsubj = parent === nothing ? "" : parent.subject
     subject = string(parentsubj, '/', desc)
-    ReTestSet(mod, parent, desc, subject, id, overall, [], 0, false,
-              verbose, NamedTuple(), nothing)
+    ReTestSet(mod, parent, desc, subject, id, overall, pastresults, [],
+              0, false, verbose, NamedTuple(), nothing)
 end
 
 # For a non-passed result, simply store the result
@@ -443,11 +445,11 @@ default_rng() = isdefined(Random, :default_rng) ?
     Random.default_rng() :
     Random.GLOBAL_RNG
 
-function make_retestset(mod, desc, id, verbose, remove_last=false)
+function make_retestset(mod, desc, id, verbose, pastresults, remove_last=false)
     _testsets = get(task_local_storage(), :__BASETESTNEXT__, Test.AbstractTestSet[])
     @assert !(remove_last && isempty(_testsets))
     testsets = @view _testsets[1:end-remove_last]
-    ReTestSet(mod, desc, id; verbose=verbose,
+    ReTestSet(mod, desc, id; verbose=verbose, pastresults=pastresults,
               parent = isempty(testsets) ? nothing : testsets[end])
 end
 
@@ -490,7 +492,8 @@ function testset_beginend(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, d
     # action (such as reporting the results)
     desc = esc(desc)
     ex = quote
-        local ts = make_retestset($mod, $desc, $id, $(options.transient_verbose))
+        local ts = make_retestset($mod, $desc, $id, $(options.transient_verbose),
+                                  $pastresults)
 
         if !$isfinal || matches($pat, ts.subject, ts)
             local ret
@@ -547,7 +550,7 @@ function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
     desc = esc(desc)
     blk = quote
         local ts0 = make_retestset($mod, $desc, $id, $(options.transient_verbose),
-                                   !first_iteration)
+                                   $pastresults, !first_iteration)
 
         if !$isfinal || matches($pat, ts0.subject, ts0)
             # Trick to handle `break` and `continue` in the test code before

test/runtests.jl

@@ -1039,9 +1039,40 @@ using ReTest
 @testset "has fails" begin
     @test false
 end
+@testset "also has passes" begin
+    @test true
+end
+@testset "unrun" begin
+    @test true
+end
+
 end # Failing
 
-@chapter Failing @test_throws Test.TestSetException retest(Failing)
+@chapter Failing begin
+    @test_throws Test.TestSetException retest(Failing)
+    retest(Failing, "passes")
+
+    check(Failing, dry=true, marks=true, -fail, [], output="""
+2| also has passes ✔
+3| unrun
+""")
+    check(Failing, dry=true, marks=true, pass, [], output="""
+2| also has passes ✔
+""")
+    check(Failing, dry=true, marks=true, [pass, fail], [], output="""
+1| has fails ✘
+2| also has passes ✔
+""")
+    check(Failing, dry=true, marks=true, -pass, -fail, [], output="""
+3| unrun
+""")
+
+    check(Failing, dry=true, marks=true, fail, [], output="1| has fails ✘")
+    check(Failing, dry=true, marks=true, -pass, [], output="""
+1| has fails ✘
+3| unrun
+""")
+end
 
 # * Duplicate ................................................................