generalize pastresults into marks

`marks` will be used to store arbitrary symbols.

Author: rfourquet

src/ReTest.jl

@@ -37,6 +37,7 @@ import InlineTest: retest, @testset_macro
 
 abstract type Pattern end
 function matches end
+function setresult end
 
 
 # * includes
@@ -69,7 +70,8 @@ mutable struct TestsetExpr
     parent::Maybe{TestsetExpr}
     children::Vector{TestsetExpr}
     strings::Vector{Union{String,Missing}}
-    pastresults::Dict{String,Bool}
+    # Union to avoid creating a vector in most cases
+    marks::Dict{String, Union{Symbol, Vector{Symbol}}} # TODO: should be a MultiDict
     # loopvalues & loopiters: when successful in evaluating loop values in resolve!,
     # we "flatten" the nested for loops into a single loop, with loopvalues
     # containing tuples of values, and loopiters the tuples of variables to which the
@@ -84,7 +86,7 @@ mutable struct TestsetExpr
 
     TestsetExpr(source, mod, desc, options, loops, parent, children=TestsetExpr[]) =
         new(0, source, mod, desc, options, loops, parent, children, String[],
-            Dict{String,Bool}())
+            Dict{String, Union{Symbol, Vector{Symbol}}}())
 end
 
 isfor(ts::TestsetExpr) = ts.loops !== nothing
@@ -99,6 +101,54 @@ function tsdepth(ts::Union{TestsetExpr,Testset.ReTestSet})
     d
 end
 
+const _pass = :__pass__
+const _fail = :__fail__
+
+# marks contains at most one of _pass, _fail
+
+# return true (pass), false (fail), or nothing (unrun)
+function pastresult(marks::Dict, subject)
+    ms = get(marks, subject, Symbol())
+    sym::Symbol = ms isa Symbol ? ms : isempty(ms) ? Symbol() : ms[1]
+    sym === _pass ? true : sym === _fail ? false : nothing
+end
+
+function setresult(marks::Dict, subject, success::Bool)
+    ms = get(marks, subject, Symbol())
+    res = success ? _pass : _fail
+    if ms isa Symbol
+        # ms could be Symbol() from get's default or delmark!
+        if ms ∈ (_pass, _fail, Symbol())
+            marks[subject] = res
+        else
+            # res always in first position
+            marks[subject] = [res, ms]
+        end
+    else # ms isa Vector
+        if !isempty(ms) && ms[1] ∈ (_pass, _fail)
+            ms[1] = res
+        else
+            pushfirst!(ms, res)
+        end
+    end
+end
+
+function delmark!(marks, subject, m::Symbol)
+    ms = get(marks, subject, Symbol())
+    if ms isa Symbol
+        if ms === m
+            marks[subject] = Symbol()
+        end
+    else
+        p = findfirst(==(m), ms)
+        if p !== nothing
+            deleteat!(ms, p)
+        end
+    end
+    nothing
+end
+
+
 struct _Invalid
     global const invalid = _Invalid.instance
 end
@@ -418,7 +468,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
     if ts.loops === nothing
         quote
             @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) #=
-            =# $(ts.pastresults) $stats $chan $body
+            =# $(ts.marks) $stats $chan $body
         end
     else
         c = count(x -> x === nothing, (ts.loopvalues, ts.loopiters))
@@ -430,7 +480,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
         end
         quote
             @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) #=
-            =# $(ts.pastresults) $stats $chan $loops $body
+            =# $(ts.marks) $stats $chan $loops $body
         end
     end
 end
@@ -1414,9 +1464,9 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
             end
         end
 
-        res = get(ts.pastresults, subject, nothing)
+        res = pastresult(ts.marks, subject)
         if clear && res !== nothing
-            delete!(ts.pastresults, subject)
+            delmark!(ts.marks, subject, res ? _pass : _fail)
         end
 
         if show
@@ -1503,7 +1553,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
                                    ts.parent, ts.children)
             beginend.run = true
             beginend.id = ts.id
-            beginend.pastresults = ts.pastresults
+            beginend.marks = ts.marks
             dryrun(mod, beginend, pat, align, parentsubj; evaldesc=false,
                    repeated=repeated, maxidw=maxidw, marks=marks, clear=clear, show=show)
         end

src/patterns.jl

@@ -127,11 +127,14 @@ end
 
 matches(d::Depth, _, ts) = d.d == tsdepth(ts)
 
-matches(::Pass, subj::AbstractString, ts) = get(ts.pastresults, subj, false)
-matches(::Fail, subj::AbstractString, ts) = !get(ts.pastresults, subj, true)
+matches(::Pass, subj::AbstractString, ts) = something(pastresult(ts.marks, subj), false)
+matches(::Fail, subj::AbstractString, ts) = !something(pastresult(ts.marks, subj), true)
 # TODO: test method below
+# instead of `isempty(ts.marks)`, a tighter test would be to check that no value in marks
+# contains the pass/fail marks; but this would be a bit more expensive, as values have to be
+# iterated over
 matches(::Union{Pass,Fail}, ::Missing, ts) =
-    isempty(ts.pastresults) ? false : missing
+    isempty(ts.marks) ? false : missing
 
 
 ## make_pattern

src/testset.jl

@@ -14,7 +14,7 @@ using Distributed: myid, nworkers
 
 import InlineTest: @testset
 
-using ..ReTest: Pattern, matches
+using ..ReTest: Pattern, matches, setresult
 
 # mostly copied from Test stdlib
 # changed from Test: pass nothing as file in ip_has_file_and_func
@@ -65,7 +65,7 @@ mutable struct ReTestSet <: AbstractTestSet
     subject::String
     id::Int64
     overall::Bool # TODO: could be conveyed by having self.mod == ""
-    pastresults::Dict{String,Bool}
+    marks::Dict{String, Union{Symbol, Vector{Symbol}}} # used in matches()
     results::Vector
     n_passed::Int
     anynonpass::Bool
@@ -76,10 +76,10 @@ end
 
 function ReTestSet(mod, desc::String, id::Integer=0;
                    overall=false, verbose=true, parent=nothing,
-                   pastresults=Dict{String,Bool}())
+                   marks=Dict{String, Union{Symbol, Vector{Symbol}}}())
     parentsubj = parent === nothing ? "" : parent.subject
     subject = string(parentsubj, '/', desc)
-    ReTestSet(mod, parent, desc, subject, id, overall, pastresults, [],
+    ReTestSet(mod, parent, desc, subject, id, overall, marks, [],
               0, false, verbose, NamedTuple(), nothing)
 end
 
@@ -447,11 +447,11 @@ default_rng() = isdefined(Random, :default_rng) ?
     Random.default_rng() :
     Random.GLOBAL_RNG
 
-function make_retestset(mod, desc, id, verbose, pastresults, remove_last=false)
+function make_retestset(mod, desc, id, verbose, marks, remove_last=false)
     _testsets = get(task_local_storage(), :__BASETESTNEXT__, Test.AbstractTestSet[])
     @assert !(remove_last && isempty(_testsets))
     testsets = @view _testsets[1:end-remove_last]
-    ReTestSet(mod, desc, id; verbose=verbose, pastresults=pastresults,
+    ReTestSet(mod, desc, id; verbose=verbose, marks=marks,
               parent = isempty(testsets) ? nothing : testsets[end])
 end
 
@@ -472,30 +472,30 @@ end
 
 # non-inline testset with regex filtering support
 macro testset(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc::Union{String,Expr},
-              options, pastresults::Dict, stats::Bool, chan, body)
+              options, marks::Dict, stats::Bool, chan, body)
     Testset.testset_beginend(mod, isfinal, pat, id, desc,
-                             options, pastresults, stats, chan, body, __source__)
+                             options, marks, stats, chan, body, __source__)
 end
 
 macro testset(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc::Union{String,Expr},
-              options, pastresults::Dict, stats::Bool, chan, loops, body)
+              options, marks::Dict, stats::Bool, chan, loops, body)
     Testset.testset_forloop(mod, isfinal, pat, id, desc,
-                            options, pastresults, stats, chan, loops, body, __source__)
+                            options, marks, stats, chan, loops, body, __source__)
 end
 
 """
 Generate the code for a `@testset` with a `begin`/`end` argument
 """
 function testset_beginend(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc, options,
-                          pastresults::Dict, stats::Bool, chan, tests, source)
+                          marks::Dict, stats::Bool, chan, tests, source)
     # Generate a block of code that initializes a new testset, adds
     # it to the task local storage, evaluates the test(s), before
     # finally removing the testset and giving it a chance to take
     # action (such as reporting the results)
     desc = esc(desc)
     ex = quote
         local ts = make_retestset($mod, $desc, $id, $(options.transient_verbose),
-                                  $pastresults)
+                                  $marks)
 
         if !$isfinal || matches($pat, ts.subject, ts)
             local ret
@@ -526,7 +526,7 @@ function testset_beginend(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, d
                                  Base.catch_stack(), $(QuoteNode(source))))
             finally
                 copy!(RNG, oldrng)
-                $pastresults[ts.subject] = !anyfailed(ts)
+                setresult($marks, ts.subject, !anyfailed(ts))
                 pop_testset()
                 ret = finish(ts, $chan)
             end
@@ -546,13 +546,13 @@ end
 Generate the code for a `@testset` with a `for` loop argument
 """
 function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
-                         desc::Union{String,Expr}, options, pastresults::Dict, stats, chan,
+                         desc::Union{String,Expr}, options, marks::Dict, stats, chan,
                          loops, tests, source)
 
     desc = esc(desc)
     blk = quote
         local ts0 = make_retestset($mod, $desc, $id, $(options.transient_verbose),
-                                   $pastresults, !first_iteration)
+                                   $marks, !first_iteration)
 
         if !$isfinal || matches($pat, ts0.subject, ts0)
             # Trick to handle `break` and `continue` in the test code before
@@ -573,13 +573,13 @@ function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
                 let
                     ts.timed = @stats $stats $(esc(tests))
                 end
-                $pastresults[ts.subject] = !anyfailed(ts)
+                setresult($marks, ts.subject, !anyfailed(ts))
             catch err
                 err isa InterruptException && rethrow()
                 # Something in the test block threw an error. Count that as an
                 # error in this test set
                 record(ts, Error(:nontest_error, Expr(:tuple), err, Base.catch_stack(), $(QuoteNode(source))))
-                $pastresults[ts.subject] = false
+                setresult($marks, ts.subject, false)
             end
         end
     end

test/test_patterns.jl

@@ -6,7 +6,7 @@ import ReTest
 
 struct MockTestset
     id
-    pastresults
+    marks
     parent
 
     MockTestset() = new(rand(1:typemax(Int)), Dict(), nothing)