filtering: add iter(::Integer) pattern constructor

Author: rfourquet

src/ReTest.jl

@@ -1,6 +1,6 @@
 module ReTest
 
-export retest, @testset, @testset_macro, not, interpolated, reachable, depth, pass, fail
+export retest, @testset, @testset_macro, not, interpolated, reachable, depth, pass, fail, iter
 
 using Distributed
 using Base.Threads: nthreads
@@ -80,6 +80,7 @@ mutable struct TestsetExpr
     loopiters::Maybe{Expr}
     hasbroken::Bool
     hasbrokenrec::Bool # recursive hasbroken, transiently
+    iter::Union{Int,UnitRange{Int}} # transient loop iteration counter
     run::Bool
     descwidth::Int # max width of self and children shown descriptions
     body::Expr
@@ -343,7 +344,8 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
         for str in parentstrs
             !strict && ts.run && break
             new = str * "/" * desc
-            hasmissing && new === missing ||
+            # string[end] == new can happen with loops when has(pat, Iter) and desc isa String
+            hasmissing && new === missing || !isempty(strings) && strings[end] === new ||
                 push!(strings, new)
             hasmissing |= new === missing # comes either from desc or str
             ts.run = ts.run || decide(new)
@@ -352,7 +354,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
     end
 
     loops = ts.loops
-    if loops === nothing || desc isa String
+    if loops === nothing || desc isa String && !has(pat, Iter)
         # TODO: maybe, for testset-for and !(desc isa String), still try this branch
         # in case the the interpolation can be resolved thanks to a global binding
         # (i.e. the description doesn't depend on loop variables)
@@ -368,9 +370,11 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
         if shown
             ts.descwidth = descwidth(desc)
         end
+        ts.iter = 1
         decide_testset!(desc, false)
 
-    else # we have a testset-for with description which needs interpolation
+    else # we have a testset-for with description which needs interpolation, or
+         # the iterator must be computed to get an iterator counter
         xs = ()
         loopiters = Expr(:tuple, (arg.args[1] for arg in loops)...)
 
@@ -398,17 +402,20 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
         catch
             @assert xs == ()
             ts.descwidth = shown ? descwidth(missing) : 0
+            ts.iter = typemax(Int)
             ts.run = ts.run || decide(missing)
             push!(strings, missing)
         end
         hasmissing = false
-        for x in xs # empty loop if eval above threw
+        for (iter, x) in enumerate(xs) # empty loop if eval above threw
             descx = eval_desc(mod, ts, x)
+            ts.iter = iter
             if shown
                 ts.descwidth = max(ts.descwidth, descwidth(descx))
             end
-            if !strict && ts.run
-                if !shown # no need to compute subsequent descx to update ts.descwidth
+            if ts.run && (!strict || ts.desc isa String)
+                if ts.desc isa String || !shown # no need to compute subsequent descx to update ts.descwidth
+                    iter = length(xs)
                     break
                 else
                     continue
@@ -420,6 +427,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
 
     run = ts.run
     ts.hasbrokenrec = ts.hasbroken
+    ts.iter = 1:ts.iter # for children, when reachable is used, set the possible range
 
     for tsc in ts.children
         runc, id = resolve!(mod, tsc, pat, force = !strict && ts.run,
@@ -465,7 +473,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
         body = make_ts(ts.body, pat, stats, chan)
     end
 
-    if ts.loops === nothing
+    if !isfor(ts)
         quote
             @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) #=
             =# $(ts.marks) $stats $chan $body
@@ -1448,7 +1456,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
     @assert ts.run
     desc = ts.desc
 
-    if ts.loops === nothing
+    if !isfor(ts)
         if evaldesc && !(desc isa String)
             try
                 desc = Core.eval(mod, desc)
@@ -1532,8 +1540,8 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
             end
             false, false, false
         end
-    else
-        function dryrun_beginend(descx, repeated=nothing)
+    else # isfor(ts)
+        function dryrun_beginend(descx; iter, repeated=nothing)
             # avoid repeating ourselves, transform this iteration into a "begin/end" testset
             if descx isa Expr
                 @assert descx.head == :string
@@ -1553,6 +1561,8 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
                                    ts.parent, ts.children)
             beginend.run = true
             beginend.id = ts.id
+            beginend.iter = iter
+            ts.iter = iter # necessary when reachable is used
             beginend.marks = ts.marks
             dryrun(mod, beginend, pat, align, parentsubj; evaldesc=false,
                    repeated=repeated, maxidw=maxidw, marks=marks, clear=clear, show=show)
@@ -1564,7 +1574,11 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
             # identitical lines (caveat: if subjects of children would change randomly)
             # but still try simply to evaluate the length of the iterator
             repeated = -1
-            if ts.desc isa String
+            if ts.desc isa String && !has(pat, Iter)
+                # when has(pat, Iter), we probably weren't able to eval the forloop-iterator
+                # in resolve!, so no need to re-try here; plus, we wouldn't be able in this
+                # branch to tell how many iterations are matching, without going thru the
+                # dryrun_beginend function in an enumerate'd loop
                 local iterlen
                 try
                     iterlen = 1
@@ -1575,7 +1589,9 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
                 catch
                 end
             end
-            dryrun_beginend(ts.desc, repeated)
+            # if iterlen was computed, then !has(pat, Iter) so the value of the iter
+            # keyword below doesn't matter
+            dryrun_beginend(ts.desc, repeated=repeated, iter=1:typemax(Int))
         else
             passes, fails, unrun = false, false, false
             for (i, x) in enumerate(loopvalues)
@@ -1587,9 +1603,10 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
                     # so we add the "repeated" annotation
                     # (it's certainly not worth it to bother being more precise about
                     # exactly which iterations are uninterpolated)
-                    lp, lf, lu = dryrun_beginend(ts.desc, length(loopvalues)-i+1)
+                    lp, lf, lu = dryrun_beginend(ts.desc, repeated=length(loopvalues)-i+1,
+                                                 iter=i:length(loopvalues))
                 else
-                    lp, lf, lu = dryrun_beginend(descx)
+                    lp, lf, lu = dryrun_beginend(descx, iter=i)
                 end
                 passes |= lp
                 fails |= lf

src/patterns.jl

@@ -65,6 +65,13 @@ struct Fail <: Pattern end
 const fail = Fail()
 
 
+### iter
+
+struct Iter <: Pattern
+    i::Int
+end
+
+
 ## alwaysmatches
 
 alwaysmatches(pat::And, d) = all(p -> alwaysmatches(p, d), pat.xs)
@@ -86,6 +93,7 @@ alwaysmatches(pat::Reachable, d) = alwaysmatches(pat.x, d)
 alwaysmatches(dep::Depth, d) = dep.d == d
 alwaysmatches(::Pass, _) = false
 alwaysmatches(::Fail, _) = false
+alwaysmatches(::Iter, _) = false
 
 
 ## matches
@@ -136,6 +144,13 @@ matches(::Fail, subj::AbstractString, ts) = !something(pastresult(ts.marks, subj
 matches(::Union{Pass,Fail}, ::Missing, ts) =
     isempty(ts.marks) ? false : missing
 
+matches(i::Iter, subj, ts) =
+    if ts.iter isa Int
+        i.i == ts.iter
+    else
+        i.i ∈ ts.iter
+    end
+
 
 ## has
 
@@ -505,3 +520,15 @@ The pattern `[pass, fail]` therefore matches any testset
 which already ran.
 """
 pass, fail
+
+"""
+    iter(i::Integer)
+
+Filtering pattern which matches only the `i`-th iteration of a testset-for.
+A non-for testset is considered to have a unique iteration.
+
+!!! warning
+
+    This is very experimental, not tested, and likely to be removed in a future version.
+"""
+iter(i::Integer) = Iter(Int(i))

src/testset.jl

@@ -66,6 +66,7 @@ mutable struct ReTestSet <: AbstractTestSet
     id::Int64
     overall::Bool # TODO: could be conveyed by having self.mod == ""
     marks::Dict{String, Union{Symbol, Vector{Symbol}}} # used in matches()
+    iter::Int # used in matches()
     results::Vector
     n_passed::Int
     anynonpass::Bool
@@ -76,10 +77,11 @@ end
 
 function ReTestSet(mod, desc::String, id::Integer=0;
                    overall=false, verbose=true, parent=nothing,
-                   marks=Dict{String, Union{Symbol, Vector{Symbol}}}())
+                   marks=Dict{String, Union{Symbol, Vector{Symbol}}}(),
+                   iter=1)
     parentsubj = parent === nothing ? "" : parent.subject
     subject = string(parentsubj, '/', desc)
-    ReTestSet(mod, parent, desc, subject, id, overall, marks, [],
+    ReTestSet(mod, parent, desc, subject, id, overall, marks, iter, [],
               0, false, verbose, NamedTuple(), nothing)
 end
 
@@ -447,11 +449,11 @@ default_rng() = isdefined(Random, :default_rng) ?
     Random.default_rng() :
     Random.GLOBAL_RNG
 
-function make_retestset(mod, desc, id, verbose, marks, remove_last=false)
+function make_retestset(mod, desc, id, verbose, marks, remove_last=false, iter=1)
     _testsets = get(task_local_storage(), :__BASETESTNEXT__, Test.AbstractTestSet[])
     @assert !(remove_last && isempty(_testsets))
     testsets = @view _testsets[1:end-remove_last]
-    ReTestSet(mod, desc, id; verbose=verbose, marks=marks,
+    ReTestSet(mod, desc, id; verbose=verbose, marks=marks, iter=iter,
               parent = isempty(testsets) ? nothing : testsets[end])
 end
 
@@ -551,8 +553,9 @@ function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
 
     desc = esc(desc)
     blk = quote
+        iter += 1
         local ts0 = make_retestset($mod, $desc, $id, $(options.transient_verbose),
-                                   $marks, !first_iteration)
+                                   $marks, !first_iteration, iter)
 
         if !$isfinal || matches($pat, ts0.subject, ts0)
             # Trick to handle `break` and `continue` in the test code before
@@ -586,6 +589,7 @@ function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
     quote
         local arr = Vector{Any}()
         local first_iteration = true
+        local iter = 0
         local ts
         local RNG = default_rng()
         local oldrng = copy(RNG)

test/test_patterns.jl

@@ -1,20 +1,21 @@
 module TestPatterns
 using Test
 
-using ReTest: and, or, not, interpolated, reachable, depth, pass, fail
+using ReTest: and, or, not, interpolated, reachable, depth, pass, fail, iter
 import ReTest
 
 struct MockTestset
     id
     marks
     parent
+    iter
 
-    MockTestset() = new(rand(1:typemax(Int)), Dict(), nothing)
+    MockTestset() = new(rand(1:typemax(Int)), Dict(), nothing, 1)
 end
 
 ReTest.tsdepth(::MockTestset) = 1
 
-const basic_patterns = [and(), or(), not(0), interpolated, 0, r"", depth(2), pass, fail]
+const basic_patterns = [and(), or(), not(0), interpolated, 0, r"", depth(2), pass, fail, iter(1)]
 VERSION >= v"1.3" && push!(basic_patterns, reachable(1))
 
 @testset "patterns: ==" begin
@@ -60,6 +61,7 @@ end
         @test ReTest.matches(a, missing, MockTestset()) isa Union{Missing, Bool}
         @test ReTest.alwaysmatches(a, 1) isa Bool
         @test ReTest.has(a, Integer) isa Bool
+        @test ReTest.has(a, ReTest.Iter) isa Bool
     end
 end