record testsets passes and fails, and show "marks" in dry mode

Author: rfourquet

src/ReTest.jl

@@ -69,6 +69,7 @@ mutable struct TestsetExpr
     parent::Maybe{TestsetExpr}
     children::Vector{TestsetExpr}
     strings::Vector{Union{String,Missing}}
+    results::Dict{String,Bool}
     # loopvalues & loopiters: when successful in evaluating loop values in resolve!,
     # we "flatten" the nested for loops into a single loop, with loopvalues
     # containing tuples of values, and loopiters the tuples of variables to which the
@@ -82,7 +83,8 @@ mutable struct TestsetExpr
     body::Expr
 
     TestsetExpr(source, mod, desc, options, loops, parent, children=TestsetExpr[]) =
-        new(0, source, mod, desc, options, loops, parent, children, String[])
+        new(0, source, mod, desc, options, loops, parent, children, String[],
+            Dict{String,Bool}())
 end
 
 isfor(ts::TestsetExpr) = ts.loops !== nothing
@@ -255,8 +257,8 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
 
     function decide(subj)
         m = matches(pat, subj, ids)
-        # For the curious, setting `s = something(static, missing)`, there are few
-        # "formulas" to compute the result without `if`, but using only
+        # For the curious reader, setting `s = something(static, missing)`, there
+        # are few "formulas" to compute the result without `if`, but using only
         # `coalesce, |, &, ==, !=, ===, !==, (a,b) -> a, (a,b) -> b, (a,b) -> !a,
         # (a,b) -> !b`. The shortest formulas involve 5 such
         # functions `fi` and are of the form
@@ -372,6 +374,7 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
             ts.hasbrokenrec |= tsc.hasbrokenrec
         end
     end
+
     pop!(ids)
     if !run || !shown
         ts.descwidth = 0
@@ -407,7 +410,8 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
 
     if ts.loops === nothing
         quote
-            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) $stats $chan $body
+            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) #=
+            =# $(ts.results) $stats $chan $body
         end
     else
         c = count(x -> x === nothing, (ts.loopvalues, ts.loopiters))
@@ -418,7 +422,8 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
             loops = ts.loops
         end
         quote
-            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) $stats $chan $loops $body
+            @testset $(ts.mod) $(isfinal(ts)) $pat $(ts.id) $(ts.desc) $(ts.options) #=
+            =# $(ts.results) $stats $chan $loops $body
         end
     end
 end
@@ -471,7 +476,7 @@ const ArgType = Union{Module,PatternX,AbstractString,AbstractArray,Tuple,Symbol,
            dry::Bool=false, stats::Bool=false, verbose::Real=true,
            [id::Bool], shuffle::Bool=false, recursive::Bool=true,
            static::Union{Bool,Nothing}=nothing, dup::Bool=false,
-           load::Bool=false, [seed::Integer])
+           load::Bool=false, [seed::Integer], marks::Bool=true)
 
 Run tests declared with [`@testset`](@ref) blocks, within modules `mod` if specified,
 or within all currently loaded modules otherwise.
@@ -512,7 +517,8 @@ Filtering `pattern`s can be specified to run only a subset of the tests.
   above), and are cached and used again on subsequent invocations.
 * If `seed` is provided, it is used to seed the global RNG before running
   the tests.
-
+* When `marks` and `dry` are `true`, "check marks" are printed next to testsets
+  which passed or failed in previous runs.
 
 ### Filtering
 
@@ -599,10 +605,11 @@ function retest(@nospecialize(args::ArgType...);
                 static::Maybe{Bool}=nothing,
                 load::Bool=false,
                 seed::Maybe{Integer}=nothing,
+                marks::Bool=true,
                 )
 
-    dry, stats, shuffle, group, verbose, recursive, id, strict, dup, static =
-        update_keywords(args, dry, stats, shuffle, group, verbose, recursive, id, strict, dup, static)
+    dry, stats, shuffle, group, verbose, recursive, id, strict, dup, static, marks =
+        update_keywords(args, dry, stats, shuffle, group, verbose, recursive, id, strict, dup, static, marks)
 
     implicitmodules, modules, verbose = process_args(args; verbose=verbose, shuffle=shuffle,
                                                      recursive=recursive, load=load)
@@ -661,7 +668,7 @@ function retest(@nospecialize(args::ArgType...);
             end
             if verbose > 0
                 foreach(ts -> dryrun(mod, ts, pat, id ? 0 : module_header*2,
-                                     maxidw = id ? maxidw[] : 0),
+                                     maxidw = id ? maxidw[] : 0, marks=marks),
                         tests)
             end
             continue
@@ -1025,7 +1032,7 @@ end
 
 # hidden feature, shortcuts for passing kwargs to retest
 function update_keywords(@nospecialize(args), dry, stats, shuffle, group, verbose,
-                         recursive, id, strict, dup, static)
+                         recursive, id, strict, dup, static, marks)
     for arg in args
         if arg isa Symbol
             for c in string(arg)
@@ -1052,13 +1059,15 @@ function update_keywords(@nospecialize(args), dry, stats, shuffle, group, verbos
                     dup = val
                 elseif c == 'c'
                     static = val
+                elseif c == 'm'
+                    marks = val
                 else
                     error("bad keyword shortcut")
                 end
             end
         end
     end
-    dry, stats, shuffle, group, verbose, recursive, id, strict, dup, static
+    dry, stats, shuffle, group, verbose, recursive, id, strict, dup, static, marks
 end
 
 function process_args(@nospecialize(args);
@@ -1307,7 +1316,7 @@ isindented(verbose, module_header, many) = (verbose > 0) & module_header
 module_summary(verbose, many) = many | iszero(verbose)
 
 function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parentsubj=""
-                ; maxidw::Int, # external calls
+                ; maxidw::Int, marks::Bool, # external calls
                 # only recursive calls:
                 evaldesc=true, repeated=nothing, ids::Vector{Int64}=Int64[])
     @assert ts.run
@@ -1334,19 +1343,26 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
         if maxidw > 0 # width (ndigits) of max id; <= 0 means ids not printed
             printstyled(lpad(ts.id, maxidw), "| ", color = :light_black, bold=true)
         end
+
         printstyled(' '^align, desc, color = desc isa String ? :normal : Base.warn_color())
 
         if repeated !== nothing
             printstyled(" (repeated",
-                        repeated == -1 ? ")" : " $repeated times)", '\n',
+                        repeated == -1 ? ")" : " $repeated times)",
                         color=:light_black)
-        else
-            println()
         end
+
+        res = get(ts.results, subject, nothing)
+        if marks && res !== nothing
+            printstyled(res ? " ✅" : " ✘", color = res ? :green : Base.error_color(),
+                        bold=true)
+        end
+        println()
+
         if ts.options.transient_verbose
             for tsc in ts.children
                 tsc.run || continue
-                dryrun(mod, tsc, pat, align + 2, subject, maxidw=maxidw, ids=ids)
+                dryrun(mod, tsc, pat, align + 2, subject, maxidw=maxidw, marks=marks, ids=ids)
             end
         end
         pop!(ids)
@@ -1372,8 +1388,9 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0, parent
                                    ts.parent, ts.children)
             beginend.run = true
             beginend.id = ts.id
+            beginend.results = ts.results
             dryrun(mod, beginend, pat, align, parentsubj; evaldesc=false,
-                   repeated=repeated, maxidw=maxidw, ids=ids)
+                   repeated=repeated, maxidw=maxidw, marks=marks, ids=ids)
         end
 
         loopvalues = ts.loopvalues

src/testset.jl

@@ -310,6 +310,8 @@ function get_test_counts(ts::ReTestSet)
     return passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken
 end
 
+anyfailed(ts::ReTestSet) = any(t -> t isa Union{Fail,Error}, ts.results)
+
 # Recursive function that prints out the results at each level of
 # the tree of test sets
 function print_counts(ts::ReTestSet, fmt::Format, depth, align,
@@ -458,22 +460,23 @@ macro testset(mod::Module, isfinal::Bool, pat::Pattern, x...)
 end
 
 # non-inline testset with regex filtering support
-macro testset(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc::Union{String,Expr}, options,
-              stats::Bool, chan, body)
-    Testset.testset_beginend(mod, isfinal, pat, id, desc, options, stats, chan,  body, __source__)
+macro testset(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc::Union{String,Expr},
+              options, results::Dict, stats::Bool, chan, body)
+    Testset.testset_beginend(mod, isfinal, pat, id, desc,
+                             options, results, stats, chan, body, __source__)
 end
 
-macro testset(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc::Union{String,Expr}, options,
-              stats::Bool, chan, loops, body)
-    Testset.testset_forloop(mod, isfinal, pat, id, desc, options,
-                            stats, chan, loops, body, __source__)
+macro testset(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc::Union{String,Expr},
+              options, results::Dict, stats::Bool, chan, loops, body)
+    Testset.testset_forloop(mod, isfinal, pat, id, desc,
+                            options, results, stats, chan, loops, body, __source__)
 end
 
 """
 Generate the code for a `@testset` with a `begin`/`end` argument
 """
 function testset_beginend(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, desc, options,
-                          stats::Bool, chan, tests, source)
+                          results::Dict, stats::Bool, chan, tests, source)
     # Generate a block of code that initializes a new testset, adds
     # it to the task local storage, evaluates the test(s), before
     # finally removing the testset and giving it a chance to take
@@ -514,6 +517,7 @@ function testset_beginend(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, d
                                  Base.catch_stack(), $(QuoteNode(source))))
             finally
                 copy!(RNG, oldrng)
+                $results[current_str] = !anyfailed(ts)
                 pop_testset()
                 ret = finish(ts, $chan)
             end
@@ -533,7 +537,8 @@ end
 Generate the code for a `@testset` with a `for` loop argument
 """
 function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
-                         desc::Union{String,Expr}, options, stats, chan, loops, tests, source)
+                         desc::Union{String,Expr}, options, results::Dict, stats, chan,
+                         loops, tests, source)
 
     desc = esc(desc)
     blk = quote
@@ -561,11 +566,13 @@ function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64,
                 let
                     ts.timed = @stats $stats $(esc(tests))
                 end
+                $results[current_str] = !anyfailed(ts)
             catch err
                 err isa InterruptException && rethrow()
                 # Something in the test block threw an error. Count that as an
                 # error in this test set
                 record(ts, Error(:nontest_error, Expr(:tuple), err, Base.catch_stack(), $(QuoteNode(source))))
+                $results[current_str] = false
             end
         end
     end

test/runtests.jl

@@ -69,9 +69,9 @@ function check(rx, list)
         seekstart(io)
         expected = map(list) do t
             if t in innertestsets
-                "  " * t
+                "  " * t * " ✅"
             else
-                t
+                t * " ✅"
             end
         end
         expected = join(expected, '\n')
@@ -865,6 +865,71 @@ end
 end
 
 
+# * Marks ....................................................................
+
+module Marks
+using ReTest, ..Trace
+
+@testset "a" begin
+    @test true
+    @testset "b" begin end # no test, counts as pass
+    @testset "c" begin
+        @test true
+    end
+    x = 1
+    @testset "d$x" begin
+        @test true
+    end
+    @testset "e" begin
+        @test false
+    end
+end
+@testset "l$i" for i=1:2
+    @testset "k$j" for j=1:2
+        @test true
+    end
+end
+end
+
+@chapter Marks begin
+    check(Marks, dry=true, marks=true, verbose=3, [], output=raw"""
+1| a
+2|   b
+3|   c
+4|   "d$(x)"
+5|   e
+6| l1
+7|   k1
+7|   k2
+6| l2
+7|   k1
+7|   k2
+""")
+    retest(Marks, "k1")
+    check(Marks, dry=true, marks=true, verbose=3, [], output=raw"""
+1| a ✅
+2|   b
+3|   c
+4|   "d$(x)"
+5|   e
+6| l1 ✅
+7|   k1 ✅
+7|   k2
+6| l2 ✅
+7|   k1 ✅
+7|   k2
+""")
+    @test_throws Test.TestSetException retest(Marks, "e")
+    check(Marks, "a", dry=true, marks=true, verbose=3, [], output=raw"""
+1| a ✅
+2|   b
+3|   c
+4|   "d$(x)"
+5|   e ✘
+""")
+end
+
+
 # * Failing ..................................................................
 
 module Failing

test/setup.jl

@@ -12,9 +12,10 @@ function trace(x)
 end
 end
 
+# NOTE: all keywords have the same defaults as `retest`, except `marks`
 function check(x...; runtests=false, output::Union{Nothing,String}=nothing,
                verbose=true, stats=false, dry=false, strict::Bool=true,
-               recursive=true, static=nothing, id=nothing, load=false)
+               recursive=true, static=nothing, id=nothing, load=false, marks::Bool=false)
     @assert !(runtests & (output !== nothing)) "unimplemented"
     args = x[1:end-1]
     expected = x[end]
@@ -26,15 +27,15 @@ function check(x...; runtests=false, output::Union{Nothing,String}=nothing,
     if runtests
         getfield(args[1], :runtests)(args[2:end]...; verbose=verbose, stats=stats, dry=dry,
                                      strict=strict, recursive=recursive, static=static,
-                                     id=id, load=load)
+                                     id=id, load=load, marks=marks)
     elseif output === nothing
         retest(args...; verbose=verbose, stats=stats, dry=dry, strict=strict,
-               recursive=recursive, static=static, id=id, load=load)
+               recursive=recursive, static=static, id=id, load=load, marks=marks)
     else
         mktemp() do path, io
             redirect_stdout(io) do
                 retest(args...; verbose=verbose, stats=stats, dry=dry, strict=strict,
-                       recursive=recursive, static=static, id=id, load=load)
+                       recursive=recursive, static=static, id=id, load=load, marks=marks)
             end
             seekstart(io)
             printed = join(map(rstrip, split(readchomp(io), '\n')), '\n')