Automatically reduce test failures General registry (#229)

Author: c42f

test/fuzz_test.jl

@@ -900,24 +900,24 @@ Reduce test case via combination of bisection and random deletion.
 This is suited to randomly generated strings, but it's surprisingly effective
 for code-like strings as well.
 """
-function rand_reduce(str)
+function rand_reduce(str, parse_failure=parser_throws_exception)
     while true
         if length(str) <= 1
             return str
         end
         m1 = thisind(str, length(str)÷2)
         m2 = nextind(str, m1)
-        if parser_throws_exception(str[1:m1])
+        if parse_failure(str[1:m1])
             str = str[1:m1]
-        elseif parser_throws_exception(str[m2:end])
+        elseif parse_failure(str[m2:end])
             str = str[m2:end]
         else
             chunklen = clamp(length(str)÷10, 1, 10)
             reduced = false
             for i = 1:100
                 m = thisind(str, rand(1:length(str)-chunklen))
                 s = str[1:m]*str[nextind(str, m+chunklen):end]
-                if parser_throws_exception(s)
+                if parse_failure(s)
                     str = s
                     reduced = true
                     break

tools/check_all_packages.jl

@@ -6,6 +6,7 @@
 using JuliaSyntax, Logging, TerminalLoggers, ProgressLogging, Serialization
 
 include("../test/test_utils.jl")
+include("../test/fuzz_test.jl")
 
 pkgspath = joinpath(@__DIR__, "pkgs")
 source_paths = find_source_in_path(pkgspath)
@@ -16,6 +17,24 @@ mismatch_count = 0
 t0 = time()
 exceptions = []
 
+function parsers_disagree(text::AbstractString)
+    fl_ex = fl_parseall(text, filename="none")
+    if Meta.isexpr(fl_ex, (:error,:incomplete)) ||
+            (Meta.isexpr(fl_ex, :toplevel) && length(fl_ex.args) >= 1 &&
+             Meta.isexpr(fl_ex.args[end], (:error,:incomplete)))
+        return false
+    end
+    try
+        ex = parseall(Expr, text, filename="none", ignore_errors=true)
+        return !exprs_roughly_equal(fl_ex, ex)
+    catch
+        @error "Reduction failed" text
+        return false
+    end
+end
+
+all_reduced_failures = String[]
+
 Logging.with_logger(TerminalLogger()) do
     global exception_count, mismatch_count, t0
     @withprogress for (ifile, fpath) in enumerate(source_paths)
@@ -29,16 +48,19 @@ Logging.with_logger(TerminalLogger()) do
             e1 = JuliaSyntax.parseall(Expr, text, filename=fpath, ignore_warnings=true)
             if !exprs_roughly_equal(e2, e1)
                 mismatch_count += 1
-                reduced_chunks = sprint(context=:color=>true) do io
+                failing_source = sprint(context=:color=>true) do io
                     for c in reduce_test(text)
                         JuliaSyntax.highlight(io, c.source, range(c), context_lines_inner=5)
                         println(io, "\n")
                     end
                 end
+                reduced_failures = rand_reduce.(sourcetext.(reduce_test(text)),
+                                                   parsers_disagree)
+                append!(all_reduced_failures, reduced_failures)
                 @error("Parsers succeed but disagree",
                        fpath,
-                       reduced_chunks=Text(reduced_chunks),
-                       # diff=Text(sprint(show_expr_text_diff, show, e1, e2)),
+                       failing_source=Text(failing_source),
+                       reduced_failures,
                        )
             end
         catch err
@@ -75,3 +97,14 @@ println()
         $(exception_count) failures compared to reference parser
         $(mismatch_count) Expr mismatches
         $(t_avg)ms per file"""
+
+open(joinpath(@__DIR__, "reduced_failures.jl"), write=true) do io
+    for str in all_reduced_failures
+        println(io, repr(str))
+    end
+    for str in all_reduced_failures
+        println(io, "#------------------------------")
+        println(io, str)
+        println(io)
+    end
+end