Hack: Test tight binding commas against JuliaSyntax ref parser

Author: c42f

test/test_utils.jl

@@ -243,8 +243,9 @@ function equals_flisp_parse(exprs_equal, tree)
     exprs_equal(fl_ex, ex)
 end
 
-function _reduce_tree(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum)
-    if equals_flisp_parse(exprs_equal, tree)
+function _reduce_tree(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum,
+                      equals_ref_parse=equals_flisp_parse)
+    if equals_ref_parse(exprs_equal, tree)
         return false
     end
     if !haschildren(tree)
@@ -257,7 +258,9 @@ function _reduce_tree(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum
             if is_trivia(child) || !haschildren(child)
                 continue
             end
-            had_failing_subtrees |= _reduce_tree(failing_subtrees, child; exprs_equal=exprs_equal)
+            had_failing_subtrees |= _reduce_tree(failing_subtrees, child;
+                                                 exprs_equal=exprs_equal,
+                                                 equals_ref_parse=equals_ref_parse)
         end
     end
     if !had_failing_subtrees

tools/check_all_packages.jl

@@ -5,6 +5,8 @@
 
 using JuliaSyntax, Logging, TerminalLoggers, ProgressLogging, Serialization
 
+using JuliaSyntax: GreenNode
+
 include("../test/test_utils.jl")
 include("../test/fuzz_test.jl")
 
@@ -19,26 +21,41 @@ exceptions = []
 
 all_reduced_failures = String[]
 
+function _lowprec_commas_equiv(exprs_equal, tree)
+    node_text = sourcetext(tree)
+    e1 = parseall(GreenNode, node_text, ignore_errors=true)
+    e2 = parseall(GreenNode, node_text, ignore_errors=true,
+                  low_precedence_comma_in_brackets=true)
+    e1 == e2
+end
+
 Logging.with_logger(TerminalLogger()) do
     global exception_count, mismatch_count, t0
     @withprogress for (ifile, fpath) in enumerate(source_paths)
         @logprogress ifile/file_count time_ms=round((time() - t0)/ifile*1000, digits = 2)
         text = read(fpath, String)
         expr_cache = fpath*".Expr"
-        #e2 = JuliaSyntax.fl_parseall(text)
-        e2 = open(deserialize, fpath*".Expr")
-        @assert Meta.isexpr(e2, :toplevel)
+        #e_ref = JuliaSyntax.fl_parseall(text)
+        #e_ref = open(deserialize, fpath*".Expr")
+        #@assert Meta.isexpr(e_ref, :toplevel)
+        e_ref = try
+            JuliaSyntax.parseall(GreenNode, text, filename=fpath, ignore_warnings=true)
+        catch
+            continue
+        end
         try
-            e1 = JuliaSyntax.parseall(Expr, text, filename=fpath, ignore_warnings=true)
-            if !exprs_roughly_equal(e2, e1)
+            e1 = JuliaSyntax.parseall(GreenNode, text, filename=fpath, ignore_warnings=true, low_precedence_comma_in_brackets=true)
+            if e1 != e_ref
+                source = SourceFile(text, filename=fpath)
+                e1sn = SyntaxNode(source, e1)
                 mismatch_count += 1
                 failing_source = sprint(context=:color=>true) do io
-                    for c in reduce_tree(parseall(SyntaxNode, text))
+                    for c in reduce_tree(e1sn, equals_ref_parse=_lowprec_commas_equiv)
                         JuliaSyntax.highlight(io, c.source, range(c), context_lines_inner=5)
                         println(io, "\n")
                     end
                 end
-                reduced_failures = reduce_text.(reduce_tree(text),
+                reduced_failures = reduce_text.(reduce_tree(text, equals_ref_parse=_lowprec_commas_equiv),
                                                 parsers_fuzzy_disagree)
                 append!(all_reduced_failures, reduced_failures)
                 @error("Parsers succeed but disagree",
@@ -51,15 +68,6 @@ Logging.with_logger(TerminalLogger()) do
             err isa InterruptException && rethrow()
             ex = (err, catch_backtrace())
             push!(exceptions, ex)
-            ref_parse = "success"
-            if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete))
-                ref_parse = "fail"
-                if err isa JuliaSyntax.ParseError
-                    # Both parsers agree that there's an error, and
-                    # JuliaSyntax didn't have an internal error.
-                    continue
-                end
-            end
 
             exception_count += 1
             parse_to_syntax = "success"