Fixes for parse check of General registry (#135)

* Fix for new `parseall()` API
* Reduce false positives by not reporting cases where reference parser
  also fails.
* Ensure we use the reference parser via `fl_parseall`, as Meta.parse
  may have been substituted
* Separate out untarring
* Also delete old hacky parser conversion tool

Author: c42f

tools/check_all_packages.jl

@@ -5,50 +5,13 @@
 
 using JuliaSyntax, Logging
 
-# like Meta.parseall, but throws
-function parseall_throws(str)
-    pos = firstindex(str)
-    exs = []
-    while pos <= lastindex(str)
-        ex, pos = Meta.parse(str, pos)
-        push!(exs, ex)
-    end
-    if length(exs) == 0
-        throw(Meta.ParseError("end of input"))
-    elseif length(exs) == 1
-        return exs[1]
-    else
-        return Expr(:toplevel, exs...)
-    end
-end
-
 logio = open(joinpath(@__DIR__, "logs.txt"), "w")
 logger = Logging.ConsoleLogger(logio)
 
 pkgspath = joinpath(@__DIR__, "pkgs")
 
-parallel = 50
 exceptions = []
 Logging.with_logger(logger) do
-    for tars in Iterators.partition(readdir(pkgspath), parallel)
-        @sync for tar in tars
-            endswith(tar, ".tgz") || continue
-            @async begin
-                dir = joinpath(@__DIR__, "pkgs", replace(tar, r"\.tgz$" => ""))
-                if !isdir(dir) || !isdir(joinpath(dir, "src"))
-                    rm(dir; recursive=true, force=true)
-                    mkpath(dir)
-                    tar_path = joinpath(@__DIR__, "pkgs", tar)
-                    try
-                        run(`tar -xf $tar_path -C $dir`)
-                    catch err
-                        @error "could not untar $tar_path"
-                    end
-                end
-            end
-        end
-    end
-
     t = time()
     i = 0
     iob = IOBuffer()
@@ -60,35 +23,31 @@ Logging.with_logger(logger) do
             if isfile(fpath)
                 file = read(fpath, String)
                 try
-                    e1 = JuliaSyntax.parse(Expr, file)
+                    e1 = JuliaSyntax.parseall(Expr, file)
                 catch err
                     err isa InterruptException && rethrow()
-                    ex_count += 1
                     ex = (err, catch_backtrace())
                     push!(exceptions, ex)
-                    meta_parse = "success"
-                    try
-                        parseall_throws(file)
-                    catch err2
-                        meta_parse = "fail"
-                        ex_count -= 1
+                    ref_parse = "success"
+                    e2 = JuliaSyntax.fl_parseall(file)
+                    @assert Meta.isexpr(e2, :toplevel)
+                    if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete))
+                        ref_parse = "fail"
+                        if err isa JuliaSyntax.ParseError
+                            # Both parsers agree that there's an error, and
+                            # JuliaSyntax didn't have an internal error.
+                            continue
+                        end
                     end
+
+                    ex_count += 1
                     parse_to_syntax = "success"
                     try
-                        JuliaSyntax.parse(JuliaSyntax.SyntaxNode, file)
+                        JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, file)
                     catch err2
                         parse_to_syntax = "fail"
                     end
-                    severity = parse_to_syntax == "fail" ? "error" :
-                        meta_parse == "fail" ? "warn" : "error"
-                    println(logio, """
-                    [$(severity)] $(fpath)
-                      parse-to-expr: fail
-                      parse-to-syntaxtree: $(parse_to_syntax)
-                      reference: $(meta_parse)
-                    """)
-                    @error "" exception = ex
-                    flush(logio)
+                    @error "Parse failed" fpath exception=ex parse_to_syntax
                 end
             end
             i += 1
@@ -97,7 +56,7 @@ Logging.with_logger(logger) do
                 avg = round(runtime/i*1000, digits = 2)
                 print(iob, "\e[2J\e[0;0H")
                 println(iob, "$i files parsed")
-                println(iob, "> $(ex_count) failures compared to Meta.parse")
+                println(iob, "> $(ex_count) failures compared to reference parser")
                 println(iob, "> $(length(exceptions)) errors in total")
                 println(iob, "> $(avg)ms per file, $(round(Int, runtime))s in total")
                 println(stderr, String(take!(iob)))

tools/flisp_defines_to_julia.jl

@@ -0,0 +1,22 @@
+
+pkgspath = joinpath(@__DIR__, "pkgs")
+
+for tars in Iterators.partition(readdir(pkgspath), 50)
+    @sync for tar in tars
+        endswith(tar, ".tgz") || continue
+        @async begin
+            dir = joinpath(@__DIR__, "pkgs", replace(tar, r"\.tgz$" => ""))
+            if !isdir(dir) || !isdir(joinpath(dir, "src"))
+                rm(dir; recursive=true, force=true)
+                mkpath(dir)
+                tar_path = joinpath(@__DIR__, "pkgs", tar)
+                try
+                    run(`tar -xf $tar_path -C $dir`)
+                catch err
+                    @error "could not untar $tar_path"
+                end
+            end
+        end
+    end
+end
+