Skip to content

Commit 997d964

Browse files
authored
Fixes for parse check of General registry (#135)
* Fix for new `parseall()` API * Reduce false positives by not reporting cases where reference parser also fails. * Ensure we use the reference parser via `fl_parseall`, as Meta.parse may have been substituted * Separate out untarring * Also delete old hacky parser conversion tool
1 parent 9f5c892 commit 997d964

File tree

3 files changed

+38
-131
lines changed

3 files changed

+38
-131
lines changed

tools/check_all_packages.jl

Lines changed: 16 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -5,50 +5,13 @@
55

66
using JuliaSyntax, Logging
77

8-
# like Meta.parseall, but throws
9-
function parseall_throws(str)
10-
pos = firstindex(str)
11-
exs = []
12-
while pos <= lastindex(str)
13-
ex, pos = Meta.parse(str, pos)
14-
push!(exs, ex)
15-
end
16-
if length(exs) == 0
17-
throw(Meta.ParseError("end of input"))
18-
elseif length(exs) == 1
19-
return exs[1]
20-
else
21-
return Expr(:toplevel, exs...)
22-
end
23-
end
24-
258
logio = open(joinpath(@__DIR__, "logs.txt"), "w")
269
logger = Logging.ConsoleLogger(logio)
2710

2811
pkgspath = joinpath(@__DIR__, "pkgs")
2912

30-
parallel = 50
3113
exceptions = []
3214
Logging.with_logger(logger) do
33-
for tars in Iterators.partition(readdir(pkgspath), parallel)
34-
@sync for tar in tars
35-
endswith(tar, ".tgz") || continue
36-
@async begin
37-
dir = joinpath(@__DIR__, "pkgs", replace(tar, r"\.tgz$" => ""))
38-
if !isdir(dir) || !isdir(joinpath(dir, "src"))
39-
rm(dir; recursive=true, force=true)
40-
mkpath(dir)
41-
tar_path = joinpath(@__DIR__, "pkgs", tar)
42-
try
43-
run(`tar -xf $tar_path -C $dir`)
44-
catch err
45-
@error "could not untar $tar_path"
46-
end
47-
end
48-
end
49-
end
50-
end
51-
5215
t = time()
5316
i = 0
5417
iob = IOBuffer()
@@ -60,35 +23,31 @@ Logging.with_logger(logger) do
6023
if isfile(fpath)
6124
file = read(fpath, String)
6225
try
63-
e1 = JuliaSyntax.parse(Expr, file)
26+
e1 = JuliaSyntax.parseall(Expr, file)
6427
catch err
6528
err isa InterruptException && rethrow()
66-
ex_count += 1
6729
ex = (err, catch_backtrace())
6830
push!(exceptions, ex)
69-
meta_parse = "success"
70-
try
71-
parseall_throws(file)
72-
catch err2
73-
meta_parse = "fail"
74-
ex_count -= 1
31+
ref_parse = "success"
32+
e2 = JuliaSyntax.fl_parseall(file)
33+
@assert Meta.isexpr(e2, :toplevel)
34+
if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete))
35+
ref_parse = "fail"
36+
if err isa JuliaSyntax.ParseError
37+
# Both parsers agree that there's an error, and
38+
# JuliaSyntax didn't have an internal error.
39+
continue
40+
end
7541
end
42+
43+
ex_count += 1
7644
parse_to_syntax = "success"
7745
try
78-
JuliaSyntax.parse(JuliaSyntax.SyntaxNode, file)
46+
JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, file)
7947
catch err2
8048
parse_to_syntax = "fail"
8149
end
82-
severity = parse_to_syntax == "fail" ? "error" :
83-
meta_parse == "fail" ? "warn" : "error"
84-
println(logio, """
85-
[$(severity)] $(fpath)
86-
parse-to-expr: fail
87-
parse-to-syntaxtree: $(parse_to_syntax)
88-
reference: $(meta_parse)
89-
""")
90-
@error "" exception = ex
91-
flush(logio)
50+
@error "Parse failed" fpath exception=ex parse_to_syntax
9251
end
9352
end
9453
i += 1
@@ -97,7 +56,7 @@ Logging.with_logger(logger) do
9756
avg = round(runtime/i*1000, digits = 2)
9857
print(iob, "\e[2J\e[0;0H")
9958
println(iob, "$i files parsed")
100-
println(iob, "> $(ex_count) failures compared to Meta.parse")
59+
println(iob, "> $(ex_count) failures compared to reference parser")
10160
println(iob, "> $(length(exceptions)) errors in total")
10261
println(iob, "> $(avg)ms per file, $(round(Int, runtime))s in total")
10362
println(stderr, String(take!(iob)))

tools/flisp_defines_to_julia.jl

Lines changed: 0 additions & 74 deletions
This file was deleted.

tools/untar_packages.jl

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
2+
pkgspath = joinpath(@__DIR__, "pkgs")
3+
4+
for tars in Iterators.partition(readdir(pkgspath), 50)
5+
@sync for tar in tars
6+
endswith(tar, ".tgz") || continue
7+
@async begin
8+
dir = joinpath(@__DIR__, "pkgs", replace(tar, r"\.tgz$" => ""))
9+
if !isdir(dir) || !isdir(joinpath(dir, "src"))
10+
rm(dir; recursive=true, force=true)
11+
mkpath(dir)
12+
tar_path = joinpath(@__DIR__, "pkgs", tar)
13+
try
14+
run(`tar -xf $tar_path -C $dir`)
15+
catch err
16+
@error "could not untar $tar_path"
17+
end
18+
end
19+
end
20+
end
21+
end
22+

0 commit comments

Comments
 (0)