Skip to content

Commit 3233676

Browse files
c42fc42f
authored
Merge pull request #214 from JuliaLang/c42f/various-fixes
Add basic benchmark script + other rearrangments
2 parents 97e2825 + 59e9176 commit 3233676

File tree

4 files changed

+131
-88
lines changed

4 files changed

+131
-88
lines changed

‎src/hooks.jl

Lines changed: 76 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -233,28 +233,6 @@ function _core_parser_hook(code, filename, lineno, offset, options)
233233
end
234234
end
235235

236-
# Call the flisp parser
237-
function _fl_parse_hook(code, filename, lineno, offset, options)
238-
@static if VERSION >= v"1.8.0-DEV.1370" # https://github.com/JuliaLang/julia/pull/43876
239-
return Core.Compiler.fl_parse(code, filename, lineno, offset, options)
240-
elseif VERSION >= v"1.6"
241-
return Core.Compiler.fl_parse(code, filename, offset, options)
242-
else
243-
if options === :all
244-
ex = Base.parse_input_line(String(code), filename=filename, depwarn=false)
245-
if !Meta.isexpr(ex, :toplevel)
246-
ex = Expr(:toplevel, ex)
247-
end
248-
return ex, sizeof(code)
249-
elseif options === :statement || options == :atom
250-
ex, pos = Meta.parse(code, offset+1, greedy=options==:statement, raise=false)
251-
return ex, pos-1
252-
else
253-
error("Unknown parse options $options")
254-
end
255-
end
256-
end
257-
258236
# Hack:
259237
# Meta.parse() attempts to construct a ParseError from a string if it receives
260238
# `Expr(:error)`. Add an override to the ParseError constructor to prevent this.
@@ -292,3 +270,79 @@ function enable_in_core!(enable=true; freeze_world_age = true,
292270
_set_core_parse_hook(enable ? core_parser_hook : _default_parser)
293271
nothing
294272
end
273+
274+
275+
#-------------------------------------------------------------------------------
276+
# Tools to call the reference flisp parser
277+
#
278+
# Call the flisp parser
279+
function _fl_parse_hook(code, filename, lineno, offset, options)
280+
@static if VERSION >= v"1.8.0-DEV.1370" # https://github.com/JuliaLang/julia/pull/43876
281+
return Core.Compiler.fl_parse(code, filename, lineno, offset, options)
282+
elseif VERSION >= v"1.6"
283+
return Core.Compiler.fl_parse(code, filename, offset, options)
284+
else
285+
if options === :all
286+
ex = Base.parse_input_line(String(code), filename=filename, depwarn=false)
287+
if !Meta.isexpr(ex, :toplevel)
288+
ex = Expr(:toplevel, ex)
289+
end
290+
return ex, sizeof(code)
291+
elseif options === :statement || options == :atom
292+
ex, pos = Meta.parse(code, offset+1, greedy=options==:statement, raise=false)
293+
return ex, pos-1
294+
else
295+
error("Unknown parse options $options")
296+
end
297+
end
298+
end
299+
300+
#------------------------------------------------
301+
# Copy of the Meta.parse() API, but ensuring that we call the flisp parser
302+
# rather than using Meta.parse() which may be using the JuliaSyntax parser.
303+
304+
"""
305+
Like Meta.parse() but always call the flisp reference parser.
306+
"""
307+
function fl_parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
308+
ex, pos = fl_parse(str, 1, greedy=true, raise=raise, depwarn=depwarn)
309+
if isa(ex,Expr) && ex.head === :error
310+
return ex
311+
end
312+
if pos <= ncodeunits(str)
313+
raise && throw(Meta.ParseError("extra token after end of expression"))
314+
return Expr(:error, "extra token after end of expression")
315+
end
316+
return ex
317+
end
318+
319+
function fl_parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
320+
depwarn::Bool=true)
321+
ex, pos = _fl_parse_string(str, "none", 1, pos, greedy ? :statement : :atom)
322+
if raise && isa(ex,Expr) && ex.head === :error
323+
throw(Meta.ParseError(ex.args[1]))
324+
end
325+
return ex, pos
326+
end
327+
328+
"""
329+
Like Meta.parseall() but always call the flisp reference parser.
330+
"""
331+
function fl_parseall(text::AbstractString; filename="none", lineno=1)
332+
ex,_ = _fl_parse_string(text, String(filename), lineno, 1, :all)
333+
return ex
334+
end
335+
336+
function _fl_parse_string(text::AbstractString, filename::AbstractString,
337+
lineno::Integer, index::Integer, options)
338+
if index < 1 || index > ncodeunits(text) + 1
339+
throw(BoundsError(text, index))
340+
end
341+
ex, offset::Int = _fl_parse_hook(text, filename, lineno, index-1, options)
342+
ex, offset+1
343+
end
344+
345+
# Convenience functions to mirror `JuliaSyntax.parse(Expr, ...)` in simple cases.
346+
fl_parse(::Type{Expr}, args...; kws...) = fl_parse(args...; kws...)
347+
fl_parseall(::Type{Expr}, args...; kws...) = fl_parseall(args...; kws...)
348+

‎src/parser_api.jl

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ _parse_docs = """
104104
version=VERSION,
105105
ignore_trivia=true,
106106
filename=nothing,
107-
ignore_warnings=false)
107+
ignore_errors=false,
108+
ignore_warnings=ignore_errors)
108109
109110
# Or, with the same arguments
110111
parseall(...)
@@ -131,7 +132,8 @@ tree, if applicable. This will also annotate errors and warnings with the
131132
source file name.
132133
133134
A `ParseError` will be thrown if any errors or warnings occurred during
134-
parsing. To avoid exceptions due to warnings, use `ignore_warnings=true`.
135+
parsing. To avoid exceptions due to warnings, use `ignore_warnings=true`. To
136+
also avoid exceptions due to errors, use `ignore_errors=true`.
135137
"""
136138

137139
parse(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:statement, true, T, text; kws...)[1]

‎src/utils.jl

Lines changed: 17 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,23 @@ macro check(ex, msgs...)
3434
return :($(esc(ex)) ? nothing : internal_error($msg))
3535
end
3636

37+
# Really remove line numbers, even from Expr(:toplevel)
38+
remove_linenums!(ex) = ex
39+
function remove_linenums!(ex::Expr)
40+
if ex.head === :block || ex.head === :quote || ex.head === :toplevel
41+
filter!(ex.args) do x
42+
!(isa(x, Expr) && x.head === :line || isa(x, LineNumberNode))
43+
end
44+
end
45+
for subex in ex.args
46+
subex isa Expr && remove_linenums!(subex)
47+
end
48+
return ex
49+
end
50+
51+
52+
#-------------------------------------------------------------------------------
53+
# Text printing/display utils
3754

3855
"""
3956
Like printstyled, but allows providing RGB colors for true color terminals
@@ -61,67 +78,3 @@ function _printstyled(io::IO, text; fgcolor=nothing, bgcolor=nothing)
6178
end
6279
end
6380

64-
# Really remove line numbers, even from Expr(:toplevel)
65-
remove_linenums!(ex) = ex
66-
function remove_linenums!(ex::Expr)
67-
if ex.head === :block || ex.head === :quote || ex.head === :toplevel
68-
filter!(ex.args) do x
69-
!(isa(x, Expr) && x.head === :line || isa(x, LineNumberNode))
70-
end
71-
end
72-
for subex in ex.args
73-
subex isa Expr && remove_linenums!(subex)
74-
end
75-
return ex
76-
end
77-
78-
79-
#-------------------------------------------------------------------------------
80-
# Copy of the Meta.parse() API, but ensuring that we call the flisp parser
81-
# rather than using Meta.parse() which may be using the JuliaSyntax parser.
82-
83-
"""
84-
Like Meta.parse() but always call the flisp reference parser.
85-
"""
86-
function fl_parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
87-
ex, pos = fl_parse(str, 1, greedy=true, raise=raise, depwarn=depwarn)
88-
if isa(ex,Expr) && ex.head === :error
89-
return ex
90-
end
91-
if pos <= ncodeunits(str)
92-
raise && throw(Meta.ParseError("extra token after end of expression"))
93-
return Expr(:error, "extra token after end of expression")
94-
end
95-
return ex
96-
end
97-
98-
function fl_parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
99-
depwarn::Bool=true)
100-
ex, pos = _fl_parse_string(str, "none", 1, pos, greedy ? :statement : :atom)
101-
if raise && isa(ex,Expr) && ex.head === :error
102-
throw(Meta.ParseError(ex.args[1]))
103-
end
104-
return ex, pos
105-
end
106-
107-
"""
108-
Like Meta.parseall() but always call the flisp reference parser.
109-
"""
110-
function fl_parseall(text::AbstractString; filename="none", lineno=1)
111-
ex,_ = _fl_parse_string(text, String(filename), lineno, 1, :all)
112-
return ex
113-
end
114-
115-
function _fl_parse_string(text::AbstractString, filename::AbstractString,
116-
lineno::Integer, index::Integer, options)
117-
if index < 1 || index > ncodeunits(text) + 1
118-
throw(BoundsError(text, index))
119-
end
120-
ex, offset::Int = _fl_parse_hook(text, filename, lineno, index-1, options)
121-
ex, offset+1
122-
end
123-
124-
# Convenience functions to mirror `JuliaSyntax.parse(Expr, ...)` in simple cases.
125-
fl_parse(::Type{Expr}, args...; kws...) = fl_parse(args...; kws...)
126-
fl_parseall(::Type{Expr}, args...; kws...) = fl_parseall(args...; kws...)
127-

‎test/benchmark.jl

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using BenchmarkTools
2+
using JuliaSyntax
3+
4+
include("test_utils.jl")
5+
6+
function concat_base()
7+
basedir = joinpath(Sys.BINDIR, "..", "share", "julia", "base")
8+
io = IOBuffer()
9+
for f in find_source_in_path(basedir)
10+
write(io, read(f, String))
11+
println(io)
12+
end
13+
return String(take!(io))
14+
end
15+
16+
all_base_code = concat_base()
17+
18+
b_ParseStream = @benchmark JuliaSyntax.parse!(JuliaSyntax.ParseStream(all_base_code), rule=:toplevel)
19+
b_GreenNode = @benchmark JuliaSyntax.parseall(JuliaSyntax.GreenNode, all_base_code)
20+
b_SyntaxNode = @benchmark JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, all_base_code)
21+
b_Expr = @benchmark JuliaSyntax.parseall(Expr, all_base_code)
22+
23+
@info "Benchmarks" ParseStream=b_ParseStream GreenNode=b_GreenNode SyntaxNode=b_SyntaxNode Expr=b_Expr
24+
25+
26+
# Allocation profiling
27+
#
28+
# using Profile.Allocs
29+
# using PProf
30+
# Allocs.clear()
31+
# stream = JuliaSyntax.ParseStream(text);
32+
# JuliaSyntax.peek(stream);
33+
# Allocs.@profile sample_rate=1 JuliaSyntax.parse(stream)
34+
# PProf.Allocs.pprof()

0 commit comments

Comments
 (0)