Skip to content

Commit 0976ded

Browse files
c42fc42f
committed
Generate line numbers in Expr conversion
This should give proper line number information when using JuliaSyntax via the Julia runtime. And hopefully allow tools like Revise.jl to work reliably.
1 parent 0f2a1bb commit 0976ded

File tree

4 files changed

+193
-40
lines changed

4 files changed

+193
-40
lines changed

‎src/expr.jl

Lines changed: 49 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ function is_eventually_call(ex)
66
is_eventually_call(ex.args[1]))
77
end
88

9-
function _to_expr(node::SyntaxNode, iteration_spec=false)
9+
function _to_expr(node::SyntaxNode, iteration_spec=false, need_linenodes=true)
1010
if !haschildren(node)
1111
if node.val isa Union{Int128,UInt128,BigInt}
1212
# Ignore the values of large integers and convert them back to
@@ -27,12 +27,32 @@ function _to_expr(node::SyntaxNode, iteration_spec=false)
2727
headsym = !isnothing(headstr) ? Symbol(headstr) :
2828
error("Can't untokenize head of kind $(kind(node))")
2929
node_args = children(node)
30-
args = Vector{Any}(undef, length(node_args))
30+
insert_linenums = (headsym == :block || headsym == :toplevel) && need_linenodes
31+
args = Vector{Any}(undef, length(node_args)*(insert_linenums ? 2 : 1))
3132
if headsym == :for && length(node_args) == 2
32-
args[1] = _to_expr(node_args[1], true)
33-
args[2] = _to_expr(node_args[2], false)
33+
# No line numbers in for loop iteration spec
34+
args[1] = _to_expr(node_args[1], true, false)
35+
args[2] = _to_expr(node_args[2])
36+
elseif headsym == :let && length(node_args) == 2
37+
# No line numbers in let statement binding list
38+
args[1] = _to_expr(node_args[1], false, false)
39+
args[2] = _to_expr(node_args[2])
3440
else
35-
map!(_to_expr, args, node_args)
41+
if insert_linenums
42+
if isempty(node_args)
43+
push!(args, source_location(LineNumberNode, node.source, node.position))
44+
else
45+
for i in 1:length(node_args)
46+
n = node_args[i]
47+
args[2*i-1] = source_location(LineNumberNode, n.source, n.position)
48+
args[2*i] = _to_expr(n)
49+
end
50+
end
51+
else
52+
for i in 1:length(node_args)
53+
args[i] = _to_expr(node_args[i])
54+
end
55+
end
3656
end
3757
# Julia's standard `Expr` ASTs have children stored in a canonical
3858
# order which is often not always source order. We permute the children
@@ -136,34 +156,41 @@ function _to_expr(node::SyntaxNode, iteration_spec=false)
136156
# Strip string from interpolations in 1.5 and lower to preserve
137157
# "hi$("ho")" ==> (string "hi" "ho")
138158
elseif headsym == :(=)
139-
if is_eventually_call(args[1]) && !iteration_spec
140-
if Meta.isexpr(args[2], :block)
141-
pushfirst!(args[2].args, loc)
142-
else
143-
# Add block for short form function locations
144-
args[2] = Expr(:block, loc, args[2])
145-
end
159+
if is_eventually_call(args[1]) && !iteration_spec && !Meta.isexpr(args[2], :block)
160+
# Add block for short form function locations
161+
args[2] = Expr(:block, loc, args[2])
146162
end
147163
elseif headsym == :elseif
148-
# Compat wart: add a block for the elseif conditional
149-
args[1] = Expr(:block, args[1])
164+
# Block for conditional's source location
165+
args[1] = Expr(:block, loc, args[1])
150166
elseif headsym == :(->)
151167
if Meta.isexpr(args[2], :block)
152-
pushfirst!(args[2].args, loc)
168+
if node.parent isa SyntaxNode && kind(node.parent) != K"do"
169+
pushfirst!(args[2].args, loc)
170+
end
153171
else
154172
# Add block for source locations
155173
args[2] = Expr(:block, loc, args[2])
156174
end
157175
elseif headsym == :function
158-
if length(args) > 1 && Meta.isexpr(args[1], :tuple)
159-
# Convert to weird Expr forms for long-form anonymous functions.
160-
#
161-
# (function (tuple (... xs)) body) ==> (function (... xs) body)
162-
if length(args[1].args) == 1 && Meta.isexpr(args[1].args[1], :...)
163-
# function (xs...) \n body end
164-
args[1] = args[1].args[1]
176+
if length(args) > 1
177+
if Meta.isexpr(args[1], :tuple)
178+
# Convert to weird Expr forms for long-form anonymous functions.
179+
#
180+
# (function (tuple (... xs)) body) ==> (function (... xs) body)
181+
if length(args[1].args) == 1 && Meta.isexpr(args[1].args[1], :...)
182+
# function (xs...) \n body end
183+
args[1] = args[1].args[1]
184+
end
165185
end
186+
pushfirst!(args[2].args, loc)
187+
end
188+
elseif headsym == :macro
189+
if length(args) > 1
190+
pushfirst!(args[2].args, loc)
166191
end
192+
elseif headsym == :module
193+
pushfirst!(args[3].args, loc)
167194
end
168195
if headsym == :inert || (headsym == :quote && length(args) == 1 &&
169196
!(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||

‎test/expr.jl

Lines changed: 115 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,114 @@
99
@test parseall(Expr, ":true", rule=:atom) == Expr(:quote, true)
1010
end
1111

12+
@testset "Line numbers" begin
13+
@testset "Blocks" begin
14+
@test parseall(Expr, "begin a\nb\n\nc\nend", rule=:statement) ==
15+
Expr(:block,
16+
LineNumberNode(1),
17+
:a,
18+
LineNumberNode(2),
19+
:b,
20+
LineNumberNode(4),
21+
:c,
22+
)
23+
@test parseall(Expr, "begin end", rule=:statement) ==
24+
Expr(:block,
25+
LineNumberNode(1)
26+
)
27+
28+
@test parseall(Expr, "a\n\nb") ==
29+
Expr(:toplevel,
30+
LineNumberNode(1),
31+
:a,
32+
LineNumberNode(3),
33+
:b,
34+
)
35+
36+
@test parseall(Expr, "module A\n\nbody\nend", rule=:statement) ==
37+
Expr(:module,
38+
true,
39+
:A,
40+
Expr(:block,
41+
LineNumberNode(1),
42+
LineNumberNode(3),
43+
:body,
44+
),
45+
)
46+
end
47+
48+
@testset "Function definition lines" begin
49+
@test parseall(Expr, "function f()\na\n\nb\nend", rule=:statement) ==
50+
Expr(:function,
51+
Expr(:call, :f),
52+
Expr(:block,
53+
LineNumberNode(1),
54+
LineNumberNode(2),
55+
:a,
56+
LineNumberNode(4),
57+
:b,
58+
)
59+
)
60+
@test parseall(Expr, "f() = 1", rule=:statement) ==
61+
Expr(:(=),
62+
Expr(:call, :f),
63+
Expr(:block,
64+
LineNumberNode(1),
65+
1
66+
)
67+
)
68+
69+
# function/macro without methods
70+
@test parseall(Expr, "function f end", rule=:statement) ==
71+
Expr(:function, :f)
72+
@test parseall(Expr, "macro f end", rule=:statement) ==
73+
Expr(:macro, :f)
74+
end
75+
76+
@testset "elseif" begin
77+
@test parseall(Expr, "if a\nb\nelseif c\n d\nend", rule=:statement) ==
78+
Expr(:if,
79+
:a,
80+
Expr(:block,
81+
LineNumberNode(2),
82+
:b),
83+
Expr(:elseif,
84+
Expr(:block,
85+
LineNumberNode(3), # Line number for elseif condition
86+
:c),
87+
Expr(:block,
88+
LineNumberNode(4),
89+
:d),
90+
)
91+
)
92+
end
93+
94+
@testset "No line numbers in for/let bindings" begin
95+
@test parseall(Expr, "for i=is, j=js\nbody\nend", rule=:statement) ==
96+
Expr(:for,
97+
Expr(:block,
98+
Expr(:(=), :i, :is),
99+
Expr(:(=), :j, :js),
100+
),
101+
Expr(:block,
102+
LineNumberNode(2),
103+
:body
104+
)
105+
)
106+
@test parseall(Expr, "let i=is, j=js\nbody\nend", rule=:statement) ==
107+
Expr(:let,
108+
Expr(:block,
109+
Expr(:(=), :i, :is),
110+
Expr(:(=), :j, :js),
111+
),
112+
Expr(:block,
113+
LineNumberNode(2),
114+
:body
115+
)
116+
)
117+
end
118+
end
119+
12120
@testset "Short form function line numbers" begin
13121
# A block is added to hold the line number node
14122
@test parseall(Expr, "f() = xs", rule=:statement) ==
@@ -22,13 +130,18 @@
22130
@test parseall(Expr, "for f() = xs\nend", rule=:statement) ==
23131
Expr(:for,
24132
Expr(:(=), Expr(:call, :f), :xs),
25-
Expr(:block))
133+
Expr(:block,
134+
LineNumberNode(1)
135+
))
26136
end
27137

28138
@testset "Long form anonymous functions" begin
29139
@test parseall(Expr, "function (xs...)\nbody end", rule=:statement) ==
30140
Expr(:function,
31141
Expr(:..., :xs),
32-
Expr(:block, :body))
142+
Expr(:block,
143+
LineNumberNode(1),
144+
LineNumberNode(2),
145+
:body))
33146
end
34147
end

‎test/parser_api.jl

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
@testset "parser API" begin
22
@testset "String and buffer input" begin
33
# String
4-
@test parse(Expr, "x+y\nz") == (Expr(:toplevel, :(x+y), :z), [], 6)
4+
let
5+
ex,diag,pos = parse(Expr, "x+y\nz")
6+
@test JuliaSyntax.remove_linenums!(ex) == Expr(:toplevel, :(x+y), :z)
7+
@test diag == []
8+
@test pos == 6
9+
end
510
@test parse(Expr, "x+y\nz", rule=:statement) == (:(x+y), [], 4)
611
@test parse(Expr, "x+y\nz", rule=:atom) == (:x, [], 2)
712
@test parse(Expr, "x+y\nz", 5, rule=:atom) == (:z, [], 6)
@@ -56,7 +61,7 @@
5661
end
5762

5863
@testset "parseall" begin
59-
@test parseall(Expr, " x ") == Expr(:toplevel, :x)
64+
@test JuliaSyntax.remove_linenums!(parseall(Expr, " x ")) == Expr(:toplevel, :x)
6065
@test parseall(Expr, " x ", rule=:statement) == :x
6166
@test parseall(Expr, " x ", rule=:atom) == :x
6267
# TODO: Fix this situation with trivia here; the brackets are trivia, but

‎test/test_utils.jl

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,24 @@ function remove_all_linenums!(ex)
3838
remove_macro_linenums!(ex)
3939
end
4040

41-
function parsers_agree_on_file(filename)
41+
function show_expr_text_diff(showfunc, ex, f_ex; context=2)
42+
if Sys.isunix()
43+
mktemp() do path1, io1
44+
mktemp() do path2, io2
45+
showfunc(io1, ex); close(io1)
46+
showfunc(io2, f_ex); close(io2)
47+
run(ignorestatus(`diff -U$context --color=always $path1 $path2`))
48+
end
49+
end
50+
else
51+
showfunc(stdout, ex)
52+
println("------------------------------------")
53+
showfunc(stdout, f_ex)
54+
end
55+
end
56+
57+
58+
function parsers_agree_on_file(filename; show_diff=false)
4259
text = try
4360
read(filename, String)
4461
catch
@@ -55,6 +72,9 @@ function parsers_agree_on_file(filename)
5572
end
5673
try
5774
ex, diagnostics, _ = parse(Expr, text, filename=filename)
75+
if show_diff && ex != fl_ex
76+
show_expr_text_diff(show, ex, fl_ex)
77+
end
5878
return !JuliaSyntax.any_error(diagnostics) &&
5979
JuliaSyntax.remove_linenums!(ex) ==
6080
JuliaSyntax.remove_linenums!(fl_ex)
@@ -223,19 +243,7 @@ function itest_parse(production, code; version::VersionNumber=v"1.6")
223243
show(stdout, MIME"text/plain"(), f_ex)
224244

225245
printstyled(stdout, "\n\n# Diff of AST dump:\n", color=:red)
226-
if Sys.isunix()
227-
mktemp() do path1, io1
228-
mktemp() do path2, io2
229-
dump(io1, ex); close(io1)
230-
dump(io2, f_ex); close(io2)
231-
run(ignorestatus(`diff -U10 --color=always $path1 $path2`))
232-
end
233-
end
234-
else
235-
dump(ex)
236-
println("------------------------------------")
237-
dump(f_ex)
238-
end
246+
show_expr_text_diff(showfunc, ex, f_ex, context=10)
239247
# return (ex, f_ex)
240248
# return (code, stream, t, s, ex)
241249
end

0 commit comments

Comments
 (0)