Skip to content

Commit 6d446bb

Browse files
c42fc42f
authored
Merge pull request #27 from JuliaLang/cjf/expr-cleanup
Move Expr-related code to expr.jl
2 parents c50e1fe + c78756e commit 6d446bb

File tree

4 files changed

+198
-191
lines changed

4 files changed

+198
-191
lines changed

‎src/JuliaSyntax.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ include("value_parsing.jl")
2929
# Tree data structures
3030
include("green_tree.jl")
3131
include("syntax_tree.jl")
32+
include("expr.jl")
3233

3334
# Hooks to integrate the parser with Base
3435
include("hooks.jl")

‎src/expr.jl

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
#-------------------------------------------------------------------------------
2+
# Conversion to Base.Expr
3+
4+
function is_eventually_call(ex)
5+
return Meta.isexpr(ex, :call) || (Meta.isexpr(ex, (:where, :(::))) &&
6+
is_eventually_call(ex.args[1]))
7+
end
8+
9+
function _to_expr(node::SyntaxNode, iteration_spec=false)
10+
if !haschildren(node)
11+
if node.val isa Union{Int128,UInt128,BigInt}
12+
# Ignore the values of large integers and convert them back to
13+
# symbolic/textural form for compatibility with the Expr
14+
# representation of these.
15+
str = replace(sourcetext(node), '_'=>"")
16+
headsym = :macrocall
17+
k = kind(node)
18+
macname = node.val isa Int128 ? Symbol("@int128_str") :
19+
node.val isa UInt128 ? Symbol("@uint128_str") :
20+
Symbol("@big_str")
21+
return Expr(:macrocall, GlobalRef(Core, macname), nothing, str)
22+
else
23+
return node.val
24+
end
25+
end
26+
headstr = untokenize(head(node), include_flag_suff=false)
27+
headsym = !isnothing(headstr) ? Symbol(headstr) :
28+
error("Can't untokenize head of kind $(kind(node))")
29+
node_args = children(node)
30+
args = Vector{Any}(undef, length(node_args))
31+
if headsym == :for && length(node_args) == 2
32+
args[1] = _to_expr(node_args[1], true)
33+
args[2] = _to_expr(node_args[2], false)
34+
else
35+
map!(_to_expr, args, node_args)
36+
end
37+
# Julia's standard `Expr` ASTs have children stored in a canonical
38+
# order which is often not always source order. We permute the children
39+
# here as necessary to get the canonical order.
40+
if is_infix(node.raw)
41+
args[2], args[1] = args[1], args[2]
42+
end
43+
loc = source_location(LineNumberNode, node.source, node.position)
44+
# Convert elements
45+
if headsym == :macrocall
46+
insert!(args, 2, loc)
47+
elseif headsym in (:call, :ref)
48+
# Move parameters block to args[2]
49+
if length(args) > 1 && Meta.isexpr(args[end], :parameters)
50+
insert!(args, 2, args[end])
51+
pop!(args)
52+
end
53+
elseif headsym in (:tuple, :parameters, :vect)
54+
# Move parameters blocks to args[1]
55+
if length(args) > 1 && Meta.isexpr(args[end], :parameters)
56+
pushfirst!(args, args[end])
57+
pop!(args)
58+
end
59+
elseif headsym == :try
60+
# Try children in source order:
61+
# try_block catch_var catch_block else_block finally_block
62+
# Expr ordering:
63+
# try_block catch_var catch_block [finally_block] [else_block]
64+
catch_ = nothing
65+
if has_flags(node, TRY_CATCH_AFTER_FINALLY_FLAG)
66+
catch_ = pop!(args)
67+
catch_var = pop!(args)
68+
end
69+
finally_ = pop!(args)
70+
else_ = pop!(args)
71+
if has_flags(node, TRY_CATCH_AFTER_FINALLY_FLAG)
72+
pop!(args)
73+
pop!(args)
74+
push!(args, catch_var)
75+
push!(args, catch_)
76+
end
77+
# At this point args is
78+
# [try_block catch_var catch_block]
79+
if finally_ !== false
80+
push!(args, finally_)
81+
end
82+
if else_ !== false
83+
push!(args, else_)
84+
end
85+
elseif headsym == :filter
86+
pushfirst!(args, last(args))
87+
pop!(args)
88+
elseif headsym == :flatten
89+
# The order of nodes inside the generators in Julia's flatten AST
90+
# is noncontiguous in the source text, so need to reconstruct
91+
# Julia's AST here from our alternative `flatten` expression.
92+
gen = Expr(:generator, args[1], args[end])
93+
for i in length(args)-1:-1:2
94+
gen = Expr(:flatten, Expr(:generator, gen, args[i]))
95+
end
96+
return gen
97+
elseif headsym in (:nrow, :ncat)
98+
# For lack of a better place, the dimension argument to nrow/ncat
99+
# is stored in the flags
100+
pushfirst!(args, numeric_flags(flags(node)))
101+
elseif headsym == :typed_ncat
102+
insert!(args, 2, numeric_flags(flags(node)))
103+
elseif headsym == :string && length(args) > 1
104+
# Julia string literals may be interspersed with trivia in two situations:
105+
# 1. Triple quoted string indentation is trivia
106+
# 2. An \ before newline removes the newline and any following indentation
107+
#
108+
# Such trivia is eagerly removed by the reference parser, so here we
109+
# concatenate adjacent string chunks together for compatibility.
110+
#
111+
# TODO: Manage the non-interpolation cases with String and CmdString
112+
# kinds instead?
113+
args2 = Vector{Any}()
114+
i = 1
115+
while i <= length(args)
116+
if args[i] isa String && i < length(args) && args[i+1] isa String
117+
buf = IOBuffer()
118+
while i <= length(args) && args[i] isa String
119+
write(buf, args[i])
120+
i += 1
121+
end
122+
push!(args2, String(take!(buf)))
123+
else
124+
push!(args2, args[i])
125+
i += 1
126+
end
127+
end
128+
args = args2
129+
if length(args2) == 1 && args2[1] isa String
130+
# If there's a single string remaining after joining we unwrap to
131+
# give a string literal.
132+
# """\n a\n b""" ==> "a\nb"
133+
return args2[1]
134+
end
135+
# elseif headsym == :string && length(args) == 1 && version <= (1,5)
136+
# Strip string from interpolations in 1.5 and lower to preserve
137+
# "hi$("ho")" ==> (string "hi" "ho")
138+
elseif headsym == :(=)
139+
if is_eventually_call(args[1]) && !iteration_spec
140+
if Meta.isexpr(args[2], :block)
141+
pushfirst!(args[2].args, loc)
142+
else
143+
# Add block for short form function locations
144+
args[2] = Expr(:block, loc, args[2])
145+
end
146+
end
147+
elseif headsym == :(->)
148+
if Meta.isexpr(args[2], :block)
149+
pushfirst!(args[2].args, loc)
150+
else
151+
# Add block for source locations
152+
args[2] = Expr(:block, loc, args[2])
153+
end
154+
elseif headsym == :function
155+
if length(args) > 1 && Meta.isexpr(args[1], :tuple)
156+
# Convert to weird Expr forms for long-form anonymous functions.
157+
#
158+
# (function (tuple (... xs)) body) ==> (function (... xs) body)
159+
if length(args[1].args) == 1 && Meta.isexpr(args[1].args[1], :...)
160+
# function (xs...) \n body end
161+
args[1] = args[1].args[1]
162+
end
163+
end
164+
end
165+
if headsym == :inert || (headsym == :quote && length(args) == 1 &&
166+
!(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
167+
a1 isa Bool # <- compat hack, Julia 1.4+
168+
))
169+
return QuoteNode(only(args))
170+
else
171+
return Expr(headsym, args...)
172+
end
173+
end
174+
175+
Base.Expr(node::SyntaxNode) = _to_expr(node)
176+
177+
function build_tree(::Type{Expr}, stream::ParseStream; kws...)
178+
Expr(build_tree(SyntaxNode, stream; kws...))
179+
end
180+

‎src/syntax_tree.jl

Lines changed: 6 additions & 190 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,12 @@ function Base.push!(node::SyntaxNode, child::SyntaxNode)
181181
push!(args, child)
182182
end
183183

184+
function build_tree(::Type{SyntaxNode}, stream::ParseStream; filename=nothing, kws...)
185+
green_tree = build_tree(GreenNode, stream; kws...)
186+
source = SourceFile(sourcetext(stream), filename=filename)
187+
SyntaxNode(source, green_tree, first_byte(stream))
188+
end
189+
184190
#-------------------------------------------------------------------------------
185191
# Tree utilities
186192

@@ -251,193 +257,3 @@ function highlight(code::String, node, path::Int...; color=(40,40,70))
251257
print(stdout, code[q:end])
252258
end
253259

254-
255-
#-------------------------------------------------------------------------------
256-
# Conversion to Base.Expr
257-
258-
function is_eventually_call(ex)
259-
return Meta.isexpr(ex, :call) || (Meta.isexpr(ex, (:where, :(::))) &&
260-
is_eventually_call(ex.args[1]))
261-
end
262-
263-
function _to_expr(node::SyntaxNode, iteration_spec=false)
264-
if !haschildren(node)
265-
if node.val isa Union{Int128,UInt128,BigInt}
266-
# Ignore the values of large integers and convert them back to
267-
# symbolic/textural form for compatibility with the Expr
268-
# representation of these.
269-
str = replace(sourcetext(node), '_'=>"")
270-
headsym = :macrocall
271-
k = kind(node)
272-
macname = node.val isa Int128 ? Symbol("@int128_str") :
273-
node.val isa UInt128 ? Symbol("@uint128_str") :
274-
Symbol("@big_str")
275-
return Expr(:macrocall, GlobalRef(Core, macname), nothing, str)
276-
else
277-
return node.val
278-
end
279-
end
280-
headstr = untokenize(head(node), include_flag_suff=false)
281-
headsym = !isnothing(headstr) ? Symbol(headstr) :
282-
error("Can't untokenize head of kind $(kind(node))")
283-
node_args = children(node)
284-
args = Vector{Any}(undef, length(node_args))
285-
if headsym == :for && length(node_args) == 2
286-
args[1] = _to_expr(node_args[1], true)
287-
args[2] = _to_expr(node_args[2], false)
288-
else
289-
map!(_to_expr, args, node_args)
290-
end
291-
# Julia's standard `Expr` ASTs have children stored in a canonical
292-
# order which is often not always source order. We permute the children
293-
# here as necessary to get the canonical order.
294-
if is_infix(node.raw)
295-
args[2], args[1] = args[1], args[2]
296-
end
297-
loc = source_location(LineNumberNode, node.source, node.position)
298-
# Convert elements
299-
if headsym == :macrocall
300-
insert!(args, 2, loc)
301-
elseif headsym in (:call, :ref)
302-
# Move parameters block to args[2]
303-
if length(args) > 1 && Meta.isexpr(args[end], :parameters)
304-
insert!(args, 2, args[end])
305-
pop!(args)
306-
end
307-
elseif headsym in (:tuple, :parameters, :vect)
308-
# Move parameters blocks to args[1]
309-
if length(args) > 1 && Meta.isexpr(args[end], :parameters)
310-
pushfirst!(args, args[end])
311-
pop!(args)
312-
end
313-
elseif headsym == :try
314-
# Try children in source order:
315-
# try_block catch_var catch_block else_block finally_block
316-
# Expr ordering:
317-
# try_block catch_var catch_block [finally_block] [else_block]
318-
catch_ = nothing
319-
if has_flags(node, TRY_CATCH_AFTER_FINALLY_FLAG)
320-
catch_ = pop!(args)
321-
catch_var = pop!(args)
322-
end
323-
finally_ = pop!(args)
324-
else_ = pop!(args)
325-
if has_flags(node, TRY_CATCH_AFTER_FINALLY_FLAG)
326-
pop!(args)
327-
pop!(args)
328-
push!(args, catch_var)
329-
push!(args, catch_)
330-
end
331-
# At this point args is
332-
# [try_block catch_var catch_block]
333-
if finally_ !== false
334-
push!(args, finally_)
335-
end
336-
if else_ !== false
337-
push!(args, else_)
338-
end
339-
elseif headsym == :filter
340-
pushfirst!(args, last(args))
341-
pop!(args)
342-
elseif headsym == :flatten
343-
# The order of nodes inside the generators in Julia's flatten AST
344-
# is noncontiguous in the source text, so need to reconstruct
345-
# Julia's AST here from our alternative `flatten` expression.
346-
gen = Expr(:generator, args[1], args[end])
347-
for i in length(args)-1:-1:2
348-
gen = Expr(:flatten, Expr(:generator, gen, args[i]))
349-
end
350-
return gen
351-
elseif headsym in (:nrow, :ncat)
352-
# For lack of a better place, the dimension argument to nrow/ncat
353-
# is stored in the flags
354-
pushfirst!(args, numeric_flags(flags(node)))
355-
elseif headsym == :typed_ncat
356-
insert!(args, 2, numeric_flags(flags(node)))
357-
elseif headsym == :string && length(args) > 1
358-
# Julia string literals may be interspersed with trivia in two situations:
359-
# 1. Triple quoted string indentation is trivia
360-
# 2. An \ before newline removes the newline and any following indentation
361-
#
362-
# Such trivia is eagerly removed by the reference parser, so here we
363-
# concatenate adjacent string chunks together for compatibility.
364-
#
365-
# TODO: Manage the non-interpolation cases with String and CmdString
366-
# kinds instead?
367-
args2 = Vector{Any}()
368-
i = 1
369-
while i <= length(args)
370-
if args[i] isa String && i < length(args) && args[i+1] isa String
371-
buf = IOBuffer()
372-
while i <= length(args) && args[i] isa String
373-
write(buf, args[i])
374-
i += 1
375-
end
376-
push!(args2, String(take!(buf)))
377-
else
378-
push!(args2, args[i])
379-
i += 1
380-
end
381-
end
382-
args = args2
383-
if length(args2) == 1 && args2[1] isa String
384-
# If there's a single string remaining after joining we unwrap to
385-
# give a string literal.
386-
# """\n a\n b""" ==> "a\nb"
387-
return args2[1]
388-
end
389-
# elseif headsym == :string && length(args) == 1 && version <= (1,5)
390-
# Strip string from interpolations in 1.5 and lower to preserve
391-
# "hi$("ho")" ==> (string "hi" "ho")
392-
elseif headsym == :(=)
393-
if is_eventually_call(args[1]) && !iteration_spec
394-
if Meta.isexpr(args[2], :block)
395-
pushfirst!(args[2].args, loc)
396-
else
397-
# Add block for short form function locations
398-
args[2] = Expr(:block, loc, args[2])
399-
end
400-
end
401-
elseif headsym == :(->)
402-
if Meta.isexpr(args[2], :block)
403-
pushfirst!(args[2].args, loc)
404-
else
405-
# Add block for source locations
406-
args[2] = Expr(:block, loc, args[2])
407-
end
408-
elseif headsym == :function
409-
if length(args) > 1 && Meta.isexpr(args[1], :tuple)
410-
# Convert to weird Expr forms for long-form anonymous functions.
411-
#
412-
# (function (tuple (... xs)) body) ==> (function (... xs) body)
413-
if length(args[1].args) == 1 && Meta.isexpr(args[1].args[1], :...)
414-
# function (xs...) \n body end
415-
args[1] = args[1].args[1]
416-
end
417-
end
418-
end
419-
if headsym == :inert || (headsym == :quote && length(args) == 1 &&
420-
!(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
421-
a1 isa Bool # <- compat hack, Julia 1.4+
422-
))
423-
return QuoteNode(only(args))
424-
else
425-
return Expr(headsym, args...)
426-
end
427-
end
428-
429-
Base.Expr(node::SyntaxNode) = _to_expr(node)
430-
431-
432-
#-------------------------------------------------------------------------------
433-
434-
function build_tree(::Type{SyntaxNode}, stream::ParseStream; filename=nothing, kws...)
435-
green_tree = build_tree(GreenNode, stream; kws...)
436-
source = SourceFile(sourcetext(stream), filename=filename)
437-
SyntaxNode(source, green_tree, first_byte(stream))
438-
end
439-
440-
function build_tree(::Type{Expr}, stream::ParseStream; kws...)
441-
Expr(build_tree(SyntaxNode, stream; kws...))
442-
end
443-

0 commit comments

Comments
 (0)