Skip to content

Commit bc8bd78

Browse files
c42fc42f
authored
Always parse standalone dotted operators as (. op) (#240)
Change the parsing of standalone dotted operators such as `.+` to always parse as `(. +)`, regardless of where they appear syntactically. This removes the last of the cases where operators were coalesced with their leading `.` into a single symbol. For example, `.+(x,y)` is now parsed as `(call (. +) x y)`, rather than `(call .+ x y)`. The reference parser concatenates the `.` and the operator in some cases, so for compatibility we convert back to that less consistent form during `Expr` conversion.
1 parent a00d5bd commit bc8bd78

File tree

4 files changed

+83
-60
lines changed

4 files changed

+83
-60
lines changed

‎src/expr.jl

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ function reorder_parameters!(args, params_pos)
3131
end
3232

3333
function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
34-
eq_to_kw=false, map_kw_in_params=false)
34+
eq_to_kw=false, map_kw_in_params=false, coalesce_dot=false)
3535
nodekind = kind(node)
3636
if !haschildren(node)
3737
val = node.val
@@ -150,8 +150,13 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
150150
args[2*i-1] = source_location(LineNumberNode, n.source, n.position)
151151
end
152152
eq_to_kw = eq_to_kw_in_call && i > 1 || eq_to_kw_all
153+
coalesce_dot_with_ops = i==1 &&
154+
(nodekind in KSet"call dotcall curly" ||
155+
nodekind == K"quote" && flags(node) == COLON_QUOTE)
153156
args[insert_linenums ? 2*i : i] =
154-
_to_expr(n, eq_to_kw=eq_to_kw, map_kw_in_params=in_vcbr)
157+
_to_expr(n, eq_to_kw=eq_to_kw,
158+
map_kw_in_params=in_vcbr,
159+
coalesce_dot=coalesce_dot_with_ops)
155160
end
156161
if nodekind == K"block" && has_flags(node, PARENS_FLAG)
157162
popfirst!(args)
@@ -190,6 +195,10 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
190195
args[1] = Symbol(".", args[1])
191196
end
192197
end
198+
elseif headsym === :. && length(args) == 1 &&
199+
is_operator(kind(node[1])) &&
200+
(coalesce_dot || is_syntactic_operator(kind(node[1])))
201+
return Symbol(".", args[1])
193202
elseif headsym in (:ref, :curly)
194203
# Move parameters blocks to args[2]
195204
reorder_parameters!(args, 2)
@@ -303,11 +312,15 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
303312
elseif headsym === :module
304313
pushfirst!(args, !has_flags(node, BARE_MODULE_FLAG))
305314
pushfirst!(args[3].args, loc)
306-
elseif headsym === :inert || (headsym === :quote && length(args) == 1 &&
307-
!(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
308-
a1 isa Bool # <- compat hack, Julia 1.4+
309-
))
315+
elseif headsym === :inert
310316
return QuoteNode(only(args))
317+
elseif (headsym === :quote && length(args) == 1)
318+
a1 = only(args)
319+
if !(a1 isa Expr || a1 isa QuoteNode || a1 isa Bool)
320+
# Flisp parser does an optimization here: simple values are stored
321+
# as inert QuoteNode rather than in `Expr(:quote)` quasiquote
322+
return QuoteNode(a1)
323+
end
311324
elseif headsym === :do
312325
@check length(args) == 3
313326
return Expr(:do, args[1], Expr(:->, args[2], args[3]))

‎src/parser.jl

Lines changed: 30 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,7 +1151,7 @@ function parse_unary(ps::ParseState)
11511151
bump_trivia(ps)
11521152
op_t = peek_token(ps)
11531153
op_k = kind(op_t)
1154-
if (
1154+
if (
11551155
!is_operator(op_k) ||
11561156
is_word_operator(op_k) ||
11571157
(op_k in KSet": ' .'") ||
@@ -1194,19 +1194,13 @@ function parse_unary(ps::ParseState)
11941194
end
11951195
end
11961196
if is_closing_token(ps, k2) || k2 in KSet"NewlineWs ="
1197-
if is_dotted(op_t)
1198-
# Standalone dotted operators are parsed as (|.| op)
1199-
# .+ ==> (. +)
1200-
# .+\n ==> (. +)
1201-
# .+ = ==> (. +)
1202-
# .+) ==> (. +)
1203-
# .& ==> (. &)
1204-
bump_dotsplit(ps, emit_dot_node=true)
1205-
else
1206-
# Standalone non-dotted operators
1207-
# +) ==> +
1208-
bump(ps)
1209-
end
1197+
# Standalone operators parsed as `op` or `(. op)`
1198+
# +) ==> +
1199+
# +\n ==> +
1200+
# + = ==> +
1201+
# .+ ==> (. +)
1202+
# .& ==> (. &)
1203+
parse_atom(ps)
12101204
elseif k2 == K"{" || (!is_unary_op(op_t) && k2 == K"(")
12111205
# Call with type parameters or non-unary prefix call
12121206
# +{T}(x::T) ==> (call (curly + T) (:: x T))
@@ -1267,28 +1261,13 @@ function parse_unary(ps::ParseState)
12671261
emit(ps, mark, op_k)
12681262
reset_node!(ps, op_pos, flags=TRIVIA_FLAG)
12691263
else
1270-
if is_dotted(op_t)
1271-
# Ugly hack to undo the split in bump_dotsplit
1272-
# .+(a,) ==> (call .+ a)
1273-
reset_node!(ps, op_pos, kind=K"TOMBSTONE")
1274-
tb1 = ps.stream.tokens[op_pos.token_index-1]
1275-
ps.stream.tokens[op_pos.token_index-1] =
1276-
SyntaxToken(SyntaxHead(K"TOMBSTONE", EMPTY_FLAGS),
1277-
K"TOMBSTONE", tb1.preceding_whitespace,
1278-
tb1.next_byte-1)
1279-
tb0 = ps.stream.tokens[op_pos.token_index]
1280-
ps.stream.tokens[op_pos.token_index] =
1281-
SyntaxToken(SyntaxHead(kind(tb0), flags(tb0)),
1282-
tb0.orig_kind, tb0.preceding_whitespace,
1283-
tb0.next_byte)
1284-
end
12851264
emit(ps, mark, K"call")
12861265
end
12871266
parse_call_chain(ps, mark)
12881267
parse_factor_with_initial_ex(ps, mark)
12891268
else
12901269
# Unary function calls with brackets as grouping, not an arglist
1291-
# .+(a) ==> (dotcall-pre (. +) (parens a))
1270+
# .+(a) ==> (dotcall-pre + (parens a))
12921271
if opts.is_block
12931272
# +(a;b) ==> (call-pre + (block-p a b))
12941273
emit(ps, mark_before_paren, K"block", PARENS_FLAG)
@@ -1627,6 +1606,10 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
16271606
parse_atom(ps)
16281607
emit(ps, m, K"$")
16291608
macro_name_position = position(ps)
1609+
# We need `inert` rather than `quote` here for subtle reasons:
1610+
# We need the expression expander to "see through" the quote
1611+
# around the `$x` in `:(f.$x)`, so that the `$x` is expanded
1612+
# even though it's double quoted.
16301613
emit(ps, m, K"inert")
16311614
emit(ps, mark, K".")
16321615
elseif k == K"@"
@@ -2307,6 +2290,10 @@ function parse_do(ps::ParseState, mark)
23072290
emit(ps, mark, K"do")
23082291
end
23092292

2293+
function _is_valid_macro_name(peektok)
2294+
return !is_error(peektok.kind) && (peektok.is_leaf || peektok.kind == K"var")
2295+
end
2296+
23102297
function fix_macro_name_kind!(ps::ParseState, macro_name_position, name_kind=nothing)
23112298
k = peek_behind(ps, macro_name_position).kind
23122299
if k == K"var"
@@ -2321,7 +2308,8 @@ function fix_macro_name_kind!(ps::ParseState, macro_name_position, name_kind=not
23212308
return
23222309
end
23232310
if isnothing(name_kind)
2324-
name_kind = (k == K"Identifier") ? K"MacroName" : K"error"
2311+
name_kind = _is_valid_macro_name(peek_behind(ps, macro_name_position)) ?
2312+
K"MacroName" : K"error"
23252313
if name_kind == K"error"
23262314
# TODO: This isn't quite accurate
23272315
emit_diagnostic(ps, macro_name_position, macro_name_position,
@@ -2343,11 +2331,11 @@ function parse_macro_name(ps::ParseState)
23432331
bump_disallowed_space(ps)
23442332
mark = position(ps)
23452333
parse_atom(ps, false)
2346-
kb = peek_behind(ps, position(ps)).kind
2347-
if kb == K"parens"
2334+
b = peek_behind(ps, position(ps))
2335+
if b.kind == K"parens"
23482336
emit_diagnostic(ps, mark,
23492337
warning="parenthesizing macro names is unnecessary")
2350-
elseif !(kb in KSet"Identifier var")
2338+
elseif !_is_valid_macro_name(b)
23512339
# @[x] y z ==> (macrocall (error (vect x)) y z)
23522340
emit(ps, mark, K"error", error="invalid macro name")
23532341
end
@@ -3448,20 +3436,18 @@ function parse_atom(ps::ParseState, check_identifiers=true)
34483436
# x₁ ==> x₁
34493437
bump(ps)
34503438
elseif is_operator(leading_kind)
3439+
# + ==> +
3440+
# .+ ==> (. +)
3441+
# .= ==> (. =)
3442+
bump_dotsplit(ps, emit_dot_node=true)
34513443
if check_identifiers && !is_valid_identifier(leading_kind)
34523444
# += ==> (error +=)
3453-
# .+= ==> (error .+=)
3454-
bump(ps, error="invalid identifier")
3445+
# ? ==> (error ?)
3446+
# .+= ==> (error (. +=))
3447+
emit(ps, mark, K"error", error="invalid identifier")
34553448
else
3456-
# + ==> +
3457-
# ~ ==> ~
34583449
# Quoted syntactic operators allowed
3459-
# :+= ==> (quote +=)
3460-
# :.= ==> (quote .=)
3461-
# Remap the kind here to K"Identifier", as operators parsed in this
3462-
# branch should be in "identifier-like" positions (I guess this is
3463-
# correct? is it convenient?)
3464-
bump(ps, remap_kind=K"Identifier")
3450+
# :+= ==> (quote-: +=)
34653451
end
34663452
elseif is_keyword(leading_kind)
34673453
if leading_kind == K"var" && (t = peek_token(ps,2);

‎test/expr.jl

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,16 +241,36 @@
241241
Expr(:tuple, Expr(:parameters, Expr(:kw, :a, 1)))
242242
end
243243

244-
@testset "dotcall" begin
244+
@testset "dotcall / dotted operators" begin
245245
@test parsestmt(Expr, "f.(x,y)") == Expr(:., :f, Expr(:tuple, :x, :y))
246246
@test parsestmt(Expr, "f.(x=1)") == Expr(:., :f, Expr(:tuple, Expr(:kw, :x, 1)))
247247
@test parsestmt(Expr, "x .+ y") == Expr(:call, Symbol(".+"), :x, :y)
248248
@test parsestmt(Expr, "(x=1) .+ y") == Expr(:call, Symbol(".+"), Expr(:(=), :x, 1), :y)
249249
@test parsestmt(Expr, "a .< b .< c") == Expr(:comparison, :a, Symbol(".<"),
250250
:b, Symbol(".<"), :c)
251-
@test parsestmt(Expr, ".*(x)") == Expr(:call, Symbol(".*"), :x)
252-
@test parsestmt(Expr, ".+(x)") == Expr(:call, Symbol(".+"), :x)
253-
@test parsestmt(Expr, ".+x") == Expr(:call, Symbol(".+"), :x)
251+
@test parsestmt(Expr, ".*(x)") == Expr(:call, Symbol(".*"), :x)
252+
@test parsestmt(Expr, ".+(x)") == Expr(:call, Symbol(".+"), :x)
253+
@test parsestmt(Expr, ".+x") == Expr(:call, Symbol(".+"), :x)
254+
@test parsestmt(Expr, "(.+)(x)") == Expr(:call, Expr(:., :+), :x)
255+
@test parsestmt(Expr, "(.+).(x)") == Expr(:., Expr(:., :+), Expr(:tuple, :x))
256+
257+
@test parsestmt(Expr, ".+") == Expr(:., :+)
258+
@test parsestmt(Expr, ":.+") == QuoteNode(Symbol(".+"))
259+
@test parsestmt(Expr, ":(.+)") == Expr(:quote, (Expr(:., :+)))
260+
@test parsestmt(Expr, "quote .+ end") == Expr(:quote,
261+
Expr(:block,
262+
LineNumberNode(1),
263+
Expr(:., :+)))
264+
@test parsestmt(Expr, ".+{x}") == Expr(:curly, Symbol(".+"), :x)
265+
266+
# Quoted syntactic ops act different when in parens
267+
@test parsestmt(Expr, ":.=") == QuoteNode(Symbol(".="))
268+
@test parsestmt(Expr, ":(.=)") == QuoteNode(Symbol(".="))
269+
270+
# A few other cases of bare dotted ops
271+
@test parsestmt(Expr, "f(.+)") == Expr(:call, :f, Expr(:., :+))
272+
@test parsestmt(Expr, "(a, .+)") == Expr(:tuple, :a, Expr(:., :+))
273+
@test parsestmt(Expr, "A.:.+") == Expr(:., :A, QuoteNode(Symbol(".+")))
254274
end
255275

256276
@testset "where" begin
@@ -361,6 +381,7 @@
361381
end
362382

363383
@testset "import" begin
384+
@test parsestmt(Expr, "import A") == Expr(:import, Expr(:., :A))
364385
@test parsestmt(Expr, "import A.(:b).:c: x.:z", ignore_warnings=true) ==
365386
Expr(:import, Expr(Symbol(":"), Expr(:., :A, :b, :c), Expr(:., :x, :z)))
366387
# Stupid parens and quotes in import paths

‎test/parser.jl

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,10 @@ tests = [
222222
# Call with type parameters or non-unary prefix call
223223
"+{T}(x::T)" => "(call (curly + T) (::-i x T))"
224224
"*(x)" => "(call * x)"
225-
".*(x)" => "(call .* x)"
225+
".*(x)" => "(call (. *) x)"
226226
# Prefix function calls for operators which are both binary and unary
227227
"+(a,b)" => "(call + a b)"
228-
".+(a,)" => "(call .+ a)"
228+
".+(a,)" => "(call (. +) a)"
229229
"(.+)(a)" => "(call (parens (. +)) a)"
230230
"+(a=1,)" => "(call + (= a 1))" => Expr(:call, :+, Expr(:kw, :a, 1))
231231
"+(a...)" => "(call + (... a))"
@@ -304,7 +304,7 @@ tests = [
304304
# parse_call
305305
"f(x)" => "(call f x)"
306306
"\$f(x)" => "(call (\$ f) x)"
307-
".&(x,y)" => "(call .& x y)"
307+
".&(x,y)" => "(call (. &) x y)"
308308
# parse_call_chain
309309
"f(a).g(b)" => "(call (. (call f a) (quote g)) b)"
310310
"\$A.@x" => "(macrocall (. (\$ A) (quote @x)))"
@@ -394,6 +394,7 @@ tests = [
394394
"f. (x)" => "(dotcall f (error-t) x)"
395395
# Other dotted syntax
396396
"A.:+" => "(. A (quote-: +))"
397+
"A.:.+" => "(. A (quote-: (. +)))"
397398
"A.: +" => "(. A (quote-: (error-t) +))"
398399
"f.\$x" => "(. f (inert (\$ x)))"
399400
"f.\$(x+y)" => "(. f (inert (\$ (parens (call-i x + y)))))"
@@ -738,15 +739,17 @@ tests = [
738739
"""var"x"end""" => "(var x (error-t))"
739740
"""var"x"1""" => "(var x (error-t))"
740741
"""var"x"y""" => "(var x (error-t))"
741-
# Syntactic operators
742+
# Standalone syntactic operators are errors
742743
"+=" => "(error +=)"
743-
".+=" => "(error .+=)"
744+
"?" => "(error ?)"
745+
".+=" => "(error (. +=))"
744746
# Normal operators
745747
"+" => "+"
746748
"~" => "~"
747749
# Quoted syntactic operators allowed
748750
":+=" => "(quote-: +=)"
749-
":.=" => "(quote-: .=)"
751+
":.=" => "(quote-: (. =))"
752+
":.&&" => "(quote-: (. &&))"
750753
# Special symbols quoted
751754
":end" => "(quote-: end)"
752755
":(end)" => "(quote-: (parens (error-t)))"

0 commit comments

Comments
 (0)