Skip to content

Commit eb3259f

Browse files
c42fc42f
authored
Fixes for macro paths and call chain parsing (#144)
Correctly parse dots and other same-precedence chaining after macro calls: @A().x @A{a}.x @A[a].x Also allow general forms in macro paths, such as calls: f().@x This is allowed by the reference parser but seemed weird so it was disallowed here. But it turns out this doesn't prevent computed macro paths as the `A` in `A.@x` is looked up dynamically - so just simplify things and allow this.
1 parent b1f16e2 commit eb3259f

File tree

2 files changed

+79
-76
lines changed

2 files changed

+79
-76
lines changed

‎src/parser.jl‎

Lines changed: 36 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1380,18 +1380,8 @@ function parse_identifier_or_interpolate(ps::ParseState)
13801380
end
13811381
end
13821382

1383-
function finish_macroname(ps, mark, valid_macroname, macro_name_position,
1384-
name_kind=nothing)
1385-
if valid_macroname
1386-
fix_macro_name_kind!(ps, macro_name_position, name_kind)
1387-
else
1388-
emit(ps, mark, K"error", error="not a valid macro name or macro module path")
1389-
end
1390-
end
1391-
13921383
# Parses a chain of sufficies at function call precedence, leftmost binding
13931384
# tightest.
1394-
# f(a,b) ==> (call f a b)
13951385
# f(a).g(b) ==> (call (. (call f a) (quote g)) b)
13961386
#
13971387
# flisp: parse-call-chain, parse-call-with-initial-ex
@@ -1404,25 +1394,24 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
14041394
# source range of the @-prefixed part of a macro
14051395
macro_atname_range = nothing
14061396
# $A.@x ==> (macrocall (. ($ A) (quote @x)))
1407-
# A.@var"#" ==> (macrocall (. A (quote @x)))
1408-
valid_macroname = peek_behind(ps, skip_trivia=false).kind in KSet"Identifier var . $"
1397+
maybe_strmac = true
14091398
# We record the last component of chains of dot-separated identifiers so we
14101399
# know which identifier was the macro name.
14111400
macro_name_position = position(ps) # points to same output span as peek_behind
14121401
while true
1413-
this_iter_valid_macroname = false
1402+
maybe_strmac_1 = false
14141403
t = peek_token(ps)
14151404
k = kind(t)
14161405
if is_macrocall && (preceding_whitespace(t) || is_closing_token(ps, k))
14171406
# Macro calls with space-separated arguments
14181407
# @foo a b ==> (macrocall @foo a b)
14191408
# @foo (x) ==> (macrocall @foo x)
14201409
# @foo (x,y) ==> (macrocall @foo (tuple x y))
1421-
# a().@x y ==> (macrocall (error (. (call a) (quote x))) y)
14221410
# [@foo x] ==> (vect (macrocall @foo x))
14231411
# @var"#" a ==> (macrocall (var @#) a)
1412+
# A.@x y ==> (macrocall (. A (quote @x)) y)
14241413
# A.@var"#" a ==> (macrocall (. A (quote (var @#))) a)
1425-
finish_macroname(ps, mark, valid_macroname, macro_name_position)
1414+
fix_macro_name_kind!(ps, macro_name_position)
14261415
let ps = with_space_sensitive(ps)
14271416
# Space separated macro arguments
14281417
# A.@foo a b ==> (macrocall (. A (quote @foo)) a b)
@@ -1455,12 +1444,9 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
14551444
# [f x] ==> (hcat f x)
14561445
break
14571446
elseif k == K"("
1458-
if is_macrocall
1459-
# a().@x(y) ==> (macrocall (error (. (call a) (quote x))) y)
1460-
finish_macroname(ps, mark, valid_macroname, macro_name_position)
1461-
end
14621447
# f(a,b) ==> (call f a b)
1463-
# f(a; b=1) ==> (call f a (parameters (b 1)))
1448+
# f(a=1; b=2) ==> (call f (= a 1) (parameters (= b 2)))
1449+
# f(a; b; c) ==> (call f a (parameters b) (parameters c))
14641450
# (a=1)() ==> (call (= a 1))
14651451
# f (a) ==> (call f (error-t) a b)
14661452
bump_disallowed_space(ps)
@@ -1472,13 +1458,13 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
14721458
parse_do(ps, mark)
14731459
end
14741460
if is_macrocall
1475-
break
1461+
# A.@x(y) ==> (macrocall (. A (quote @x)) y)
1462+
# A.@x(y).z ==> (. (macrocall (. A (quote @x)) y) (quote z))
1463+
fix_macro_name_kind!(ps, macro_name_position)
1464+
is_macrocall = false
1465+
macro_atname_range = nothing
14761466
end
14771467
elseif k == K"["
1478-
if is_macrocall
1479-
# a().@x[1] ==> (macrocall (error (. (call a) (quote x))) (vect 1))
1480-
finish_macroname(ps, mark, valid_macroname, macro_name_position)
1481-
end
14821468
m = position(ps)
14831469
# a [i] ==> (ref a (error-t) i)
14841470
bump_disallowed_space(ps)
@@ -1489,12 +1475,16 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
14891475
# @S[a,b] ==> (macrocall @S (vect a b))
14901476
# @S[a b] ==> (macrocall @S (hcat a b))
14911477
# @S[a; b] ==> (macrocall @S (vcat a b))
1478+
# A.@S[a] ==> (macrocall (. A (quote @S)) (vect a))
1479+
# @S[a].b ==> (. (macrocall @S (vect a)) (quote b))
14921480
#v1.7: @S[a ;; b] ==> (macrocall @S (ncat-2 a b))
14931481
#v1.6: @S[a ;; b] ==> (macrocall @S (error (ncat-2 a b)))
1482+
fix_macro_name_kind!(ps, macro_name_position)
14941483
emit(ps, m, ckind, cflags)
14951484
check_ncat_compat(ps, m, ckind)
14961485
emit(ps, mark, K"macrocall")
1497-
break
1486+
is_macrocall = false
1487+
macro_atname_range = nothing
14981488
else
14991489
# a[i] ==> (ref a i)
15001490
# a[i,j] ==> (ref a i j)
@@ -1516,16 +1506,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
15161506
elseif k == K"."
15171507
# x .y ==> (. x (error-t) (quote y))
15181508
bump_disallowed_space(ps)
1519-
if peek(ps, 2) == K"'"
1520-
# f.' => f (error-t . ')
1521-
emark = position(ps)
1522-
bump(ps)
1523-
bump(ps)
1524-
emit(ps, emark, K"error", TRIVIA_FLAG,
1525-
error="the .' operator for transpose is discontinued")
1526-
valid_macroname = false
1527-
continue
1528-
end
1509+
emark = position(ps)
15291510
if !isnothing(macro_atname_range)
15301511
# Allow `@` in macrocall only in first and last position
15311512
# A.B.@x ==> (macrocall (. (. A (quote B)) (quote @x)))
@@ -1564,21 +1545,20 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
15641545
elseif k == K"$"
15651546
# f.$x ==> (. f (inert ($ x)))
15661547
# f.$(x+y) ==> (. f (inert ($ (call + x y))))
1548+
# A.$B.@x ==> (macrocall (. (. A (inert ($ B))) (quote @x)))
15671549
m = position(ps)
15681550
bump(ps, TRIVIA_FLAG)
15691551
parse_atom(ps)
15701552
emit(ps, m, K"$")
15711553
emit(ps, m, K"inert")
15721554
emit(ps, mark, K".")
1573-
# A.$B.@x ==> (macrocall (. (. A (inert ($ B))) (quote @x)))
1574-
this_iter_valid_macroname = true
15751555
elseif k == K"@"
15761556
# A macro call after some prefix A has been consumed
15771557
# A.@x ==> (macrocall (. A (quote @x)))
15781558
# A.@x a ==> (macrocall (. A (quote @x)) a)
15791559
m = position(ps)
15801560
if is_macrocall
1581-
# @A.B.@x a ==> (macrocall (error (. A (quote x))) a)
1561+
# @A.B.@x a ==> (macrocall (. (. A (quote B)) (quote (error-t) @x)) a)
15821562
bump(ps, TRIVIA_FLAG, error="repeated `@` in macro module path")
15831563
else
15841564
bump(ps, TRIVIA_FLAG)
@@ -1589,16 +1569,21 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
15891569
macro_atname_range = (m, macro_name_position)
15901570
emit(ps, m, K"quote")
15911571
emit(ps, mark, K".")
1592-
this_iter_valid_macroname = true
1572+
elseif k == K"'"
1573+
# TODO: Reclaim dotted postfix operators :-)
1574+
# f.' => f (error-t ')
1575+
bump(ps)
1576+
emit(ps, emark, K"error", TRIVIA_FLAG,
1577+
error="the .' operator for transpose is discontinued")
15931578
else
15941579
# Field/property syntax
15951580
# f.x.y ==> (. (. f (quote x)) (quote y))
15961581
m = position(ps)
15971582
parse_atom(ps, false)
15981583
macro_name_position = position(ps)
1584+
maybe_strmac_1 = true
15991585
emit(ps, m, K"quote")
16001586
emit(ps, mark, K".")
1601-
this_iter_valid_macroname = true
16021587
end
16031588
elseif k == K"'" && !preceding_whitespace(t)
16041589
# f' ==> (call-post f ')
@@ -1607,27 +1592,28 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
16071592
emit(ps, mark, K"call", POSTFIX_OP_FLAG)
16081593
elseif k == K"{"
16091594
# Type parameter curlies and macro calls
1610-
if is_macrocall
1611-
# a().@x{y} ==> (macrocall (error (. (call a) (quote x))) (braces y))
1612-
finish_macroname(ps, mark, valid_macroname, macro_name_position)
1613-
end
16141595
m = position(ps)
16151596
# S {a} ==> (curly S (error-t) a)
16161597
bump_disallowed_space(ps)
16171598
bump(ps, TRIVIA_FLAG)
16181599
parse_call_arglist(ps, K"}")
16191600
if is_macrocall
16201601
# @S{a,b} ==> (macrocall S (braces a b))
1602+
# A.@S{a} ==> (macrocall (. A (quote @S)) (braces a))
1603+
# @S{a}.b ==> (. (macrocall @S (braces a)) (quote b))
1604+
fix_macro_name_kind!(ps, macro_name_position)
16211605
emit(ps, m, K"braces")
16221606
emit(ps, mark, K"macrocall")
16231607
min_supported_version(v"1.6", ps, mark, "macro call without space before `{}`")
1624-
break
1608+
is_macrocall = false
1609+
macro_atname_range = nothing
16251610
else
16261611
# S{a,b} ==> (curly S a b)
16271612
emit(ps, mark, K"curly")
16281613
end
16291614
elseif k in KSet" \" \"\"\" ` ``` " &&
1630-
!preceding_whitespace(t) && valid_macroname
1615+
!preceding_whitespace(t) &&
1616+
maybe_strmac && peek_behind(ps, macro_name_position).kind == K"Identifier"
16311617
# Custom string and command literals
16321618
# x"str" ==> (macrocall @x_str (string-r "str"))
16331619
# x`str` ==> (macrocall @x_cmd (cmdstring-r "str"))
@@ -1641,7 +1627,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
16411627
# Use a special token kind for string and cmd macro names so the
16421628
# names can be expanded later as necessary.
16431629
outk = is_string_delim(k) ? K"StringMacroName" : K"CmdMacroName"
1644-
finish_macroname(ps, mark, valid_macroname, macro_name_position, outk)
1630+
fix_macro_name_kind!(ps, macro_name_position, outk)
16451631
parse_string(ps, true)
16461632
t = peek_token(ps)
16471633
k = kind(t)
@@ -1660,7 +1646,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
16601646
else
16611647
break
16621648
end
1663-
valid_macroname &= this_iter_valid_macroname
1649+
maybe_strmac = maybe_strmac_1
16641650
end
16651651
end
16661652

@@ -2267,6 +2253,7 @@ function parse_macro_name(ps::ParseState)
22672253
mark = position(ps)
22682254
k = peek(ps)
22692255
if k == K"."
2256+
# TODO: deal with __dot__ lowering in Expr conversion?
22702257
# @. y ==> (macrocall @__dot__ y)
22712258
bump(ps)
22722259
else

‎test/parser.jl‎

Lines changed: 43 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -240,33 +240,23 @@ tests = [
240240
"\$\$a" => "(\$ (\$ a))"
241241
],
242242
JuliaSyntax.parse_call => [
243-
# Mostly parse_call_chain
243+
# parse_call
244244
"f(x)" => "(call f x)"
245245
"\$f(x)" => "(call (\$ f) x)"
246-
"f(a,b)" => "(call f a b)"
247-
"f(a=1; b=2)" => "(call f (= a 1) (parameters (= b 2)))" =>
248-
Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))
249-
"f(a; b; c)" => "(call f a (parameters b) (parameters c))" =>
250-
Expr(:call, :f, Expr(:parameters, Expr(:parameters, :c), :b), :a)
251-
"(a=1)()" => "(call (= a 1))" => Expr(:call, Expr(:(=), :a, 1))
252-
"f (a)" => "(call f (error-t) a)"
246+
# parse_call_chain
253247
"f(a).g(b)" => "(call (. (call f a) (quote g)) b)"
254248
"\$A.@x" => "(macrocall (. (\$ A) (quote @x)))"
255-
# do
256-
"f() do\nend" => "(do (call f) (tuple) (block))"
257-
"f() do ; body end" => "(do (call f) (tuple) (block body))"
258-
"f() do x, y\n body end" => "(do (call f) (tuple x y) (block body))"
259-
"f(x) do y body end" => "(do (call f x) (tuple y) (block body))"
249+
250+
# space separated macro calls
260251
"@foo a b" => "(macrocall @foo a b)"
261252
"@foo (x)" => "(macrocall @foo x)"
262253
"@foo (x,y)" => "(macrocall @foo (tuple x y))"
263254
"A.@foo a b" => "(macrocall (. A (quote @foo)) a b)"
264255
"@A.foo a b" => "(macrocall (. A (quote @foo)) a b)"
265256
"[@foo x]" => "(vect (macrocall @foo x))"
266257
"@var\"#\" a" => "(macrocall (var @#) a)" => Expr(:macrocall, Symbol("@#"), LineNumberNode(1), :a)
258+
"A.@x y" => "(macrocall (. A (quote @x)) y)"
267259
"A.@var\"#\" a"=> "(macrocall (. A (quote (var @#))) a)" => Expr(:macrocall, Expr(:., :A, QuoteNode(Symbol("@#"))), LineNumberNode(1), :a)
268-
"[f (x)]" => "(hcat f x)"
269-
"[f x]" => "(hcat f x)"
270260
# Macro names
271261
"@! x" => "(macrocall @! x)"
272262
"@.. x" => "(macrocall @.. x)"
@@ -278,24 +268,35 @@ tests = [
278268
"@doc x y\nz" => "(macrocall @doc x y)"
279269
"@doc x\n\ny" => "(macrocall @doc x)"
280270
"@doc x\nend" => "(macrocall @doc x)"
281-
# .' discontinued
282-
"f.'" => "f (error-t . ')"
283-
# Allow `@` in macrocall only in first and last position
284-
"A.B.@x" => "(macrocall (. (. A (quote B)) (quote @x)))"
285-
"@A.B.x" => "(macrocall (. (. A (quote B)) (quote @x)))"
286-
"[email protected]" => "(macrocall (. (. A (quote B)) (error-t) (quote @x)))"
287-
"A.@. y" => "(macrocall (. A (quote @__dot__)) y)"
288-
"a().@x(y)" => "(macrocall (error (. (call a) (quote x))) y)"
289-
"a().@x y" => "(macrocall (error (. (call a) (quote x))) y)"
290-
"a().@x{y}" => "(macrocall (error (. (call a) (quote x))) (braces y))"
271+
272+
# non-errors in space sensitive contexts
273+
"[f (x)]" => "(hcat f x)"
274+
"[f x]" => "(hcat f x)"
275+
# calls with brackets
276+
"f(a,b)" => "(call f a b)"
277+
"f(a=1; b=2)" => "(call f (= a 1) (parameters (= b 2)))" =>
278+
Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))
279+
"f(a; b; c)" => "(call f a (parameters b) (parameters c))" =>
280+
Expr(:call, :f, Expr(:parameters, Expr(:parameters, :c), :b), :a)
281+
"(a=1)()" => "(call (= a 1))" => Expr(:call, Expr(:(=), :a, 1))
282+
"f (a)" => "(call f (error-t) a)"
283+
"A.@x(y)" => "(macrocall (. A (quote @x)) y)"
284+
"A.@x(y).z" => "(. (macrocall (. A (quote @x)) y) (quote z))"
285+
# do
286+
"f() do\nend" => "(do (call f) (tuple) (block))"
287+
"f() do ; body end" => "(do (call f) (tuple) (block body))"
288+
"f() do x, y\n body end" => "(do (call f) (tuple x y) (block body))"
289+
"f(x) do y body end" => "(do (call f x) (tuple y) (block body))"
290+
291291
# square brackets
292-
"a().@x[1]" => "(macrocall (error (. (call a) (quote x))) (vect 1))"
293292
"@S[a,b]" => "(macrocall @S (vect a b))" =>
294293
Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:vect, :a, :b))
295294
"@S[a b]" => "(macrocall @S (hcat a b))" =>
296295
Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:hcat, :a, :b))
297296
"@S[a; b]" => "(macrocall @S (vcat a b))" =>
298297
Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:vcat, :a, :b))
298+
"A.@S[a]" => "(macrocall (. A (quote @S)) (vect a))"
299+
"@S[a].b" => "(. (macrocall @S (vect a)) (quote b))"
299300
((v=v"1.7",), "@S[a ;; b]") => "(macrocall @S (ncat-2 a b))" =>
300301
Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:ncat, 2, :a, :b))
301302
((v=v"1.6",), "@S[a ;; b]") => "(macrocall @S (error (ncat-2 a b)))"
@@ -308,6 +309,13 @@ tests = [
308309
"T[a b; c d]" => "(typed_vcat T (row a b) (row c d))"
309310
"T[x for x in xs]" => "(typed_comprehension T (generator x (= x xs)))"
310311
((v=v"1.8",), "T[a ; b ;; c ; d]") => "(typed_ncat-2 T (nrow-1 a b) (nrow-1 c d))"
312+
313+
# Dotted forms
314+
# Allow `@` in macrocall only in first and last position
315+
"A.B.@x" => "(macrocall (. (. A (quote B)) (quote @x)))"
316+
"@A.B.x" => "(macrocall (. (. A (quote B)) (quote @x)))"
317+
"[email protected]" => "(macrocall (. (. A (quote B)) (error-t) (quote @x)))"
318+
"@. y" => "(macrocall @__dot__ y)"
311319
"f.(a,b)" => "(. f (tuple a b))"
312320
"f.(a=1; b=2)" => "(. f (tuple (= a 1) (parameters (= b 2))))" =>
313321
Expr(:., :f, Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1)))
@@ -319,16 +327,24 @@ tests = [
319327
"f.\$x" => "(. f (inert (\$ x)))"
320328
"f.\$(x+y)" => "(. f (inert (\$ (call-i x + y))))"
321329
"A.\$B.@x" => "(macrocall (. (. A (inert (\$ B))) (quote @x)))"
330+
"A.@x" => "(macrocall (. A (quote @x)))"
331+
"A.@x a" => "(macrocall (. A (quote @x)) a)"
332+
"@A.B.@x a" => "(macrocall (. (. A (quote B)) (quote (error-t) @x)) a)"
333+
# .' discontinued
334+
"f.'" => "f (error-t ')"
322335
# Field/property syntax
323336
"f.x.y" => "(. (. f (quote x)) (quote y))"
324337
"x .y" => "(. x (error-t) (quote y))"
325338
# Adjoint
326339
"f'" => "(call-post f ')"
327340
"f'ᵀ" => "(call-post f 'ᵀ)"
328341
# Curly calls
342+
"S {a}" => "(curly S (error-t) a)"
343+
"A.@S{a}" => "(macrocall (. A (quote @S)) (braces a))"
329344
"@S{a,b}" => "(macrocall @S (braces a b))"
345+
"A.@S{a}" => "(macrocall (. A (quote @S)) (braces a))"
346+
"@S{a}.b" => "(. (macrocall @S (braces a)) (quote b))"
330347
"S{a,b}" => "(curly S a b)"
331-
"S {a}" => "(curly S (error-t) a)"
332348
# String macros
333349
"x\"str\"" => """(macrocall @x_str (string-r "str"))"""
334350
"x`str`" => """(macrocall @x_cmd (cmdstring-r "str"))"""

0 commit comments

Comments
 (0)