Skip to content

Commit 1bf1787

Browse files
c42fc42f
authored
Wrap var"" nonstandard identifiers in var nodes (#127)
Useful to hold associated trivia (delimiter and var prefix) and indicate in a clean way that var"" was used.
1 parent ff21c47 commit 1bf1787

File tree

3 files changed

+72
-50
lines changed

3 files changed

+72
-50
lines changed

‎src/expr.jl‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,14 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
3030
end
3131
end
3232
nodekind = kind(node)
33-
if nodekind == K"?"
33+
node_args = children(node)
34+
if nodekind == K"var"
35+
@check length(node_args) == 1
36+
return _to_expr(node_args[1])
37+
elseif nodekind == K"char"
38+
@check length(node_args) == 1
39+
return _to_expr(node_args[1])
40+
elseif nodekind == K"?"
3441
headsym = :if
3542
elseif nodekind == K"=" && !is_decorated(node) && eq_to_kw
3643
headsym = :kw
@@ -39,7 +46,6 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
3946
headsym = !isnothing(headstr) ? Symbol(headstr) :
4047
error("Can't untokenize head of kind $(nodekind)")
4148
end
42-
node_args = children(node)
4349
if headsym == :string || headsym == :cmdstring
4450
# Julia string literals may be interspersed with trivia in two situations:
4551
# 1. Triple quoted string indentation is trivia
@@ -240,9 +246,6 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
240246
elseif headsym == :do
241247
@check length(args) == 3
242248
return Expr(:do, args[1], Expr(:->, args[2], args[3]))
243-
elseif headsym == :char
244-
@check length(args) == 1
245-
return args[1]
246249
elseif headsym == :let
247250
@check Meta.isexpr(args[1], :block)
248251
a1 = args[1].args

‎src/parser.jl‎

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,10 @@ function textbuf(ps::ParseState)
125125
textbuf(ps.stream)
126126
end
127127

128+
function first_child_position(ps::ParseState, pos::ParseStreamPosition)
129+
first_child_position(ps.stream, pos)
130+
end
131+
128132
#-------------------------------------------------------------------------------
129133
# Parser Utils
130134

@@ -315,7 +319,7 @@ function was_eventually_call(ps::ParseState)
315319
if kb == K"call"
316320
return true
317321
elseif kb == K"where" || kb == K"::"
318-
p = first_child_position(ps.stream, p)
322+
p = first_child_position(ps, p)
319323
else
320324
return false
321325
end
@@ -1363,7 +1367,7 @@ function parse_identifier_or_interpolate(ps::ParseState)
13631367
# export outer ==> (export outer)
13641368
# export ($f) ==> (export ($ f))
13651369
ok = (b.is_leaf && (b.kind == K"Identifier" || is_operator(b.kind))) ||
1366-
(!b.is_leaf && b.kind == K"$")
1370+
(!b.is_leaf && b.kind in KSet"$ var")
13671371
if !ok
13681372
emit(ps, mark, K"error", error="Expected identifier")
13691373
end
@@ -1372,10 +1376,7 @@ end
13721376
function finish_macroname(ps, mark, valid_macroname, macro_name_position,
13731377
name_kind=nothing)
13741378
if valid_macroname
1375-
if isnothing(name_kind)
1376-
name_kind = macro_name_kind(peek_behind(ps, macro_name_position).kind)
1377-
end
1378-
reset_node!(ps, macro_name_position, kind = name_kind)
1379+
fix_macro_name_kind!(ps, macro_name_position, name_kind)
13791380
else
13801381
emit(ps, mark, K"error", error="not a valid macro name or macro module path")
13811382
end
@@ -1396,7 +1397,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
13961397
# source range of the @-prefixed part of a macro
13971398
macro_atname_range = nothing
13981399
# $A.@x ==> (macrocall (. ($ A) (quote @x)))
1399-
valid_macroname = peek_behind(ps, skip_trivia=false).kind in KSet"Identifier . $"
1400+
# A.@var"#" ==> (macrocall (. A (quote @x)))
1401+
valid_macroname = peek_behind(ps, skip_trivia=false).kind in KSet"Identifier var . $"
14001402
# We record the last component of chains of dot-separated identifiers so we
14011403
# know which identifier was the macro name.
14021404
macro_name_position = position(ps) # points to same output span as peek_behind
@@ -1411,6 +1413,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
14111413
# @foo (x,y) ==> (macrocall @foo (tuple x y))
14121414
# a().@x y ==> (macrocall (error (. (call a) (quote x))) y)
14131415
# [@foo x] ==> (vect (macrocall @foo x))
1416+
# @var"#" a ==> (macrocall (var @#) a)
1417+
# A.@var"#" a ==> (macrocall (. A (quote (var @#))) a)
14141418
finish_macroname(ps, mark, valid_macroname, macro_name_position)
14151419
let ps = with_space_sensitive(ps)
14161420
# Space separated macro arguments
@@ -2229,10 +2233,18 @@ function parse_do(ps::ParseState, mark)
22292233
emit(ps, mark, K"do")
22302234
end
22312235

2232-
function macro_name_kind(k)
2233-
return k == K"Identifier" ? K"MacroName" :
2234-
k == K"." ? K"@." :
2235-
internal_error("unrecognized source kind for macro name ", k)
2236+
function fix_macro_name_kind!(ps::ParseState, macro_name_position, name_kind=nothing)
2237+
k = peek_behind(ps, macro_name_position).kind
2238+
if k == K"var"
2239+
macro_name_position = first_child_position(ps, macro_name_position)
2240+
k = peek_behind(ps, macro_name_position).kind
2241+
end
2242+
if isnothing(name_kind)
2243+
name_kind = k == K"Identifier" ? K"MacroName" :
2244+
k == K"." ? K"@." :
2245+
internal_error("unrecognized source kind for macro name ", k)
2246+
end
2247+
reset_node!(ps, macro_name_position, kind=name_kind)
22362248
end
22372249

22382250
# If remap_kind is false, the kind will be remapped by parse_call_chain after
@@ -2250,23 +2262,25 @@ function parse_macro_name(ps::ParseState)
22502262
# @! x ==> (macrocall @! x)
22512263
# @.. x ==> (macrocall @.. x)
22522264
# @$ x ==> (macrocall @$ x)
2265+
# @var"#" x ==> (macrocall (var #) @$ x)
22532266
let ps = with_space_sensitive(ps)
22542267
parse_atom(ps, false)
22552268
end
22562269
end
22572270
end
22582271

2259-
# Parse an identifier, interpolation of @-prefixed symbol
2272+
# Parse an identifier, interpolation or @-prefixed symbol
22602273
#
22612274
# flisp: parse-atsym
22622275
function parse_atsym(ps::ParseState)
22632276
bump_trivia(ps)
22642277
if peek(ps) == K"@"
2265-
# export @a ==> (export @a)
2266-
# export a, \n @b ==> (export a @b)
2278+
# export @a ==> (export @a)
2279+
# export @var"'" ==> (export (var @'))
2280+
# export a, \n @b ==> (export a @b)
22672281
bump(ps, TRIVIA_FLAG)
22682282
parse_macro_name(ps)
2269-
reset_node!(ps, position(ps), kind=macro_name_kind(peek_behind(ps).kind))
2283+
fix_macro_name_kind!(ps, position(ps))
22702284
else
22712285
# export a ==> (export a)
22722286
# export \n a ==> (export a)
@@ -3322,12 +3336,12 @@ function parse_atom(ps::ParseState, check_identifiers=true)
33223336
elseif is_keyword(leading_kind)
33233337
if leading_kind == K"var" && (t = peek_token(ps,2);
33243338
kind(t) == K"\"" && !preceding_whitespace(t))
3325-
# var"x" ==> x
3339+
# var"x" ==> (var x)
33263340
# Raw mode unescaping
3327-
# var"" ==>
3328-
# var"\"" ==> "
3329-
# var"\\"" ==> \"
3330-
# var"\\x" ==> \\x
3341+
# var"" ==> (var )
3342+
# var"\"" ==> (var ")
3343+
# var"\\"" ==> (var \")
3344+
# var"\\x" ==> (var \\x)
33313345
#
33323346
# NB: Triple quoted var identifiers are not implemented, but with
33333347
# the complex deindentation rules they seem like a misfeature
@@ -3344,7 +3358,7 @@ function parse_atom(ps::ParseState, check_identifiers=true)
33443358
bump(ps, TRIVIA_FLAG)
33453359
else
33463360
bump_invisible(ps, K"error", TRIVIA_FLAG,
3347-
error="unterminated string literal")
3361+
error="unterminated `var\"\"` identifier")
33483362
end
33493363
t = peek_token(ps)
33503364
k = kind(t)
@@ -3354,11 +3368,12 @@ function parse_atom(ps::ParseState, check_identifiers=true)
33543368
# var"x") ==> x
33553369
# var"x"( ==> x
33563370
else
3357-
# var"x"end ==> (error (end))
3358-
# var"x"1 ==> (error 1)
3359-
# var"x"y ==> (error y)
3360-
bump(ps, error="suffix not allowed after var\"...\" syntax")
3371+
# var"x"end ==> (var x (error-t))
3372+
# var"x"1 ==> (var x (error-t))
3373+
# var"x"y ==> (var x (error-t))
3374+
bump(ps, TRIVIA_FLAG, error="suffix not allowed after var\"...\" syntax")
33613375
end
3376+
emit(ps, mark, K"var")
33623377
elseif check_identifiers && is_closing_token(ps, leading_kind)
33633378
# :(end) ==> (quote (error end))
33643379
bump(ps, error="invalid identifier")

‎test/parser.jl‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,9 @@ tests = [
258258
"@foo (x,y)" => "(macrocall @foo (tuple x y))"
259259
"A.@foo a b" => "(macrocall (. A (quote @foo)) a b)"
260260
"@A.foo a b" => "(macrocall (. A (quote @foo)) a b)"
261-
"[@foo x]" => "(vect (macrocall @foo x))"
261+
"[@foo x]" => "(vect (macrocall @foo x))"
262+
"@var\"#\" a" => "(macrocall (var @#) a)" => Expr(:macrocall, Symbol("@#"), LineNumberNode(1), :a)
263+
"A.@var\"#\" a"=> "(macrocall (. A (quote (var @#))) a)" => Expr(:macrocall, Expr(:., :A, QuoteNode(Symbol("@#"))), LineNumberNode(1), :a)
262264
"[f (x)]" => "(hcat f x)"
263265
"[f x]" => "(hcat f x)"
264266
# Macro names
@@ -402,15 +404,16 @@ tests = [
402404
"module A \n a \n b \n end" => "(module true A (block a b))"
403405
"""module A \n "x"\na\n end""" => """(module true A (block (macrocall :(Core.var"@doc") (string "x") a)))"""
404406
# export
405-
"export a" => "(export a)"
406-
"export @a" => "(export @a)"
407-
"export a, \n @b" => "(export a @b)"
408-
"export +, ==" => "(export + ==)"
409-
"export \n a" => "(export a)"
410-
"export \$a, \$(a*b)" => "(export (\$ a) (\$ (call-i a * b)))"
407+
"export a" => "(export a)" => Expr(:export, :a)
408+
"export @a" => "(export @a)" => Expr(:export, Symbol("@a"))
409+
"export @var\"'\"" => "(export (var @'))" => Expr(:export, Symbol("@'"))
410+
"export a, \n @b" => "(export a @b)" => Expr(:export, :a, Symbol("@b"))
411+
"export +, ==" => "(export + ==)" => Expr(:export, :+, :(==))
412+
"export \n a" => "(export a)" => Expr(:export, :a)
413+
"export \$a, \$(a*b)" => "(export (\$ a) (\$ (call-i a * b)))" => Expr(:export, Expr(:$, :a), Expr(:$, Expr(:call, :*, :a, :b)))
411414
"export (x::T)" => "(export (error (:: x T)))"
412-
"export outer" => "(export outer)"
413-
"export (\$f)" => "(export (\$ f))"
415+
"export outer" => "(export outer)" => Expr(:export, :outer)
416+
"export (\$f)" => "(export (\$ f))" => Expr(:export, Expr(:$, :f))
414417
],
415418
JuliaSyntax.parse_if_elseif => [
416419
"if a xx elseif b yy else zz end" => "(if a (block xx) (elseif b (block yy) (block zz)))"
@@ -612,18 +615,19 @@ tests = [
612615
"xx" => "xx"
613616
"x₁" => "x₁"
614617
# var syntax
615-
"""var"x" """ => "x"
616-
"""var"x"+""" => "x"
617-
"""var"x")""" => "x"
618-
"""var"x"(""" => "x"
619-
"""var"x"end""" => "x (error (end))"
620-
"""var"x"1""" => "x (error 1)"
621-
"""var"x"y""" => "x (error y)"
618+
"""var"x" """ => "(var x)" => :x
622619
# var syntax raw string unescaping
623-
"var\"\"" => ""
624-
"var\"\\\"\"" => "\""
625-
"var\"\\\\\\\"\"" => "\\\""
626-
"var\"\\\\x\"" => "\\\\x"
620+
"var\"\"" => "(var )" => Symbol("")
621+
"var\"\\\"\"" => "(var \")" => Symbol("\"")
622+
"var\"\\\\\\\"\"" => "(var \\\")" => Symbol("\\\"")
623+
"var\"\\\\x\"" => "(var \\\\x)" => Symbol("\\\\x")
624+
# trailing syntax after var
625+
"""var"x"+""" => "(var x)" => :x
626+
"""var"x")""" => "(var x)" => :x
627+
"""var"x"(""" => "(var x)" => :x
628+
"""var"x"end""" => "(var x (error-t))"
629+
"""var"x"1""" => "(var x (error-t))"
630+
"""var"x"y""" => "(var x (error-t))"
627631
# Syntactic operators
628632
"+=" => "(error +=)"
629633
".+=" => "(error .+=)"

0 commit comments

Comments
 (0)