diff --git a/src/expr.jl b/src/expr.jl index b18b9c0e..2f6ee29e 100644 --- a/src/expr.jl +++ b/src/expr.jl @@ -197,6 +197,9 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true, end elseif headsym === :where reorder_parameters!(args, 2) + elseif headsym == :parens + # parens are used for grouping and don't appear in the Expr AST + return only(args) elseif headsym in (:try, :try_finally_catch) # Try children in source order: # try_block catch_var catch_block else_block finally_block diff --git a/src/parse_stream.jl b/src/parse_stream.jl index 8db4489f..0ff02159 100644 --- a/src/parse_stream.jl +++ b/src/parse_stream.jl @@ -582,26 +582,25 @@ function first_child_position(stream::ParseStream, pos::ParseStreamPosition) end function peek_behind(stream::ParseStream; skip_trivia::Bool=true) - pos = position(stream) - if !skip_trivia || !token_is_last(stream, pos) - return peek_behind(stream, pos) - else - token_index = lastindex(stream.tokens) - range_index = lastindex(stream.ranges) - last_token_in_nonterminal = isempty(stream.ranges) ? 0 : - stream.ranges[range_index].last_token - while token_index > last_token_in_nonterminal - t = stream.tokens[token_index] - if !is_trivia(t) && kind(t) != K"TOMBSTONE" - break - end - token_index -= 1 - end - if token_index > 0 - return peek_behind(stream, ParseStreamPosition(token_index, range_index)) - else - internal_error("Can't peek behind at start of stream") + token_index = lastindex(stream.tokens) + range_index = lastindex(stream.ranges) + while range_index >= firstindex(stream.ranges) && + kind(stream.ranges[range_index]) == K"parens" + range_index -= 1 + end + last_token_in_nonterminal = range_index == 0 ? 0 : + stream.ranges[range_index].last_token + while token_index > last_token_in_nonterminal + t = stream.tokens[token_index] + if kind(t) != K"TOMBSTONE" && (!skip_trivia || !is_trivia(t)) + break end + token_index -= 1 + end + if token_index > 0 + return peek_behind(stream, ParseStreamPosition(token_index, range_index)) + else + internal_error("Can't peek behind at start of stream") end end diff --git a/src/parser.jl b/src/parser.jl index 46a51bf0..89936013 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -315,8 +315,8 @@ function was_eventually_call(ps::ParseState) b = peek_behind(stream, p) if b.kind == K"call" return true - elseif b.kind == K"where" || (b.kind == K"::" && - has_flags(b.flags, INFIX_FLAG)) + elseif b.kind == K"where" || b.kind == K"parens" || + (b.kind == K"::" && has_flags(b.flags, INFIX_FLAG)) p = first_child_position(ps, p) else return false @@ -885,7 +885,7 @@ function parse_range(ps::ParseState) if had_newline # Error message for people coming from python # 1:\n2 ==> (call-i 1 : (error)) - # (1:\n2) ==> (call-i 1 : 2) + # (1:\n2) ==> (parens (call-i 1 : 2)) emit_diagnostic(ps, whitespace=true, error="line break after `:` in range expression") bump_invisible(ps, K"error") @@ -1021,7 +1021,7 @@ function parse_unary_subtype(ps::ParseState) elseif k2 in KSet"{ (" # parse <:{T}(x::T) or <:(x::T) like other unary operators # <:{T}(x::T) ==> (call (curly <: T) (:: x T)) - # <:(x::T) ==> (<:-pre (:: x T)) + # <:(x::T) ==> (<:-pre (parens (:: x T))) parse_where(ps, parse_juxtapose) else # <: x ==> (<:-pre x) @@ -1108,9 +1108,9 @@ end # Juxtoposition. Ugh! But so useful for units and Field identities like `im` # # 2x ==> (juxtapose 2 x) -# 2(x) ==> (juxtapose 2 x) -# (2)(3)x ==> (juxtapose 2 3 x) -# (x-1)y ==> (juxtapose (call-i x - 1) y) +# 2(x) ==> (juxtapose 2 (parens x)) +# (2)(3)x ==> (juxtapose (parens 2) (parens 3) x) +# (x-1)y ==> (juxtapose (parens (call-i x - 1)) y) # x'y ==> (juxtapose (call-post x ') y) # # flisp: parse-juxtapose @@ -1239,9 +1239,9 @@ function parse_unary(ps::ParseState) mark_before_paren = position(ps) bump(ps, TRIVIA_FLAG) # ( - initial_semi = peek(ps) == K";" + _is_paren_call = peek(ps, skip_newlines=true) in KSet"; )" opts = parse_brackets(ps, K")") do had_commas, had_splat, num_semis, num_subexprs - is_paren_call = had_commas || had_splat || initial_semi + is_paren_call = had_commas || had_splat || _is_paren_call return (needs_parameters=is_paren_call, is_paren_call=is_paren_call, is_block=!is_paren_call && num_semis > 0) @@ -1263,6 +1263,7 @@ function parse_unary(ps::ParseState) # +(a...) ==> (call + (... a)) # +(a;b,c) ==> (call + a (parameters b c)) # +(;a) ==> (call + (parameters a)) + # +() ==> (call +) # Prefix calls have higher precedence than ^ # +(a,b)^2 ==> (call-i (call + a b) ^ 2) # +(a,b)(x)^2 ==> (call-i (call (call + a b) x) ^ 2) @@ -1292,21 +1293,23 @@ function parse_unary(ps::ParseState) parse_factor_with_initial_ex(ps, mark) else # Unary function calls with brackets as grouping, not an arglist - # .+(a) ==> (dotcall-pre (. +) a) + # .+(a) ==> (dotcall-pre (. +) (parens a)) if opts.is_block # +(a;b) ==> (call-pre + (block-p a b)) emit(ps, mark_before_paren, K"block", PARENS_FLAG) + else + emit(ps, mark_before_paren, K"parens") end # Not a prefix operator call but a block; `=` is not `kw` - # +(a=1) ==> (call-pre + (= a 1)) + # +(a=1) ==> (call-pre + (parens (= a 1))) # Unary operators have lower precedence than ^ - # +(a)^2 ==> (call-pre + (call-i a ^ 2)) - # .+(a)^2 ==> (dotcall-pre + (call-i a ^ 2)) - # +(a)(x,y)^2 ==> (call-pre + (call-i (call a x y) ^ 2)) + # +(a)^2 ==> (call-pre + (call-i (parens a) ^ 2)) + # .+(a)^2 ==> (dotcall-pre + (call-i (parens a) ^ 2)) + # +(a)(x,y)^2 ==> (call-pre + (call-i (call (parens a) x y) ^ 2)) parse_call_chain(ps, mark_before_paren) parse_factor_with_initial_ex(ps, mark_before_paren) if is_type_operator(op_t) - # <:(a) ==> (<:-pre a) + # <:(a) ==> (<:-pre (parens a)) emit(ps, mark, op_k, PREFIX_OP_FLAG) reset_node!(ps, op_pos, flags=TRIVIA_FLAG) else @@ -1451,7 +1454,7 @@ function parse_identifier_or_interpolate(ps::ParseState) mark = position(ps) parse_unary_prefix(ps) b = peek_behind(ps) - # export (x::T) ==> (export (error (::-i x T))) + # export (x::T) ==> (export (error (parens (::-i x T)))) # export outer ==> (export outer) # export ($f) ==> (export ($ f)) ok = (b.is_leaf && (b.kind == K"Identifier" || is_operator(b.kind))) || @@ -1491,13 +1494,13 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) k = kind(t) if !is_macrocall && ps.space_sensitive && preceding_whitespace(t) && k in KSet"( [ { \" \"\"\" ` ```" - # [f (x)] ==> (hcat f x) + # [f (x)] ==> (hcat f (parens x)) # [f x] ==> (hcat f x) break elseif is_macrocall && (preceding_whitespace(t) || !(k in KSet"( [ { ' .")) # Macro calls with space-separated arguments # @foo a b ==> (macrocall @foo a b) - # @foo (x) ==> (macrocall @foo x) + # @foo (x) ==> (macrocall @foo (parens x)) # @foo (x,y) ==> (macrocall @foo (tuple-p x y)) # [@foo x] ==> (vect (macrocall @foo x)) # [@foo] ==> (vect (macrocall @foo)) @@ -1537,8 +1540,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) # f(a,b) ==> (call f a b) # f(a=1; b=2) ==> (call f (= a 1) (parameters (= b 2))) # f(a; b; c) ==> (call f a (parameters b) (parameters c)) - # (a=1)() ==> (call (= a 1)) - # f (a) ==> (call f (error-t) a b) + # (a=1)() ==> (call (parens (= a 1))) + # f (a) ==> (call f (error-t) a) bump_disallowed_space(ps) bump(ps, TRIVIA_FLAG) parse_call_arglist(ps, K")") @@ -1580,7 +1583,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) else # a[i] ==> (ref a i) # a[i,j] ==> (ref a i j) - # (a=1)[] ==> (ref (= a 1)) + # (a=1)[] ==> (ref (parens (= a 1))) + # a[end] ==> (ref a end) # T[x y] ==> (typed_hcat T x y) # T[x ; y] ==> (typed_vcat T x y) # T[a b; c d] ==> (typed_vcat T (row a b) (row c d)) @@ -1905,7 +1909,7 @@ function parse_resword(ps::ParseState) else # Function/macro definition with no methods # function f end ==> (function f) - # (function f \n end) ==> (function f) + # (function f \n end) ==> (parens (function f)) # function f \n\n end ==> (function f) # function $f end ==> (function ($ f)) # macro f end ==> (macro f) @@ -2007,7 +2011,7 @@ function parse_resword(ps::ParseState) # export a, \n @b ==> (export a @b) # export +, == ==> (export + ==) # export \n a ==> (export a) - # export \$a, \$(a*b) ==> (export (\$ a) (\$ (call-i a * b))) + # export \$a, \$(a*b) ==> (export (\$ a) (\$ (parens (call-i a * b)))) bump(ps, TRIVIA_FLAG) parse_comma_separated(ps, parse_atsym) emit(ps, mark, K"export") @@ -2105,10 +2109,10 @@ function parse_function_signature(ps::ParseState, is_function::Bool) emit(ps, mark, K"error", error="Invalid macro name") else # macro f() end ==> (macro (call f) (block)) - # macro (:)(ex) end ==> (macro (call : ex) (block)) - # macro (type)(ex) end ==> (macro (call type ex) (block)) + # macro (:)(ex) end ==> (macro (call (parens :) ex) (block)) + # macro (type)(ex) end ==> (macro (call (parens type) ex) (block)) # macro $f() end ==> (macro (call ($ f)) (block)) - # macro ($f)() end ==> (macro (call ($ f)) (block)) + # macro ($f)() end ==> (macro (call (parens ($ f))) (block)) end else if peek(ps) == K"(" @@ -2145,10 +2149,11 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function ()(x) end ==> (function (call (tuple-p) x) (block)) emit(ps, mark, K"tuple", PARENS_FLAG) else - # function (A).f() end ==> (function (call (. A (quote f))) (block)) - # function (:)() end ==> (function (call :) (block)) - # function (x::T)() end ==> (function (call (::-i x T)) (block)) - # function (::T)() end ==> (function (call (::-pre T)) (block)) + # function (A).f() end ==> (function (call (. (parens A) (quote f))) (block)) + # function (:)() end ==> (function (call (parens :)) (block)) + # function (x::T)() end ==> (function (call (parens (::-i x T))) (block)) + # function (::T)() end ==> (function (call (parens (::-pre T))) (block)) + emit(ps, mark, K"parens", PARENS_FLAG) end else parse_unary_prefix(ps) @@ -2163,8 +2168,7 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function type() end ==> (function (call type) (block)) # function \n f() end ==> (function (call f) (block)) # function $f() end ==> (function (call ($ f)) (block)) - # function (:)() end ==> (function (call :) (block)) - # function (::Type{T})(x) end ==> (function (call (::-pre (curly Type T)) x) (block)) + # function (::Type{T})(x) end ==> (function (call (parens (::-pre (curly Type T))) x) (block)) end end end @@ -2205,8 +2209,8 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function (f() where T) end ==> (function (where (call f) T) (block)) # function (f()) where T end ==> (function (where (call f) T) (block)) # function (f() where T) where U end ==> (function (where (where (call f) T) U) (block)) - # function (f()::S) end ==> (function (::-i (call f) S) (block)) - # function ((f()::S) where T) end ==> (function (where (::-i (call f) S) T) (block)) + # function (f()::S) end ==> (function (parens (::-i (call f) S)) (block)) + # function ((f()::S) where T) end ==> (function (where (parens (::-i (call f) S)) T) (block)) # # TODO: Warn for use of parens? The precedence of `::` and # `where` don't work inside parens so this is a bit of a syntax @@ -2401,7 +2405,7 @@ function parse_atsym(ps::ParseState) else # export a ==> (export a) # export \n a ==> (export a) - # export $a, $(a*b) ==> (export ($ a) ($ (call * a b))) + # export $a, $(a*b) ==> (export ($ a) (parens ($ (call * a b)))) parse_identifier_or_interpolate(ps) end end @@ -2706,7 +2710,7 @@ end function parse_generator(ps::ParseState, mark, flatten=false) t = peek_token(ps) if !preceding_whitespace(t) - # [(x)for x in xs] ==> (comprehension (generator x (error) (= x xs))) + # [(x)for x in xs] ==> (comprehension (generator (parens x) (error) (= x xs))) bump_invisible(ps, K"error", TRIVIA_FLAG, error="Expected space before `for` in generator") end @@ -2715,21 +2719,21 @@ function parse_generator(ps::ParseState, mark, flatten=false) filter_mark = position(ps) parse_comma_separated(ps, parse_iteration_spec) if peek(ps) == K"if" - # (a for x in xs if cond) ==> (generator a (filter (= x xs) cond)) + # (a for x in xs if cond) ==> (parens (generator a (filter (= x xs) cond))) bump(ps, TRIVIA_FLAG) parse_cond(ps) emit(ps, filter_mark, K"filter") end t = peek_token(ps) if kind(t) == K"for" - # (xy for x in xs for y in ys) ==> (flatten xy (= x xs) (= y ys)) - # (xy for x in xs for y in ys for z in zs) ==> (flatten xy (= x xs) (= y ys) (= z zs)) + # (xy for x in xs for y in ys) ==> (parens (flatten xy (= x xs) (= y ys))) + # (xy for x in xs for y in ys for z in zs) ==> (parens (flatten xy (= x xs) (= y ys) (= z zs))) parse_generator(ps, mark, true) if !flatten emit(ps, mark, K"flatten") end elseif !flatten - # (x for a in as) ==> (generator x (= a as)) + # (x for a in as) ==> (parens (generator x (= a as))) emit(ps, mark, K"generator") end end @@ -3071,10 +3075,11 @@ function parse_paren(ps::ParseState, check_identifiers=true) emit(ps, mark, K"block", PARENS_FLAG) else # Parentheses used for grouping - # (a * b) ==> (call-i * a b) - # (a=1) ==> (= a 1) - # (x) ==> x - # (a...) ==> (... a) + # (a * b) ==> (parens (call-i * a b)) + # (a=1) ==> (parens (= a 1)) + # (x) ==> (parens x) + # (a...) ==> (parens (... a)) + emit(ps, mark, K"parens") end end end @@ -3144,8 +3149,8 @@ function parse_brackets(after_parse::Function, continue elseif k == K"for" # Generator syntax - # (x for a in as) ==> (generator x (= a as)) - # (x \n\n for a in as) ==> (generator x (= a as)) + # (x for a in as) ==> (parens (generator x (= a as))) + # (x \n\n for a in as) ==> (parens (generator x (= a as))) parse_generator(ps, mark) else # Error - recovery done when consuming closing_kind @@ -3203,8 +3208,8 @@ function parse_string(ps::ParseState, raw::Bool) bump(ps, TRIVIA_FLAG) k = peek(ps) if k == K"(" - # "a $(x + y) b" ==> (string "a " (call-i x + y) " b") - # "hi$("ho")" ==> (string "hi" (string "ho")) + # "a $(x + y) b" ==> (string "a " (parens (call-i x + y)) " b") + # "hi$("ho")" ==> (string "hi" (parens (string "ho"))) parse_atom(ps) elseif k == K"var" # var identifiers disabled in strings @@ -3346,7 +3351,7 @@ function parse_string(ps::ParseState, raw::Bool) end # String interpolations # "$x$y$z" ==> (string x y z) - # "$(x)" ==> (string x) + # "$(x)" ==> (string (parens x)) # "$x" ==> (string x) # """$x""" ==> (string-s x) # @@ -3440,7 +3445,7 @@ function parse_atom(ps::ParseState, check_identifiers=true) # Being inside quote makes keywords into identifiers at the # first level of nesting # :end ==> (quote end) - # :(end) ==> (quote (error (end))) + # :(end) ==> (quote (parens (error-t))) # Being inside quote makes end non-special again (issue #27690) # a[:(end)] ==> (ref a (quote (error-t end))) parse_atom(ParseState(ps, end_symbol=false), false) diff --git a/test/parser.jl b/test/parser.jl index cc3c02c7..6adddbfa 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -181,9 +181,9 @@ tests = [ JuliaSyntax.parse_juxtapose => [ "2x" => "(juxtapose 2 x)" "2x" => "(juxtapose 2 x)" - "2(x)" => "(juxtapose 2 x)" - "(2)(3)x" => "(juxtapose 2 3 x)" - "(x-1)y" => "(juxtapose (call-i x - 1) y)" + "2(x)" => "(juxtapose 2 (parens x))" + "(2)(3)x" => "(juxtapose (parens 2) (parens 3) x)" + "(x-1)y" => "(juxtapose (parens (call-i x - 1)) y)" "x'y" => "(juxtapose (call-post x ') y)" # errors "\"a\"\"b\"" => "(juxtapose (string \"a\") (error-t) (string \"b\"))" @@ -226,11 +226,14 @@ tests = [ # Prefix function calls for operators which are both binary and unary "+(a,b)" => "(call + a b)" ".+(a,)" => "(call .+ a)" - "(.+)(a)" => "(call (. +) a)" + "(.+)(a)" => "(call (parens (. +)) a)" "+(a=1,)" => "(call + (= a 1))" => Expr(:call, :+, Expr(:kw, :a, 1)) "+(a...)" => "(call + (... a))" "+(a;b,c)" => "(call + a (parameters b c))" "+(;a)" => "(call + (parameters a))" + "+()" => "(call +)" + "+(\n;a)" => "(call + (parameters a))" + "+(\n)" => "(call +)" # Whitespace not allowed before prefix function call bracket "+ (a,b)" => "(call + (error) a b)" # Prefix calls have higher precedence than ^ @@ -238,14 +241,14 @@ tests = [ "+(a,b)(x)^2" => "(call-i (call (call + a b) x) ^ 2)" "<:(a,)" => "(<: a)" # Unary function calls with brackets as grouping, not an arglist - ".+(a)" => "(dotcall-pre + a)" + ".+(a)" => "(dotcall-pre + (parens a))" "+(a;b)" => "(call-pre + (block-p a b))" - "+(a=1)" => "(call-pre + (= a 1))" => Expr(:call, :+, Expr(:(=), :a, 1)) + "+(a=1)" => "(call-pre + (parens (= a 1)))" => Expr(:call, :+, Expr(:(=), :a, 1)) # Unary operators have lower precedence than ^ - "+(a)^2" => "(call-pre + (call-i a ^ 2))" - ".+(a)^2" => "(dotcall-pre + (call-i a ^ 2))" - "+(a)(x,y)^2" => "(call-pre + (call-i (call a x y) ^ 2))" - "<:(a)" => "(<:-pre a)" + "+(a)^2" => "(call-pre + (call-i (parens a) ^ 2))" + ".+(a)^2" => "(dotcall-pre + (call-i (parens a) ^ 2))" + "+(a)(x,y)^2" => "(call-pre + (call-i (call (parens a) x y) ^ 2))" + "<:(a)" => "(<:-pre (parens a))" # Normal unary calls "+x" => "(call-pre + x)" "√x" => "(call-pre √ x)" @@ -276,7 +279,7 @@ tests = [ "<: \n" => "<:" "<: =" => "<:" "<:{T}(x::T)" => "(call (curly <: T) (::-i x T))" - "<:(x::T)" => "(<:-pre (::-i x T))" + "<:(x::T)" => "(<:-pre (parens (::-i x T)))" "<: x" => "(<:-pre x)" "<: <: x" => "(<:-pre (<:-pre x))" "<: A where B" => "(<:-pre (where A B))" @@ -307,11 +310,11 @@ tests = [ "\$A.@x" => "(macrocall (. (\$ A) (quote @x)))" # non-errors in space sensitive contexts - "[f (x)]" => "(hcat f x)" + "[f (x)]" => "(hcat f (parens x))" "[f x]" => "(hcat f x)" # space separated macro calls "@foo a b" => "(macrocall @foo a b)" - "@foo (x)" => "(macrocall @foo x)" + "@foo (x)" => "(macrocall @foo (parens x))" "@foo (x,y)" => "(macrocall @foo (tuple-p x y))" "A.@foo a b" => "(macrocall (. A (quote @foo)) a b)" "@A.foo a b" => "(macrocall (. A (quote @foo)) a b)" @@ -341,7 +344,7 @@ tests = [ Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1)) "f(a; b; c)" => "(call f a (parameters b) (parameters c))" => Expr(:call, :f, Expr(:parameters, Expr(:parameters, :c), :b), :a) - "(a=1)()" => "(call (= a 1))" => Expr(:call, Expr(:(=), :a, 1)) + "(a=1)()" => "(call (parens (= a 1)))" => Expr(:call, Expr(:(=), :a, 1)) "f (a)" => "(call f (error-t) a)" "@x(a, b)" => "(macrocall-p @x a b)" "A.@x(y)" => "(macrocall-p (. A (quote @x)) y)" @@ -367,7 +370,10 @@ tests = [ "a[i]" => "(ref a i)" "a [i]" => "(ref a (error-t) i)" "a[i,j]" => "(ref a i j)" - "(a=1)[]" => "(ref (= a 1))" => Expr(:ref, Expr(:(=), :a, 1)) + "(a=1)[]" => "(ref (parens (= a 1)))" => Expr(:ref, Expr(:(=), :a, 1)) + "a[end]" => "(ref a end)" + "a[begin]" => "(ref a begin)" + "a[:(end)]" => "(typed_hcat a (quote (parens (error-t))) (error-t))" "T[x y]" => "(typed_hcat T x y)" "T[x ; y]" => "(typed_vcat T x y)" "T[a b; c d]" => "(typed_vcat T (row a b) (row c d))" @@ -382,13 +388,13 @@ tests = [ "f.(a,b)" => "(dotcall f a b)" "f.(a=1; b=2)" => "(dotcall f (= a 1) (parameters (= b 2)))" => Expr(:., :f, Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))) - "(a=1).()" => "(dotcall (= a 1))" => Expr(:., Expr(:(=), :a, 1), Expr(:tuple)) + "(a=1).()" => "(dotcall (parens (= a 1)))" => Expr(:., Expr(:(=), :a, 1), Expr(:tuple)) "f. (x)" => "(dotcall f (error-t) x)" # Other dotted syntax "A.:+" => "(. A (quote +))" "A.: +" => "(. A (quote (error-t) +))" "f.\$x" => "(. f (inert (\$ x)))" - "f.\$(x+y)" => "(. f (inert (\$ (call-i x + y))))" + "f.\$(x+y)" => "(. f (inert (\$ (parens (call-i x + y)))))" "A.\$B.@x" => "(macrocall (. (. A (inert (\$ B))) (quote @x)))" "@A.\$x a" => "(macrocall (. A (inert (error x))) a)" "A.@x" => "(macrocall (. A (quote @x)))" @@ -496,10 +502,10 @@ tests = [ "export a, \n @b" => "(export a @b)" => Expr(:export, :a, Symbol("@b")) "export +, ==" => "(export + ==)" => Expr(:export, :+, :(==)) "export \n a" => "(export a)" => Expr(:export, :a) - "export \$a, \$(a*b)" => "(export (\$ a) (\$ (call-i a * b)))" => Expr(:export, Expr(:$, :a), Expr(:$, Expr(:call, :*, :a, :b))) - "export (x::T)" => "(export (error (::-i x T)))" + "export \$a, \$(a*b)" => "(export (\$ a) (\$ (parens (call-i a * b))))" => Expr(:export, Expr(:$, :a), Expr(:$, Expr(:call, :*, :a, :b))) + "export (x::T)" => "(export (error (parens (::-i x T))))" "export outer" => "(export outer)" => Expr(:export, :outer) - "export (\$f)" => "(export (\$ f))" => Expr(:export, Expr(:$, :f)) + "export (\$f)" => "(export (parens (\$ f)))" => Expr(:export, Expr(:$, :f)) ], JuliaSyntax.parse_if_elseif => [ "if a xx elseif b yy else zz end" => "(if a (block xx) (elseif b (block yy) (block zz)))" @@ -537,28 +543,27 @@ tests = [ # Macros and functions "macro while(ex) end" => "(macro (call (error while) ex) (block))" "macro f() end" => "(macro (call f) (block))" - "macro (:)(ex) end" => "(macro (call : ex) (block))" - "macro (type)(ex) end" => "(macro (call type ex) (block))" + "macro (:)(ex) end" => "(macro (call (parens :) ex) (block))" + "macro (type)(ex) end" => "(macro (call (parens type) ex) (block))" "macro \$f() end" => "(macro (call (\$ f)) (block))" - "macro (\$f)() end" => "(macro (call (\$ f)) (block))" + "macro (\$f)() end" => "(macro (call (parens (\$ f))) (block))" "function (x) body end"=> "(function (tuple-p x) (block body))" "function (x,y) end" => "(function (tuple-p x y) (block))" "function (x=1) end" => "(function (tuple-p (= x 1)) (block))" "function (;x=1) end" => "(function (tuple-p (parameters (= x 1))) (block))" "function ()(x) end" => "(function (call (tuple-p) x) (block))" - "function (A).f() end" => "(function (call (. A (quote f))) (block))" - "function (:)() end" => "(function (call :) (block))" - "function (x::T)() end"=> "(function (call (::-i x T)) (block))" - "function (::g(x))() end" => "(function (call (::-pre (call g x))) (block))" - "function (f::T{g(i)})() end" => "(function (call (::-i f (curly T (call g i)))) (block))" - "function (::T)() end" => "(function (call (::-pre T)) (block))" + "function (A).f() end" => "(function (call (. (parens A) (quote f))) (block))" + "function (:)() end" => "(function (call (parens :)) (block))" + "function (x::T)() end"=> "(function (call (parens (::-i x T))) (block))" + "function (::g(x))() end" => "(function (call (parens (::-pre (call g x)))) (block))" + "function (f::T{g(i)})() end" => "(function (call (parens (::-i f (curly T (call g i))))) (block))" + "function (::T)() end" => "(function (call (parens (::-pre T))) (block))" "function begin() end" => "(function (call (error begin)) (block))" "function f() end" => "(function (call f) (block))" "function type() end" => "(function (call type) (block))" "function \n f() end" => "(function (call f) (block))" "function \$f() end" => "(function (call (\$ f)) (block))" - "function (:)() end" => "(function (call :) (block))" - "function (::Type{T})(x) end" => "(function (call (::-pre (curly Type T)) x) (block))" + "function (::Type{T})(x) end" => "(function (call (parens (::-pre (curly Type T))) x) (block))" # Function/macro definition with no methods "function f end" => "(function f)" "function f \n\n end" => "(function f)" @@ -576,11 +581,11 @@ tests = [ "function f()::S where T end" => "(function (where (::-i (call f) S) T) (block))" # Ugly cases for compat where extra parentheses existed and we've # already parsed at least the call part of the signature - "function (f() where T) end" => "(function (where (call f) T) (block))" => Expr(:function, Expr(:where, Expr(:call, :f), :T), Expr(:block)) - "function (f()) where T end" => "(function (where (call f) T) (block))" - "function (f() where T) where U end" => "(function (where (where (call f) T) U) (block))" - "function (f()::S) end"=> "(function (::-i (call f) S) (block))" => Expr(:function, Expr(:(::), Expr(:call, :f), :S), Expr(:block)) - "function ((f()::S) where T) end" => "(function (where (::-i (call f) S) T) (block))" + "function (f() where T) end" => "(function (parens (where (call f) T)) (block))" => Expr(:function, Expr(:where, Expr(:call, :f), :T), Expr(:block)) + "function (f()) where T end" => "(function (where (parens (call f)) T) (block))" + "function (f() where T) where U end" => "(function (where (parens (where (call f) T)) U) (block))" + "function (f()::S) end"=> "(function (parens (::-i (call f) S)) (block))" => Expr(:function, Expr(:(::), Expr(:call, :f), :S), Expr(:block)) + "function ((f()::S) where T) end" => "(function (parens (where (parens (::-i (call f) S)) T)) (block))" # body "function f() \n a \n b end" => "(function (call f) (block a b))" "function f() end" => "(function (call f) (block))" @@ -682,16 +687,16 @@ tests = [ "(a;b;;c)" => "(block-p a b c)" "(a=1; b=2)" => "(block-p (= a 1) (= b 2))" # Parentheses used for grouping - "(a * b)" => "(call-i a * b)" - "(a=1)" => "(= a 1)" - "(x)" => "x" - "(a...)" => "(... a)" + "(a * b)" => "(parens (call-i a * b))" + "(a=1)" => "(parens (= a 1))" + "(x)" => "(parens x)" + "(a...)" => "(parens (... a))" # Generators - "(x for a in as)" => "(generator x (= a as))" - "(x \n\n for a in as)" => "(generator x (= a as))" + "(x for a in as)" => "(parens (generator x (= a as)))" + "(x \n\n for a in as)" => "(parens (generator x (= a as)))" # Range parsing in parens - "(1:\n2)" => "(call-i 1 : 2)" - "(1:2)" => "(call-i 1 : 2)" + "(1:\n2)" => "(parens (call-i 1 : 2))" + "(1:2)" => "(parens (call-i 1 : 2))" ], JuliaSyntax.parse_atom => [ # char literal @@ -741,7 +746,7 @@ tests = [ ":.=" => "(quote .=)" # Special symbols quoted ":end" => "(quote end)" - ":(end)" => "(quote (error-t))" + ":(end)" => "(quote (parens (error-t)))" ":<:" => "(quote <:)" # unexpect = "=" => "(error =)" @@ -759,11 +764,11 @@ tests = [ # parse_generator "[x for a = as for b = bs if cond1 for c = cs if cond2]" => "(comprehension (flatten x (= a as) (filter (= b bs) cond1) (filter (= c cs) cond2)))" "[x for a = as if begin cond2 end]" => "(comprehension (generator x (filter (= a as) (block cond2))))" - "[(x)for x in xs]" => "(comprehension (generator x (error-t) (= x xs)))" - "(a for x in xs if cond)" => "(generator a (filter (= x xs) cond))" - "(xy for x in xs for y in ys)" => "(flatten xy (= x xs) (= y ys))" - "(xy for x in xs for y in ys for z in zs)" => "(flatten xy (= x xs) (= y ys) (= z zs))" - "(x for a in as)" => "(generator x (= a as))" + "[(x)for x in xs]" => "(comprehension (generator (parens x) (error-t) (= x xs)))" + "(a for x in xs if cond)" => "(parens (generator a (filter (= x xs) cond)))" + "(xy for x in xs for y in ys)" => "(parens (flatten xy (= x xs) (= y ys)))" + "(xy for x in xs for y in ys for z in zs)" => "(parens (flatten xy (= x xs) (= y ys) (= z zs)))" + "(x for a in as)" => "(parens (generator x (= a as)))" # parse_vect "[x, y]" => "(vect x y)" "[x, y]" => "(vect x y)" @@ -776,7 +781,7 @@ tests = [ # parse_paren ":(=)" => "(quote =)" ":(::)" => "(quote ::)" - "(function f \n end)" => "(function f)" + "(function f \n end)" => "(parens (function f))" # braces "{x y}" => "(bracescat (row x y))" ((v=v"1.7",), "{x ;;; y}") => "(bracescat (nrow-3 x y))" @@ -845,8 +850,8 @@ tests = [ ((v=v"1.7",), "[;;]") => "(ncat-2 (error))" # parse_string "\"\"\"\n\$x\n a\"\"\"" => "(string-s x \"\\n\" \" a\")" - "\"a \$(x + y) b\"" => "(string \"a \" (call-i x + y) \" b\")" - "\"hi\$(\"ho\")\"" => "(string \"hi\" (string \"ho\"))" + "\"a \$(x + y) b\"" => "(string \"a \" (parens (call-i x + y)) \" b\")" + "\"hi\$(\"ho\")\"" => "(string \"hi\" (parens (string \"ho\")))" "\"a \$foo b\"" => "(string \"a \" foo \" b\")" "\"\$var\"" => "(string var)" "\"\$outer\"" => "(string outer)" @@ -888,7 +893,7 @@ tests = [ "\"str" => "(string \"str\" (error-t))" # String interpolations "\"\$x\$y\$z\"" => "(string x y z)" - "\"\$(x)\"" => "(string x)" + "\"\$(x)\"" => "(string (parens x))" "\"\$x\"" => "(string x)" # Strings with embedded whitespace trivia "\"a\\\nb\"" => raw"""(string "a" "b")""" @@ -931,11 +936,11 @@ end parseall_test_specs = [ # whitespace before keywords in space-insensitive mode - "(y::\nif x z end)" => "(toplevel (::-i y (if x (block z))))" + "(y::\nif x z end)" => "(toplevel (parens (::-i y (if x (block z)))))" # The following may not be ideal error recovery! But at least the parser # shouldn't crash - "@(x y)" => "(toplevel (macrocall x (error-t @y)))" + "@(x y)" => "(toplevel (macrocall (error x (error-t y))))" "|(&\nfunction" => "(toplevel (call | (& (function (error (error)) (block (error)) (error-t))) (error-t)))" ] diff --git a/test/parser_api.jl b/test/parser_api.jl index bc6a1aa8..719af8bb 100644 --- a/test/parser_api.jl +++ b/test/parser_api.jl @@ -3,11 +3,7 @@ @test parse(Expr, " x ") == :x @test JuliaSyntax.remove_linenums!(parseall(Expr, " x ")) == Expr(:toplevel, :x) @test parseatom(Expr, " x ") == :x - # TODO: Fix this situation with trivia here; the brackets are trivia, but - # must be parsed to discover the atom inside. But in GreenTree we only - # place trivia as siblings of the leaf node with identifier `x`, not as - # children. - @test_broken parseatom(Expr, "(x)") == :x + @test parseatom(Expr, "(x)") == :x # SubString @test parse(Expr, SubString("x+y")) == :(x+y)