Skip to content

Add TRAILING_COMMA_FLAG to distinguish (a,b) vs (a,b,) #521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 22, 2024
Merged

Add TRAILING_COMMA_FLAG to distinguish (a,b) vs (a,b,) #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/src/api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ JuliaSyntax.has_flags
JuliaSyntax.TRIPLE_STRING_FLAG
JuliaSyntax.RAW_STRING_FLAG
JuliaSyntax.PARENS_FLAG
JuliaSyntax.TRAILING_COMMA_FLAG
JuliaSyntax.COLON_QUOTE
JuliaSyntax.TOPLEVEL_SEMICOLONS_FLAG
JuliaSyntax.MUTABLE_FLAG
Expand Down
26 changes: 19 additions & 7 deletions src/parse_stream.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ Set for K"tuple", K"block" or K"macrocall" which are delimited by parentheses
"""
const PARENS_FLAG = RawFlags(1<<5)

"""
Set for various delimited constructs when they contains a trailing comma. For
example, to distinguish `(a,b,)` vs `(a,b)`, and `f(a)` vs `f(a,)`. Kinds where
this applies are: `tuple call dotcall macrocall vect curly braces <: >:`.
"""
const TRAILING_COMMA_FLAG = RawFlags(1<<6)

"""
Set for K"quote" for the short form `:x` as opposed to long form `quote x end`
"""
Expand Down Expand Up @@ -139,22 +146,27 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
is_prefix_op_call(head) && (str = str*"-pre")
is_postfix_op_call(head) && (str = str*"-post")

if kind(head) in KSet"string cmdstring Identifier"
k = kind(head)
if k in KSet"string cmdstring Identifier"
has_flags(head, TRIPLE_STRING_FLAG) && (str = str*"-s")
has_flags(head, RAW_STRING_FLAG) && (str = str*"-r")
elseif kind(head) in KSet"tuple block macrocall"
elseif k in KSet"tuple block macrocall"
has_flags(head, PARENS_FLAG) && (str = str*"-p")
elseif kind(head) == K"quote"
elseif k == K"quote"
has_flags(head, COLON_QUOTE) && (str = str*"-:")
elseif kind(head) == K"toplevel"
elseif k == K"toplevel"
has_flags(head, TOPLEVEL_SEMICOLONS_FLAG) && (str = str*"-;")
elseif kind(head) == K"function"
elseif k == K"function"
has_flags(head, SHORT_FORM_FUNCTION_FLAG) && (str = str*"-=")
elseif kind(head) == K"struct"
elseif k == K"struct"
has_flags(head, MUTABLE_FLAG) && (str = str*"-mut")
elseif kind(head) == K"module"
elseif k == K"module"
has_flags(head, BARE_MODULE_FLAG) && (str = str*"-bare")
end
if k in KSet"tuple call dotcall macrocall vect curly braces <: >:" &&
has_flags(head, TRAILING_COMMA_FLAG)
str *= "-,"
end
is_suffixed(head) && (str = str*"-suf")
n = numeric_flags(head)
n != 0 && (str = str*"-"*string(n))
Expand Down
56 changes: 34 additions & 22 deletions src/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1302,10 +1302,10 @@ function parse_unary(ps::ParseState)
# +(a,b)(x)^2 ==> (call-i (call (call + a b) x) ^ 2)
if is_type_operator(op_t)
# <:(a,) ==> (<: a)
emit(ps, mark, op_k)
emit(ps, mark, op_k, opts.delim_flags)
reset_node!(ps, op_pos, flags=TRIVIA_FLAG)
else
emit(ps, mark, K"call")
emit(ps, mark, K"call", opts.delim_flags)
end
parse_call_chain(ps, mark)
parse_factor_with_initial_ex(ps, mark)
Expand Down Expand Up @@ -1552,13 +1552,14 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
# f (a) ==> (call f (error-t) a)
bump_disallowed_space(ps)
bump(ps, TRIVIA_FLAG)
parse_call_arglist(ps, K")")
opts = parse_call_arglist(ps, K")")
if peek(ps) == K"do"
# f(x) do y body end ==> (call f x (do (tuple y) (block body)))
parse_do(ps)
end
emit(ps, mark, is_macrocall ? K"macrocall" : K"call",
is_macrocall ? PARENS_FLAG : EMPTY_FLAGS)
# TODO: Add PARENS_FLAG to all calls which use them?
(is_macrocall ? PARENS_FLAG : EMPTY_FLAGS)|opts.delim_flags)
if is_macrocall
# @x(a, b) ==> (macrocall-p @x a b)
# A.@x(y) ==> (macrocall-p (. A @x) y)
Expand Down Expand Up @@ -1634,8 +1635,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
# f. (x) ==> (dotcall f (error-t) x)
bump_disallowed_space(ps)
bump(ps, TRIVIA_FLAG)
parse_call_arglist(ps, K")")
emit(ps, mark, K"dotcall")
opts = parse_call_arglist(ps, K")")
emit(ps, mark, K"dotcall", opts.delim_flags)
elseif k == K":"
# A.:+ ==> (. A (quote-: +))
# A.: + ==> (. A (error-t) (quote-: +))
Expand Down Expand Up @@ -1697,20 +1698,20 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
# S {a} ==> (curly S (error-t) a)
bump_disallowed_space(ps)
bump(ps, TRIVIA_FLAG)
parse_call_arglist(ps, K"}")
opts = parse_call_arglist(ps, K"}")
if is_macrocall
# @S{a,b} ==> (macrocall S (braces a b))
# A.@S{a} ==> (macrocall (. A @S) (braces a))
# @S{a}.b ==> (. (macrocall @S (braces a)) b)
fix_macro_name_kind!(ps, macro_name_position)
emit(ps, m, K"braces")
emit(ps, m, K"braces", opts.delim_flags)
emit(ps, mark, K"macrocall")
min_supported_version(v"1.6", ps, mark, "macro call without space before `{}`")
is_macrocall = false
macro_atname_range = nothing
else
# S{a,b} ==> (curly S a b)
emit(ps, mark, K"curly")
emit(ps, mark, K"curly", opts.delim_flags)
end
elseif k in KSet" \" \"\"\" ` ``` " &&
!preceding_whitespace(t) && maybe_strmac &&
Expand Down Expand Up @@ -2151,7 +2152,7 @@ function parse_function_signature(ps::ParseState, is_function::Bool)
# function (f(x),) end ==> (function (tuple-p (call f x)) (block))
ambiguous_parens = opts.maybe_grouping_parens &&
peek_behind(ps).kind in KSet"macrocall $"
emit(ps, mark, K"tuple", PARENS_FLAG)
emit(ps, mark, K"tuple", PARENS_FLAG|opts.delim_flags)
if ambiguous_parens
# Got something like `(@f(x))`. Is it anon `(@f(x),)` or named sig `@f(x)` ??
emit(ps, mark, K"error", error="Ambiguous signature. Add a trailing comma if this is a 1-argument anonymous function; remove parentheses if this is a macro call acting as function signature.")
Expand Down Expand Up @@ -2716,16 +2717,21 @@ end
# surrounding brackets.
#
# flisp: parse-vect
function parse_vect(ps::ParseState, closer)
function parse_vect(ps::ParseState, closer, prefix_trailing_comma)
# [x, y] ==> (vect x y)
# [x, y] ==> (vect x y)
# [x,y ; z] ==> (vect x y (parameters z))
# [x=1, y=2] ==> (vect (= x 1) (= y 2))
# [x=1, ; y=2] ==> (vect (= x 1) (parameters (= y 2)))
parse_brackets(ps, closer) do _, _, _, _
return (needs_parameters=true,)
opts = parse_brackets(ps, closer) do _, _, _, num_subexprs
return (needs_parameters=true,
num_subexprs=num_subexprs)
end
delim_flags = opts.delim_flags
if opts.num_subexprs == 0 && prefix_trailing_comma
delim_flags |= TRAILING_COMMA_FLAG
end
return (K"vect", EMPTY_FLAGS)
return (K"vect", delim_flags)
end

# Parse generators
Expand Down Expand Up @@ -2988,7 +2994,7 @@ function parse_cat(ps::ParseState, closer, end_is_symbol)
mark = position(ps)
if k == closer
# [] ==> (vect)
return parse_vect(ps, closer)
return parse_vect(ps, closer, false)
elseif k == K";"
#v1.8: [;] ==> (ncat-1)
#v1.8: [;;] ==> (ncat-2)
Expand All @@ -3003,14 +3009,15 @@ function parse_cat(ps::ParseState, closer, end_is_symbol)
parse_eq_star(ps)
k = peek(ps, skip_newlines=true)
if k == K"," || (is_closing_token(ps, k) && k != K";")
if k == K","
prefix_trailing_comma = k == K","
if prefix_trailing_comma
# [x,] ==> (vect x)
bump(ps, TRIVIA_FLAG; skip_newlines = true)
end
# [x] ==> (vect x)
# [x \n ] ==> (vect x)
# [x ==> (vect x (error-t))
parse_vect(ps, closer)
parse_vect(ps, closer, prefix_trailing_comma)
elseif k == K"for"
# [x for a in as] ==> (comprehension (generator x (iteration (in a as))))
# [x \n\n for a in as] ==> (comprehension (generator x (iteration (in a as))))
Expand Down Expand Up @@ -3087,7 +3094,7 @@ function parse_paren(ps::ParseState, check_identifiers=true)
# (; a=1; b=2) ==> (tuple-p (parameters (= a 1)) (parameters (= b 2)))
# (a; b; c,d) ==> (tuple-p a (parameters b) (parameters c d))
# (a=1, b=2; c=3) ==> (tuple-p (= a 1) (= b 2) (parameters (= c 3)))
emit(ps, mark, K"tuple", PARENS_FLAG)
emit(ps, mark, K"tuple", PARENS_FLAG|opts.delim_flags)
elseif opts.is_block
# Blocks
# (;;) ==> (block-p)
Expand Down Expand Up @@ -3135,6 +3142,7 @@ function parse_brackets(after_parse::Function,
had_commas = false
had_splat = false
param_start = nothing
trailing_comma = false
while true
k = peek(ps)
if k == closing_kind
Expand All @@ -3150,11 +3158,13 @@ function parse_brackets(after_parse::Function,
bump(ps, TRIVIA_FLAG)
bump_trivia(ps)
elseif is_closing_token(ps, k)
trailing_comma = false
# Error; handled below in bump_closing_token
break
else
mark = position(ps)
parse_eq_star(ps)
trailing_comma = false
num_subexprs += 1
if num_subexprs == 1
had_splat = peek_behind(ps).kind == K"..."
Expand All @@ -3172,6 +3182,7 @@ function parse_brackets(after_parse::Function,
if k == K","
had_commas = true
bump(ps, TRIVIA_FLAG)
trailing_comma = true
elseif k == K";" || k == closing_kind
# Handled above
continue
Expand All @@ -3193,7 +3204,7 @@ function parse_brackets(after_parse::Function,
end
release_positions(ps.stream, params_positions)
bump_closing_token(ps, closing_kind, " or `,`")
return opts
return (; opts..., delim_flags=trailing_comma ? TRAILING_COMMA_FLAG : EMPTY_FLAGS)
end

_is_indentation(b::UInt8) = (b == u8" " || b == u8"\t")
Expand Down Expand Up @@ -3420,14 +3431,15 @@ end
function emit_braces(ps, mark, ckind, cflags)
if ckind == K"hcat"
# {x y} ==> (bracescat (row x y))
emit(ps, mark, K"row", cflags)
emit(ps, mark, K"row", cflags & ~TRAILING_COMMA_FLAG)
elseif ckind == K"ncat"
# {x ;;; y} ==> (bracescat (nrow-3 x y))
emit(ps, mark, K"nrow", cflags)
emit(ps, mark, K"nrow", cflags & ~TRAILING_COMMA_FLAG)
end
check_ncat_compat(ps, mark, ckind)
outk = ckind in KSet"vect comprehension" ? K"braces" : K"bracescat"
emit(ps, mark, outk)
delim_flags = outk == K"braces" ? (cflags & TRAILING_COMMA_FLAG) : EMPTY_FLAGS
emit(ps, mark, outk, delim_flags)
end

# parse numbers, identifiers, parenthesized expressions, lists, vectors, etc.
Expand Down
29 changes: 19 additions & 10 deletions test/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,9 +234,10 @@ tests = [
".*(x)" => "(call (. *) x)"
# Prefix function calls for operators which are both binary and unary
"+(a,b)" => "(call + a b)"
".+(a,)" => "(call (. +) a)"
"+(a,)" => "(call-, + a)"
".+(a,)" => "(call-, (. +) a)"
"(.+)(a)" => "(call (parens (. +)) a)"
"+(a=1,)" => "(call + (= a 1))"
"+(a=1,)" => "(call-, + (= a 1))"
"+(a...)" => "(call + (... a))"
"+(a;b,c)" => "(call + a (parameters b c))"
"+(;a)" => "(call + (parameters a))"
Expand All @@ -251,7 +252,7 @@ tests = [
# Prefix calls have higher precedence than ^
"+(a,b)^2" => "(call-i (call + a b) ^ 2)"
"+(a,b)(x)^2" => "(call-i (call (call + a b) x) ^ 2)"
"<:(a,)" => "(<: a)"
"<:(a,)" => "(<:-, a)"
# Unary function calls with brackets as grouping, not an arglist
".+(a)" => "(dotcall-pre + (parens a))"
"+(a;b)" => "(call-pre + (block-p a b))"
Expand Down Expand Up @@ -306,6 +307,7 @@ tests = [
# Really for parse_where
"x where \n {T}" => "(where x (braces T))"
"x where {T,S}" => "(where x (braces T S))"
"x where {T,S,}" => "(where x (braces-, T S))"
"x where {T S}" => "(where x (bracescat (row T S)))"
"x where {y for y in ys}" => "(where x (braces (generator y (iteration (in y ys)))))"
"x where T" => "(where x T)"
Expand Down Expand Up @@ -364,11 +366,13 @@ tests = [

# calls with brackets
"f(a,b)" => "(call f a b)"
"f(a,)" => "(call-, f a)"
"f(a=1; b=2)" => "(call f (= a 1) (parameters (= b 2)))"
"f(a; b; c)" => "(call f a (parameters b) (parameters c))"
"(a=1)()" => "(call (parens (= a 1)))"
"f (a)" => "(call f (error-t) a)"
"@x(a, b)" => "(macrocall-p @x a b)"
"@x(a, b,)" => "(macrocall-p-, @x a b)"
"A.@x(y)" => "(macrocall-p (. A @x) y)"
"A.@x(y).z" => "(. (macrocall-p (. A @x) y) z)"
"f(y for x = xs; a)" => "(call f (generator y (iteration (in x xs))) (parameters a))"
Expand Down Expand Up @@ -407,6 +411,7 @@ tests = [
"[email protected]" => "(macrocall (. (. A B) (error-t) @x))"
"@M.(x)" => "(macrocall (dotcall @M (error-t) x))"
"f.(a,b)" => "(dotcall f a b)"
"f.(a,b,)" => "(dotcall-, f a b)"
"f.(a=1; b=2)" => "(dotcall f (= a 1) (parameters (= b 2)))"
"(a=1).()" => "(dotcall (parens (= a 1)))"
"f. (x)" => "(dotcall f (error-t) x)"
Expand Down Expand Up @@ -577,9 +582,10 @@ tests = [
"macro (\$f)() end" => "(macro (call (parens (\$ f))) (block))"
"function (x) body end"=> "(function (tuple-p x) (block body))"
"function (x,y) end" => "(function (tuple-p x y) (block))"
"function (x,y,) end" => "(function (tuple-p-, x y) (block))"
"function (x=1) end" => "(function (tuple-p (= x 1)) (block))"
"function (;x=1) end" => "(function (tuple-p (parameters (= x 1))) (block))"
"function (f(x),) end" => "(function (tuple-p (call f x)) (block))"
"function (f(x),) end" => "(function (tuple-p-, (call f x)) (block))"
"function (@f(x);) end" => "(function (tuple-p (macrocall-p @f x) (parameters)) (block))"
"function (@f(x)...) end" => "(function (tuple-p (... (macrocall-p @f x))) (block))"
"function (@f(x)) end" => "(function (error (tuple-p (macrocall-p @f x))) (block))"
Expand Down Expand Up @@ -715,7 +721,7 @@ tests = [
JuliaSyntax.parse_paren => [
# Tuple syntax with commas
"()" => "(tuple-p)"
"(x,)" => "(tuple-p x)"
"(x,)" => "(tuple-p-, x)"
"(x,y)" => "(tuple-p x y)"
"(x=1, y=2)" => "(tuple-p (= x 1) (= y 2))"
# Named tuples with initial semicolon
Expand Down Expand Up @@ -827,11 +833,12 @@ tests = [
"=" => "(error =)"
# parse_cat
"[]" => "(vect)"
"[x,]" => "(vect x)"
"[x\n,,]" => "(vect x (error-t ✘))"
"[x,]" => "(vect-, x)"
"[x,y,]" => "(vect-, x y)"
"[x\n,,]" => "(vect-, x (error-t ✘))"
"[x]" => "(vect x)"
"[x \n ]" => "(vect x)"
"[x \n, ]" => "(vect x)"
"[x \n, ]" => "(vect-, x)"
"[x" => "(vect x (error-t))"
"[x \n\n ]" => "(vect x)"
"[x for a in as]" => "(comprehension (generator x (iteration (in a as))))"
Expand All @@ -849,10 +856,10 @@ tests = [
"(x for a in as if z)" => "(parens (generator x (filter (iteration (in a as)) z)))"
# parse_vect
"[x, y]" => "(vect x y)"
"[x, y]" => "(vect x y)"
"[x, y,]" => "(vect-, x y)"
"[x,\n y]" => "(vect x y)"
"[x\n, y]" => "(vect x y)"
"[x\n,, y]" => "(vect x (error-t ✘ y))"
"[x\n,, y]" => "(vect-, x (error-t ✘ y))"
"[x,y ; z]" => "(vect x y (parameters z))"
"[x=1, y=2]" => "(vect (= x 1) (= y 2))"
"[x=1, ; y=2]" => "(vect (= x 1) (parameters (= y 2)))"
Expand All @@ -862,6 +869,8 @@ tests = [
":(::\n)" => "(quote-: (parens ::))"
"(function f \n end)" => "(parens (function f))"
# braces
"{x,y}" => "(braces x y)"
"{x,y,}" => "(braces-, x y)"
"{x y}" => "(bracescat (row x y))"
((v=v"1.7",), "{x ;;; y}") => "(bracescat (nrow-3 x y))"
# Macro names can be keywords
Expand Down
Loading