Make low precedence comma a feature flag, off by default

Also enable it within concatenation syntax for consistency

Author: c42f

src/parse_stream.jl

@@ -236,12 +236,16 @@ mutable struct ParseStream
     diagnostics::Vector{Diagnostic}
     # Counter for number of peek()s we've done without making progress via a bump()
     peek_count::Int
+
+    # Feature flags
     # (major,minor) version of Julia we're parsing this code for.
     # May be different from VERSION!
     version::Tuple{Int,Int}
+    # Comma binds looser than macrocall in bracketed expressions
+    low_precedence_comma_in_brackets::Bool
 
-    function ParseStream(text_buf::Vector{UInt8}, text_root, next_byte::Integer,
-                         version::VersionNumber)
+    function ParseStream(text_buf::Vector{UInt8}, text_root, next_byte::Integer;
+                         version=VERSION, low_precedence_comma_in_brackets=false)
         io = IOBuffer(text_buf)
         seek(io, next_byte-1)
         lexer = Tokenize.Lexer(io)
@@ -264,44 +268,45 @@ mutable struct ParseStream
             Vector{TaggedRange}(),
             Vector{Diagnostic}(),
             0,
-            ver)
+            ver,
+            low_precedence_comma_in_brackets)
     end
 end
 
-function ParseStream(text::Vector{UInt8}, index::Integer=1; version=VERSION)
-    ParseStream(text, text, index, version)
+function ParseStream(text::Vector{UInt8}, index::Integer=1; kws...)
+    ParseStream(text, text, index; kws...)
 end
 
 # Buffer with unknown owner. Not exactly recommended, but good for C interop
-function ParseStream(ptr::Ptr{UInt8}, len::Integer, index::Integer=1; version=VERSION)
-    ParseStream(unsafe_wrap(Vector{UInt8}, ptr, len), nothing, index, version)
+function ParseStream(ptr::Ptr{UInt8}, len::Integer, index::Integer=1; kws...)
+    ParseStream(unsafe_wrap(Vector{UInt8}, ptr, len), nothing, index; kws...)
 end
 
 # Buffers originating from strings
-function ParseStream(text::String, index::Integer=1; version=VERSION)
+function ParseStream(text::String, index::Integer=1; kws...)
     ParseStream(unsafe_wrap(Vector{UInt8}, text),
-                text, index, version)
+                text, index; kws...)
 end
-function ParseStream(text::SubString, index::Integer=1; version=VERSION)
+function ParseStream(text::SubString, index::Integer=1; kws...)
     # See also IOBuffer(SubString("x"))
     ParseStream(unsafe_wrap(Vector{UInt8}, pointer(text), sizeof(text)),
-                text, index, version)
+                text, index; kws...)
 end
-function ParseStream(text::AbstractString, index::Integer=1; version=VERSION)
-    ParseStream(String(text), index; version=version)
+function ParseStream(text::AbstractString, index::Integer=1; kws...)
+    ParseStream(String(text), index; kws...)
 end
 
 # IO-based cases
-function ParseStream(io::IOBuffer; version=VERSION)
-    ParseStream(io.data, io, position(io)+1, version)
+function ParseStream(io::IOBuffer; kws...)
+    ParseStream(io.data, io, position(io)+1; kws...)
 end
-function ParseStream(io::Base.GenericIOBuffer; version=VERSION)
+function ParseStream(io::Base.GenericIOBuffer; kws...)
     textbuf = unsafe_wrap(Vector{UInt8}, pointer(io.data), length(io.data))
-    ParseStream(textbuf, io, position(io)+1, version)
+    ParseStream(textbuf, io, position(io)+1; kws...)
 end
-function ParseStream(io::IO; version=VERSION)
+function ParseStream(io::IO; kws...)
     textbuf = read(io)
-    ParseStream(textbuf, textbuf, 1, version)
+    ParseStream(textbuf, textbuf, 1; kws...)
 end
 
 function Base.show(io::IO, mime::MIME"text/plain", stream::ParseStream)

src/parser.jl

@@ -21,7 +21,7 @@ struct ParseState
     whitespace_newline::Bool
     # Enable parsing `where` with high precedence
     where_enabled::Bool
-    # Comma special
+    # Comma binds looser than macro calls (for use in brackets)
     low_precedence_comma::Bool
 end
 
@@ -41,7 +41,8 @@ function ParseState(ps::ParseState; range_colon_enabled=nothing,
         end_symbol === nothing ? ps.end_symbol : end_symbol,
         whitespace_newline === nothing ? ps.whitespace_newline : whitespace_newline,
         where_enabled === nothing ? ps.where_enabled : where_enabled,
-        low_precedence_comma === nothing ? ps.low_precedence_comma : low_precedence_comma)
+        low_precedence_comma === nothing ? ps.low_precedence_comma :
+            low_precedence_comma && ps.stream.low_precedence_comma_in_brackets)
 end
 
 # Functions to change parse state
@@ -2937,7 +2938,8 @@ function parse_cat(ps::ParseState, closer, end_is_symbol)
                     space_sensitive=true,
                     where_enabled=true,
                     whitespace_newline=false,
-                    for_generator=true)
+                    for_generator=true,
+                    low_precedence_comma=true)
     k = peek(ps, skip_newlines=true)
     mark = position(ps)
     if k == closer

src/parser_api.jl

@@ -76,9 +76,12 @@ function parse!(::Type{TreeType}, io::IO;
 end
 
 function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version=VERSION,
+                low_precedence_comma_in_brackets=false,
                 ignore_trivia=true, filename=nothing, first_line=1, ignore_errors=false,
                 ignore_warnings=ignore_errors, kws...) where {T}
-    stream = ParseStream(text, index; version=version)
+    stream = ParseStream(text, index;
+         version=version,
+         low_precedence_comma_in_brackets=low_precedence_comma_in_brackets)
     if ignore_trivia && rule != :all
         bump_trivia(stream, skip_newlines=true)
         empty!(stream)

test/expr.jl

@@ -399,6 +399,15 @@
             Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:ncat, 2, :a, :b))
     end
 
+    @testset "var" begin
+        @test parsestmt("var\"x\"") == :x
+        @test parsestmt("var\"\"")         == Symbol("")
+        @test parsestmt("var\"\\\"\"")     == Symbol("\"")
+        @test parsestmt("var\"\\\\\\\"\"") == Symbol("\\\"")
+        @test parsestmt("var\"\\\\x\"")    == Symbol("\\\\x")
+        @test parsestmt("var\"x\"+y")      == Expr(:call, :+, :x, :y)
+    end
+
     @testset "vect" begin
         @test parsestmt("[x,y ; z]") == Expr(:vect, Expr(:parameters, :z), :x, :y)
     end

test/parser.jl

@@ -1,8 +1,8 @@
 """
 Parse string to SyntaxNode tree and show as an sexpression
 """
-function parse_to_sexpr_str(production, code::AbstractString; v=v"1.6", expr=false)
-    stream = ParseStream(code, version=v)
+function parse_to_sexpr_str(production, code::AbstractString; v=v"1.6", expr=false, kws...)
+    stream = ParseStream(code; version=v, kws...)
     production(ParseState(stream))
     JuliaSyntax.validate_tokens(stream)
     t = build_tree(GreenNode, stream, wrap_toplevel_as_kind=K"None")
@@ -25,12 +25,7 @@ function test_parse(production, input, output)
     else
         opts = NamedTuple()
     end
-    if output isa Pair
-        @test parse_to_sexpr_str(production, input; opts...) == output[1]
-        @test parse_to_sexpr_str(production, input; opts..., expr=true) == output[2]
-    else
-        @test parse_to_sexpr_str(production, input; opts...) == output
-    end
+    @test parse_to_sexpr_str(production, input; opts...) == output
 end
 
 function test_parse(inout::Pair)
@@ -435,6 +430,13 @@ tests = [
         "x\"s\"2"    => """(macrocall @x_str (string-r "s") 2)"""
         "x\"s\"10.0" => """(macrocall @x_str (string-r "s") 10.0)"""
         # 
+        "f(@x a, b)" => "(call f (macrocall @x (tuple a b)))"
+        ((low_precedence_comma_in_brackets=true,), "f(@x a, b)") =>
+            "(call f (macrocall @x a) b)"
+        ((low_precedence_comma_in_brackets=true,), "(@x a, b)") =>
+            "(tuple-p (macrocall @x a) b)"
+        ((low_precedence_comma_in_brackets=true,), "[@x a, b]") =>
+            "(vect (macrocall @x a) b)"
     ],
     JuliaSyntax.parse_resword => [
         # In normal_context
@@ -724,16 +726,16 @@ tests = [
         "xx"     => "xx"
         "x₁"     => "x₁"
         # var syntax
-        """var"x" """   =>  "(var x)"  => :x
+        """var"x" """   =>  "(var x)"
         # var syntax raw string unescaping
-        "var\"\""          =>  "(var )"      => Symbol("")
-        "var\"\\\"\""      =>  "(var \")"    => Symbol("\"")
-        "var\"\\\\\\\"\""  =>  "(var \\\")"  => Symbol("\\\"")
-        "var\"\\\\x\""     =>  "(var \\\\x)" => Symbol("\\\\x")
+        "var\"\""          =>  "(var )"
+        "var\"\\\"\""      =>  "(var \")"
+        "var\"\\\\\\\"\""  =>  "(var \\\")"
+        "var\"\\\\x\""     =>  "(var \\\\x)"
         # trailing syntax after var
-        """var"x"+"""   =>  "(var x)" => :x
-        """var"x")"""   =>  "(var x)" => :x
-        """var"x"("""   =>  "(var x)" => :x
+        """var"x"+"""   =>  "(var x)"
+        """var"x")"""   =>  "(var x)"
+        """var"x"("""   =>  "(var x)"
         """var"x"end""" =>  "(var x (error-t))"
         """var"x"1"""   =>  "(var x (error-t))"
         """var"x"y"""   =>  "(var x (error-t))"