Merge pull request #347 from julia-vscode/sp/fix-generator-for-parsing

Fix generator for parsing

Author: pfitzseb

src/CSTParser.jl

@@ -52,7 +52,7 @@ function parse_expression(ps::ParseState, esc_on_error = false; allow_const_fiel
         elseif kindof(ps.t) === Tokens.LPAREN
             ret = parse_paren(ps)
         elseif kindof(ps.t) === Tokens.LSQUARE
-            ret = @default ps parse_array(ps)
+            ret = @closer ps :for_generator @default ps parse_array(ps)
         elseif kindof(ps.t) === Tokens.LBRACE
             ret = @default ps @closebrace ps parse_braces(ps)
         elseif isinstance(ps.t) || isoperator(ps.t)
@@ -120,7 +120,7 @@ function parse_compound(ps::ParseState, ret::EXPR)
         ret = parse_prefixed_string_cmd(ps, ret)
     elseif kindof(ps.nt) === Tokens.LPAREN
         no_ws = !isemptyws(ps.ws)
-        ret = @closeparen ps parse_call(ps, ret)
+        ret = @closer ps :for_generator @closeparen ps parse_call(ps, ret)
         if no_ws && !isunarycall(ret)
             ret = mErrorToken(ps, ret, UnexpectedWhiteSpace)
         end
@@ -131,7 +131,7 @@ function parse_compound(ps::ParseState, ret::EXPR)
             ret = mErrorToken(ps, (@default ps @nocloser ps :inwhere @closebrace ps parse_curly(ps, ret)), UnexpectedWhiteSpace)
         end
     elseif kindof(ps.nt) === Tokens.LSQUARE && isemptyws(ps.ws) && !isoperator(ret)
-        ret = @default ps @nocloser ps :block parse_ref(ps, ret)
+        ret = @closer ps :for_generator @default ps @nocloser ps :block parse_ref(ps, ret)
     elseif iscomma(ps.nt)
         ret = parse_tuple(ps, ret)
     elseif isunaryop(ret) && kindof(ps.nt) != Tokens.EQ

src/components/internals.jl

@@ -310,8 +310,7 @@ function parse_macrocall(ps::ParseState)
                 push!(args, a)
                 break
             end
-
-            if (insquare || ps.closer.paren || ps.closer.square) && kindof(ps.nt) === Tokens.FOR
+            if ps.closer.for_generator && kindof(ps.nt) === Tokens.FOR
                 break
             end
             prevpos = loop_check(ps, prevpos)

src/lexer.jl

@@ -24,9 +24,10 @@ mutable struct Closer
     ws::Bool
     wsop::Bool
     unary::Bool
+    for_generator::Bool
     precedence::Int
 end
-Closer() = Closer(true, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, -1)
+Closer() = Closer(true, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, -1)
 
 mutable struct ParseState
     l::Lexer{Base.GenericIOBuffer{Array{UInt8,1}},RawToken}

test/parser.jl

@@ -452,7 +452,10 @@ end
         @test "@inline get_chunks_id(i::Integer) = _div64(Int(i)-1)+1, _mod64(Int(i) -1)" |> test_expr
         @test "@inline f() = (), ()" |> test_expr
         @test "@sprintf(\"%08d\", id)" |> test_expr
-        @test "[@m @n a for a in A]" |> test_expr # ensure closer.insquare propogates
+        @test "[@m @n a for a in A]" |> test_expr
+        @test ":(@foo bar baz bat)" |> test_expr
+        @test ":(@foo bar for i in j end)" |> test_expr
+        @test "(@foo bar for i in j end)" |> test_expr
         @test CSTParser.parse("@__DIR__\n\nx", true)[1].span == 8
 
         if VERSION >= v"1.8.0-"