Merge pull request #291 from julia-vscode/sp/assorted-fixes

Fix Base parsing

Author: davidanthoff

Project.toml

@@ -11,7 +11,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 julia = "1"
-Tokenize = "0.5.13"
+Tokenize = "0.5.18"
 
 [targets]
 test = ["Test"]

src/CSTParser.jl

@@ -59,7 +59,15 @@ function parse_expression(ps::ParseState, esc_on_error = false)
             if both_symbol_and_op(ps.t)
                 ret = EXPR(:IDENTIFIER, ps)
             else
-                ret = INSTANCE(ps)
+                if ps.t.dotop && closer(ps) && !isassignmentop(ps.t)
+                    # Split dotted operator into dot-call
+                    v = val(ps.t, ps)[2:end]
+                    dot = EXPR(:OPERATOR, 1, 1, ".")
+                    op = EXPR(:OPERATOR, ps.nt.startbyte - ps.t.startbyte - 1, ps.t.endbyte - ps.t.startbyte, v)
+                    ret = EXPR(dot, EXPR[op], nothing)
+                else
+                    ret = INSTANCE(ps)
+                end
             end
             if is_colon(ret) && !(iscomma(ps.nt) || kindof(ps.ws) == SemiColonWS)
                 ret = parse_unary(ps, ret)
@@ -96,8 +104,13 @@ function parse_compound(ps::ParseState, ret::EXPR)
         end
         ret = parse_operator(ps, ret, EXPR(:OPERATOR, 0, 0, "*"))
     elseif issuffixableliteral(ps, ret)
-        arg = EXPR(:IDENTIFIER, next(ps))
-        push!(ret, EXPR(:STRING, arg.fullspan, arg.span, val(ps.t, ps)))
+        if isnumberliteral(ps.nt)
+            arg = mLITERAL(next(ps))
+            push!(ret, arg)
+        else
+            arg = EXPR(:IDENTIFIER, next(ps))
+            push!(ret, EXPR(:STRING, arg.fullspan, arg.span, val(ps.t, ps)))
+        end
     elseif (isidentifier(ret) || is_getfield(ret)) && isemptyws(ps.ws) && isprefixableliteral(ps.nt)
         ret = parse_prefixed_string_cmd(ps, ret)
     elseif kindof(ps.nt) === Tokens.LPAREN

src/components/internals.jl

@@ -50,6 +50,9 @@ function adjust_iter(x::EXPR)
     # Assumes x is a valid iterator
     if x.head === :call # isoperator(x.args[1]) && x.args[1].val in ("in", "∈")
         EXPR(EXPR(:OPERATOR, 0, 0, "="), EXPR[x.args[2], x.args[3]], EXPR[x.args[1]])
+    elseif isassignment(x) && length(x.args) == 2 && x.args[1].head == :call && x.args[2].head == :block && length(x.args[2].args) == 1
+        x.args[2] = setparent!(x.args[2].args[1], x)
+        x
     else
         x
     end
@@ -63,7 +66,7 @@ Is `x` a valid iterator for use in `for` loops or generators?
 is_range(x::EXPR) = isassignment(x) || (x.head === :call && (is_in(x.args[1]) || is_elof(x.args[1])))
 
 function parse_outer(ps)
-    if kindof(ps.nt) === Tokens.OUTER && kindof(ps.nws) !== EmptyWS && !Tokens.isoperator(kindof(ps.nnt))
+    if kindof(ps.nt) === Tokens.OUTER && kindof(ps.nws) !== EmptyWS && !(Tokens.isoperator(kindof(ps.nnt)) && isbinaryop(ps.nnt) && kindof(ps.nnws) !== EmptyWS)
         EXPR(next(ps))
     end
 end
@@ -229,7 +232,14 @@ function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR
         isfirst = true
     end
     if !isempty(args1)
-        insert!(args, insert_params_at, EXPR(:parameters, args1, trivia))
+        # this happens when multiple semi-colons are used
+        # Julia will error during lowering, so these shenanigans are just for matching
+        # kwarg order with Base's parser
+        if length(args) >= 1 && args[1] isa EXPR && args[1].head == :kw && usekw
+            insert!(args, max(insert_params_at - 1, 1), EXPR(:parameters, args1, trivia))
+        else
+            insert!(args, insert_params_at, EXPR(:parameters, args1, trivia))
+        end
     end
     return
 end
@@ -291,7 +301,6 @@ function parse_macrocall(ps::ParseState)
         next(ps)
         return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), @default ps parse_array(ps)], nothing)
     else
-        #TODO add special hndling for @doc
         args = EXPR[mname, EXPR(:NOTHING, 0, 0)]
         insquare = ps.closer.insquare
         prevpos = position(ps)
@@ -302,6 +311,13 @@ function parse_macrocall(ps::ParseState)
                 a = @closer ps :inmacro @closer ps :ws @closer ps :wsop parse_expression(ps)
             end
             push!(args, a)
+
+            if valof(mname) == "@doc" && ps.t.endpos[1] + 1 == ps.nt.startpos[1] && length(args) == 3
+                a = parse_expression(ps)
+                push!(args, a)
+                break
+            end
+
             if (insquare || ps.closer.paren || ps.closer.square) && kindof(ps.nt) === Tokens.FOR
                 break
             end

src/components/keywords.jl

@@ -102,13 +102,13 @@ function parse_local_global(ps::ParseState, islocal = true)
         EXPR(:const, EXPR[EXPR(islocal ? :local : :global, EXPR[arg1.args[1]], nothing)], EXPR[kw, arg1.trivia[1]])
     else
         args, trivia = EXPR[], EXPR[kw]
-        @closer ps :comma while !closer(ps)
-            push!(args, parse_expression(ps))
-            if iscomma(ps.nt)
-                accept_comma(ps, trivia)
-            else
-                break
-            end
+        arg = parse_expression(ps)
+        if isassignment(unwrapbracket(arg))
+            push!(args, arg)
+        elseif arg.head === :tuple
+            append!(args, arg.args)
+        else
+            push!(args, arg)
         end
         EXPR(islocal ? :local : :global, args, trivia)
     end
@@ -172,7 +172,7 @@ function parse_imports(ps::ParseState)
         ret = EXPR(kwt, EXPR[arg], EXPR[kw])
     elseif iscolon(ps.nt)
         ret = EXPR(kwt, EXPR[EXPR(EXPR(:OPERATOR, next(ps)), EXPR[arg])], EXPR[kw])
-        
+
         arg = parse_dot_mod(ps, true, true)
         push!(ret.args[1], arg)
         prevpos = position(ps)
@@ -301,7 +301,7 @@ end
 """
     parse_blockexpr(ps::ParseState, head)
 
-General function for parsing block expressions comprised of a series of statements 
+General function for parsing block expressions comprised of a series of statements
 terminated by an `end`.
 """
 function parse_blockexpr(ps::ParseState, head)