Generate Expr(:incomplete) for errors which hit EOF (#102)

This allows REPL completion to work correctly. It works by pattern matching the parse tree, rather than hard coding incomplete expression detection into the parser itself. There's still more changes from #88 which would help make this nicer but for now it works.

The tests here are somewhat derived from Base, but had to be reviewed and tweaked because they turn out to not really be consistent. For example, "begin;" and "begin" are both the prefix of a block construct, one of them shouldn't come out as `:other`.

Author: c42f

src/hooks.jl

@@ -1,6 +1,76 @@
 # This file provides an adaptor to match the API expected by the Julia runtime
 # code in the binding Core._parse
 
+# Find the first error in a SyntaxNode tree, returning the index of the error
+# within its parent and the node itself.
+function _first_error(t::SyntaxNode)
+    if is_error(t)
+        return 0,t
+    end
+    if haschildren(t)
+        for (i,c) in enumerate(children(t))
+            if is_error(c)
+                return i,c
+            else
+                x = _first_error(c)
+                if x != (0,nothing)
+                    return x
+                end
+            end
+        end
+    end
+    return 0,nothing
+end
+
+# Classify an incomplete expression, returning a Symbol compatible with
+# Base.incomplete_tag().
+#
+# Roughly, the intention here is to classify which expression head is expected
+# next if the incomplete stream was to continue. (Though this is just rough. In
+# practice several categories are combined for the purposes of the REPL -
+# perhaps we can/should do something more precise in the future.)
+function _incomplete_tag(n::SyntaxNode)
+    i,c = _first_error(n)
+    if isnothing(c)
+        return :none
+    end
+    # TODO: Check error hits last character
+    if kind(c) == K"error" && begin
+                cs = children(c)
+                length(cs) > 0
+            end
+        k1 = kind(cs[1])
+        if k1 == K"ErrorEofMultiComment"
+            return :comment
+        elseif k1 == K"ErrorEofChar"
+            # TODO: Make this case into an internal node
+            return :char
+        end
+        for cc in cs
+            if kind(cc) == K"error"
+                return :other
+            end
+        end
+    end
+    kp = kind(c.parent)
+    if kp == K"string"
+        return :string
+    elseif kp == K"cmdstring"
+        return :cmd
+    elseif kp in KSet"block quote let try"
+        return :block
+    elseif kp in KSet"for while function if"
+        return i == 1 ? :other : :block
+    elseif kp in KSet"module struct"
+        return i == 2 ? :other : :block
+    elseif kp == K"do"
+        return i < 3  ? :other : :block
+    else
+        return :other
+    end
+end
+
+#-------------------------------------------------------------------------------
 @static if isdefined(Core, :_setparser!)
     const _set_core_parse_hook = Core._setparser!
 elseif isdefined(Core, :set_parser)
@@ -93,11 +163,29 @@ function _core_parser_hook(code, filename, lineno, offset, options)
         end
 
         if any_error(stream)
-            e = Expr(:error, ParseError(SourceFile(code, filename=filename), stream.diagnostics))
-            ex = options === :all ? Expr(:toplevel, e) : e
+            tree = build_tree(SyntaxNode, stream, wrap_toplevel_as_kind=K"None")
+            _,err = _first_error(tree)
+            # In the flisp parser errors are normally `Expr(:error, msg)` where
+            # `msg` is a String. By using a ParseError for msg we can do fancy
+            # error reporting instead.
+            if last_byte(err) == lastindex(code)
+                tag = _incomplete_tag(tree)
+                # Here we replicate the particular messages 
+                msg =
+                    tag === :string  ? "incomplete: invalid string syntax"     :
+                    tag === :comment ? "incomplete: unterminated multi-line comment #= ... =#" :
+                    tag === :block   ? "incomplete: construct requires end"    :
+                    tag === :cmd     ? "incomplete: invalid \"`\" syntax"      :
+                    tag === :char    ? "incomplete: invalid character literal" :
+                                       "incomplete: premature end of input"
+                error_ex = Expr(:incomplete, msg)
+            else
+                error_ex = Expr(:error, ParseError(stream, filename=filename))
+            end
+            ex = options === :all ? Expr(:toplevel, error_ex) : error_ex
         else
             # FIXME: Add support to lineno to this tree build (via SourceFile?)
-            ex = build_tree(Expr, stream, filename=filename, wrap_toplevel_as_kind=K"None")
+            ex = build_tree(Expr, stream; filename=filename, wrap_toplevel_as_kind=K"None")
             if Meta.isexpr(ex, :None)
                 # The None wrapping is only to give somewhere for trivia to be
                 # attached; unwrap!

src/parser_api.jl

@@ -46,6 +46,11 @@ struct ParseError <: Exception
     diagnostics::Vector{Diagnostic}
 end
 
+function ParseError(stream::ParseStream; filename=nothing)
+    source = SourceFile(sourcetext(stream), filename=filename)
+    ParseError(source, stream.diagnostics)
+end
+
 function Base.showerror(io::IO, err::ParseError, bt; backtrace=false)
     println(io, "ParseError:")
     show_diagnostics(io, err.diagnostics, err.source)
@@ -156,8 +161,7 @@ function parseall(::Type{T}, input...; rule=:toplevel, version=VERSION,
         emit_diagnostic(stream, error="unexpected text after parsing $rule")
     end
     if any_error(stream.diagnostics)
-        source = SourceFile(sourcetext(stream, steal_textbuf=true), filename=filename)
-        throw(ParseError(source, stream.diagnostics))
+        throw(ParseError(stream, filename=filename))
     end
     # TODO: Figure out a more satisfying solution to the wrap_toplevel_as_kind
     # mess that we've got here.

src/syntax_tree.jl

@@ -120,6 +120,9 @@ children(node::SyntaxNode) = haschildren(node) ? node.val::Vector{SyntaxNode} :
 
 span(node::SyntaxNode) = span(node.raw)
 
+first_byte(node::SyntaxNode) = node.position
+last_byte(node::SyntaxNode)  = node.position + span(node) - 1
+
 """
     sourcetext(node)
 
@@ -138,7 +141,7 @@ end
 function _show_syntax_node(io, current_filename, node::SyntaxNode, indent)
     fname = node.source.filename
     line, col = source_location(node.source, node.position)
-    posstr = "$(lpad(line, 4)):$(rpad(col,3))│$(lpad(node.position,6)):$(rpad(node.position+span(node)-1,6))│"
+    posstr = "$(lpad(line, 4)):$(rpad(col,3))│$(lpad(first_byte(node),6)):$(rpad(last_byte(node),6))│"
     val = node.val
     nodestr = haschildren(node) ? "[$(untokenize(head(node)))]" :
               isa(val, Symbol) ? string(val) : repr(val)

test/hooks.jl

@@ -31,8 +31,69 @@
 
         # Check that Meta.parse throws the JuliaSyntax.ParseError rather than
         # Meta.ParseError when Core integration is enabled.
-        @test_throws JuliaSyntax.ParseError Meta.parse("[x")
+        @test_throws JuliaSyntax.ParseError Meta.parse("[x)")
 
         JuliaSyntax.enable_in_core!(false)
     end
+
+    @testset "Expr(:incomplete)" begin
+        JuliaSyntax.enable_in_core!()
+
+        @test Meta.isexpr(Meta.parse("[x"), :incomplete)
+
+        for (str, tag) in [
+                ""             => :none
+                "\""           => :string
+                "\"\$foo"      => :string
+                "#="           => :comment
+                "'"            => :char
+                "'a"           => :char
+                "`"            => :cmd
+                "("            => :other
+                "["            => :other
+                "begin"        => :block
+                "quote"        => :block
+                "let"          => :block
+                "let;"         => :block
+                "for"          => :other
+                "for x=xs"     => :block
+                "function"     => :other
+                "function f()" => :block
+                "macro"        => :other
+                "macro f()"    => :block
+                "f() do"       => :other
+                "f() do x"     => :block
+                "module"       => :other
+                "module X"     => :block
+                "baremodule"   => :other
+                "baremodule X" => :block
+                "mutable struct"    => :other
+                "mutable struct X"  => :block
+                "struct"       => :other
+                "struct X"     => :block
+                "if"           => :other
+                "if x"         => :block
+                "while"        => :other
+                "while x"      => :block
+                "try"          => :block
+                # could be `try x catch exc body end` or `try x catch ; body end`
+                "try x catch"  => :block
+                "using"        => :other
+                "import"       => :other
+                "local"        => :other
+                "global"       => :other
+
+                "1 == 2 ?"     => :other
+                "1 == 2 ? 3 :" => :other
+                "1,"           => :other
+                "1, "          => :other
+                "1,\n"         => :other
+                "1, \n"        => :other
+            ]
+            @testset "$(repr(str))" begin
+                @test Base.incomplete_tag(Meta.parse(str, raise=false)) == tag
+            end
+        end
+        JuliaSyntax.enable_in_core!(false)
+    end
 end