Improvements to core parsing hooks (#12)

* Make ParseError work with Meta.parse()
* Make incremental parsing of statements work
* Add some basic tests

Author: c42f

src/hooks.jl

@@ -1,5 +1,5 @@
 # Adaptor for the API/ABI expected by the Julia runtime code.
-function core_parser_hook(code, filename, offset, options)
+function core_parser_hook(code, filename, lineno, offset, options)
     try
         # TODO: Check that we do all this input wrangling without copying the
         # code buffer
@@ -13,19 +13,32 @@ function core_parser_hook(code, filename, offset, options)
 
         stream = ParseStream(io)
         rule = options === :all ? :toplevel : options
+        if rule !== :toplevel
+            # To copy the flisp parser driver, we ignore leading trivia when
+            # parsing statements or atoms
+            bump_trivia(stream)
+        end
         JuliaSyntax.parse(stream; rule=rule)
 
-        ex = if any_error(stream)
+        if any_error(stream)
             e = Expr(:error, ParseError(SourceFile(code), stream.diagnostics))
-            options === :all ? Expr(:toplevel, e) : e
+            ex = options === :all ? Expr(:toplevel, e) : e
         else
-            build_tree(Expr, stream)
+            ex = build_tree(Expr, stream, wrap_toplevel_as_kind=K"None")
+            if Meta.isexpr(ex, :None)
+                # The None wrapping is only to give somewhere for trivia to be
+                # attached; unwrap!
+                ex = only(ex.args)
+            end
         end
 
-        pos = last_byte(stream) - 1
+        # Note the next byte in 1-based indexing is `last_byte(stream) + 1` but
+        # the Core hook must return an offset (ie, it's 0-based) so the factors
+        # of one cancel here.
+        last_offset = last_byte(stream)
 
         # Rewrap result in an svec for use by the C code
-        return Core.svec(ex, pos)
+        return Core.svec(ex, last_offset)
     catch exc
         @error("JuliaSyntax parser failed — falling back to flisp!",
                exception=(exc,catch_backtrace()),
@@ -35,6 +48,18 @@ function core_parser_hook(code, filename, offset, options)
     return Core.Compiler.fl_parse(code, filename, offset, options)
 end
 
+# Core._parse gained a `lineno` argument in
+# https://github.com/JuliaLang/julia/pull/43876
+# Prior to this, the following signature was needed:
+function core_parser_hook(code, filename, offset, options)
+    core_parser_hook(code, filename, LineNumberNode(0), offset, options)
+end
+
+# Hack:
+# Meta.parse() attempts to construct a ParseError from a string if it receives
+# `Expr(:error)`.
+Base.Meta.ParseError(e::JuliaSyntax.ParseError) = e
+
 """
 Connect the JuliaSyntax parser to the Julia runtime so that it replaces the
 flisp parser for all parsing work.

src/parser.jl

@@ -2969,11 +2969,11 @@ function parse_brackets(after_parse::Function,
                 # (x \n\n for a in as)  ==>  (generator x (= a as))
                 parse_generator(ps, mark)
             else
-                k_str = untokenize(k)
-                ck_str = untokenize(closing_kind)
                 if is_closing_token(ps, k)
+                    k_str = untokenize(k, unique=false)
                     emit_diagnostic(ps, error="unexpected `$k_str` in bracketed list")
                 else
+                    ck_str = untokenize(closing_kind)
                     emit_diagnostic(ps, error="missing comma or $ck_str in bracketed list")
                 end
                 # Recovery done after loop

test/hooks.jl

@@ -0,0 +1,16 @@
+@testset "Hooks for Core integration" begin
+    JuliaSyntax.enable_in_core!()
+
+    @test Meta.parse("x + 1") == :(x + 1)
+    @test Meta.parse("x + 1", 1) == (:(x + 1), 6)
+
+    # Test that parsing statements incrementally works
+    @test Meta.parse("x + 1\n(y)", 1) == (:(x + 1), 6)
+    @test Meta.parse("x + 1\n(y)", 6) == (:y, 10)
+
+    # Check that Meta.parse throws the JuliaSyntax.ParseError rather than
+    # Meta.ParseError when Core integration is enabled.
+    @test_throws JuliaSyntax.ParseError Meta.parse("[x")
+
+    JuliaSyntax.disable_in_core!()
+end

test/runtests.jl

@@ -19,11 +19,10 @@ include("parse_stream.jl")
 include("parser.jl")
 include("parser_api.jl")
 include("syntax_tree.jl")
-
 @testset "Parsing values from strings" begin
     include("value_parsing.jl")
 end
-
+include("hooks.jl")
 include("parse_packages.jl")
 
 # Prototypes