Ignore trailing trivia in Core parser hook

For compatibility with the flisp parser, we must consume any trailing
whitespace at the end of a string.  Otherwise Meta.parse() will fail
unnecessarily when parsing a single statement.

Author: c42f

src/hooks.jl

@@ -25,11 +25,14 @@ function core_parser_hook(code, filename, lineno, offset, options)
         stream = ParseStream(io)
         rule = options === :all ? :toplevel : options
         if rule !== :toplevel
-            # To copy the flisp parser driver, we ignore leading trivia when
-            # parsing statements or atoms
+            # To copy the flisp parser driver, we ignore leading and trailing
+            # trivia when parsing statements or atoms
             bump_trivia(stream)
         end
         JuliaSyntax.parse(stream; rule=rule)
+        if rule !== :toplevel
+            bump_trivia(stream)
+        end
 
         if any_error(stream)
             e = Expr(:error, ParseError(SourceFile(code), stream.diagnostics))

test/hooks.jl

@@ -4,9 +4,11 @@
     @test Meta.parse("x + 1") == :(x + 1)
     @test Meta.parse("x + 1", 1) == (:(x + 1), 6)
 
-    # Test that parsing statements incrementally works
-    @test Meta.parse("x + 1\n(y)", 1) == (:(x + 1), 6)
-    @test Meta.parse("x + 1\n(y)", 6) == (:y, 10)
+    # Test that parsing statements incrementally works and stops after
+    # whitespace / comment trivia
+    @test Meta.parse("x + 1\n(y)\n", 1) == (:(x + 1), 7)
+    @test Meta.parse("x + 1\n(y)\n", 7) == (:y, 11)
+    @test Meta.parse(" x#==#", 1) == (:x, 7)
 
     # Check that Meta.parse throws the JuliaSyntax.ParseError rather than
     # Meta.ParseError when Core integration is enabled.