Fix hooks: whitespace parsing with Meta.parseatom() (#48)

Previously, `Meta.parseatom()` would consume trailing whitespace which breaks usages like in `Base.shell_parse()`

Author: c42f

src/hooks.jl

@@ -24,9 +24,10 @@ function core_parser_hook(code, filename, lineno, offset, options)
 
         stream = ParseStream(io)
         rule = options === :all ? :toplevel : options
-        if rule !== :toplevel
-            # To copy the flisp parser driver, we ignore leading and trailing
-            # trivia when parsing statements or atoms
+        if rule === :statement || rule === :atom
+            # To copy the flisp parser driver:
+            # * Parsing atoms      consumes leading trivia
+            # * Parsing statements consumes leading+trailing trivia
             bump_trivia(stream)
             if peek(stream) == K"EndMarker"
                 # If we're at the end of stream after skipping whitespace, just
@@ -36,7 +37,7 @@ function core_parser_hook(code, filename, lineno, offset, options)
             end
         end
         JuliaSyntax.parse(stream; rule=rule)
-        if rule !== :toplevel
+        if rule === :statement
             bump_trivia(stream)
         end
 

test/hooks.jl

@@ -1,10 +1,13 @@
 @testset "Hooks for Core integration" begin
-    @testset "parsing empty strings" begin
+    @testset "whitespace parsing" begin
         @test JuliaSyntax.core_parser_hook("", "somefile", 0, :statement) == Core.svec(nothing, 0)
         @test JuliaSyntax.core_parser_hook("", "somefile", 0, :statement) == Core.svec(nothing, 0)
 
         @test JuliaSyntax.core_parser_hook("  ", "somefile", 2, :statement) == Core.svec(nothing,2)
         @test JuliaSyntax.core_parser_hook(" #==# ", "somefile", 6, :statement) == Core.svec(nothing,6)
+
+        @test JuliaSyntax.core_parser_hook(" x \n", "somefile", 0, :statement) == Core.svec(:x,4)
+        @test JuliaSyntax.core_parser_hook(" x \n", "somefile", 0, :atom)      == Core.svec(:x,2)
     end
 
     @testset "filename is used" begin