Merge pull request #101 from JuliaLang/c42f/minor-bugfixes

Some minor fixes

Author: c42f

src/hooks.jl

@@ -46,7 +46,7 @@ end
 # https://github.com/JuliaLang/julia/pull/43876
 # Prior to this, the following signature was needed:
 function core_parser_hook(code, filename, offset, options)
-    core_parser_hook(code, filename, LineNumberNode(0), offset, options)
+    core_parser_hook(code, filename, 1, offset, options)
 end
 
 # Debug log file for dumping parsed code
@@ -96,6 +96,7 @@ function _core_parser_hook(code, filename, lineno, offset, options)
             e = Expr(:error, ParseError(SourceFile(code, filename=filename), stream.diagnostics))
             ex = options === :all ? Expr(:toplevel, e) : e
         else
+            # FIXME: Add support to lineno to this tree build (via SourceFile?)
             ex = build_tree(Expr, stream, filename=filename, wrap_toplevel_as_kind=K"None")
             if Meta.isexpr(ex, :None)
                 # The None wrapping is only to give somewhere for trivia to be

src/tokenize.jl

@@ -98,13 +98,13 @@ function Lexer(io::IO)
         c2 = read(io, Char)
         p2 = position(io)
         if eof(io)
-            c3, p3 = EOF_CHAR, p1
-            c4, p4 = EOF_CHAR, p1
+            c3, p3 = EOF_CHAR, p2
+            c4, p4 = EOF_CHAR, p2
         else
             c3 = read(io, Char)
             p3 = position(io)
             if eof(io)
-                c4, p4 = EOF_CHAR, p1
+                c4, p4 = EOF_CHAR, p3
             else
                 c4 = read(io, Char)
                 p4 = position(io)

test/tokenize.jl

@@ -897,6 +897,12 @@ end
     ]
 end
 
+@testset "lexer initialization" begin
+    # Ranges of EndMarker
+    @test (t = last(collect(tokenize("+"))); (t.startbyte, t.endbyte)) == (1,0)
+    @test (t = last(collect(tokenize("+*"))); (t.startbyte, t.endbyte)) == (2,1)
+end
+
 @testset "dotop miscellanea" begin
     @test strtok("a .-> b")  ==  ["a", " ", ".-", ">", " ", "b", ""]
     @test strtok(".>: b")    ==  [".>:", " ", "b", ""]