Fix tokenization of numbers followed by .. (#143)

* `.1..` is `.1` followed by `..`
* `0x01..` is `0x01` followed by `..`

Also fixes these cases when followed by `...`

Author: c42f

src/tokenize.jl

@@ -763,12 +763,13 @@ function lex_digit(l::Lexer, kind)
     accept_number(l, isdigit)
     pc,ppc = dpeekchar(l)
     if pc == '.'
-        if kind === K"Float"
+        if ppc == '.'
+            # Number followed by K".." or K"..."
+            return emit(l, kind)
+        elseif kind === K"Float"
             # If we enter the function with kind == K"Float" then a '.' has been parsed.
             readchar(l)
             return emit_error(l, K"ErrorInvalidNumericConstant")
-        elseif ppc == '.'
-            return emit(l, kind)
         elseif is_operator_start_char(ppc) && ppc !== ':'
             readchar(l)
             return emit_error(l)
@@ -838,7 +839,9 @@ function lex_digit(l::Lexer, kind)
             readchar(l)
             !(ishex(ppc) || ppc == '.') && return emit_error(l, K"ErrorInvalidNumericConstant")
             accept_number(l, ishex)
-            if accept(l, '.')
+            pc,ppc = dpeekchar(l)
+            if pc == '.' && ppc != '.'
+                readchar(l)
                 accept_number(l, ishex)
                 isfloat = true
             end

test/tokenize.jl

@@ -21,6 +21,13 @@ tok(str, i = 1) = collect(tokenize(str))[i]
 
 strtok(str) = untokenize.(collect(tokenize(str)), str)
 
+function toks(str)
+    ts = [untokenize(t, str)=>kind(t) for t in tokenize(str)]
+    @test ts[end] == (""=>K"EndMarker")
+    pop!(ts)
+    ts
+end
+
 @testset "tokens" begin
     for s in ["a", IOBuffer("a")]
         l = tokenize(s)
@@ -553,7 +560,7 @@ end
     @test kind(tok("1234x", 2))        == K"Identifier"
 end
 
-@testset "floats with trailing `.` " begin
+@testset "numbers with trailing `.` " begin
     @test tok("1.0").kind == K"Float"
     @test tok("1.a").kind == K"Float"
     @test tok("1.(").kind == K"Float"
@@ -569,7 +576,10 @@ end
     @test tok("1.").kind == K"Float"
     @test tok("1.\"text\" ").kind == K"Float"
 
-    @test tok("1..").kind  == K"Integer"
+    @test toks("1..")    == ["1"=>K"Integer",   ".."=>K".."]
+    @test toks(".1..")   == [".1"=>K"Float",    ".."=>K".."]
+    @test toks("0x01..") == ["0x01"=>K"HexInt", ".."=>K".."]
+
     @test kind.(collect(tokenize("1f0./1"))) == [K"Float", K"/", K"Integer", K"EndMarker"]
 end