Use specific error token kinds, not K"error"

Author: c42f

src/kinds.jl

@@ -16,10 +16,12 @@ const _kind_names =
         # Tokenization errors
         "ErrorEofMultiComment"
         "ErrorInvalidNumericConstant"
+        "ErrorAmbiguousNumericConstant"
         "ErrorInvalidInterpolationTerminator"
         "ErrorNumericOverflow"
         "ErrorInvalidEscapeSequence"
         "ErrorOverLongCharacter"
+        "ErrorUnknownCharacter"
         # Generic error
         "error"
     "END_ERRORS"
@@ -1014,10 +1016,12 @@ const _nonunique_kind_names = Set([
 
     K"ErrorEofMultiComment"
     K"ErrorInvalidNumericConstant"
+    K"ErrorAmbiguousNumericConstant"
     K"ErrorInvalidInterpolationTerminator"
     K"ErrorNumericOverflow"
     K"ErrorInvalidEscapeSequence"
     K"ErrorOverLongCharacter"
+    K"ErrorUnknownCharacter"
     K"ErrorInvalidOperator"
 
     K"Integer"
@@ -1053,6 +1057,20 @@ function untokenize(k::Kind; unique=true)
     end
 end
 
+# Error kind => description
+_token_error_descriptions = Dict{Kind, String}(
+    K"ErrorEofMultiComment" => "unterminated multi-line comment #= ... =#",
+    K"ErrorInvalidNumericConstant" => "invalid numeric constant",
+    K"ErrorAmbiguousNumericConstant" => "ambiguous `.` syntax; add whitespace to clarify (eg `1.+2` might be `1.0+2` or `1 .+ 2`)",
+    K"ErrorInvalidInterpolationTerminator" => "interpolated variable ends with invalid character; use `\$(...)` instead",
+    K"ErrorNumericOverflow"=>"overflow in numeric literal",
+    K"ErrorInvalidEscapeSequence"=>"invalid string escape sequence",
+    K"ErrorOverLongCharacter"=>"character literal contains multiple characters",
+    K"ErrorUnknownCharacter"=>"unknown unicode character",
+    K"ErrorInvalidOperator" => "invalid operator",
+    K"Error**" => "use `x^y` instead of `x**y` for exponentiation, and `x...` instead of `**x` for splatting",
+    K"error" => "unknown error token",
+)
 
 #-------------------------------------------------------------------------------
 # Predicates

src/parse_stream.jl

@@ -913,7 +913,7 @@ function validate_tokens(stream::ParseStream)
         elseif is_error(k) && k != K"error"
             # Emit messages for non-generic token errors
             emit_diagnostic(stream, fbyte, lbyte,
-                            error=Tokenize.TOKEN_ERROR_DESCRIPTION[k])
+                            error=_token_error_descriptions[k])
         end
         if error_kind != K"None"
             toks[i] = SyntaxToken(SyntaxHead(error_kind, EMPTY_FLAGS),

src/tokenize.jl

@@ -12,19 +12,6 @@ include("tokenize_utils.jl")
 #-------------------------------------------------------------------------------
 # Tokens
 
-# Error kind => description
-TOKEN_ERROR_DESCRIPTION = Dict{Kind, String}(
-    K"ErrorEofMultiComment" => "unterminated multi-line comment #= ... =#",
-    K"ErrorInvalidNumericConstant" => "invalid numeric constant",
-    K"ErrorInvalidInterpolationTerminator" => "interpolated variable ends with invalid character; use `\$(...)` instead",
-    K"ErrorNumericOverflow"=>"overflow in numeric literal",
-    K"ErrorInvalidEscapeSequence"=>"invalid string escape sequence",
-    K"ErrorOverLongCharacter"=>"character literal contains multiple characters",
-    K"ErrorInvalidOperator" => "invalid operator",
-    K"Error**" => "use `x^y` instead of `x**y` for exponentiation, and `x...` instead of `**x` for splatting",
-    K"error" => "unknown error token",
-)
-
 struct Token
     kind::Kind
     # Offsets into a string or buffer
@@ -283,11 +270,11 @@ function emit(l::Lexer, kind::Kind, maybe_op=true)
 end
 
 """
-    emit_error(l::Lexer, err::Kind=K"error")
+    emit_error(l::Lexer, err::Kind)
 
 Returns an `K"error"` token with error `err` and starts a new `Token`.
 """
-function emit_error(l::Lexer, err::Kind = K"error")
+function emit_error(l::Lexer, err::Kind)
     @assert is_error(err)
     return emit(l, err)
 end
@@ -387,7 +374,7 @@ function _next_token(l::Lexer, c)
     elseif (k = get(UNICODE_OPS, c, K"error")) != K"error"
         return emit(l, k)
     else
-        emit_error(l)
+        emit_error(l, K"ErrorUnknownCharacter")
     end
 end
 
@@ -785,7 +772,7 @@ function lex_digit(l::Lexer, kind)
             return emit_error(l, K"ErrorInvalidNumericConstant")
         elseif is_operator_start_char(ppc) && ppc !== ':'
             readchar(l)
-            return emit_error(l)
+            return emit_error(l, K"ErrorAmbiguousNumericConstant")
         elseif (!(isdigit(ppc) ||
             iswhitespace(ppc) ||
             is_identifier_start_char(ppc)
@@ -824,7 +811,7 @@ function lex_digit(l::Lexer, kind)
                     return emit_error(l, K"ErrorInvalidNumericConstant")
                 end
             else
-                return emit_error(l)
+                return emit_error(l, K"ErrorInvalidNumericConstant")
             end
         elseif pc == '.' && (is_identifier_start_char(ppc) || ppc == EOF_CHAR)
             readchar(l)
@@ -842,7 +829,7 @@ function lex_digit(l::Lexer, kind)
                 return emit_error(l, K"ErrorInvalidNumericConstant")
             end
         else
-            return emit_error(l)
+            return emit_error(l, K"ErrorInvalidNumericConstant")
         end
     elseif position(l) - startpos(l) == 1 && l.chars[1] == '0'
         kind == K"Integer"

test/tokenize.jl

@@ -769,10 +769,21 @@ end
     test_error(tok("0op",1),     K"ErrorInvalidNumericConstant")
     test_error(tok("--",1),      K"ErrorInvalidOperator")
 
+    @test toks("1e+")   == ["1e+"=>K"ErrorInvalidNumericConstant"]
+    @test toks("1.0e+") == ["1.0e+"=>K"ErrorInvalidNumericConstant"]
+    @test toks("0x.")   == ["0x."=>K"ErrorInvalidNumericConstant"]
+
     @test toks("1**2") == ["1"=>K"Integer", "**"=>K"Error**", "2"=>K"Integer"]
     @test toks("a<---b") == ["a"=>K"Identifier", "<---"=>K"ErrorInvalidOperator", "b"=>K"Identifier"]
     @test toks("a..+b") == ["a"=>K"Identifier", "..+"=>K"ErrorInvalidOperator", "b"=>K"Identifier"]
     @test toks("a..−b") == ["a"=>K"Identifier", "..−"=>K"ErrorInvalidOperator", "b"=>K"Identifier"]
+
+    @test toks("1.+2") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+", "2"=>K"Integer"]
+    @test toks("1.+ ") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+", " "=>K"Whitespace"]
+    @test toks("1.⤋")  == ["1."=>K"ErrorAmbiguousNumericConstant", "⤋"=>K"⤋"]
+    @test toks("1.?")  == ["1."=>K"ErrorAmbiguousNumericConstant", "?"=>K"?"]
+
+    @test toks("\x00") == ["\x00"=>K"ErrorUnknownCharacter"]
 end
 
 @testset "hat suffix" begin
@@ -786,21 +797,6 @@ end
     @test untokenize(collect(tokenize(s))[1], s) == s
 end
 
-@testset "invalid float juxt" begin
-    s = "1.+2"
-    @test tok(s, 1).kind == K"error"
-    @test is_operator(tok(s, 2).kind)
-    test_roundtrip("1234.+1", K"error", "1234.")
-    @test tok("1.+ ").kind == K"error"
-    @test tok("1.⤋").kind  == K"error"
-    @test tok("1.?").kind == K"error"
-end
-
-@testset "invalid hexadecimal" begin
-    s = "0x."
-    tok(s, 1).kind === K"error"
-end
-
 @testset "circ arrow right op" begin
     s = "↻"
     @test collect(tokenize(s))[1].kind == K"↻"