Merge pull request #181 from JuliaLang/c42f/token-error-fixes

Fixes for General registry testing

Author: c42f

src/kinds.jl

@@ -16,10 +16,12 @@ const _kind_names =
         # Tokenization errors
         "ErrorEofMultiComment"
         "ErrorInvalidNumericConstant"
+        "ErrorAmbiguousNumericConstant"
         "ErrorInvalidInterpolationTerminator"
         "ErrorNumericOverflow"
         "ErrorInvalidEscapeSequence"
         "ErrorOverLongCharacter"
+        "ErrorUnknownCharacter"
         # Generic error
         "error"
     "END_ERRORS"
@@ -1014,10 +1016,12 @@ const _nonunique_kind_names = Set([
 
     K"ErrorEofMultiComment"
     K"ErrorInvalidNumericConstant"
+    K"ErrorAmbiguousNumericConstant"
     K"ErrorInvalidInterpolationTerminator"
     K"ErrorNumericOverflow"
     K"ErrorInvalidEscapeSequence"
     K"ErrorOverLongCharacter"
+    K"ErrorUnknownCharacter"
     K"ErrorInvalidOperator"
 
     K"Integer"
@@ -1053,6 +1057,20 @@ function untokenize(k::Kind; unique=true)
     end
 end
 
+# Error kind => description
+_token_error_descriptions = Dict{Kind, String}(
+    K"ErrorEofMultiComment" => "unterminated multi-line comment #= ... =#",
+    K"ErrorInvalidNumericConstant" => "invalid numeric constant",
+    K"ErrorAmbiguousNumericConstant" => "ambiguous `.` syntax; add whitespace to clarify (eg `1.+2` might be `1.0+2` or `1 .+ 2`)",
+    K"ErrorInvalidInterpolationTerminator" => "interpolated variable ends with invalid character; use `\$(...)` instead",
+    K"ErrorNumericOverflow"=>"overflow in numeric literal",
+    K"ErrorInvalidEscapeSequence"=>"invalid string escape sequence",
+    K"ErrorOverLongCharacter"=>"character literal contains multiple characters",
+    K"ErrorUnknownCharacter"=>"unknown unicode character",
+    K"ErrorInvalidOperator" => "invalid operator",
+    K"Error**" => "use `x^y` instead of `x**y` for exponentiation, and `x...` instead of `**x` for splatting",
+    K"error" => "unknown error token",
+)
 
 #-------------------------------------------------------------------------------
 # Predicates

src/parse_stream.jl

@@ -913,7 +913,7 @@ function validate_tokens(stream::ParseStream)
         elseif is_error(k) && k != K"error"
             # Emit messages for non-generic token errors
             emit_diagnostic(stream, fbyte, lbyte,
-                            error=Tokenize.TOKEN_ERROR_DESCRIPTION[k])
+                            error=_token_error_descriptions[k])
         end
         if error_kind != K"None"
             toks[i] = SyntaxToken(SyntaxHead(error_kind, EMPTY_FLAGS),

src/tokenize.jl

@@ -12,19 +12,6 @@ include("tokenize_utils.jl")
 #-------------------------------------------------------------------------------
 # Tokens
 
-# Error kind => description
-TOKEN_ERROR_DESCRIPTION = Dict{Kind, String}(
-    K"ErrorEofMultiComment" => "unterminated multi-line comment #= ... =#",
-    K"ErrorInvalidNumericConstant" => "invalid numeric constant",
-    K"ErrorInvalidInterpolationTerminator" => "interpolated variable ends with invalid character; use `\$(...)` instead",
-    K"ErrorNumericOverflow"=>"overflow in numeric literal",
-    K"ErrorInvalidEscapeSequence"=>"invalid string escape sequence",
-    K"ErrorOverLongCharacter"=>"character literal contains multiple characters",
-    K"ErrorInvalidOperator" => "invalid operator",
-    K"Error**" => "use `x^y` instead of `x**y` for exponentiation, and `x...` instead of `**x` for splatting",
-    K"error" => "unknown error token",
-)
-
 struct Token
     kind::Kind
     # Offsets into a string or buffer
@@ -283,11 +270,11 @@ function emit(l::Lexer, kind::Kind, maybe_op=true)
 end
 
 """
-    emit_error(l::Lexer, err::Kind=K"error")
+    emit_error(l::Lexer, err::Kind)
 
 Returns an `K"error"` token with error `err` and starts a new `Token`.
 """
-function emit_error(l::Lexer, err::Kind = K"error")
+function emit_error(l::Lexer, err::Kind)
     @assert is_error(err)
     return emit(l, err)
 end
@@ -387,7 +374,7 @@ function _next_token(l::Lexer, c)
     elseif (k = get(UNICODE_OPS, c, K"error")) != K"error"
         return emit(l, k)
     else
-        emit_error(l)
+        emit_error(l, K"ErrorUnknownCharacter")
     end
 end
 
@@ -785,7 +772,7 @@ function lex_digit(l::Lexer, kind)
             return emit_error(l, K"ErrorInvalidNumericConstant")
         elseif is_operator_start_char(ppc) && ppc !== ':'
             readchar(l)
-            return emit_error(l)
+            return emit_error(l, K"ErrorAmbiguousNumericConstant")
         elseif (!(isdigit(ppc) ||
             iswhitespace(ppc) ||
             is_identifier_start_char(ppc)
@@ -824,7 +811,7 @@ function lex_digit(l::Lexer, kind)
                     return emit_error(l, K"ErrorInvalidNumericConstant")
                 end
             else
-                return emit_error(l)
+                return emit_error(l, K"ErrorInvalidNumericConstant")
             end
         elseif pc == '.' && (is_identifier_start_char(ppc) || ppc == EOF_CHAR)
             readchar(l)
@@ -842,7 +829,7 @@ function lex_digit(l::Lexer, kind)
                 return emit_error(l, K"ErrorInvalidNumericConstant")
             end
         else
-            return emit_error(l)
+            return emit_error(l, K"ErrorInvalidNumericConstant")
         end
     elseif position(l) - startpos(l) == 1 && l.chars[1] == '0'
         kind == K"Integer"

test/tokenize.jl

@@ -769,10 +769,21 @@ end
     test_error(tok("0op",1),     K"ErrorInvalidNumericConstant")
     test_error(tok("--",1),      K"ErrorInvalidOperator")
 
+    @test toks("1e+")   == ["1e+"=>K"ErrorInvalidNumericConstant"]
+    @test toks("1.0e+") == ["1.0e+"=>K"ErrorInvalidNumericConstant"]
+    @test toks("0x.")   == ["0x."=>K"ErrorInvalidNumericConstant"]
+
     @test toks("1**2") == ["1"=>K"Integer", "**"=>K"Error**", "2"=>K"Integer"]
     @test toks("a<---b") == ["a"=>K"Identifier", "<---"=>K"ErrorInvalidOperator", "b"=>K"Identifier"]
     @test toks("a..+b") == ["a"=>K"Identifier", "..+"=>K"ErrorInvalidOperator", "b"=>K"Identifier"]
     @test toks("a..−b") == ["a"=>K"Identifier", "..−"=>K"ErrorInvalidOperator", "b"=>K"Identifier"]
+
+    @test toks("1.+2") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+", "2"=>K"Integer"]
+    @test toks("1.+ ") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+", " "=>K"Whitespace"]
+    @test toks("1.⤋")  == ["1."=>K"ErrorAmbiguousNumericConstant", "⤋"=>K"⤋"]
+    @test toks("1.?")  == ["1."=>K"ErrorAmbiguousNumericConstant", "?"=>K"?"]
+
+    @test toks("\x00") == ["\x00"=>K"ErrorUnknownCharacter"]
 end
 
 @testset "hat suffix" begin
@@ -786,21 +797,6 @@ end
     @test untokenize(collect(tokenize(s))[1], s) == s
 end
 
-@testset "invalid float juxt" begin
-    s = "1.+2"
-    @test tok(s, 1).kind == K"error"
-    @test is_operator(tok(s, 2).kind)
-    test_roundtrip("1234.+1", K"error", "1234.")
-    @test tok("1.+ ").kind == K"error"
-    @test tok("1.⤋").kind  == K"error"
-    @test tok("1.?").kind == K"error"
-end
-
-@testset "invalid hexadecimal" begin
-    s = "0x."
-    tok(s, 1).kind === K"error"
-end
-
 @testset "circ arrow right op" begin
     s = "↻"
     @test collect(tokenize(s))[1].kind == K"↻"

tools/check_all_packages.jl

@@ -12,68 +12,73 @@ logger = Logging.ConsoleLogger(logio)
 
 pkgspath = joinpath(@__DIR__, "pkgs")
 
+exception_count = 0
+mismatch_count = 0
+file_count = 0
+t0 = time()
 exceptions = []
+
 Logging.with_logger(logger) do
-    t = time()
-    i = 0
-    iob = IOBuffer()
-    exception_count = 0
-    mismatch_count = 0
+    global exception_count, mismatch_count, file_count, t0
     for (r, _, files) in walkdir(pkgspath)
         for f in files
             endswith(f, ".jl") || continue
             fpath = joinpath(r, f)
-            if isfile(fpath)
-                code = read(fpath, String)
-                expr_cache = fpath*".Expr"
-                #e2 = JuliaSyntax.fl_parseall(code)
-                e2 = open(deserialize, fpath*".Expr")
-                @assert Meta.isexpr(e2, :toplevel)
-                try
-                    e1 = JuliaSyntax.parseall(Expr, code, filename=fpath)
-                    if !exprs_roughly_equal(e2, e1)
-                        mismatch_count += 1
-                        @error("Parsers succeed but disagree",
-                               fpath,
-                               diff=Text(sprint(show_expr_text_diff, show, e1, e2)),
-                               )
-                    end
-                catch err
-                    err isa InterruptException && rethrow()
-                    ex = (err, catch_backtrace())
-                    push!(exceptions, ex)
-                    ref_parse = "success"
-                    if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete))
-                        ref_parse = "fail"
-                        if err isa JuliaSyntax.ParseError
-                            # Both parsers agree that there's an error, and
-                            # JuliaSyntax didn't have an internal error.
-                            continue
-                        end
-                    end
+            isfile(fpath) || continue
 
-                    exception_count += 1
-                    parse_to_syntax = "success"
-                    try
-                        JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, code)
-                    catch err2
-                        parse_to_syntax = "fail"
+            code = read(fpath, String)
+            expr_cache = fpath*".Expr"
+            #e2 = JuliaSyntax.fl_parseall(code)
+            e2 = open(deserialize, fpath*".Expr")
+            @assert Meta.isexpr(e2, :toplevel)
+            try
+                e1 = JuliaSyntax.parseall(Expr, code, filename=fpath, ignore_warnings=true)
+                if !exprs_roughly_equal(e2, e1)
+                    mismatch_count += 1
+                    @error("Parsers succeed but disagree",
+                           fpath,
+                           diff=Text(sprint(show_expr_text_diff, show, e1, e2)),
+                           )
+                end
+            catch err
+                err isa InterruptException && rethrow()
+                ex = (err, catch_backtrace())
+                push!(exceptions, ex)
+                ref_parse = "success"
+                if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete))
+                    ref_parse = "fail"
+                    if err isa JuliaSyntax.ParseError
+                        # Both parsers agree that there's an error, and
+                        # JuliaSyntax didn't have an internal error.
+                        continue
                     end
-                    @error "Parse failed" fpath exception=ex parse_to_syntax
                 end
+
+                exception_count += 1
+                parse_to_syntax = "success"
+                try
+                    JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, code)
+                catch err2
+                    parse_to_syntax = "fail"
+                end
+                @error "Parse failed" fpath exception=ex parse_to_syntax
             end
-            i += 1
-            if i % 100 == 0
-                runtime = time() - t
-                avg = round(runtime/i*1000, digits = 2)
-                print(iob, "\e[2J\e[0;0H")
-                println(iob, "$i files parsed")
-                println(iob, "> $(exception_count) failures compared to reference parser")
-                println(iob, "> $(mismatch_count) Expr mismatches")
-                println(iob, "> $(avg)ms per file, $(round(Int, runtime))s in total")
-                println(stderr, String(take!(iob)))
+
+            file_count += 1
+            if file_count % 100 == 0
+                t_avg = round((time() - t0)/file_count*1000, digits = 2)
+                print(stderr, "\r$file_count files parsed, $t_avg ms per file")
             end
         end
     end
 end
 close(logio)
+
+t_avg = round((time() - t0)/file_count*1000, digits = 2)
+
+println()
+@info """
+    Finished parsing $file_count files.
+        $(exception_count) failures compared to reference parser
+        $(mismatch_count) Expr mismatches
+        $(t_avg)ms per file"""