Merge pull request #225 from JuliaLang/c42f/emit-diagnostic-cleanup

Clean up emit_diagnostic stream position handling

Author: c42f

src/diagnostics.jl

@@ -39,7 +39,7 @@ end
 
 first_byte(d::Diagnostic) = d.first_byte
 last_byte(d::Diagnostic)  = d.last_byte
-is_error(d::Diagnostic)   = d.level == :error
+is_error(d::Diagnostic)   = d.level === :error
 Base.range(d::Diagnostic) = first_byte(d):last_byte(d)
 
 # Make relative path into a file URL
@@ -54,9 +54,9 @@ function _file_url(filename)
 end
 
 function show_diagnostic(io::IO, diagnostic::Diagnostic, source::SourceFile)
-    color,prefix = diagnostic.level == :error   ? (:light_red, "Error")      :
-                   diagnostic.level == :warning ? (:light_yellow, "Warning") :
-                   diagnostic.level == :note    ? (:light_blue, "Note")      :
+    color,prefix = diagnostic.level === :error   ? (:light_red, "Error")      :
+                   diagnostic.level === :warning ? (:light_yellow, "Warning") :
+                   diagnostic.level === :note    ? (:light_blue, "Note")      :
                    (:normal, "Info")
     line, col = source_location(source, first_byte(diagnostic))
     linecol = "$line:$col"
@@ -92,8 +92,8 @@ function show_diagnostics(io::IO, diagnostics::AbstractVector{Diagnostic}, text:
 end
 
 function emit_diagnostic(diagnostics::AbstractVector{Diagnostic},
-                         fbyte::Integer, lbyte::Integer; kws...)
-    push!(diagnostics, Diagnostic(fbyte, lbyte; kws...))
+                         byterange::AbstractUnitRange; kws...)
+    push!(diagnostics, Diagnostic(first(byterange), last(byterange); kws...))
 end
 
 function any_error(diagnostics::AbstractVector{Diagnostic})

src/expr.jl

@@ -76,7 +76,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
         headsym = !isnothing(headstr) ? Symbol(headstr) :
             error("Can't untokenize head of kind $(nodekind)")
     end
-    if headsym == :string || headsym == :cmdstring
+    if headsym === :string || headsym === :cmdstring
         # Julia string literals may be interspersed with trivia in two situations:
         # 1. Triple quoted string indentation is trivia
         # 2. An \ before newline removes the newline and any following indentation
@@ -100,7 +100,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
                 end
             else
                 e = _to_expr(node_args[i])
-                if e isa String && headsym == :string
+                if e isa String && headsym === :string
                     # Wrap interpolated literal strings in (string) so we can
                     # distinguish them from the surrounding text (issue #38501)
                     # Ie, "$("str")"  vs  "str"
@@ -124,22 +124,22 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
     end
 
     # Convert children
-    insert_linenums = (headsym == :block || headsym == :toplevel) && need_linenodes
+    insert_linenums = (headsym === :block || headsym === :toplevel) && need_linenodes
     args = Vector{Any}(undef, length(node_args)*(insert_linenums ? 2 : 1))
-    if headsym == :for && length(node_args) == 2
+    if headsym === :for && length(node_args) == 2
         # No line numbers in for loop iteration spec
         args[1] = _to_expr(node_args[1], iteration_spec=true, need_linenodes=false)
         args[2] = _to_expr(node_args[2])
-    elseif headsym == :let && length(node_args) == 2
+    elseif headsym === :let && length(node_args) == 2
         # No line numbers in let statement binding list
         args[1] = _to_expr(node_args[1], need_linenodes=false)
         args[2] = _to_expr(node_args[2])
     else
         eq_to_kw_in_call =
-            ((headsym == :call || headsym == :dotcall) && is_prefix_call(node)) ||
-            headsym == :ref
-        eq_to_kw_all = headsym == :parameters && !map_kw_in_params
-        in_vcbr = headsym == :vect || headsym == :curly || headsym == :braces || headsym == :ref
+            ((headsym === :call || headsym === :dotcall) && is_prefix_call(node)) ||
+            headsym === :ref
+        eq_to_kw_all = headsym === :parameters && !map_kw_in_params
+        in_vcbr = headsym === :vect || headsym === :curly || headsym === :braces || headsym === :ref
         if insert_linenums && isempty(node_args)
             push!(args, source_location(LineNumberNode, node.source, node.position))
         else
@@ -150,8 +150,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
                 end
                 eq_to_kw = eq_to_kw_in_call && i > 1 || eq_to_kw_all
                 args[insert_linenums ? 2*i : i] =
-                    _to_expr(n, eq_to_kw=eq_to_kw,
-                             map_kw_in_params=in_vcbr)
+                    _to_expr(n, eq_to_kw=eq_to_kw, map_kw_in_params=in_vcbr)
             end
             if nodekind == K"block" && has_flags(node, PARENS_FLAG)
                 popfirst!(args)
@@ -204,7 +203,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
         end
     elseif headsym === :where
         reorder_parameters!(args, 2)
-    elseif headsym == :parens
+    elseif headsym === :parens
         # parens are used for grouping and don't appear in the Expr AST
         return only(args)
     elseif headsym in (:try, :try_finally_catch)
@@ -252,15 +251,15 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
         pushfirst!(args, numeric_flags(flags(node)))
     elseif headsym === :typed_ncat
         insert!(args, 2, numeric_flags(flags(node)))
-    # elseif headsym == :string && length(args) == 1 && version <= (1,5)
+    # elseif headsym === :string && length(args) == 1 && version <= (1,5)
     #   Strip string from interpolations in 1.5 and lower to preserve
     #   "hi$("ho")" ==>  (string "hi" "ho")
     elseif headsym === :(=) && !is_decorated(node)
         if is_eventually_call(args[1]) && !iteration_spec && !Meta.isexpr(args[2], :block)
             # Add block for short form function locations
             args[2] = Expr(:block, loc, args[2])
         end
-    elseif headsym == :elseif
+    elseif headsym === :elseif
         # Block for conditional's source location
         args[1] = Expr(:block, loc, args[1])
     elseif headsym === :(->)
@@ -290,7 +289,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
     elseif headsym === :module
         pushfirst!(args, !has_flags(node, BARE_MODULE_FLAG))
         pushfirst!(args[3].args, loc)
-    elseif headsym == :inert || (headsym == :quote && length(args) == 1 &&
+    elseif headsym === :inert || (headsym === :quote && length(args) == 1 &&
                  !(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
                    a1 isa Bool  # <- compat hack, Julia 1.4+
                   ))

src/hooks.jl

@@ -288,7 +288,7 @@ function _fl_parse_hook(code, filename, lineno, offset, options)
                 ex = Expr(:toplevel, ex)
             end
             return ex, sizeof(code)
-        elseif options === :statement || options == :atom
+        elseif options === :statement || options === :atom
             ex, pos = Meta.parse(code, offset+1, greedy=options==:statement, raise=false)
             return ex, pos-1
         else

src/kinds.jl

@@ -1133,6 +1133,10 @@ function is_radical_op(x)
     kind(x) in (K"√", K"∛", K"∜")
 end
 
+"""
+Return true if `x` has whitespace or comment kind
+"""
 function is_whitespace(x)
-    kind(x) in (K"Whitespace", K"NewlineWs")
+    k = kind(x)
+    return k == K"Whitespace" || k == K"NewlineWs" || k == K"Comment"
 end

src/literal_parsing.jl

@@ -261,7 +261,7 @@ function unescape_julia_string(io::IO, str::AbstractString,
                 u = m == 4 ? 'u' : 'U'
                 msg = (m == 2) ? "invalid hex escape sequence" :
                                  "invalid unicode escape sequence"
-                emit_diagnostic(diagnostics, escstart, i, error=msg)
+                emit_diagnostic(diagnostics, escstart:i, error=msg)
                 had_error = true
             else
                 if m == 2 # \x escape sequence
@@ -279,7 +279,7 @@ function unescape_julia_string(io::IO, str::AbstractString,
                 i += 1
             end
             if n > 255
-                emit_diagnostic(diagnostics, escstart, i,
+                emit_diagnostic(diagnostics, escstart:i,
                                 error="invalid octal escape sequence")
                 had_error = true
             else
@@ -303,7 +303,7 @@ function unescape_julia_string(io::IO, str::AbstractString,
                 c == '`' ? '`' :
                 nothing
             if isnothing(u)
-                emit_diagnostic(diagnostics, escstart, i,
+                emit_diagnostic(diagnostics, escstart:i,
                                 error="invalid escape sequence")
                 had_error = true
             else

src/parse_stream.jl

@@ -359,7 +359,7 @@ function _buffer_lookahead_tokens(lexer, lookahead)
     while true
         raw = Tokenize.next_token(lexer)
         k = kind(raw)
-        was_whitespace = k in (K"Whitespace", K"Comment", K"NewlineWs")
+        was_whitespace = is_whitespace(k)
         had_whitespace |= was_whitespace
         f = EMPTY_FLAGS
         raw.dotop      && (f |= DOTOP_FLAG)
@@ -622,7 +622,7 @@ function _bump_until_n(stream::ParseStream, n::Integer, flags, remap_kind=K"None
             break
         end
         f = flags | (@__MODULE__).flags(tok)
-        is_trivia = k ∈ (K"Whitespace", K"Comment", K"NewlineWs")
+        is_trivia = is_whitespace(k)
         is_trivia && (f |= TRIVIA_FLAG)
         outk = (is_trivia || remap_kind == K"None") ? k : remap_kind
         h = SyntaxHead(outk, f)
@@ -686,7 +686,7 @@ function bump_invisible(stream::ParseStream, kind, flags=EMPTY_FLAGS;
     h = SyntaxHead(kind, flags)
     push!(stream.tokens, SyntaxToken(h, (@__MODULE__).kind(h), false, b))
     if !isnothing(error)
-        emit_diagnostic(stream, b, b-1, error=error)
+        emit_diagnostic(stream, b:b-1, error=error)
     end
     stream.peek_count = 0
     return position(stream)
@@ -797,12 +797,6 @@ function Base.position(stream::ParseStream)
     ParseStreamPosition(lastindex(stream.tokens), lastindex(stream.ranges))
 end
 
-# Get position of next item to be emitted into the output stream
-# TODO: Figure out how to remove this? It's only used with emit_diagnostic
-function next_position(stream::ParseStream)
-    ParseStreamPosition(lastindex(stream.tokens)+1, lastindex(stream.ranges)+1)
-end
-
 """
     emit(stream, mark, kind, flags = EMPTY_FLAGS; error=nothing)
 
@@ -819,14 +813,14 @@ function emit(stream::ParseStream, mark::ParseStreamPosition, kind::Kind,
         # nested.
         fbyte = token_first_byte(stream, first_token)
         lbyte = token_last_byte(stream, lastindex(stream.tokens))
-        emit_diagnostic(stream, fbyte, lbyte, error=error)
+        emit_diagnostic(stream, fbyte:lbyte, error=error)
     end
     push!(stream.ranges, range)
     return position(stream)
 end
 
-function emit_diagnostic(stream::ParseStream, fbyte::Integer, lbyte::Integer; kws...)
-    emit_diagnostic(stream.diagnostics, fbyte, lbyte; kws...)
+function emit_diagnostic(stream::ParseStream, byterange::AbstractUnitRange; kws...)
+    emit_diagnostic(stream.diagnostics, byterange; kws...)
     return nothing
 end
 
@@ -849,20 +843,30 @@ function emit_diagnostic(stream::ParseStream; whitespace=false, kws...)
     end
     fbyte = lookahead_token_first_byte(stream, begin_tok_i)
     lbyte = lookahead_token_last_byte(stream, end_tok_i)
-    emit_diagnostic(stream, fbyte, lbyte; kws...)
+    emit_diagnostic(stream, fbyte:lbyte; kws...)
     return nothing
 end
 
-function emit_diagnostic(stream::ParseStream, mark::ParseStreamPosition; kws...)
-    emit_diagnostic(stream, token_first_byte(stream, mark.token_index),
-                    _next_byte(stream) - 1; kws...)
+function emit_diagnostic(stream::ParseStream, mark::ParseStreamPosition; trim_whitespace=true, kws...)
+    i = mark.token_index
+    j = lastindex(stream.tokens)
+    if trim_whitespace
+        while i < j && is_whitespace(stream.tokens[j])
+            j -= 1
+        end
+        while i+1 < j && is_whitespace(stream.tokens[i+1])
+            i += 1
+        end
+    end
+    byterange = stream.tokens[i].next_byte:stream.tokens[j].next_byte-1
+    emit_diagnostic(stream, byterange; kws...)
 end
 
 function emit_diagnostic(stream::ParseStream, mark::ParseStreamPosition,
                          end_mark::ParseStreamPosition; kws...)
-    fbyte = token_first_byte(stream, mark.token_index)
-    lbyte = token_first_byte(stream, end_mark.token_index) - 1
-    emit_diagnostic(stream, fbyte, lbyte; kws...)
+    fbyte = stream.tokens[mark.token_index].next_byte
+    lbyte = stream.tokens[end_mark.token_index].next_byte-1
+    emit_diagnostic(stream, fbyte:lbyte; kws...)
 end
 
 #-------------------------------------------------------------------------------
@@ -890,19 +894,19 @@ function validate_tokens(stream::ParseStream)
                 # jl_strtod_c can return "underflow" even for valid cases such
                 # as `5e-324` where the source is an exact representation of
                 # `x`. So only warn when underflowing to zero.
-                underflow0 = code == :underflow && x == 0
+                underflow0 = code === :underflow && x == 0
             else
                 x, code = parse_float_literal(Float32, text, fbyte, nbyte)
-                underflow0 = code == :underflow && x == 0
+                underflow0 = code === :underflow && x == 0
             end
-            if code == :ok
+            if code === :ok
                 # pass
-            elseif code == :overflow
-                emit_diagnostic(stream, fbyte, lbyte,
+            elseif code === :overflow
+                emit_diagnostic(stream, fbyte:lbyte,
                                 error="overflow in floating point literal")
                 error_kind = K"ErrorNumericOverflow"
             elseif underflow0
-                emit_diagnostic(stream, fbyte, lbyte,
+                emit_diagnostic(stream, fbyte:lbyte,
                                 warning="underflow to zero in floating point literal")
             end
         elseif k == K"Char"
@@ -917,7 +921,7 @@ function validate_tokens(stream::ParseStream)
                 read(charbuf, Char)
                 if !eof(charbuf)
                     error_kind = K"ErrorOverLongCharacter"
-                    emit_diagnostic(stream, fbyte, lbyte,
+                    emit_diagnostic(stream, fbyte:lbyte,
                                     error="character literal contains multiple characters")
                 end
             end
@@ -929,7 +933,7 @@ function validate_tokens(stream::ParseStream)
             end
         elseif is_error(k) && k != K"error"
             # Emit messages for non-generic token errors
-            emit_diagnostic(stream, fbyte, lbyte,
+            emit_diagnostic(stream, fbyte:lbyte,
                             error=_token_error_descriptions[k])
         end
         if error_kind != K"None"