Fix token error ranges + highlighting for multibyte chars (#271)

Author: c42f

src/parse_stream.jl

@@ -888,7 +888,7 @@ function validate_tokens(stream::ParseStream)
         k = kind(t)
         fbyte = toks[i-1].next_byte
         nbyte = t.next_byte
-        lbyte = prevind(text, t.next_byte)
+        tokrange = fbyte:nbyte-1
         error_kind = K"None"
         if k in KSet"Integer BinInt OctInt HexInt"
             # The following shouldn't be able to error...
@@ -909,11 +909,11 @@ function validate_tokens(stream::ParseStream)
             if code === :ok
                 # pass
             elseif code === :overflow
-                emit_diagnostic(stream, fbyte:lbyte,
+                emit_diagnostic(stream, tokrange,
                                 error="overflow in floating point literal")
                 error_kind = K"ErrorNumericOverflow"
             elseif underflow0
-                emit_diagnostic(stream, fbyte:lbyte,
+                emit_diagnostic(stream, tokrange,
                                 warning="underflow to zero in floating point literal")
             end
         elseif k == K"Char"
@@ -928,7 +928,7 @@ function validate_tokens(stream::ParseStream)
                 read(charbuf, Char)
                 if !eof(charbuf)
                     error_kind = K"ErrorOverLongCharacter"
-                    emit_diagnostic(stream, fbyte:lbyte,
+                    emit_diagnostic(stream, tokrange,
                                     error="character literal contains multiple characters")
                 end
             end
@@ -940,7 +940,7 @@ function validate_tokens(stream::ParseStream)
             end
         elseif is_error(k) && k != K"error"
             # Emit messages for non-generic token errors
-            emit_diagnostic(stream, fbyte:lbyte,
+            emit_diagnostic(stream, tokrange,
                             error=_token_error_descriptions[k])
         end
         if error_kind != K"None"

src/source_files.jl

@@ -108,6 +108,10 @@ function Base.getindex(source::SourceFile, i::Int)
     source.code[i]
 end
 
+function Base.thisind(source::SourceFile, i::Int)
+    thisind(source.code, i)
+end
+
 Base.firstindex(source::SourceFile) = firstindex(source.code)
 Base.lastindex(source::SourceFile) = lastindex(source.code)
 
@@ -203,7 +207,7 @@ function highlight(io::IO, source::SourceFile, range::UnitRange;
         print(io, source[x:p-1])
         _printstyled(io, hitext; bgcolor=color)
         print(io, source[q+1:d])
-        source[d] == '\n' || print(io, "\n")
+        source[thisind(source, d)] == '\n' || print(io, "\n")
         _print_marker_line(io, source[a:p-1], hitext, true, true, marker_line_color, note, notecolor)
     else
         # x   --------------
@@ -232,13 +236,13 @@ function highlight(io::IO, source::SourceFile, range::UnitRange;
             _printstyled(io, source[z:q]; bgcolor=color)
         end
         print(io, source[q+1:d])
-        source[d] == '\n' || print(io, "\n")
+        source[thisind(source, d)] == '\n' || print(io, "\n")
         qline = source[c:q]
         _print_marker_line(io, "", qline, true, false, marker_line_color, note, notecolor)
     end
-    if context_lines_after > 0 && d+1 < lastindex(source)
+    if context_lines_after > 0 && d+1 <= lastindex(source)
         print(io, '\n')
-        w1 = source[w] == '\n' ? w - 1 : w
+        w1 = source[thisind(source, w)] == '\n' ? w - 1 : w
         print(io, source[d+1:w1])
     end
 end

test/diagnostics.jl

@@ -11,6 +11,10 @@ function diagnostic(str; only_first=false, allow_multiple=false)
     end
 end
 
+@testset "token errors" begin
+    @test diagnostic(":⥻") == Diagnostic(2, 4, :error, "unknown unicode character")
+end
+
 @testset "parser errors" begin
 	@test diagnostic("+ #==# (a,b)") ==
         Diagnostic(2, 7, :error, "whitespace not allowed between prefix function call and argument list")

test/source_files.jl

@@ -52,7 +52,7 @@ end
 
 
 @testset "highlight()" begin
-    src = JuliaSyntax.SourceFile("""
+    src = SourceFile("""
         abcd
         αβγδ
         +-*/""")
@@ -81,6 +81,10 @@ end
         αβγδ
         #└─┘
         +-*/"""
+    # multi-byte char at eof
+    @test sprint(highlight, SourceFile("a α"), 3:4) == "a α\n# ╙"
+    @test sprint(highlight, SourceFile("a\nα"), 1:4) == "┌\na\nα\n┘"
+    @test sprint(highlight, SourceFile("a\nb\nα"), 3:3) == "a\nb\n╙\nα"
 
     # Multi-line ranges
     @test sprint(highlight, src, 1:7) == """