Make source_location() and line_starts consistent (#552)

when the last source byte is a newline

Author: mlechu

src/source_files.jl

@@ -156,9 +156,6 @@ function SourceFile(code::AbstractString; filename=nothing, first_line=1,
         # The line is considered to start after the `\n`
         code[i] == '\n' && push!(line_starts, i+1)
     end
-    if isempty(code) || last(code) != '\n'
-        push!(line_starts, ncodeunits(code)+1)
-    end
     SourceFile(code, first_index-1, filename, first_line, line_starts)
 end
 
@@ -168,8 +165,7 @@ end
 
 # Get line number of the given byte within the code
 function _source_line_index(source::SourceFile, byte_index)
-    lineidx = searchsortedlast(source.line_starts, byte_index - source.byte_offset)
-    return (lineidx < lastindex(source.line_starts)) ? lineidx : lineidx-1
+    searchsortedlast(source.line_starts, byte_index - source.byte_offset)
 end
 _source_line(source::SourceFile, lineidx) = lineidx + source.first_line - 1
 
@@ -204,7 +200,10 @@ function source_line_range(source::SourceFile, byte_index::Integer;
                            context_lines_before=0, context_lines_after=0)
     lineidx = _source_line_index(source, byte_index)
     fbyte = source.line_starts[max(lineidx-context_lines_before, 1)]
-    lbyte = source.line_starts[min(lineidx+1+context_lines_after, end)] - 1
+    lline = lineidx + context_lines_after
+    lbyte = lline >= lastindex(source.line_starts) ?
+        ncodeunits(source.code) : source.line_starts[lline + 1] - 1
+
     return (fbyte + source.byte_offset,
             lbyte + source.byte_offset)
 end

test/source_files.jl

@@ -3,23 +3,28 @@
     @test source_location(SourceFile("a"), 2) == (1,2)
 
     @test source_location(SourceFile("a\n"), 2) == (1,2)
-    @test source_location(SourceFile("a\n"), 3) == (1,3)
+    @test source_location(SourceFile("a\n"), 3) == (2,1)
 
     @test source_location(SourceFile("a\nb\n"), 2) == (1,2)
     @test source_location(SourceFile("a\nb\n"), 3) == (2,1)
     @test source_location(SourceFile("a\nb\n"), 4) == (2,2)
-    @test source_location(SourceFile("a\nb\n"), 5) == (2,3)
+    @test source_location(SourceFile("a\nb\n"), 5) == (3,1)
+
+    @test source_location(SourceFile("\n\n"), 1) == (1,1)
+    @test source_location(SourceFile("\n\n"), 2) == (2,1)
+    @test source_location(SourceFile("\n\n"), 3) == (3,1)
 
     @test source_location(SourceFile("a"; first_line=7), 1) == (7,1)
     @test source_location(SourceFile("a"; first_line=7), 2) == (7,2)
 
     @test source_location(SourceFile("a\n"; first_line=7), 2) == (7,2)
-    @test source_location(SourceFile("a\n"; first_line=7), 3) == (7,3)
+    @test source_location(SourceFile("a\n"; first_line=7), 3) == (8,1)
 
     @test source_location(SourceFile("a\nb\n"; first_line=7), 2) == (7,2)
     @test source_location(SourceFile("a\nb\n"; first_line=7), 3) == (8,1)
     @test source_location(SourceFile("a\nb\n"; first_line=7), 4) == (8,2)
-    @test source_location(SourceFile("a\nb\n"; first_line=7), 5) == (8,3)
+    @test source_location(SourceFile("a\nb\n"; first_line=7), 5) == (9,1)
+
 
     mktemp() do path, io
         write(io, "a\n")

test/syntax_tree.jl

@@ -37,6 +37,11 @@
     e = try node.val = :q catch e e end
     @test occursin("immutable", e.msg) && occursin("SyntaxData", e.msg)
 
+    # Newline-terminated source
+    t = parsestmt(SyntaxNode, "a*b + c\n")
+    @test sprint(highlight, t[1][3]) == "a*b + c\n# ╙"
+    @test sprint(highlight, t.source, t.raw, 1, 3) == "a*b + c\n# ╙"
+
     # copy
     t = parsestmt(SyntaxNode, "a*b + c")
     ct = copy(t)