Fixes for SourceFile byte offsets

This fixes a crash formatting error messages when core_parse_hook is
used to parse a piece of broken code with a nontrivial byte offset.

* `SourceFile` held by `SyntaxNode` preserves the indexing of the original
  string passed to the `parse*()` functions.
* Fix `source_line_range` and `thisind` accordingly

Author: c42f

src/expr.jl

@@ -457,8 +457,7 @@ end
 
 function build_tree(::Type{Expr}, stream::ParseStream;
                     filename=nothing, first_line=1, kws...)
-    source = SourceFile(sourcetext(stream), first_index=first_byte(stream),
-                        filename=filename, first_line=first_line)
+    source = SourceFile(stream, filename=filename, first_line=first_line)
     txtbuf = unsafe_textbuf(stream)
     args = Any[]
     childranges = UnitRange{Int}[]

src/parse_stream.jl

@@ -1108,6 +1108,10 @@ function sourcetext(stream::ParseStream; steal_textbuf=false)
     SubString(str, first_byte(stream), thisind(str, last_byte(stream)))
 end
 
+function SourceFile(stream::ParseStream; kws...)
+    return SourceFile(sourcetext(stream); first_index=first_byte(stream), kws...)
+end
+
 """
     unsafe_textbuf(stream)
 

src/parser_api.jl

@@ -11,7 +11,7 @@ struct ParseError <: Exception
 end
 
 function ParseError(stream::ParseStream; incomplete_tag=:none, kws...)
-    source = SourceFile(sourcetext(stream); kws...)
+    source = SourceFile(stream; kws...)
     ParseError(source, stream.diagnostics, incomplete_tag)
 end
 

src/source_files.jl

@@ -76,7 +76,8 @@ function source_line_range(source::SourceFile, byte_index;
     lineidx = _source_line_index(source, byte_index)
     fbyte = source.line_starts[max(lineidx-context_lines_before, 1)]
     lbyte = source.line_starts[min(lineidx+1+context_lines_after, end)] - 1
-    fbyte,lbyte
+    return (fbyte + source.byte_offset,
+            lbyte + source.byte_offset)
 end
 
 function source_location(::Type{LineNumberNode}, source::SourceFile, byte_index)
@@ -120,7 +121,7 @@ function Base.getindex(source::SourceFile, i::Int)
 end
 
 function Base.thisind(source::SourceFile, i::Int)
-    thisind(source.code, i - source.byte_offset)
+    thisind(source.code, i - source.byte_offset) + source.byte_offset
 end
 
 Base.firstindex(source::SourceFile) = firstindex(source.code) + source.byte_offset

src/syntax_tree.jl

@@ -61,8 +61,11 @@ Base.show(io::IO, ::ErrorVal) = printstyled(io, "✘", color=:light_red)
 
 function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead};
                     keep_parens=false, position::Integer=1)
-    offset, txtbuf = _unsafe_wrap_substring(sourcetext(source))
-    _to_SyntaxNode(source, txtbuf, offset, raw, convert(Int, position), keep_parens)
+    GC.@preserve source begin
+        raw_offset, txtbuf = _unsafe_wrap_substring(source.code)
+        offset = raw_offset - source.byte_offset
+        _to_SyntaxNode(source, txtbuf, offset, raw, convert(Int, position), keep_parens)
+    end
 end
 
 function _to_SyntaxNode(source::SourceFile, txtbuf::Vector{UInt8}, offset::Int,
@@ -222,8 +225,8 @@ Base.copy(data::SyntaxData) = SyntaxData(data.source, data.raw, data.position, d
 function build_tree(::Type{SyntaxNode}, stream::ParseStream;
                     filename=nothing, first_line=1, keep_parens=false, kws...)
     green_tree = build_tree(GreenNode, stream; kws...)
-    source = SourceFile(sourcetext(stream), filename=filename, first_line=first_line)
-    SyntaxNode(source, green_tree, position=1, keep_parens=keep_parens)
+    source = SourceFile(stream, filename=filename, first_line=first_line)
+    SyntaxNode(source, green_tree, position=first_byte(stream), keep_parens=keep_parens)
 end
 
 #-------------------------------------------------------------------------------

test/hooks.jl

@@ -1,3 +1,15 @@
+function _unwrap_parse_error(core_hook_result)
+    @test Meta.isexpr(core_hook_result[1], :error, 1)
+    err = core_hook_result[1].args[1]
+    if JuliaSyntax._has_v1_10_hooks
+        @test err isa Meta.ParseError
+        return err.detail
+    else
+        @test err isa JuliaSyntax.ParseError
+        return err
+    end
+end
+
 @testset "Hooks for Core integration" begin
     @testset "whitespace parsing" begin
         @test JuliaSyntax.core_parser_hook("", "somefile", 1, 0, :statement) == Core.svec(nothing, 0)
@@ -19,26 +31,28 @@
         @test ex.args[2] == LineNumberNode(2, "otherfile")
 
         # Errors also propagate file & lineno
-        err = JuliaSyntax.core_parser_hook("[x)", "f1", 1, 0, :statement)[1].args[1]
-        if JuliaSyntax._has_v1_10_hooks
-            @test err isa Meta.ParseError
-            err = err.detail
-        else
-            @test err isa JuliaSyntax.ParseError
-        end
+        err = _unwrap_parse_error(
+            JuliaSyntax.core_parser_hook("[x)", "f1", 1, 0, :statement)
+        )
         @test err isa JuliaSyntax.ParseError
         @test err.source.filename == "f1"
         @test err.source.first_line == 1
-        err = JuliaSyntax.core_parser_hook("[x)", "f2", 2, 0, :statement)[1].args[1]
-        if JuliaSyntax._has_v1_10_hooks
-            @test err isa Meta.ParseError
-            err = err.detail
-        else
-            @test err isa JuliaSyntax.ParseError
-        end
+        err = _unwrap_parse_error(
+            JuliaSyntax.core_parser_hook("[x)", "f2", 2, 0, :statement)
+        )
         @test err isa JuliaSyntax.ParseError
         @test err.source.filename == "f2"
         @test err.source.first_line == 2
+
+        # Errors including nontrivial offset indices
+        err = _unwrap_parse_error(
+            JuliaSyntax.core_parser_hook("a\nh{x)\nb", "test.jl", 1, 2, :statement)
+        )
+        @test err isa JuliaSyntax.ParseError
+        @test err.source.first_line == 1
+        @test err.diagnostics[1].first_byte == 6
+        @test err.diagnostics[1].last_byte == 5
+        @test err.diagnostics[1].message == "Expected `}`"
     end
 
     @testset "toplevel errors" begin

test/source_files.jl

@@ -28,8 +28,14 @@
     end
 
     # byte offset
-    @test source_location(SourceFile("a\nbb\nccc\ndddd", first_index=10), 13) == (2,2)
-    @test source_line(SourceFile("a\nbb\nccc\ndddd", first_index=10), 15) == 3
+    sf = SourceFile("a\nbb\nccc\ndddd", first_index=10)
+    @test source_location(sf, 13) == (2,2)
+    @test source_line(sf, 15) == 3
+    @test source_line_range(sf, 10) == (10,11)
+    @test source_line_range(sf, 11) == (10,11)
+    @test source_line_range(sf, 12) == (12,14)
+    @test source_line_range(sf, 14) == (12,14)
+    @test source_line_range(sf, 15) == (15,18)
 
     # source_line convenience function
     @test source_line(SourceFile("a\nb\n"), 2) == 1
@@ -52,6 +58,11 @@ end
     @test sf[10:11] == "ab"
     @test view(sf, 10:11) == "ab"
 
+    @test thisind(SourceFile("xαx", first_index=10), 10) == 10
+    @test thisind(SourceFile("xαx", first_index=10), 11) == 11
+    @test thisind(SourceFile("xαx", first_index=10), 12) == 11
+    @test thisind(SourceFile("xαx", first_index=10), 13) == 13
+
     if Base.VERSION >= v"1.4"
         # Protect the `[begin` from being viewed by the parser on older Julia versions
         @test eval(Meta.parse("SourceFile(\"a\nb\n\")[begin:end]")) == "a\nb\n"

test/syntax_tree.jl

@@ -49,6 +49,12 @@
     @test length(children(node)) == 2
     node[2] = parsestmt(SyntaxNode, "y")
     @test sourcetext(child(node, 2)) == "y"
+
+    # SyntaxNode with offsets
+    t,_ = parsestmt(SyntaxNode, "begin a end\nbegin b end", 13)
+    @test t.position == 13
+    @test child(t,1).position == 19
+    @test child(t,1).val == :b
 end
 
 @testset "SyntaxNode pretty printing" begin

test/test_utils.jl

@@ -9,6 +9,7 @@ using .JuliaSyntax:
     SourceFile,
     source_location,
     source_line,
+    source_line_range,
     parse!,
     parsestmt,
     parseall,