Fix line numbers with source fragments (#310)

When parsing source code fragments incrementally with

* `Meta.parse(str, index)` or
* `parsestmt(str, index)`

we must avoid scanning the rest of `str` for line numbers for
efficiency. In this mode, the user is expected to provide `first_line`
to "manually" specify which line number we're counting from.

Admittedly this is a bit clunky and should be integrated better with
SourceFile (which should also be renamed - see issue #190) but for now
seems to be the most consistent way to approach things here.

As part of the refactoring here, switch over to using `Vector{UInt8}`
for literal parsing which makes parsing to `ParseStream` and
`GreenNode` around 10% faster.

Author: c42f

docs/src/api.md

@@ -38,6 +38,7 @@ JuliaSyntax.highlight
 JuliaSyntax.sourcetext
 JuliaSyntax.source_line
 JuliaSyntax.source_location
+JuliaSyntax.source_line_range
 ```
 
 ## Expression heads/kinds

prototypes/simple_parser.jl

@@ -140,7 +140,7 @@ end
 function parse_and_show(production::Function, code)
     st = ParseStream(code)
     production(st)
-    t = JuliaSyntax.build_tree(GreenNode, st, wrap_toplevel_as_kind=K"error")
+    t = JuliaSyntax.build_tree(GreenNode, st)
     show(stdout, MIME"text/plain"(), t, code, show_trivia=true)
     if !isempty(st.diagnostics)
         println()

src/expr.jl

@@ -68,7 +68,7 @@ function _strip_parens(ex)
     end
 end
 
-function _leaf_to_Expr(source, head, srcrange, node)
+function _leaf_to_Expr(source, txtbuf, head, srcrange, node)
     k = kind(head)
     if k == K"core_@cmd"
         return GlobalRef(Core, Symbol("@cmd"))
@@ -79,7 +79,7 @@ function _leaf_to_Expr(source, head, srcrange, node)
             Expr(:error) :
             Expr(:error, "$(_token_error_descriptions[k]): `$(source[srcrange])`")
     else
-        val = isnothing(node) ? parse_julia_literal(source, head, srcrange) : node.val
+        val = isnothing(node) ? parse_julia_literal(txtbuf, head, srcrange) : node.val
         if val isa Union{Int128,UInt128,BigInt}
             # Ignore the values of large integers and convert them back to
             # symbolic/textural form for compatibility with the Expr
@@ -457,7 +457,9 @@ end
 
 function build_tree(::Type{Expr}, stream::ParseStream;
                     filename=nothing, first_line=1, kws...)
-    source = SourceFile(sourcetext(stream), filename=filename, first_line=first_line)
+    source = SourceFile(sourcetext(stream), first_index=first_byte(stream),
+                        filename=filename, first_line=first_line)
+    txtbuf = textbuf(stream)
     args = Any[]
     childranges = UnitRange{Int}[]
     childheads = SyntaxHead[]
@@ -467,7 +469,7 @@ function build_tree(::Type{Expr}, stream::ParseStream;
         end
         k = kind(head)
         if isnothing(nodechildren)
-            ex = _leaf_to_Expr(source, head, srcrange, nothing)
+            ex = _leaf_to_Expr(source, txtbuf, head, srcrange, nothing)
         else
             resize!(childranges, length(nodechildren))
             resize!(childheads, length(nodechildren))
@@ -487,7 +489,8 @@ end
 
 function _to_expr(node::SyntaxNode)
     if !haschildren(node)
-        return _leaf_to_Expr(node.source, head(node), range(node), node)
+        offset, txtbuf = _unsafe_wrap_substring(sourcetext(node.source))
+        return _leaf_to_Expr(node.source, txtbuf, head(node), range(node) .+ offset, node)
     end
     cs = children(node)
     args = Any[_to_expr(c) for c in cs]

src/hooks.jl

@@ -157,10 +157,7 @@ function core_parser_hook(code, filename::String, lineno::Int, offset::Int, opti
             write(_debug_log[], code)
         end
 
-        io = IOBuffer(code)
-        seek(io, offset)
-
-        stream = ParseStream(io)
+        stream = ParseStream(code, offset+1)
         if options === :statement || options === :atom
             # To copy the flisp parser driver:
             # * Parsing atoms      consumes leading trivia
@@ -179,9 +176,7 @@ function core_parser_hook(code, filename::String, lineno::Int, offset::Int, opti
         end
 
         if any_error(stream)
-            tree = build_tree(SyntaxNode, stream,
-                              wrap_toplevel_as_kind=K"None", first_line=lineno,
-                              filename=filename)
+            tree = build_tree(SyntaxNode, stream, first_line=lineno, filename=filename)
             tag = _incomplete_tag(tree, lastindex(code))
             if _has_v1_10_hooks
                 exc = ParseError(stream, filename=filename, first_line=lineno,
@@ -233,13 +228,7 @@ function core_parser_hook(code, filename::String, lineno::Int, offset::Int, opti
             #
             # show_diagnostics(stdout, stream.diagnostics, code)
             #
-            ex = build_tree(Expr, stream; filename=filename,
-                            wrap_toplevel_as_kind=K"None", first_line=lineno)
-            if @isexpr(ex, :None)
-                # The None wrapping is only to give somewhere for trivia to be
-                # attached; unwrap!
-                ex = only(ex.args)
-            end
+            ex = build_tree(Expr, stream; filename=filename, first_line=lineno)
         end
 
         # Note the next byte in 1-based indexing is `last_byte(stream) + 1` but
@@ -291,15 +280,13 @@ else
     Base.Meta.ParseError(e::JuliaSyntax.ParseError) = e
 end
 
-const _default_parser = _has_v1_6_hooks ? Core._parse : nothing
-
 """
     enable_in_core!([enable=true; freeze_world_age=true, debug_filename=nothing])
 
 Connect the JuliaSyntax parser to the Julia runtime so that it replaces the
 flisp parser for all parsing work. That is, JuliaSyntax will be used for
-`include()` `Meta.parse()`, the REPL, etc. To disable, set use
-`enable_in_core!(false)`.
+`include()` `Meta.parse()`, the REPL, etc. To reset to the reference parser,
+use `enable_in_core!(false)`.
 
 Keyword arguments:
 * `freeze_world_age` - Use a fixed world age for the parser to prevent
@@ -322,7 +309,7 @@ function enable_in_core!(enable=true; freeze_world_age = true,
         world_age = freeze_world_age ? Base.get_world_counter() : typemax(UInt)
         _set_core_parse_hook(fix_world_age(core_parser_hook, world_age))
     else
-        _set_core_parse_hook(_default_parser)
+        _set_core_parse_hook(Core.Compiler.fl_parse)
     end
     nothing
 end

src/kinds.jl

@@ -914,6 +914,8 @@ const _kind_names =
         "cartesian_iterator"
         "comprehension"
         "typed_comprehension"
+        # Container for a single statement/atom plus any trivia and errors
+        "wrapper"
     "END_SYNTAX_KINDS"
 ]