Parser optimization: Reuse position arrays, outline error code paths

Author: c42f

src/parse_stream.jl

@@ -186,6 +186,8 @@ mutable struct ParseStream
     lexer::Tokenize.Lexers.Lexer{IOBuffer}
     # Lookahead buffer for already lexed tokens
     lookahead::Vector{SyntaxToken}
+    # Pool of stream positions for use as working space in parsing
+    position_pool::Vector{Vector{ParseStreamPosition}}
     # Parser output as an ordered sequence of ranges, parent nodes after children.
     ranges::Vector{TaggedRange}
     # Parsing diagnostics (errors/warnings etc)
@@ -210,6 +212,7 @@ mutable struct ParseStream
         # like an acceptable tradeoff.
         ver = (version.major, version.minor)
         new(text_buf, text_root, lexer,
+            Vector{Vector{ParseStreamPosition}}(),
             Vector{SyntaxToken}(),
             Vector{TaggedRange}(),
             Vector{Diagnostic}(),
@@ -268,6 +271,19 @@ function show_diagnostics(io::IO, stream::ParseStream, code)
     show_diagnostics(io, stream.diagnostics, code)
 end
 
+# We manage a pool of stream positions as parser working space
+function acquire_positions(stream)
+    if isempty(stream.position_pool)
+        return Vector{ParseStreamPosition}()
+    end
+    pop!(stream.position_pool)
+end
+
+function release_positions(stream, positions)
+    empty!(positions)
+    push!(stream.position_pool, positions)
+end
+
 #-------------------------------------------------------------------------------
 # Stream input interface - the peek_* family of functions
 
@@ -311,6 +327,11 @@ function _lookahead_index(stream::ParseStream, n::Integer, skip_newlines::Bool)
     end
 end
 
+@noinline function _parser_stuck_error(stream)
+    # Optimization: emit unlikely errors in a separate function
+    error("The parser seems stuck at byte $(stream.next_byte)")
+end
+
 """
     peek(stream [, n=1]; skip_newlines=false)
 
@@ -333,7 +354,7 @@ function peek_token(stream::ParseStream, n::Integer=1;
                     skip_newlines=false, skip_whitespace=true)
     stream.peek_count += 1
     if stream.peek_count > 100_000
-        error("The parser seems stuck at byte $(stream.next_byte)")
+        _parser_stuck_error(stream)
     end
     i = _lookahead_index(stream, n, skip_newlines)
     if !skip_whitespace

src/parser.jl

@@ -2940,8 +2940,8 @@ function parse_brackets(after_parse::Function,
                     space_sensitive=false,
                     where_enabled=true,
                     whitespace_newline=true)
-    params_marks = ParseStreamPosition[]
-    eq_positions = ParseStreamPosition[]
+    params_marks = acquire_positions(ps.stream)
+    eq_positions = acquire_positions(ps.stream)
     last_eq_before_semi = 0
     num_subexprs = 0
     num_semis = 0
@@ -3021,6 +3021,8 @@ function parse_brackets(after_parse::Function,
             emit(ps, mark, K"parameters")
         end
     end
+    release_positions(ps.stream, params_marks)
+    release_positions(ps.stream, eq_positions)
 end
 
 is_indentation(b::UInt8) = (b == UInt8(' ') || b == UInt8('\t'))
@@ -3036,9 +3038,7 @@ function parse_string(ps::ParseState, raw::Bool)
     string_chunk_kind = delim_k in KSet`" """` ? K"String" : K"CmdString"
     indent_ref_i = 0
     indent_ref_len = typemax(Int)
-    if triplestr
-        indent_chunks = Vector{ParseStreamPosition}()
-    end
+    indent_chunks = acquire_positions(ps.stream)
     buf = textbuf(ps)
     str_flags = (triplestr ? TRIPLE_STRING_FLAG : EMPTY_FLAGS) |
                 (raw       ? RAW_STRING_FLAG : EMPTY_FLAGS)
@@ -3191,6 +3191,7 @@ function parse_string(ps::ParseState, raw::Bool)
             end
         end
     end
+    release_positions(ps.stream, indent_chunks)
     if had_end_delim
         if n_valid_chunks == 0
             # Empty strings, or empty after triple quoted processing