Bump Prism version to 1.5.0

Author: k0kubun

lib/prism.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 # The Prism Ruby parser.
 #
@@ -59,15 +60,16 @@ def self.lex_ripper(source)
   end
 
   # :call-seq:
-  #   Prism::load(source, serialized) -> ParseResult
+  #   Prism::load(source, serialized, freeze) -> ParseResult
   #
   # Load the serialized AST using the source as a reference into a tree.
-  def self.load(source, serialized)
-    Serialize.load(source, serialized)
+  def self.load(source, serialized, freeze = false)
+    Serialize.load_parse(source, serialized, freeze)
   end
 end
 
 require_relative "prism/polyfill/byteindex"
+require_relative "prism/polyfill/warn"
 require_relative "prism/node"
 require_relative "prism/node_ext"
 require_relative "prism/parse_result"

lib/prism/desugar_compiler.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 module Prism
   class DesugarAndWriteNode # :nodoc:

lib/prism/ffi.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 # typed: ignore
 
 # This file is responsible for mirroring the API provided by the C extension by
@@ -7,6 +8,10 @@
 require "rbconfig"
 require "ffi"
 
+# We want to eagerly load this file if there are Ractors so that it does not get
+# autoloaded from within a non-main Ractor.
+require "prism/serialize" if defined?(Ractor)
+
 module Prism
   module LibRubyParser # :nodoc:
     extend FFI::Library
@@ -15,7 +20,8 @@ module LibRubyParser # :nodoc:
     # must align with the build shared library from make/rake.
     libprism_in_build = File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
     libprism_in_libdir = "#{RbConfig::CONFIG["libdir"]}/prism/libprism.#{RbConfig::CONFIG["SOEXT"]}"
-    if File.exist? libprism_in_build
+
+    if File.exist?(libprism_in_build)
       INCLUDE_DIR = File.expand_path("../../include", __dir__)
       ffi_lib libprism_in_build
     else
@@ -159,6 +165,9 @@ def self.with
     class PrismString # :nodoc:
       SIZEOF = LibRubyParser.pm_string_sizeof
 
+      PLATFORM_EXPECTS_UTF8 =
+        RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i)
+
       attr_reader :pointer, :length
 
       def initialize(pointer, length, from_string)
@@ -193,8 +202,7 @@ def self.with_file(filepath)
         # On Windows and Mac, it's expected that filepaths will be encoded in
         # UTF-8. If they are not, we need to convert them to UTF-8 before
         # passing them into pm_string_mapped_init.
-        if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
-           (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
+        if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
           filepath = filepath.encode(Encoding::UTF_8)
         end
 
@@ -223,7 +231,7 @@ def self.with_file(filepath)
   private_constant :LibRubyParser
 
   # The version constant is set by reading the result of calling pm_version.
-  VERSION = LibRubyParser.pm_version.read_string
+  VERSION = LibRubyParser.pm_version.read_string.freeze
 
   class << self
     # Mirror the Prism.dump API by using the serialization API.
@@ -357,50 +365,37 @@ def profile_file(filepath, **options)
     def dump_common(string, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
-        buffer.read
+
+        dumped = buffer.read
+        dumped.freeze if options.fetch(:freeze, false)
+
+        dumped
       end
     end
 
     def lex_common(string, code, options) # :nodoc:
-      serialized = LibRubyParser::PrismBuffer.with do |buffer|
+      LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
-        buffer.read
+        Serialize.load_lex(code, buffer.read, options.fetch(:freeze, false))
       end
-
-      Serialize.load_tokens(Source.for(code), serialized)
     end
 
     def parse_common(string, code, options) # :nodoc:
       serialized = dump_common(string, options)
-      Prism.load(code, serialized)
+      Serialize.load_parse(code, serialized, options.fetch(:freeze, false))
     end
 
     def parse_comments_common(string, code, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
-
-        source = Source.for(code)
-        loader = Serialize::Loader.new(source, buffer.read)
-
-        loader.load_header
-        loader.load_encoding
-        loader.load_start_line
-        loader.load_comments
+        Serialize.load_parse_comments(code, buffer.read, options.fetch(:freeze, false))
       end
     end
 
     def parse_lex_common(string, code, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
-
-        source = Source.for(code)
-        loader = Serialize::Loader.new(source, buffer.read)
-
-        tokens = loader.load_tokens
-        node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
-        tokens.each { |token,| token.value.force_encoding(loader.encoding) }
-
-        ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
+        Serialize.load_parse_lex(code, buffer.read, options.fetch(:freeze, false))
       end
     end
 
@@ -430,11 +425,13 @@ def dump_options_command_line(options)
     def dump_options_version(version)
       case version
       when nil, "latest"
-        0
+        0 # Handled in pm_parser_init
       when /\A3\.3(\.\d+)?\z/
         1
       when /\A3\.4(\.\d+)?\z/
-        0
+        2
+      when /\A3\.5(\.\d+)?\z/
+        3
       else
         raise ArgumentError, "invalid version: #{version}"
       end
@@ -483,15 +480,43 @@ def dump_options(options)
       template << "C"
       values << (options.fetch(:partial_script, false) ? 1 : 0)
 
+      template << "C"
+      values << (options.fetch(:freeze, false) ? 1 : 0)
+
       template << "L"
       if (scopes = options[:scopes])
         values << scopes.length
 
         scopes.each do |scope|
+          locals = nil
+          forwarding = 0
+
+          case scope
+          when Array
+            locals = scope
+          when Scope
+            locals = scope.locals
+
+            scope.forwarding.each do |forward|
+              case forward
+              when :*     then forwarding |= 0x1
+              when :**    then forwarding |= 0x2
+              when :&     then forwarding |= 0x4
+              when :"..." then forwarding |= 0x8
+              else raise ArgumentError, "invalid forwarding value: #{forward}"
+              end
+            end
+          else
+            raise TypeError, "wrong argument type #{scope.class.inspect} (expected Array or Prism::Scope)"
+          end
+
           template << "L"
-          values << scope.length
+          values << locals.length
+
+          template << "C"
+          values << forwarding
 
-          scope.each do |local|
+          locals.each do |local|
             name = local.name
             template << "L"
             values << name.bytesize

lib/prism/lex_compat.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 require "delegate"
 require "ripper"

lib/prism/node_ext.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
+# :markup: markdown
 
+#--
 # Here we are reopening the prism module to provide methods on nodes that aren't
 # templated and are meant as convenience methods.
+#++
 module Prism
   class Node
     def deprecated(*replacements) # :nodoc:
       location = caller_locations(1, 1)
       location = location[0].label if location
       suggest = replacements.map { |replacement| "#{self.class}##{replacement}" }
 
-      warn(<<~MSG, category: :deprecated)
+      warn(<<~MSG, uplevel: 1, category: :deprecated)
         [deprecation]: #{self.class}##{location} is deprecated and will be \
         removed in the next major version. Use #{suggest.join("/")} instead.
         #{(caller(1, 3) || []).join("\n")}

lib/prism/pack.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
+# :markup: markdown
 # typed: ignore
 
+#
 module Prism
   # A parser for the pack template language.
   module Pack

lib/prism/parse_result.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 module Prism
   # This represents a source of Ruby code that has been parsed. It is used in
@@ -48,6 +49,16 @@ def initialize(source, start_line = 1, offsets = [])
       @offsets = offsets # set after parsing is done
     end
 
+    # Replace the value of start_line with the given value.
+    def replace_start_line(start_line)
+      @start_line = start_line
+    end
+
+    # Replace the value of offsets with the given value.
+    def replace_offsets(offsets)
+      @offsets.replace(offsets)
+    end
+
     # Returns the encoding of the source code, which is set by parameters to the
     # parser or by the encoding magic comment.
     def encoding
@@ -132,6 +143,13 @@ def code_units_column(byte_offset, encoding)
       code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
     end
 
+    # Freeze this object and the objects it contains.
+    def deep_freeze
+      source.freeze
+      offsets.freeze
+      freeze
+    end
+
     private
 
     # Binary search through the offsets to find the line number for the given
@@ -854,5 +872,40 @@ def inspect
       location
       super
     end
+
+    # Freeze this object and the objects it contains.
+    def deep_freeze
+      value.freeze
+      location.freeze
+      freeze
+    end
+  end
+
+  # This object is passed to the various Prism.* methods that accept the
+  # `scopes` option as an element of the list. It defines both the local
+  # variables visible at that scope as well as the forwarding parameters
+  # available at that scope.
+  class Scope
+    # The list of local variables that are defined in this scope. This should be
+    # defined as an array of symbols.
+    attr_reader :locals
+
+    # The list of local variables that are forwarded to the next scope. This
+    # should by defined as an array of symbols containing the specific values of
+    # :*, :**, :&, or :"...".
+    attr_reader :forwarding
+
+    # Create a new scope object with the given locals and forwarding.
+    def initialize(locals, forwarding)
+      @locals = locals
+      @forwarding = forwarding
+    end
+  end
+
+  # Create a new scope with the given locals and forwarding options that is
+  # suitable for passing into one of the Prism.* methods that accepts the
+  # `scopes` option.
+  def self.scope(locals: [], forwarding: [])
+    Scope.new(locals, forwarding)
   end
 end

lib/prism/parse_result/comments.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 module Prism
   class ParseResult < Result

lib/prism/parse_result/errors.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 require "stringio"
 

lib/prism/parse_result/newlines.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 module Prism
   class ParseResult < Result