Add benchmarks/ractor directory with 2 JSON benchmarks

Add JSON benchmarks:
  parse_float
  parse_string

Add to ractor harness, make it possible to use gemfiles in different
directories and make it easy to do so. This way we don't have to create
directories for each benchmark.

Improve run_benchmarks.rb, allow it to take a regular expression for --name_filters option
Ex:
$ WARMUP_ITRS=10 ruby run_benchmarks.rb --category=ractor-only --name_filters=/json/

Allow it to take --skip-yjit option  to only run with interpreter.

Author: luke-gruber

benchmarks/ractor/gemfiles/Gemfile_json_parse.rb

@@ -0,0 +1 @@
+gem "json", "2.13.2"

benchmarks/ractor/gemfiles/Gemfile_json_parse.rb.lock

@@ -0,0 +1,13 @@
+GEM
+  specs:
+    json (2.13.2)
+
+PLATFORMS
+  arm64-darwin-24
+  ruby
+
+DEPENDENCIES
+  json (= 2.13.2)
+
+BUNDLED WITH
+   2.8.0.dev

benchmarks/ractor/json_parse_float.rb

@@ -0,0 +1,38 @@
+require_relative "../../harness/loader"
+
+use_ractor_gemfile("json_parse")
+puts "JSON #{JSON::VERSION}"
+
+ELEMENTS = 100_000
+
+list = ELEMENTS.times.map do
+  {
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+    rand => rand,
+  }.to_json
+end
+Ractor.make_shareable(list)
+
+# Work is divided between ractors
+run_benchmark(5, ractor_args: [list]) do |num_rs, list|
+  # num_rs: 1,list: 100_000
+  # num_rs: 2 list: 50_000
+  # num_rs: 4 list: 25_000
+  if num_rs.zero?
+    num = list.size
+  else
+    num = list.size / num_rs
+  end
+  list.each_with_index do |json, idx|
+    break if idx >= num
+    JSON.parse(json)
+  end
+end

benchmarks/ractor/json_parse_string.rb

@@ -0,0 +1,47 @@
+require_relative "../../harness/loader"
+
+use_ractor_gemfile("json_parse")
+puts "JSON #{JSON::VERSION}"
+
+ELEMENTS = 300_000
+list = ELEMENTS.times.map do |i|
+  {
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+    "string #{i}" => "value #{i}",
+  }.to_json
+end
+Ractor.make_shareable(list)
+
+# Work is divided between ractors
+run_benchmark(5, ractor_args: [list]) do |num_rs, list|
+  # num_rs: 1,list: 100_000
+  # num_rs: 2 list: 50_000
+  # num_rs: 4 list: 25_000
+  if num_rs.zero?
+    num = list.size
+  else
+    num = list.size / num_rs
+  end
+  list.each_with_index do |json, idx|
+    break if idx >= num
+    JSON.parse(json)
+  end
+end

harness-ractor/harness.rb

@@ -27,6 +27,14 @@ def join
   end
 end
 
+def use_ractor_gemfile(filename)
+  filename = File.expand_path("Gemfile_#{filename}.rb", "benchmarks/ractor/gemfiles")
+  raise "Gemfile #{filename} doesn't exist" unless  File.exist?(filename)
+  use_inline_gemfile do
+    instance_eval File.read(filename), filename, 1
+  end
+end
+
 MAX_ITERS = Integer(ENV.fetch("MAX_BENCH_ITRS", 5))
 
 def run_benchmark(num_itrs_hint, ractor_args: [], &block)
@@ -98,4 +106,4 @@ def ractor_deep_dup(args)
   end
 end
 
-Ractor.make_shareable(self)
+Ractor.make_shareable(self) # until we get Ractor.shareable_proc

harness/harness-common.rb

@@ -46,10 +46,20 @@ def use_gemfile(extra_setup_cmd: nil)
   # Need to be in the appropriate directory for this...
   require "bundler"
   # Use Bundler.setup instead of require 'bundler/setup' to avoid bundler's autoswitch restarting the
-  # process and messing with LOAD_PATH.
+  # process and messing with LOAD_PATH. Autoswitching occurs when the BUNDLED_WITH in the Gemfile.lock
+  # is a different version than the loaded version of bundler.
   Bundler.setup
 end
 
+def use_inline_gemfile(&block)
+  raise  "Block must be given" unless block
+  require "bundler/inline"
+  gemfile(install=true) do
+    source "https://rubygems.org"
+    instance_eval(&block)
+  end
+end
+
 # This returns its best estimate of the Resident Set Size in bytes.
 # That's roughly the amount of memory the process takes, including shareable resources.
 # RSS reference: https://stackoverflow.com/questions/7880784/what-is-rss-and-vsz-in-linux-memory-management

run_benchmarks.rb

@@ -183,9 +183,28 @@ def benchmark_categories(name)
 
 # Check if the name matches any of the names in a list of filters
 def match_filter(entry, categories:, name_filters:)
+  name_filters = process_name_filters(name_filters)
   name = entry.sub(/\.rb\z/, '')
   (categories.empty? || benchmark_categories(name).any? { |cat| categories.include?(cat) }) &&
-    (name_filters.empty? || name_filters.include?(name))
+    (name_filters.empty? || name_filters.any? { |filter| filter === name })
+end
+
+# process "/my_benchmark/i" into /my_benchmark/i
+def process_name_filters(name_filters)
+  name_filters.map do |name_filter|
+    if name_filter[0] == "/"
+      regexp_str = name_filter[1..-1].reverse.sub(/\A(\w*)\//, "")
+      regexp_opts = $1.to_s
+      regexp_str.reverse!
+      r = /#{regexp_str}/
+      if !regexp_opts.empty?
+        r = Regexp.compile(r.to_s, regexp_opts)
+      end
+      r
+    else
+      name_filter
+    end
+  end
 end
 
 # Resolve the pre_init file path into a form that can be required
@@ -332,6 +351,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
   graph: false,
   no_pinning: false,
   turbo: false,
+  skip_yjit: false,
 })
 
 OptionParser.new do |opts|
@@ -372,23 +392,27 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
 
   opts.on("--category=headline,other,micro,ractor", "when given, only benchmarks with specified categories will run") do |v|
     args.categories += v.split(",")
-    if args.categories == ['ractor']
-      args.harness = 'harness-ractor'
+    if args.categories == ["ractor"]
+      args.harness = "harness-ractor"
     end
   end
 
-  opts.on("--headline", "when given, headline benchmarks will be run") do |v|
+  opts.on("--headline", "when given, headline benchmarks will be run") do
     args.categories += ["headline"]
   end
 
-  opts.on("--ractor-only", "ractor-only benchmarks (benchmarks/ractor/*.rb) will be run") do |v|
+  opts.on("--ractor-only", "ractor-only benchmarks (benchmarks/ractor/*.rb) will be run") do
     args.categories = ["ractor-only"]
   end
 
   opts.on("--name_filters=x,y,z", Array, "when given, only benchmarks with names that contain one of these strings will run") do |list|
     args.name_filters = list
   end
 
+  opts.on("--skip-yjit", "Don't run with yjit after interpreter") do
+    args.skip_yjit = true
+  end
+
   opts.on("--harness=HARNESS_DIR", "which harness to use") do |v|
     v = "harness-#{v}" unless v.start_with?('harness')
     args.harness = v
@@ -446,7 +470,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
 
 # If -e is not specified, benchmark the current Ruby. Compare it with YJIT if available.
 if args.executables.empty?
-  if have_yjit?(RbConfig.ruby)
+  if have_yjit?(RbConfig.ruby) && !args.skip_yjit
     args.executables["interp"] = [RbConfig.ruby]
     args.executables["yjit"] = [RbConfig.ruby, "--yjit", *args.yjit_opts.shellsplit]
   else