Redesign the Ractor/Knucleotide benchmark

.github/workflows/test.yml

@@ -64,7 +64,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ruby, head]
+        ruby: [head]
     if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
     steps:
       - uses: actions/checkout@v3
@@ -85,7 +85,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ruby, head]
+        ruby: [head]
     if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
     steps:
       - uses: actions/checkout@v3

benchmarks.yml

@@ -91,6 +91,11 @@ graphql-native:
   desc: GraphQL gem parsing a large file, but using a native parser
 knucleotide:
   desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork
+knucleotide-ractor:
+  desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors
+  ractor: true
+  ractor_only: true
+  default_harness: harness
 lee:
   desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way
 matmul:
@@ -231,13 +236,20 @@ throw:
   ractor: true
 
 #
-# Ractor-only benchmarks
+# Ractor scaling benchmarks
 #
-ractor/knucleotide:
-  desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables. Counts groups in parallel using Ractors.
-ractor/gvl_release_acquire:
+gvl_release_acquire:
   desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors.
-ractor/json_parse_float:
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor
+json_parse_float:
   desc: test the performance of parsing multiple lists of json floats with ractors.
-ractor/json_parse_string:
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor
+json_parse_string:
   desc: test the performance of parsing multiple lists of strings with ractors.
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor

benchmarks-ractor/knucleotide/benchmark.rb → benchmarks/knucleotide-ractor/benchmark.rb

@@ -2,9 +2,11 @@
 # https://salsa.debian.org/benchmarksgame-team/benchmarksgame/
 #
 # k-nucleotide benchmark - Ractor implementation
-# Mirrors the Process.fork version structure as closely as possible
+# Mirrors the Process.fork version: spawns 7 ractors (one per task)
 
-require_relative "../../harness/loader"
+Warning[:experimental] = false
+
+require_relative '../../harness/loader'
 
 def frequency(seq, length)
   frequencies = Hash.new(0)
@@ -25,9 +27,9 @@ def sort_by_freq(seq, length)
   table.sort { |a, b|
     cmp = b[1] <=> a[1]
     cmp == 0 ? a[0] <=> b[0] : cmp
-  }.map! { |seq, count|
+  }.map { |seq, count|
     "#{seq} #{'%.3f' % ((count * 100.0) / n)}"
-  }.join("\n") << "\n\n"
+  }.join("\n") + "\n\n"
 end
 
 def find_seq(seq, s)
@@ -48,19 +50,22 @@ def generate_test_sequence(size)
   full_copies.times { sequence << alu }
   sequence << alu[0, remainder] if remainder > 0
 
-  sequence.upcase.freeze
+  sequence.upcase
 end
 
-# Make sequence shareable for Ractors
-TEST_SEQUENCE = make_shareable(generate_test_sequence(100_000))
+TEST_SEQUENCE = Ractor.make_shareable(generate_test_sequence(100_000))
 
-run_benchmark(5) do |num_ractors, ractor_args|
+run_benchmark(5) do
   freqs = [1, 2]
   nucleos = %w(GGT GGTA GGTATT GGTATTTTAATT GGTATTTTAATTTATAGT)
 
-  # Sequential version - mirrors Process version but without Workers
-  results = []
-  freqs.each { |i| results << sort_by_freq(TEST_SEQUENCE, i) }
-  nucleos.each { |s| results << find_seq(TEST_SEQUENCE, s) }
+  ractors = freqs.map { |i|
+    Ractor.new(TEST_SEQUENCE, i) { |seq, len| sort_by_freq(seq, len) }
+  }
+  ractors += nucleos.map { |s|
+    Ractor.new(TEST_SEQUENCE, s) { |seq, nucleo| find_seq(seq, nucleo) }
+  }
+
+  results = ractors.map(&:value)
   results
 end

burn_in.rb

@@ -59,14 +59,10 @@ def free_file_path(parent_dir, name_prefix)
   end
 end
 
-def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version)
-  # Determine the path to the benchmark script
-  bench_name = bench_id.sub('ractor/', '')
-  bench_dir, harness = if bench_name == bench_id
-    ['benchmarks', 'harness']
-  else
-    ['benchmarks-ractor', 'harness-ractor']
-  end
+def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata)
+  bench_dir = 'benchmarks'
+  entry = metadata[bench_name] || {}
+  harness = entry.fetch('default_harness', 'harness')
 
   script_path = File.join(bench_dir, bench_name, 'benchmark.rb')
   if not File.exist?(script_path)
@@ -153,12 +149,12 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version)
   return false
 end
 
-def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
+def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version, metadata)
   error_found = false
 
   while true
     bench_name = bench_names.sample()
-    error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version)
+    error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata)
     error_found ||= error
 
     if error_found
@@ -201,12 +197,16 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
 bench_names = []
 
 if args.categories.include?('ractor-only')
-  # Only include benchmarks with ractor/ prefix (from benchmarks-ractor directory)
-  bench_names = metadata.keys.select { |name| name.start_with?('ractor/') }
+  # Include only benchmarks with ractor_only: true
+  metadata.each do |name, entry|
+    if entry['ractor_only']
+      bench_names << name
+    end
+  end
 elsif args.categories.include?('ractor')
-  # Include both ractor/ prefixed benchmarks and those with ractor: true
+  # Include benchmarks with ractor: true or ractor_only: true
   metadata.each do |name, entry|
-    if name.start_with?('ractor/') || entry['ractor']
+    if entry['ractor'] || entry['ractor_only']
       bench_names << name
     end
   end
@@ -221,10 +221,12 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
     end
   end
 else
-  # Regular category filtering
+  # Regular category filtering - exclude ractor-only and ractor harness benchmarks
   metadata.each do |name, entry|
     category = entry.fetch('category', 'other')
-    if args.categories.include?(category)
+    is_ractor_only = entry['ractor_only'] ||
+      (entry['ractor'] && entry['default_harness'] == 'harness-ractor')
+    if args.categories.include?(category) && !is_ractor_only
       bench_names << name
     end
   end
@@ -237,7 +239,7 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
 args.num_procs.times do |i|
   pid = Process.fork do
     run_time = (i < args.num_long_runs)? (3600 * 2):10
-    test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version)
+    test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version, metadata)
   end
 end
 

lib/argument_parser.rb

@@ -8,13 +8,15 @@ class ArgumentParser
     :out_path,
     :out_override,
     :harness,
+    :harness_explicit,
     :yjit_opts,
     :categories,
     :name_filters,
     :excludes,
     :rss,
     :graph,
     :no_pinning,
+    :force_pinning,
     :turbo,
     :skip_yjit,
     :with_pre_init,
@@ -52,10 +54,8 @@ def parse(argv)
             name = name.shellsplit.first
           end
           version, *options = version.shellsplit
-          rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies"
-          unless executable = ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"].find { |path| File.executable?(path) }
-            abort "Cannot find '#{version}' in /opt/rubies or #{rubies_dir}"
-          end
+          executable = find_chruby_ruby(version)
+          abort "Cannot find '#{version}' in chruby paths" unless executable
           args.executables[name] = [executable, *options]
         end
       end
@@ -94,6 +94,7 @@ def parse(argv)
       opts.on("--harness=HARNESS_DIR", "which harness to use") do |v|
         v = "harness-#{v}" unless v.start_with?('harness')
         args.harness = v
+        args.harness_explicit = true
       end
 
       opts.on("--warmup=N", "the number of warmup iterations for the default harness (default: 15)") do |n|
@@ -140,6 +141,10 @@ def parse(argv)
         args.no_pinning = true
       end
 
+      opts.on("--force-pinning", "force pinning even for benchmarks marked no_pinning") do
+        args.force_pinning = true
+      end
+
       opts.on("--turbo", "don't disable CPU turbo boost") do
         args.turbo = true
       end
@@ -165,6 +170,15 @@ def parse(argv)
 
   private
 
+  def find_chruby_ruby(version)
+    rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies"
+    chruby_search_paths(version, rubies_dir).find { |path| File.executable?(path) }
+  end
+
+  def chruby_search_paths(version, rubies_dir)
+    ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"]
+  end
+
   def have_yjit?(ruby)
     ruby_version = `#{ruby} -v --yjit 2> #{File::NULL}`.strip
     ruby_version.downcase.include?("yjit")
@@ -176,13 +190,15 @@ def default_args
       out_path: File.expand_path("./data"),
       out_override: nil,
       harness: "harness",
+      harness_explicit: false,
       yjit_opts: "",
       categories: [],
       name_filters: [],
       excludes: [],
       rss: false,
       graph: false,
       no_pinning: false,
+      force_pinning: false,
       turbo: false,
       skip_yjit: false,
       with_pre_init: nil,

lib/benchmark_filter.rb

@@ -18,12 +18,21 @@ def match?(name)
   private
 
   def matches_category?(name)
-    return true if @categories.empty?
+    if @categories.empty?
+      return false if ractor_harness_benchmark?(name)
+      return true
+    end
 
     benchmark_categories = get_benchmark_categories(name)
     @categories.intersect?(benchmark_categories)
   end
 
+  def ractor_harness_benchmark?(name)
+    benchmark_metadata = @metadata[name] || {}
+    benchmark_metadata['ractor_only'] ||
+      (benchmark_metadata['ractor'] && benchmark_metadata['default_harness'] == 'harness-ractor')
+  end
+
   def matches_name_filter?(name)
     return true if @name_filters.empty?
 
@@ -58,7 +67,8 @@ def get_benchmark_categories(name)
     @category_cache[name] ||= begin
       benchmark_metadata = @metadata[name] || {}
       categories = [benchmark_metadata.fetch('category', 'other')]
-      categories << 'ractor' if benchmark_metadata['ractor']
+      categories << 'ractor' if benchmark_metadata['ractor'] || benchmark_metadata['ractor_only']
+      categories << 'ractor-only' if benchmark_metadata['ractor_only']
       categories
     end
   end

lib/benchmark_runner/cli.rb

@@ -34,8 +34,10 @@ def run
         excludes: args.excludes,
         out_path: args.out_path,
         harness: args.harness,
+        harness_explicit: args.harness_explicit,
         pre_init: args.with_pre_init,
-        no_pinning: args.no_pinning
+        no_pinning: args.no_pinning,
+        force_pinning: args.force_pinning
       )
 
       # Benchmark with and without YJIT