Restructure the category selection

For benchmarks like the knucleotide ractor benchmark, which use Ractors
internally so need to be included in the ractor benchmarks, but which
must not use the ractor harness

New category selection is as follows:

  - No category: Runs all benchmarks except those with `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to the `default` harness.
  - `--category=ractor`: Runs benchmarks with ractor: true or `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to `harness-ractor`.
  - `--category=ractor-only`: Runs only benchmarks with `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to `harness-ractor`.

Author: eightbitraptor

benchmarks.yml

@@ -94,7 +94,9 @@ knucleotide:
   no_pinning: true
 knucleotide-ractor:
   desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors
-  no_pinning: true
+  ractor: true
+  ractor_only: true
+  default_harness: harness
 lee:
   desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way
 matmul:
@@ -235,11 +237,20 @@ throw:
   ractor: true
 
 #
-# Ractor-only benchmarks
+# Ractor scaling benchmarks
 #
-ractor/gvl_release_acquire:
+gvl_release_acquire:
   desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors.
-ractor/json_parse_float:
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor
+json_parse_float:
   desc: test the performance of parsing multiple lists of json floats with ractors.
-ractor/json_parse_string:
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor
+json_parse_string:
   desc: test the performance of parsing multiple lists of strings with ractors.
+  ractor: true
+  ractor_only: true
+  default_harness: harness-ractor

benchmarks-ractor/gvl_release_acquire/benchmark.rb renamed to benchmarks/gvl_release_acquire/benchmark.rb

@@ -59,14 +59,10 @@ def free_file_path(parent_dir, name_prefix)
   end
 end
 
-def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version)
-  # Determine the path to the benchmark script
-  bench_name = bench_id.sub('ractor/', '')
-  bench_dir, harness = if bench_name == bench_id
-    ['benchmarks', 'harness']
-  else
-    ['benchmarks-ractor', 'harness-ractor']
-  end
+def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata)
+  bench_dir = 'benchmarks'
+  entry = metadata[bench_name] || {}
+  harness = entry.fetch('default_harness', 'harness')
 
   script_path = File.join(bench_dir, bench_name, 'benchmark.rb')
   if not File.exist?(script_path)
@@ -153,12 +149,12 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version)
   return false
 end
 
-def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
+def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version, metadata)
   error_found = false
 
   while true
     bench_name = bench_names.sample()
-    error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version)
+    error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata)
     error_found ||= error
 
     if error_found
@@ -201,12 +197,16 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
 bench_names = []
 
 if args.categories.include?('ractor-only')
-  # Only include benchmarks with ractor/ prefix (from benchmarks-ractor directory)
-  bench_names = metadata.keys.select { |name| name.start_with?('ractor/') }
+  # Include only benchmarks with ractor_only: true
+  metadata.each do |name, entry|
+    if entry['ractor_only']
+      bench_names << name
+    end
+  end
 elsif args.categories.include?('ractor')
-  # Include both ractor/ prefixed benchmarks and those with ractor: true
+  # Include benchmarks with ractor: true or ractor_only: true
   metadata.each do |name, entry|
-    if name.start_with?('ractor/') || entry['ractor']
+    if entry['ractor'] || entry['ractor_only']
       bench_names << name
     end
   end
@@ -221,10 +221,12 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
     end
   end
 else
-  # Regular category filtering
+  # Regular category filtering - exclude ractor-only and ractor harness benchmarks
   metadata.each do |name, entry|
     category = entry.fetch('category', 'other')
-    if args.categories.include?(category)
+    is_ractor_only = entry['ractor_only'] ||
+      (entry['ractor'] && entry['default_harness'] == 'harness-ractor')
+    if args.categories.include?(category) && !is_ractor_only
       bench_names << name
     end
   end
@@ -237,7 +239,7 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version)
 args.num_procs.times do |i|
   pid = Process.fork do
     run_time = (i < args.num_long_runs)? (3600 * 2):10
-    test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version)
+    test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version, metadata)
   end
 end
 

benchmarks-ractor/json_parse_float/Gemfile renamed to benchmarks/json_parse_float/Gemfile

@@ -8,6 +8,7 @@ class ArgumentParser
     :out_path,
     :out_override,
     :harness,
+    :harness_explicit,
     :yjit_opts,
     :categories,
     :name_filters,
@@ -93,6 +94,7 @@ def parse(argv)
       opts.on("--harness=HARNESS_DIR", "which harness to use") do |v|
         v = "harness-#{v}" unless v.start_with?('harness')
         args.harness = v
+        args.harness_explicit = true
       end
 
       opts.on("--warmup=N", "the number of warmup iterations for the default harness (default: 15)") do |n|
@@ -188,6 +190,7 @@ def default_args
       out_path: File.expand_path("./data"),
       out_override: nil,
       harness: "harness",
+      harness_explicit: false,
       yjit_opts: "",
       categories: [],
       name_filters: [],