Movel all hardness to a single directory

rafaelfranca · rafaelfranca · commit eb606d6c12dd · 2025-12-01T17:32:51.000Z
Use file based harness instead of directory based harness
diff --git a/README.md b/README.md
@@ -211,22 +211,38 @@ This file will then be passed to the underlying Ruby interpreter with
 
 ## Harnesses
 
-You can find several test harnesses in this repository:
+You can find several test harnesses in the `harness/` directory:
 
-* harness - the normal default harness, with duration controlled by warmup iterations and time/count limits
-* harness-bips - a harness that measures iterations/second until stable
-* harness-continuous - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches
-* harness-once - a simplified harness that simply runs once
-* harness-perf - a simplified harness that runs for exactly the hinted number of iterations
-* harness-stackprof - a harness to profile the benchmark with stackprof
-* harness-stats - count method calls and loop iterations
-* harness-vernier - a harness to profile the benchmark with vernier
-* harness-warmup - a harness which runs as long as needed to find warmed up (peak) performance
+* `harness` - the normal default harness, with duration controlled by warmup iterations and time/count limits
+* `bips` - a harness that measures iterations/second until stable
+* `continuous` - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches
+* `once` - a simplified harness that simply runs once
+* `perf` - a simplified harness that runs for exactly the hinted number of iterations
+* `stackprof` - a harness to profile the benchmark with stackprof
+* `stats` - count method calls and loop iterations
+* `vernier` - a harness to profile the benchmark with vernier
+* `warmup` - a harness which runs as long as needed to find warmed up (peak) performance
+* `chain` - a harness to chain multiple harnesses together
+* `mplr` - a harness for multiple iterations with time limits
 
-To use it, run a benchmark script directly, specifying a harness directory with `-I`:
+To use a specific harness, run a benchmark script directly with `-I` to add the harness directory to the load path, and `-r` to require the specific harness:
 
 ```
+# Use default harness
 ruby -Iharness benchmarks/railsbench/benchmark.rb
+
+# Use the 'once' harness
+ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb
+
+# Use the 'perf' harness
+ruby -Iharness -rperf benchmarks/railsbench/benchmark.rb
+```
+
+When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option:
+
+```
+./run_benchmarks.rb --harness=once
+./run_benchmarks.rb --harness=perf
 ```
 
 There is also a robust but complex CI harness in [the yjit-metrics repo](https://github.com/Shopify/yjit-metrics).
@@ -265,10 +281,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w
 
 ```sh
 # Use `perf record` for both warmup and benchmark
-perf record ruby --yjit-perf=map -Iharness-perf benchmarks/railsbench/benchmark.rb
+perf record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb
 
 # Use `perf record` only for benchmark
-PERF=record ruby --yjit-perf=map -Iharness-perf benchmarks/railsbench/benchmark.rb
+PERF=record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb
 ```
 
 This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`.
diff --git a/harness/bips.rb b/harness/bips.rb
@@ -1,5 +1,5 @@
 require 'benchmark/ips'
-require_relative "../harness/harness-common"
+require_relative "harness-common"
 
 puts RUBY_DESCRIPTION
 
diff --git a/harness/chain.rb b/harness/chain.rb
@@ -1,4 +1,4 @@
-require_relative '../harness/harness-common'
+require_relative 'harness-common'
 
 # Ex: HARNESS_CHAIN="vernier,ractor"
 # Wraps the ractor harness in ther vernier harness
@@ -10,7 +10,7 @@
 end
 
 if CHAIN.include?("vernier") && CHAIN.last != "vernier"
-  require_relative "../harness/harness-extra"
+  require_relative "harness-extra"
   def run_enough_to_profile(n, **kwargs, &block)
     block.call
   end
@@ -30,12 +30,12 @@ def self.method_added(name)
 def run_benchmark(n, **kwargs, &block)
   CHAIN.each do |h|
     begin
-      path = "../harness-#{h}/harness"
+      path = "#{h}"
       $current_harness = h
       require_relative path
     rescue LoadError => e
       if e.path == path
-        $stderr.puts "Can't find harness harness-#{h}/harness.rb. Exiting."
+        $stderr.puts "Can't find harness #{h}.rb in harness/. Exiting."
         exit 1
       end
       raise
diff --git a/harness/continuous.rb b/harness/continuous.rb
@@ -1,4 +1,4 @@
-require_relative "../harness/harness-common"
+require_relative "harness-common"
 
 puts RUBY_DESCRIPTION
 
diff --git a/harness/loader.rb b/harness/loader.rb
@@ -1,5 +1,8 @@
-# Use harness/harness.rb by default. You can change it with -I option.
-# i.e. ruby -Iharness benchmarks/railsbench/benchmark.rb
+# Use harness/harness.rb by default. You can change it with -I and -r options.
+# Examples:
+#   ruby -Iharness benchmarks/railsbench/benchmark.rb           # uses harness/harness.rb
+#   ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb    # uses harness/once.rb
+#   ruby -Iharness -rractor benchmarks/railsbench/benchmark.rb  # uses harness/ractor.rb
 retries = 0
 begin
   require "harness"
diff --git a/harness/mplr.rb b/harness/mplr.rb
@@ -1,5 +1,5 @@
 require 'benchmark'
-require_relative "../harness/harness-common"
+require_relative "harness-common"
 
 # Minimum number of benchmarking iterations
 MAX_BENCH_ITRS = Integer(ENV.fetch('MAX_BENCH_ITRS', 1000))
diff --git a/harness/once.rb b/harness/once.rb
@@ -5,7 +5,7 @@
 # Intended only for checking whether the benchmark can set itself up properly
 # and can run to completion.
 
-require_relative '../harness/harness-common'
+require_relative 'harness-common'
 
 def run_benchmark(_hint, **)
   yield
diff --git a/harness/perf.rb b/harness/perf.rb
@@ -3,14 +3,14 @@
 # This is a relatively minimal harness meant for use with Linux perf(1).
 # Example usage:
 #
-#    $ PERF='record -e cycles' ruby -Iharness-perf benchmarks/fib.rb
+#    $ PERF='record -e cycles' ruby -Iharness -rperf benchmarks/fib.rb
 #
 # When recording with perf(1), make sure the benchmark runs long enough; you
 # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race
 # condition is possible where the benchmark finishes before the perf(1)
 # subprocess has a chance to attach, in which case perf outputs no profile.
 
-require_relative "../harness/harness-common"
+require_relative "harness-common"
 
 # Run $WARMUP_ITRS or 10 iterations of a given block. Then run $MIN_BENCH_ITRS
 # or `num_itrs_int` iterations of the block, attaching a perf command to the
diff --git a/harness/ractor.rb b/harness/ractor.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require_relative '../harness/harness-common'
+require_relative 'harness-common'
 
 Warning[:experimental] = false
 ENV["RUBY_BENCH_RACTOR_HARNESS"] = "1"
diff --git a/harness/stackprof.rb b/harness/stackprof.rb
@@ -3,11 +3,11 @@
 # Profile the benchmark (ignoring initialization code) with stackprof.
 # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'.
 # Usage:
-# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb
-# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb
+# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb
+# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb
 
-require_relative "../harness/harness-common"
-require_relative "../harness/harness-extra"
+require_relative "harness-common"
+require_relative "harness-extra"
 
 ensure_global_gem("stackprof")
 
diff --git a/harness/stats.rb b/harness/stats.rb
@@ -1,4 +1,4 @@
-require_relative '../harness/harness'
+require_relative 'harness'
 
 # Using Module#prepend to enable TracePoint right before #run_benchmark
 # while also reusing the original implementation.
diff --git a/harness/vernier.rb b/harness/vernier.rb
@@ -3,11 +3,11 @@
 # Profile the benchmark (ignoring initialization code) using vernier and display the profile.
 # Set NO_VIERWER=1 to disable automatically opening the profile in a browser.
 # Usage:
-# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/...
-# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/...
+# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/...
+# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/...
 
-require_relative "../harness/harness-common"
-require_relative "../harness/harness-extra"
+require_relative "harness-common"
+require_relative "harness-extra"
 
 ensure_global_gem("vernier")
 ensure_global_gem_exe("profile-viewer")
diff --git a/harness/warmup.rb b/harness/warmup.rb
@@ -1,5 +1,5 @@
 require 'benchmark'
-require_relative '../harness/harness-common'
+require_relative 'harness-common'
 require_relative '../misc/stats'
 
 MIN_ITERS = Integer(ENV['MIN_ITERS'] || 10)
diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb
@@ -71,7 +71,7 @@ def parse(argv)
       opts.on("--category=headline,other,micro,ractor", "when given, only benchmarks with specified categories will run") do |v|
         args.categories += v.split(",")
         if args.categories == ["ractor"]
-          args.harness = "harness-ractor"
+          args.harness = "ractor"
         end
       end
 
@@ -91,8 +91,9 @@ def parse(argv)
         args.skip_yjit = true
       end
 
-      opts.on("--harness=HARNESS_DIR", "which harness to use") do |v|
-        v = "harness-#{v}" unless v.start_with?('harness')
+      opts.on("--harness=HARNESS_NAME", "which harness to use (default: harness, options: once, bips, perf, ractor, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |v|
+        # Strip 'harness-' prefix if provided for backward compatibility
+        v = v.sub(/^harness-/, '')
         args.harness = v
       end
 
diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb
@@ -17,7 +17,8 @@ class BenchmarkSuite
   RACTOR_BENCHMARKS_DIR = "benchmarks-ractor"
   RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze
   RACTOR_CATEGORY = ["ractor"].freeze
-  RACTOR_HARNESS = "harness-ractor"
+  RACTOR_HARNESS = "ractor"
+  HARNESS_DIR = File.expand_path("../harness", __dir__)
 
   attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir
 
@@ -150,9 +151,18 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env)
     ENV["RESULT_JSON_PATH"] = result_json_path
 
     # Set up the benchmarking command
+    # If harness is 'harness', use default (no -r needed)
+    # Otherwise use -r to load the specific harness file
+    harness_args = if harness == "harness"
+      []
+    else
+      ["-r", harness]
+    end
+
     cmd = cmd_prefix + [
       *ruby,
-      "-I", harness,
+      "-I", HARNESS_DIR,
+      *harness_args,
       *pre_init,
       script_path,
     ].compact
diff --git a/run_once.sh b/run_once.sh
@@ -7,12 +7,12 @@
 #   ./run_once.sh benchmarks-ractor/optcarrot/benchmark.rb
 
 # Detect if any argument contains benchmarks-ractor/ to determine harness
-HARNESS="./harness"
+HARNESS_ARGS=()
 for arg in "$@"; do
     if [[ "$arg" == *"benchmarks-ractor/"* ]]; then
-        HARNESS="./harness-ractor"
+        HARNESS_ARGS=("-r" "ractor")
         break
     fi
 done
 
-WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"$HARNESS" "$@"
+WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"./harness" "${HARNESS_ARGS[@]}" "$@"
diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb
@@ -281,12 +281,12 @@ def setup_mock_ruby(path)
         assert_equal ['headline', 'micro'], args.categories
       end
 
-      it 'sets harness to harness-ractor when category is ractor' do
+      it 'sets harness to ractor when category is ractor' do
         parser = ArgumentParser.new
         args = parser.parse(['--category=ractor'])
 
         assert_equal ['ractor'], args.categories
-        assert_equal 'harness-ractor', args.harness
+        assert_equal 'ractor', args.harness
       end
 
       it 'allows multiple category flags' do
@@ -339,18 +339,18 @@ def setup_mock_ruby(path)
     end
 
     describe '--harness option' do
-      it 'sets harness directory' do
+      it 'sets harness name' do
         parser = ArgumentParser.new
         args = parser.parse(['--harness=once'])
 
-        assert_equal 'harness-once', args.harness
+        assert_equal 'once', args.harness
       end
 
-      it 'accepts harness- prefix' do
+      it 'strips harness- prefix for backward compatibility' do
         parser = ArgumentParser.new
         args = parser.parse(['--harness=harness-stats'])
 
-        assert_equal 'harness-stats', args.harness
+        assert_equal 'stats', args.harness
       end
     end
 
diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb
@@ -97,7 +97,7 @@
 
       assert_equal 'benchmarks-ractor', suite.bench_dir
       assert_equal 'benchmarks-ractor', suite.ractor_bench_dir
-      assert_equal 'harness-ractor', suite.harness
+      assert_equal 'ractor', suite.harness
       assert_equal [], suite.categories
     end
 
@@ -261,8 +261,8 @@
       assert_includes bench_data, 'ractor_test'
       assert_empty bench_failures
 
-      # harness should be updated to harness-ractor
-      assert_equal 'harness-ractor', suite.harness
+      # harness should be updated to ractor
+      assert_equal 'ractor', suite.harness
     end
 
     it 'includes both regular and ractor benchmarks with ractor category' do

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-require_relative "../harness/harness-common"`
	`1`	`+require_relative "harness-common"`
`2`	`2`
`3`	`3`	`puts RUBY_DESCRIPTION`
`4`	`4`