From eb606d6c12dd23ceda1720f4715dcb0ad8573b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Fri, 21 Nov 2025 21:59:36 +0000 Subject: [PATCH 1/6] Movel all hardness to a single directory Use file based harness instead of directory based harness --- README.md | 42 +++++++++++++------ harness-bips/harness.rb => harness/bips.rb | 2 +- harness-chain/harness.rb => harness/chain.rb | 8 ++-- .../harness.rb => harness/continuous.rb | 2 +- harness/loader.rb | 7 +++- harness-mplr/harness.rb => harness/mplr.rb | 2 +- harness-once/harness.rb => harness/once.rb | 2 +- harness-perf/harness.rb => harness/perf.rb | 4 +- .../harness.rb => harness/ractor.rb | 2 +- .../harness.rb => harness/stackprof.rb | 8 ++-- harness-stats/harness.rb => harness/stats.rb | 2 +- .../harness.rb => harness/vernier.rb | 8 ++-- .../harness.rb => harness/warmup.rb | 2 +- lib/argument_parser.rb | 7 ++-- lib/benchmark_suite.rb | 14 ++++++- run_once.sh | 6 +-- test/argument_parser_test.rb | 12 +++--- test/benchmark_suite_test.rb | 6 +-- 18 files changed, 83 insertions(+), 53 deletions(-) rename harness-bips/harness.rb => harness/bips.rb (79%) rename harness-chain/harness.rb => harness/chain.rb (85%) rename harness-continuous/harness.rb => harness/continuous.rb (94%) rename harness-mplr/harness.rb => harness/mplr.rb (93%) rename harness-once/harness.rb => harness/once.rb (88%) rename harness-perf/harness.rb => harness/perf.rb (94%) rename harness-ractor/harness.rb => harness/ractor.rb (98%) rename harness-stackprof/harness.rb => harness/stackprof.rb (91%) rename harness-stats/harness.rb => harness/stats.rb (98%) rename harness-vernier/harness.rb => harness/vernier.rb (71%) rename harness-warmup/harness.rb => harness/warmup.rb (97%) diff --git a/README.md b/README.md index bba0d1ef..de077427 100644 --- a/README.md +++ b/README.md @@ -211,22 +211,38 @@ This file will then be passed to the underlying Ruby interpreter with ## Harnesses -You can find several test harnesses in this repository: +You can find several test harnesses in the `harness/` directory: -* harness - the normal default harness, with duration controlled by warmup iterations and time/count limits -* harness-bips - a harness that measures iterations/second until stable -* harness-continuous - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches -* harness-once - a simplified harness that simply runs once -* harness-perf - a simplified harness that runs for exactly the hinted number of iterations -* harness-stackprof - a harness to profile the benchmark with stackprof -* harness-stats - count method calls and loop iterations -* harness-vernier - a harness to profile the benchmark with vernier -* harness-warmup - a harness which runs as long as needed to find warmed up (peak) performance +* `harness` - the normal default harness, with duration controlled by warmup iterations and time/count limits +* `bips` - a harness that measures iterations/second until stable +* `continuous` - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches +* `once` - a simplified harness that simply runs once +* `perf` - a simplified harness that runs for exactly the hinted number of iterations +* `stackprof` - a harness to profile the benchmark with stackprof +* `stats` - count method calls and loop iterations +* `vernier` - a harness to profile the benchmark with vernier +* `warmup` - a harness which runs as long as needed to find warmed up (peak) performance +* `chain` - a harness to chain multiple harnesses together +* `mplr` - a harness for multiple iterations with time limits -To use it, run a benchmark script directly, specifying a harness directory with `-I`: +To use a specific harness, run a benchmark script directly with `-I` to add the harness directory to the load path, and `-r` to require the specific harness: ``` +# Use default harness ruby -Iharness benchmarks/railsbench/benchmark.rb + +# Use the 'once' harness +ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb + +# Use the 'perf' harness +ruby -Iharness -rperf benchmarks/railsbench/benchmark.rb +``` + +When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: + +``` +./run_benchmarks.rb --harness=once +./run_benchmarks.rb --harness=perf ``` There is also a robust but complex CI harness in [the yjit-metrics repo](https://github.com/Shopify/yjit-metrics). @@ -265,10 +281,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w ```sh # Use `perf record` for both warmup and benchmark -perf record ruby --yjit-perf=map -Iharness-perf benchmarks/railsbench/benchmark.rb +perf record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ruby --yjit-perf=map -Iharness-perf benchmarks/railsbench/benchmark.rb +PERF=record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb ``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. diff --git a/harness-bips/harness.rb b/harness/bips.rb similarity index 79% rename from harness-bips/harness.rb rename to harness/bips.rb index d1319a19..b583fa0f 100644 --- a/harness-bips/harness.rb +++ b/harness/bips.rb @@ -1,5 +1,5 @@ require 'benchmark/ips' -require_relative "../harness/harness-common" +require_relative "harness-common" puts RUBY_DESCRIPTION diff --git a/harness-chain/harness.rb b/harness/chain.rb similarity index 85% rename from harness-chain/harness.rb rename to harness/chain.rb index 48e6b0f9..eef9d77b 100644 --- a/harness-chain/harness.rb +++ b/harness/chain.rb @@ -1,4 +1,4 @@ -require_relative '../harness/harness-common' +require_relative 'harness-common' # Ex: HARNESS_CHAIN="vernier,ractor" # Wraps the ractor harness in ther vernier harness @@ -10,7 +10,7 @@ end if CHAIN.include?("vernier") && CHAIN.last != "vernier" - require_relative "../harness/harness-extra" + require_relative "harness-extra" def run_enough_to_profile(n, **kwargs, &block) block.call end @@ -30,12 +30,12 @@ def self.method_added(name) def run_benchmark(n, **kwargs, &block) CHAIN.each do |h| begin - path = "../harness-#{h}/harness" + path = "#{h}" $current_harness = h require_relative path rescue LoadError => e if e.path == path - $stderr.puts "Can't find harness harness-#{h}/harness.rb. Exiting." + $stderr.puts "Can't find harness #{h}.rb in harness/. Exiting." exit 1 end raise diff --git a/harness-continuous/harness.rb b/harness/continuous.rb similarity index 94% rename from harness-continuous/harness.rb rename to harness/continuous.rb index 6606b182..f76f535b 100644 --- a/harness-continuous/harness.rb +++ b/harness/continuous.rb @@ -1,4 +1,4 @@ -require_relative "../harness/harness-common" +require_relative "harness-common" puts RUBY_DESCRIPTION diff --git a/harness/loader.rb b/harness/loader.rb index 84de6383..12f3abe0 100644 --- a/harness/loader.rb +++ b/harness/loader.rb @@ -1,5 +1,8 @@ -# Use harness/harness.rb by default. You can change it with -I option. -# i.e. ruby -Iharness benchmarks/railsbench/benchmark.rb +# Use harness/harness.rb by default. You can change it with -I and -r options. +# Examples: +# ruby -Iharness benchmarks/railsbench/benchmark.rb # uses harness/harness.rb +# ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ruby -Iharness -rractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb retries = 0 begin require "harness" diff --git a/harness-mplr/harness.rb b/harness/mplr.rb similarity index 93% rename from harness-mplr/harness.rb rename to harness/mplr.rb index 73234934..4473a204 100644 --- a/harness-mplr/harness.rb +++ b/harness/mplr.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative "../harness/harness-common" +require_relative "harness-common" # Minimum number of benchmarking iterations MAX_BENCH_ITRS = Integer(ENV.fetch('MAX_BENCH_ITRS', 1000)) diff --git a/harness-once/harness.rb b/harness/once.rb similarity index 88% rename from harness-once/harness.rb rename to harness/once.rb index c03d9db5..dbf44e60 100644 --- a/harness-once/harness.rb +++ b/harness/once.rb @@ -5,7 +5,7 @@ # Intended only for checking whether the benchmark can set itself up properly # and can run to completion. -require_relative '../harness/harness-common' +require_relative 'harness-common' def run_benchmark(_hint, **) yield diff --git a/harness-perf/harness.rb b/harness/perf.rb similarity index 94% rename from harness-perf/harness.rb rename to harness/perf.rb index 4991cb54..a778526e 100644 --- a/harness-perf/harness.rb +++ b/harness/perf.rb @@ -3,14 +3,14 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ruby -Iharness-perf benchmarks/fib.rb +# $ PERF='record -e cycles' ruby -Iharness -rperf benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race # condition is possible where the benchmark finishes before the perf(1) # subprocess has a chance to attach, in which case perf outputs no profile. -require_relative "../harness/harness-common" +require_relative "harness-common" # Run $WARMUP_ITRS or 10 iterations of a given block. Then run $MIN_BENCH_ITRS # or `num_itrs_int` iterations of the block, attaching a perf command to the diff --git a/harness-ractor/harness.rb b/harness/ractor.rb similarity index 98% rename from harness-ractor/harness.rb rename to harness/ractor.rb index fbfb7fb9..5e133636 100644 --- a/harness-ractor/harness.rb +++ b/harness/ractor.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require_relative '../harness/harness-common' +require_relative 'harness-common' Warning[:experimental] = false ENV["RUBY_BENCH_RACTOR_HARNESS"] = "1" diff --git a/harness-stackprof/harness.rb b/harness/stackprof.rb similarity index 91% rename from harness-stackprof/harness.rb rename to harness/stackprof.rb index 6a63d1c5..c9bf7a81 100644 --- a/harness-stackprof/harness.rb +++ b/harness/stackprof.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb -require_relative "../harness/harness-common" -require_relative "../harness/harness-extra" +require_relative "harness-common" +require_relative "harness-extra" ensure_global_gem("stackprof") diff --git a/harness-stats/harness.rb b/harness/stats.rb similarity index 98% rename from harness-stats/harness.rb rename to harness/stats.rb index a0b9a2a3..b9b216ae 100644 --- a/harness-stats/harness.rb +++ b/harness/stats.rb @@ -1,4 +1,4 @@ -require_relative '../harness/harness' +require_relative 'harness' # Using Module#prepend to enable TracePoint right before #run_benchmark # while also reusing the original implementation. diff --git a/harness-vernier/harness.rb b/harness/vernier.rb similarity index 71% rename from harness-vernier/harness.rb rename to harness/vernier.rb index fe32ff1c..2b7c93ff 100644 --- a/harness-vernier/harness.rb +++ b/harness/vernier.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... -# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... +# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... +# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... -require_relative "../harness/harness-common" -require_relative "../harness/harness-extra" +require_relative "harness-common" +require_relative "harness-extra" ensure_global_gem("vernier") ensure_global_gem_exe("profile-viewer") diff --git a/harness-warmup/harness.rb b/harness/warmup.rb similarity index 97% rename from harness-warmup/harness.rb rename to harness/warmup.rb index 21045227..33ae5e43 100644 --- a/harness-warmup/harness.rb +++ b/harness/warmup.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative '../harness/harness-common' +require_relative 'harness-common' require_relative '../misc/stats' MIN_ITERS = Integer(ENV['MIN_ITERS'] || 10) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 35de87c1..761fa6a8 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -71,7 +71,7 @@ def parse(argv) opts.on("--category=headline,other,micro,ractor", "when given, only benchmarks with specified categories will run") do |v| args.categories += v.split(",") if args.categories == ["ractor"] - args.harness = "harness-ractor" + args.harness = "ractor" end end @@ -91,8 +91,9 @@ def parse(argv) args.skip_yjit = true end - opts.on("--harness=HARNESS_DIR", "which harness to use") do |v| - v = "harness-#{v}" unless v.start_with?('harness') + opts.on("--harness=HARNESS_NAME", "which harness to use (default: harness, options: once, bips, perf, ractor, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |v| + # Strip 'harness-' prefix if provided for backward compatibility + v = v.sub(/^harness-/, '') args.harness = v end diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 30c4a853..f94fbca9 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -17,7 +17,8 @@ class BenchmarkSuite RACTOR_BENCHMARKS_DIR = "benchmarks-ractor" RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_CATEGORY = ["ractor"].freeze - RACTOR_HARNESS = "harness-ractor" + RACTOR_HARNESS = "ractor" + HARNESS_DIR = File.expand_path("../harness", __dir__) attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir @@ -150,9 +151,18 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command + # If harness is 'harness', use default (no -r needed) + # Otherwise use -r to load the specific harness file + harness_args = if harness == "harness" + [] + else + ["-r", harness] + end + cmd = cmd_prefix + [ *ruby, - "-I", harness, + "-I", HARNESS_DIR, + *harness_args, *pre_init, script_path, ].compact diff --git a/run_once.sh b/run_once.sh index d2035e8d..a5240fce 100755 --- a/run_once.sh +++ b/run_once.sh @@ -7,12 +7,12 @@ # ./run_once.sh benchmarks-ractor/optcarrot/benchmark.rb # Detect if any argument contains benchmarks-ractor/ to determine harness -HARNESS="./harness" +HARNESS_ARGS=() for arg in "$@"; do if [[ "$arg" == *"benchmarks-ractor/"* ]]; then - HARNESS="./harness-ractor" + HARNESS_ARGS=("-r" "ractor") break fi done -WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"$HARNESS" "$@" \ No newline at end of file +WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"./harness" "${HARNESS_ARGS[@]}" "$@" \ No newline at end of file diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 7008dcef..e093e69e 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -281,12 +281,12 @@ def setup_mock_ruby(path) assert_equal ['headline', 'micro'], args.categories end - it 'sets harness to harness-ractor when category is ractor' do + it 'sets harness to ractor when category is ractor' do parser = ArgumentParser.new args = parser.parse(['--category=ractor']) assert_equal ['ractor'], args.categories - assert_equal 'harness-ractor', args.harness + assert_equal 'ractor', args.harness end it 'allows multiple category flags' do @@ -339,18 +339,18 @@ def setup_mock_ruby(path) end describe '--harness option' do - it 'sets harness directory' do + it 'sets harness name' do parser = ArgumentParser.new args = parser.parse(['--harness=once']) - assert_equal 'harness-once', args.harness + assert_equal 'once', args.harness end - it 'accepts harness- prefix' do + it 'strips harness- prefix for backward compatibility' do parser = ArgumentParser.new args = parser.parse(['--harness=harness-stats']) - assert_equal 'harness-stats', args.harness + assert_equal 'stats', args.harness end end diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index cd9eee3b..2cb820ef 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -97,7 +97,7 @@ assert_equal 'benchmarks-ractor', suite.bench_dir assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness-ractor', suite.harness + assert_equal 'ractor', suite.harness assert_equal [], suite.categories end @@ -261,8 +261,8 @@ assert_includes bench_data, 'ractor_test' assert_empty bench_failures - # harness should be updated to harness-ractor - assert_equal 'harness-ractor', suite.harness + # harness should be updated to ractor + assert_equal 'ractor', suite.harness end it 'includes both regular and ractor benchmarks with ractor category' do From 4c84b90156508f003fe1cda0e96e5111308bde4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Fri, 21 Nov 2025 22:20:13 +0000 Subject: [PATCH 2/6] Keep the harness implementation separated from the harness framework --- README.md | 32 ++++++++----- .../gvl_release_acquire/benchmark.rb | 2 +- .../json_parse_float/benchmark.rb | 2 +- .../json_parse_string/benchmark.rb | 2 +- benchmarks-ractor/knucleotide/benchmark.rb | 2 +- benchmarks/30k_ifelse.rb | 2 +- benchmarks/30k_methods.rb | 2 +- benchmarks/activerecord/benchmark.rb | 2 +- benchmarks/addressable/equality.rb | 2 +- benchmarks/addressable/getters.rb | 2 +- benchmarks/addressable/join.rb | 2 +- benchmarks/addressable/merge.rb | 2 +- benchmarks/addressable/new.rb | 2 +- benchmarks/addressable/normalize.rb | 2 +- benchmarks/addressable/parse.rb | 2 +- benchmarks/addressable/setters.rb | 2 +- benchmarks/addressable/to-s.rb | 2 +- benchmarks/attr_accessor.rb | 2 +- benchmarks/binarytrees/benchmark.rb | 2 +- benchmarks/blurhash/benchmark.rb | 2 +- benchmarks/cfunc_itself.rb | 2 +- benchmarks/chunky-png/benchmark.rb | 2 +- benchmarks/erubi-rails/benchmark.rb | 2 +- benchmarks/erubi/benchmark.rb | 2 +- benchmarks/etanni/benchmark.rb | 2 +- benchmarks/fannkuchredux/benchmark.rb | 2 +- benchmarks/fib.rb | 2 +- benchmarks/fluentd/benchmark.rb | 2 +- benchmarks/getivar.rb | 2 +- benchmarks/graphql-native/benchmark.rb | 2 +- benchmarks/graphql/benchmark.rb | 2 +- benchmarks/hexapdf/benchmark.rb | 2 +- benchmarks/keyword_args.rb | 2 +- benchmarks/knucleotide/benchmark.rb | 2 +- benchmarks/lee/benchmark.rb | 2 +- benchmarks/liquid-c/benchmark.rb | 2 +- benchmarks/liquid-compile/benchmark.rb | 2 +- benchmarks/liquid-render/benchmark.rb | 2 +- benchmarks/lobsters/benchmark.rb | 2 +- benchmarks/loops-times.rb | 2 +- benchmarks/mail/benchmark.rb | 2 +- benchmarks/matmul.rb | 2 +- benchmarks/nbody/benchmark.rb | 2 +- benchmarks/nqueens.rb | 2 +- benchmarks/object-new.rb | 2 +- benchmarks/optcarrot/benchmark.rb | 2 +- benchmarks/protoboeuf-encode/benchmark.rb | 2 +- benchmarks/protoboeuf/benchmark.rb | 2 +- benchmarks/psych-load/benchmark.rb | 2 +- benchmarks/rack/benchmark.rb | 2 +- benchmarks/railsbench/benchmark.rb | 2 +- benchmarks/respond_to.rb | 2 +- benchmarks/rubocop/benchmark.rb | 2 +- benchmarks/ruby-json/benchmark.rb | 2 +- benchmarks/ruby-lsp/benchmark.rb | 2 +- benchmarks/ruby-xor.rb | 2 +- benchmarks/rubyboy/benchmark.rb | 2 +- benchmarks/rubykon/benchmark.rb | 2 +- benchmarks/send_bmethod.rb | 2 +- benchmarks/send_cfunc_block.rb | 2 +- benchmarks/send_rubyfunc_block.rb | 2 +- benchmarks/sequel/benchmark.rb | 2 +- benchmarks/setivar.rb | 2 +- benchmarks/setivar_object.rb | 2 +- benchmarks/setivar_young.rb | 2 +- benchmarks/shipit/benchmark.rb | 2 +- benchmarks/str_concat.rb | 2 +- benchmarks/structaref.rb | 2 +- benchmarks/structaset.rb | 2 +- benchmarks/sudoku.rb | 2 +- benchmarks/throw.rb | 2 +- benchmarks/tinygql/benchmark.rb | 2 +- burn_in.rb | 7 +-- harness/bips.rb | 2 +- harness/chain.rb | 4 +- harness/continuous.rb | 2 +- harness/{harness.rb => default.rb} | 2 +- harness/loader.rb | 16 ------- harness/mplr.rb | 2 +- harness/once.rb | 2 +- harness/perf.rb | 4 +- harness/ractor.rb | 2 +- harness/stackprof.rb | 8 ++-- harness/stats.rb | 2 +- harness/vernier.rb | 8 ++-- harness/warmup.rb | 2 +- lib/argument_parser.rb | 2 +- lib/benchmark_suite.rb | 10 ++-- harness/harness-common.rb => lib/harness.rb | 0 .../harness-extra.rb => lib/harness/extra.rb | 5 +- lib/harness/loader.rb | 22 +++++++++ run_once.sh | 4 +- test/argument_parser_test.rb | 2 +- test/benchmark_runner_cli_test.rb | 2 +- test/benchmark_suite_test.rb | 48 +++++++++---------- 95 files changed, 173 insertions(+), 159 deletions(-) rename harness/{harness.rb => default.rb} (98%) delete mode 100644 harness/loader.rb rename harness/harness-common.rb => lib/harness.rb (100%) rename harness/harness-extra.rb => lib/harness/extra.rb (95%) create mode 100644 lib/harness/loader.rb diff --git a/README.md b/README.md index de077427..62da5852 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Small set of benchmarks and scripts for the Ruby programming language. The benchmarks are found in the `benchmarks` directory. Individual Ruby files in `benchmarks` are microbenchmarks. Subdirectories under `benchmarks` are larger macrobenchmarks. Each benchmark relies on a harness found in -[./harness/harness.rb](harness/harness.rb). The harness controls the number of times a benchmark is +[./harness/default.rb](harness/default.rb). The harness controls the number of times a benchmark is run, and writes timing values into an output file. The `run_benchmarks.rb` script (optional) traverses the `benchmarks` directory and @@ -138,19 +138,19 @@ There are two Ractor-related categories: * **`--category ractor`** - Runs both regular benchmarks marked with `ractor: true` in `benchmarks.yml` AND all benchmarks from the `benchmarks-ractor` - directory. The `harness-ractor` harness is used for both types of benchmark. + directory. The `ractor` harness is used for both types of benchmark. * **`--category ractor-only`** - Runs ONLY benchmarks from the `benchmarks-ractor` directory, ignoring regular benchmarks even if they are marked with `ractor: true`. This category also automatically uses the - `harness-ractor` harness. + `ractor` harness. ### Directory Structure The `benchmarks-ractor/` directory sits at the same level as the main `benchmarks` directory, and contains Ractor-specific benchmark implementations that are designed to test Ractor functionality. They are not -intended to be used with any harness except `harness-ractor`. +intended to be used with any harness except `ractor`. ### Usage Examples @@ -162,7 +162,7 @@ intended to be used with any harness except `harness-ractor`. ./run_benchmarks.rb --category ractor-only ``` -Note: The `harness-ractor` harness is automatically selected when using these +Note: The `ractor` harness is automatically selected when using these categories, so there's no need to specify `--harness` manually. ## Ruby options @@ -213,7 +213,7 @@ This file will then be passed to the underlying Ruby interpreter with You can find several test harnesses in the `harness/` directory: -* `harness` - the normal default harness, with duration controlled by warmup iterations and time/count limits +* `default` - the normal default harness, with duration controlled by warmup iterations and time/count limits * `bips` - a harness that measures iterations/second until stable * `continuous` - a harness that adjusts the batch sizes of iterations to run in stable iteration size batches * `once` - a simplified harness that simply runs once @@ -225,17 +225,23 @@ You can find several test harnesses in the `harness/` directory: * `chain` - a harness to chain multiple harnesses together * `mplr` - a harness for multiple iterations with time limits -To use a specific harness, run a benchmark script directly with `-I` to add the harness directory to the load path, and `-r` to require the specific harness: +To use a specific harness, run a benchmark script directly with `-r` to require the harness: ``` -# Use default harness -ruby -Iharness benchmarks/railsbench/benchmark.rb +# Use default harness (loads harness/default.rb automatically) +ruby benchmarks/railsbench/benchmark.rb # Use the 'once' harness -ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb +ruby -r./harness/once benchmarks/railsbench/benchmark.rb # Use the 'perf' harness -ruby -Iharness -rperf benchmarks/railsbench/benchmark.rb +ruby -r./harness/perf benchmarks/railsbench/benchmark.rb + +# Use the 'stackprof' harness +ruby -r./harness/stackprof benchmarks/railsbench/benchmark.rb + +# Use the 'vernier' harness +ruby -r./harness/vernier benchmarks/railsbench/benchmark.rb ``` When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: @@ -281,10 +287,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w ```sh # Use `perf record` for both warmup and benchmark -perf record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb +perf record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ruby --yjit-perf=map -Iharness -rperf benchmarks/railsbench/benchmark.rb +PERF=record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb ``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. diff --git a/benchmarks-ractor/gvl_release_acquire/benchmark.rb b/benchmarks-ractor/gvl_release_acquire/benchmark.rb index 3c368477..8963b54a 100644 --- a/benchmarks-ractor/gvl_release_acquire/benchmark.rb +++ b/benchmarks-ractor/gvl_release_acquire/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" run_benchmark(5) do |num_rs, ractor_args| output = File.open("/dev/null", "wb") diff --git a/benchmarks-ractor/json_parse_float/benchmark.rb b/benchmarks-ractor/json_parse_float/benchmark.rb index ef81bc49..2aef033f 100644 --- a/benchmarks-ractor/json_parse_float/benchmark.rb +++ b/benchmarks-ractor/json_parse_float/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks-ractor/json_parse_string/benchmark.rb b/benchmarks-ractor/json_parse_string/benchmark.rb index a579f805..71417cd3 100644 --- a/benchmarks-ractor/json_parse_string/benchmark.rb +++ b/benchmarks-ractor/json_parse_string/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks-ractor/knucleotide/benchmark.rb b/benchmarks-ractor/knucleotide/benchmark.rb index ca5b9cd9..b8fbb34a 100644 --- a/benchmarks-ractor/knucleotide/benchmark.rb +++ b/benchmarks-ractor/knucleotide/benchmark.rb @@ -4,7 +4,7 @@ # k-nucleotide benchmark - Ractor implementation # Mirrors the Process.fork version structure as closely as possible -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" def frequency(seq, length) frequencies = Hash.new(0) diff --git a/benchmarks/30k_ifelse.rb b/benchmarks/30k_ifelse.rb index ff9f32c3..a525fda4 100644 --- a/benchmarks/30k_ifelse.rb +++ b/benchmarks/30k_ifelse.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' if ENV["RUBY_BENCH_RACTOR_HARNESS"] eval_recv = Object.new eval_meth = :instance_eval diff --git a/benchmarks/30k_methods.rb b/benchmarks/30k_methods.rb index f3d75f61..4d7bdccd 100644 --- a/benchmarks/30k_methods.rb +++ b/benchmarks/30k_methods.rb @@ -121011,7 +121011,7 @@ def fun_l29_n999() end end -require_relative '../harness/loader' +require_relative '../lib/harness/loader' INTERNAL_ITRS = Integer(ENV.fetch("INTERNAL_ITRS", 200)) diff --git a/benchmarks/activerecord/benchmark.rb b/benchmarks/activerecord/benchmark.rb index e820d5ae..4f336daa 100644 --- a/benchmarks/activerecord/benchmark.rb +++ b/benchmarks/activerecord/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/equality.rb b/benchmarks/addressable/equality.rb index ca48dffe..95141423 100644 --- a/benchmarks/addressable/equality.rb +++ b/benchmarks/addressable/equality.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/getters.rb b/benchmarks/addressable/getters.rb index 301439f2..43d0d03a 100644 --- a/benchmarks/addressable/getters.rb +++ b/benchmarks/addressable/getters.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/join.rb b/benchmarks/addressable/join.rb index a4df7b39..4d600795 100644 --- a/benchmarks/addressable/join.rb +++ b/benchmarks/addressable/join.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/merge.rb b/benchmarks/addressable/merge.rb index 39c07f2d..20f33283 100644 --- a/benchmarks/addressable/merge.rb +++ b/benchmarks/addressable/merge.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/new.rb b/benchmarks/addressable/new.rb index 5fd9731e..ae5d44ae 100644 --- a/benchmarks/addressable/new.rb +++ b/benchmarks/addressable/new.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/normalize.rb b/benchmarks/addressable/normalize.rb index 121a8ad0..014c81fa 100644 --- a/benchmarks/addressable/normalize.rb +++ b/benchmarks/addressable/normalize.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/parse.rb b/benchmarks/addressable/parse.rb index 02d29a83..d95f1094 100644 --- a/benchmarks/addressable/parse.rb +++ b/benchmarks/addressable/parse.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/setters.rb b/benchmarks/addressable/setters.rb index 41ee88b3..3431bba3 100644 --- a/benchmarks/addressable/setters.rb +++ b/benchmarks/addressable/setters.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/addressable/to-s.rb b/benchmarks/addressable/to-s.rb index 355d931f..9f337ff7 100644 --- a/benchmarks/addressable/to-s.rb +++ b/benchmarks/addressable/to-s.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/attr_accessor.rb b/benchmarks/attr_accessor.rb index d5e397dc..1ab4ae7f 100644 --- a/benchmarks/attr_accessor.rb +++ b/benchmarks/attr_accessor.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass attr_accessor :levar diff --git a/benchmarks/binarytrees/benchmark.rb b/benchmarks/binarytrees/benchmark.rb index 4e54528d..ca460c07 100644 --- a/benchmarks/binarytrees/benchmark.rb +++ b/benchmarks/binarytrees/benchmark.rb @@ -22,7 +22,7 @@ def bottom_up_tree(depth) MAX_DEPTH = MIN_DEPTH + 2 if MIN_DEPTH + 2 > MAX_DEPTH STRETCH_DEPTH = MAX_DEPTH + 1 -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' run_benchmark(60) do max_depth = MAX_DEPTH diff --git a/benchmarks/blurhash/benchmark.rb b/benchmarks/blurhash/benchmark.rb index 1f081f4b..327fc225 100644 --- a/benchmarks/blurhash/benchmark.rb +++ b/benchmarks/blurhash/benchmark.rb @@ -172,7 +172,7 @@ def blurHashForPixels(xComponents, yComponents, width, height, rgb, bytesPerRow) # :startdoc: end -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" FILE = File.join(__dir__, "test.bin") diff --git a/benchmarks/cfunc_itself.rb b/benchmarks/cfunc_itself.rb index 2c39137e..8560da66 100644 --- a/benchmarks/cfunc_itself.rb +++ b/benchmarks/cfunc_itself.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' run_benchmark(500) do # 500K calls diff --git a/benchmarks/chunky-png/benchmark.rb b/benchmarks/chunky-png/benchmark.rb index ef511893..29054d29 100644 --- a/benchmarks/chunky-png/benchmark.rb +++ b/benchmarks/chunky-png/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/erubi-rails/benchmark.rb b/benchmarks/erubi-rails/benchmark.rb index 38c22634..1817a0f8 100644 --- a/benchmarks/erubi-rails/benchmark.rb +++ b/benchmarks/erubi-rails/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/erubi/benchmark.rb b/benchmarks/erubi/benchmark.rb index 0a75f1bb..f62a2f3d 100644 --- a/benchmarks/erubi/benchmark.rb +++ b/benchmarks/erubi/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/etanni/benchmark.rb b/benchmarks/etanni/benchmark.rb index 77e86386..48c41f1a 100644 --- a/benchmarks/etanni/benchmark.rb +++ b/benchmarks/etanni/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ # This is an Etanni translation of the Erb template in the Erubi diff --git a/benchmarks/fannkuchredux/benchmark.rb b/benchmarks/fannkuchredux/benchmark.rb index e73dc0ee..2e44c7d1 100644 --- a/benchmarks/fannkuchredux/benchmark.rb +++ b/benchmarks/fannkuchredux/benchmark.rb @@ -56,7 +56,7 @@ def fannkuch(n) #n = (ARGV[0] || 1).to_i N = 9 # Benchmarks Game uses n = 12, but it's too slow -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' run_benchmark(10) do sum, flips = fannkuch(N) diff --git a/benchmarks/fib.rb b/benchmarks/fib.rb index 4e5b3428..02be31b8 100644 --- a/benchmarks/fib.rb +++ b/benchmarks/fib.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def fib(n) if n < 2 diff --git a/benchmarks/fluentd/benchmark.rb b/benchmarks/fluentd/benchmark.rb index c503a653..4e3f9a6d 100644 --- a/benchmarks/fluentd/benchmark.rb +++ b/benchmarks/fluentd/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/getivar.rb b/benchmarks/getivar.rb index 80acfae1..d77cc33d 100644 --- a/benchmarks/getivar.rb +++ b/benchmarks/getivar.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/graphql-native/benchmark.rb b/benchmarks/graphql-native/benchmark.rb index d4fd7077..ff5858c9 100644 --- a/benchmarks/graphql-native/benchmark.rb +++ b/benchmarks/graphql-native/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/graphql/benchmark.rb b/benchmarks/graphql/benchmark.rb index 85b5da1f..26c1feb6 100644 --- a/benchmarks/graphql/benchmark.rb +++ b/benchmarks/graphql/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/hexapdf/benchmark.rb b/benchmarks/hexapdf/benchmark.rb index 9609139b..0195d733 100644 --- a/benchmarks/hexapdf/benchmark.rb +++ b/benchmarks/hexapdf/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/keyword_args.rb b/benchmarks/keyword_args.rb index a9b035f3..386ef894 100644 --- a/benchmarks/keyword_args.rb +++ b/benchmarks/keyword_args.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def add(left:, right:) left + right diff --git a/benchmarks/knucleotide/benchmark.rb b/benchmarks/knucleotide/benchmark.rb index cf6234b4..affd44f1 100644 --- a/benchmarks/knucleotide/benchmark.rb +++ b/benchmarks/knucleotide/benchmark.rb @@ -4,7 +4,7 @@ # k-nucleotide benchmark - Fastest implementation # Based on Ruby #1 with byteslice optimization -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' def frequency(seq, length) frequencies = Hash.new(0) diff --git a/benchmarks/lee/benchmark.rb b/benchmarks/lee/benchmark.rb index 2abdd945..7e052339 100644 --- a/benchmarks/lee/benchmark.rb +++ b/benchmarks/lee/benchmark.rb @@ -83,7 +83,7 @@ def lay(depth, solution) end end -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-c/benchmark.rb b/benchmarks/liquid-c/benchmark.rb index e074e33c..4a649dff 100644 --- a/benchmarks/liquid-c/benchmark.rb +++ b/benchmarks/liquid-c/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-compile/benchmark.rb b/benchmarks/liquid-compile/benchmark.rb index 0e0d0f1c..956e0fb8 100644 --- a/benchmarks/liquid-compile/benchmark.rb +++ b/benchmarks/liquid-compile/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/liquid-render/benchmark.rb b/benchmarks/liquid-render/benchmark.rb index 661015a8..b851474c 100644 --- a/benchmarks/liquid-render/benchmark.rb +++ b/benchmarks/liquid-render/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/lobsters/benchmark.rb b/benchmarks/lobsters/benchmark.rb index 3a28fe81..63c36100 100644 --- a/benchmarks/lobsters/benchmark.rb +++ b/benchmarks/lobsters/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' ENV['RAILS_ENV'] ||= 'production' ENV['DISABLE_DATABASE_ENVIRONMENT_CHECK'] = '1' # Benchmarks don't really have 'production', so trash it at will. diff --git a/benchmarks/loops-times.rb b/benchmarks/loops-times.rb index 653fe989..22c196bd 100644 --- a/benchmarks/loops-times.rb +++ b/benchmarks/loops-times.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' # Fix these values for determinism U = 5 diff --git a/benchmarks/mail/benchmark.rb b/benchmarks/mail/benchmark.rb index 0d625650..9bef261d 100644 --- a/benchmarks/mail/benchmark.rb +++ b/benchmarks/mail/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/matmul.rb b/benchmarks/matmul.rb index c4087dbb..80187ade 100644 --- a/benchmarks/matmul.rb +++ b/benchmarks/matmul.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def matgen(n) tmp = 1.0 / n / n diff --git a/benchmarks/nbody/benchmark.rb b/benchmarks/nbody/benchmark.rb index 737e298b..86a11a3c 100644 --- a/benchmarks/nbody/benchmark.rb +++ b/benchmarks/nbody/benchmark.rb @@ -5,7 +5,7 @@ # modified by Jesse Millikan # modified by Yusuke Endoh -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' SOLAR_MASS = 4 * Math::PI**2 DAYS_PER_YEAR = 365.24 diff --git a/benchmarks/nqueens.rb b/benchmarks/nqueens.rb index 9a1b7a8c..0912a64a 100644 --- a/benchmarks/nqueens.rb +++ b/benchmarks/nqueens.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def nq_solve(n) a = Array.new(n, -1) diff --git a/benchmarks/object-new.rb b/benchmarks/object-new.rb index b079c393..ed3b37b8 100644 --- a/benchmarks/object-new.rb +++ b/benchmarks/object-new.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' run_benchmark(100) do i = 0 diff --git a/benchmarks/optcarrot/benchmark.rb b/benchmarks/optcarrot/benchmark.rb index 3c48515f..29e6e58f 100755 --- a/benchmarks/optcarrot/benchmark.rb +++ b/benchmarks/optcarrot/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' require_relative "lib/optcarrot" if ENV["RUBY_BENCH_RACTOR_HARNESS"] diff --git a/benchmarks/protoboeuf-encode/benchmark.rb b/benchmarks/protoboeuf-encode/benchmark.rb index a7afcdb7..16551acc 100644 --- a/benchmarks/protoboeuf-encode/benchmark.rb +++ b/benchmarks/protoboeuf-encode/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' # Protoboeuf decoder require_relative 'benchmark_pb' diff --git a/benchmarks/protoboeuf/benchmark.rb b/benchmarks/protoboeuf/benchmark.rb index 6e3078f4..9f477b79 100644 --- a/benchmarks/protoboeuf/benchmark.rb +++ b/benchmarks/protoboeuf/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' # Protoboeuf decoder require_relative 'benchmark_pb' diff --git a/benchmarks/psych-load/benchmark.rb b/benchmarks/psych-load/benchmark.rb index 795135f3..08b99c64 100644 --- a/benchmarks/psych-load/benchmark.rb +++ b/benchmarks/psych-load/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' Dir.chdir __dir__ use_gemfile diff --git a/benchmarks/rack/benchmark.rb b/benchmarks/rack/benchmark.rb index a3fde656..57c25052 100644 --- a/benchmarks/rack/benchmark.rb +++ b/benchmarks/rack/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/railsbench/benchmark.rb b/benchmarks/railsbench/benchmark.rb index 89561d48..46b3a270 100644 --- a/benchmarks/railsbench/benchmark.rb +++ b/benchmarks/railsbench/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' ENV['RAILS_ENV'] ||= 'production' diff --git a/benchmarks/respond_to.rb b/benchmarks/respond_to.rb index 9a419060..55f49f70 100644 --- a/benchmarks/respond_to.rb +++ b/benchmarks/respond_to.rb @@ -1,7 +1,7 @@ # Microbenchmark to test the performance of respond_to? # This is one of the top most called methods in rack/railsbench -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class A def foo diff --git a/benchmarks/rubocop/benchmark.rb b/benchmarks/rubocop/benchmark.rb index 60b49d05..12f415a0 100644 --- a/benchmarks/rubocop/benchmark.rb +++ b/benchmarks/rubocop/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/ruby-json/benchmark.rb b/benchmarks/ruby-json/benchmark.rb index 3c2d3e4e..f8c085b8 100644 --- a/benchmarks/ruby-json/benchmark.rb +++ b/benchmarks/ruby-json/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" require "json" require "strscan" diff --git a/benchmarks/ruby-lsp/benchmark.rb b/benchmarks/ruby-lsp/benchmark.rb index 232a0e8d..0bbc304b 100644 --- a/benchmarks/ruby-lsp/benchmark.rb +++ b/benchmarks/ruby-lsp/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/ruby-xor.rb b/benchmarks/ruby-xor.rb index 5352c76a..2a9f9d1b 100644 --- a/benchmarks/ruby-xor.rb +++ b/benchmarks/ruby-xor.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' # XOR strings in place # Expected to behave like Xorcist.xor! from the xorcist gem diff --git a/benchmarks/rubyboy/benchmark.rb b/benchmarks/rubyboy/benchmark.rb index 54b0d5e1..36d68f96 100644 --- a/benchmarks/rubyboy/benchmark.rb +++ b/benchmarks/rubyboy/benchmark.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir(__dir__) use_gemfile diff --git a/benchmarks/rubykon/benchmark.rb b/benchmarks/rubykon/benchmark.rb index e8f0e527..d312daaf 100644 --- a/benchmarks/rubykon/benchmark.rb +++ b/benchmarks/rubykon/benchmark.rb @@ -2,7 +2,7 @@ # https://github.com/benchmark-driver/sky2-bench/blob/master/benchmark/rubykon-benchmark.rb, # part of benchmark-driver's default benchmarking suite, by Takashi Kokubun. -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' require_relative 'lib/rubykon' diff --git a/benchmarks/send_bmethod.rb b/benchmarks/send_bmethod.rb index 649df891..07774ea0 100644 --- a/benchmarks/send_bmethod.rb +++ b/benchmarks/send_bmethod.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' # Call with and without args define_method(:zero) { :b } diff --git a/benchmarks/send_cfunc_block.rb b/benchmarks/send_cfunc_block.rb index d6013f29..f0c954f7 100644 --- a/benchmarks/send_cfunc_block.rb +++ b/benchmarks/send_cfunc_block.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' ARR = [] diff --git a/benchmarks/send_rubyfunc_block.rb b/benchmarks/send_rubyfunc_block.rb index 6cdeba9b..22b18d19 100644 --- a/benchmarks/send_rubyfunc_block.rb +++ b/benchmarks/send_rubyfunc_block.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class C def ruby_func diff --git a/benchmarks/sequel/benchmark.rb b/benchmarks/sequel/benchmark.rb index 81b07c67..7c2491b4 100644 --- a/benchmarks/sequel/benchmark.rb +++ b/benchmarks/sequel/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" require "securerandom" # Provides `Random::Formatter` in Ruby 2.6+ Dir.chdir __dir__ diff --git a/benchmarks/setivar.rb b/benchmarks/setivar.rb index e64dca4d..9e9ebf3a 100644 --- a/benchmarks/setivar.rb +++ b/benchmarks/setivar.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/setivar_object.rb b/benchmarks/setivar_object.rb index 6f34cba6..0c316833 100644 --- a/benchmarks/setivar_object.rb +++ b/benchmarks/setivar_object.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/setivar_young.rb b/benchmarks/setivar_young.rb index 2f673756..d463d811 100644 --- a/benchmarks/setivar_young.rb +++ b/benchmarks/setivar_young.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' class TheClass def initialize diff --git a/benchmarks/shipit/benchmark.rb b/benchmarks/shipit/benchmark.rb index 607498d0..9e88b22d 100644 --- a/benchmarks/shipit/benchmark.rb +++ b/benchmarks/shipit/benchmark.rb @@ -1,4 +1,4 @@ -require_relative '../../harness/loader' +require_relative '../../lib/harness/loader' ENV['RAILS_ENV'] ||= 'production' ENV['DISABLE_DATABASE_ENVIRONMENT_CHECK'] = '1' # Benchmarks don't really have 'production', so trash it at will. diff --git a/benchmarks/str_concat.rb b/benchmarks/str_concat.rb index ee514cc4..9bbeaf2b 100644 --- a/benchmarks/str_concat.rb +++ b/benchmarks/str_concat.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require_relative '../harness/loader' +require_relative '../lib/harness/loader' NUM_ITERS = 10 * 1024 TEST_STR = 'sssssséé'.freeze diff --git a/benchmarks/structaref.rb b/benchmarks/structaref.rb index 95bb3575..b319092e 100644 --- a/benchmarks/structaref.rb +++ b/benchmarks/structaref.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' TheClass = Struct.new(:v0, :v1, :v2, :levar) diff --git a/benchmarks/structaset.rb b/benchmarks/structaset.rb index 5d73035e..bcff9b0e 100644 --- a/benchmarks/structaset.rb +++ b/benchmarks/structaset.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' TheClass = Struct.new(:v0, :v1, :v2, :levar) diff --git a/benchmarks/sudoku.rb b/benchmarks/sudoku.rb index 62308b36..e8a298e0 100644 --- a/benchmarks/sudoku.rb +++ b/benchmarks/sudoku.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def sd_genmat mr = Array.new(324) { [] } diff --git a/benchmarks/throw.rb b/benchmarks/throw.rb index 3d3a7b2b..e3aa8b9c 100644 --- a/benchmarks/throw.rb +++ b/benchmarks/throw.rb @@ -1,4 +1,4 @@ -require_relative '../harness/loader' +require_relative '../lib/harness/loader' def foo yield diff --git a/benchmarks/tinygql/benchmark.rb b/benchmarks/tinygql/benchmark.rb index 7195a9ed..8457772d 100644 --- a/benchmarks/tinygql/benchmark.rb +++ b/benchmarks/tinygql/benchmark.rb @@ -1,4 +1,4 @@ -require_relative "../../harness/loader" +require_relative "../../lib/harness/loader" Dir.chdir __dir__ use_gemfile diff --git a/burn_in.rb b/burn_in.rb index ea305eb8..f1d66b46 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -63,9 +63,9 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) # Determine the path to the benchmark script bench_name = bench_id.sub('ractor/', '') bench_dir, harness = if bench_name == bench_id - ['benchmarks', 'harness'] + ['benchmarks', 'default'] else - ['benchmarks-ractor', 'harness-ractor'] + ['benchmarks-ractor', 'ractor'] end script_path = File.join(bench_dir, bench_name, 'benchmark.rb') @@ -108,7 +108,8 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) cmd = [ 'ruby', *test_options, - "-I#{harness}", + "-Iharness", + "-r#{harness}", script_path, ].compact cmd_str = cmd.shelljoin diff --git a/harness/bips.rb b/harness/bips.rb index b583fa0f..dddb546e 100644 --- a/harness/bips.rb +++ b/harness/bips.rb @@ -1,5 +1,5 @@ require 'benchmark/ips' -require_relative "harness-common" +require_relative "../lib/harness" puts RUBY_DESCRIPTION diff --git a/harness/chain.rb b/harness/chain.rb index eef9d77b..62d50712 100644 --- a/harness/chain.rb +++ b/harness/chain.rb @@ -1,4 +1,4 @@ -require_relative 'harness-common' +require_relative '../lib/harness' # Ex: HARNESS_CHAIN="vernier,ractor" # Wraps the ractor harness in ther vernier harness @@ -10,7 +10,7 @@ end if CHAIN.include?("vernier") && CHAIN.last != "vernier" - require_relative "harness-extra" + require_relative "../lib/harness/extra" def run_enough_to_profile(n, **kwargs, &block) block.call end diff --git a/harness/continuous.rb b/harness/continuous.rb index f76f535b..05a9162f 100644 --- a/harness/continuous.rb +++ b/harness/continuous.rb @@ -1,4 +1,4 @@ -require_relative "harness-common" +require_relative "../lib/harness" puts RUBY_DESCRIPTION diff --git a/harness/harness.rb b/harness/default.rb similarity index 98% rename from harness/harness.rb rename to harness/default.rb index 8e660296..077e7601 100644 --- a/harness/harness.rb +++ b/harness/default.rb @@ -1,4 +1,4 @@ -require_relative "./harness-common" +require_relative "../lib/harness" # Warmup iterations WARMUP_ITRS = Integer(ENV.fetch('WARMUP_ITRS', 15)) diff --git a/harness/loader.rb b/harness/loader.rb deleted file mode 100644 index 12f3abe0..00000000 --- a/harness/loader.rb +++ /dev/null @@ -1,16 +0,0 @@ -# Use harness/harness.rb by default. You can change it with -I and -r options. -# Examples: -# ruby -Iharness benchmarks/railsbench/benchmark.rb # uses harness/harness.rb -# ruby -Iharness -ronce benchmarks/railsbench/benchmark.rb # uses harness/once.rb -# ruby -Iharness -rractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb -retries = 0 -begin - require "harness" -rescue LoadError => e - if retries == 0 && e.path == "harness" - retries += 1 - $LOAD_PATH << File.expand_path(__dir__) - retry - end - raise -end diff --git a/harness/mplr.rb b/harness/mplr.rb index 4473a204..f91bfd94 100644 --- a/harness/mplr.rb +++ b/harness/mplr.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative "harness-common" +require_relative "../lib/harness" # Minimum number of benchmarking iterations MAX_BENCH_ITRS = Integer(ENV.fetch('MAX_BENCH_ITRS', 1000)) diff --git a/harness/once.rb b/harness/once.rb index dbf44e60..29b3340a 100644 --- a/harness/once.rb +++ b/harness/once.rb @@ -5,7 +5,7 @@ # Intended only for checking whether the benchmark can set itself up properly # and can run to completion. -require_relative 'harness-common' +require_relative '../lib/harness' def run_benchmark(_hint, **) yield diff --git a/harness/perf.rb b/harness/perf.rb index a778526e..afc339ef 100644 --- a/harness/perf.rb +++ b/harness/perf.rb @@ -3,14 +3,14 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ruby -Iharness -rperf benchmarks/fib.rb +# $ PERF='record -e cycles' ruby -r./harness/perf benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race # condition is possible where the benchmark finishes before the perf(1) # subprocess has a chance to attach, in which case perf outputs no profile. -require_relative "harness-common" +require_relative "../lib/harness" # Run $WARMUP_ITRS or 10 iterations of a given block. Then run $MIN_BENCH_ITRS # or `num_itrs_int` iterations of the block, attaching a perf command to the diff --git a/harness/ractor.rb b/harness/ractor.rb index 5e133636..a2858349 100644 --- a/harness/ractor.rb +++ b/harness/ractor.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require_relative 'harness-common' +require_relative '../lib/harness' Warning[:experimental] = false ENV["RUBY_BENCH_RACTOR_HARNESS"] = "1" diff --git a/harness/stackprof.rb b/harness/stackprof.rb index c9bf7a81..d11860a6 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -Iharness -rstackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb -require_relative "harness-common" -require_relative "harness-extra" +require_relative "../lib/harness" +require_relative "../lib/harness/extra" ensure_global_gem("stackprof") diff --git a/harness/stats.rb b/harness/stats.rb index b9b216ae..2971c667 100644 --- a/harness/stats.rb +++ b/harness/stats.rb @@ -1,4 +1,4 @@ -require_relative 'harness' +require_relative 'default' # Using Module#prepend to enable TracePoint right before #run_benchmark # while also reusing the original implementation. diff --git a/harness/vernier.rb b/harness/vernier.rb index 2b7c93ff..747ce223 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -3,11 +3,11 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... -# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -Iharness -rvernier benchmarks/... +# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... +# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... -require_relative "harness-common" -require_relative "harness-extra" +require_relative "../lib/harness" +require_relative "../lib/harness/extra" ensure_global_gem("vernier") ensure_global_gem_exe("profile-viewer") diff --git a/harness/warmup.rb b/harness/warmup.rb index 33ae5e43..fbcc6bbf 100644 --- a/harness/warmup.rb +++ b/harness/warmup.rb @@ -1,5 +1,5 @@ require 'benchmark' -require_relative 'harness-common' +require_relative '../lib/harness' require_relative '../misc/stats' MIN_ITERS = Integer(ENV['MIN_ITERS'] || 10) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 761fa6a8..e79ce7a1 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -176,7 +176,7 @@ def default_args executables: {}, out_path: File.expand_path("./data"), out_override: nil, - harness: "harness", + harness: "default", yjit_opts: "", categories: [], name_filters: [], diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index f94fbca9..202da66e 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -151,17 +151,17 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command - # If harness is 'harness', use default (no -r needed) - # Otherwise use -r to load the specific harness file - harness_args = if harness == "harness" + # If harness is 'default', use default (no -r needed) + # Otherwise use -r to load the specific harness file with full path + harness_args = if harness == "default" [] else - ["-r", harness] + harness_path = File.join(HARNESS_DIR, harness) + ["-r", harness_path] end cmd = cmd_prefix + [ *ruby, - "-I", HARNESS_DIR, *harness_args, *pre_init, script_path, diff --git a/harness/harness-common.rb b/lib/harness.rb similarity index 100% rename from harness/harness-common.rb rename to lib/harness.rb diff --git a/harness/harness-extra.rb b/lib/harness/extra.rb similarity index 95% rename from harness/harness-extra.rb rename to lib/harness/extra.rb index 18a3654f..99f3cbf4 100644 --- a/harness/harness-extra.rb +++ b/lib/harness/extra.rb @@ -33,11 +33,12 @@ def benchmark_name # Get name of harness (stackprof, vernier, etc) from the file path of the loaded harness. def harness_name $LOADED_FEATURES.reverse_each do |feat| - if m = feat.match(%r{/harness-([^/]+)/harness\.rb$}) + if m = feat.match(%r{/harness/([^/]+)\.rb$}) return m[1] end end - raise "Unable to determine harness name" + # Default to 'harness' if we can't determine the name + 'harness' end # Share a single timestamp for everything from this execution. diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb new file mode 100644 index 00000000..5ee4d79b --- /dev/null +++ b/lib/harness/loader.rb @@ -0,0 +1,22 @@ +# Use harness/default.rb by default. You can change it with -r option. +# The -r option should be specified BEFORE requiring loader. +# Examples: +# ruby benchmarks/railsbench/benchmark.rb # uses harness/default.rb +# ruby -r./harness/once benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ruby -r./harness/ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb + +# Only load the default harness if no other harness has defined run_benchmark +unless defined?(run_benchmark) + retries = 0 + begin + require "default" + rescue LoadError => e + if retries == 0 && e.path == "default" + retries += 1 + # Add the harness directory to the load path + $LOAD_PATH << File.expand_path("../../harness", __dir__) + retry + end + raise + end +end diff --git a/run_once.sh b/run_once.sh index a5240fce..bef1d8bd 100755 --- a/run_once.sh +++ b/run_once.sh @@ -10,9 +10,9 @@ HARNESS_ARGS=() for arg in "$@"; do if [[ "$arg" == *"benchmarks-ractor/"* ]]; then - HARNESS_ARGS=("-r" "ractor") + HARNESS_ARGS=("-r" "./harness/ractor") break fi done -WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby -I"./harness" "${HARNESS_ARGS[@]}" "$@" \ No newline at end of file +WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby "${HARNESS_ARGS[@]}" "$@" diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index e093e69e..8512a7c2 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -44,7 +44,7 @@ def setup_mock_ruby(path) assert_equal({ 'ruby' => [mock_ruby] }, args.executables) assert_equal File.expand_path("./data"), args.out_path assert_nil args.out_override - assert_equal "harness", args.harness + assert_equal "default", args.harness assert_equal "", args.yjit_opts assert_equal [], args.categories assert_equal [], args.name_filters diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 72452429..fd66dec6 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -42,7 +42,7 @@ def create_args(overrides = {}) executables: executables, out_path: nil, out_override: nil, - harness: 'harness', + harness: 'default', yjit_opts: '', categories: [], name_filters: [], diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index 2cb820ef..1a3fdb1d 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -50,13 +50,13 @@ categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal ['micro'], suite.categories assert_equal [], suite.name_filters assert_equal @out_path, suite.out_path - assert_equal 'harness', suite.harness + assert_equal 'default', suite.harness assert_nil suite.pre_init assert_equal false, suite.no_pinning end @@ -66,7 +66,7 @@ categories: [], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -78,12 +78,12 @@ categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal 'benchmarks', suite.bench_dir assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness', suite.harness + assert_equal 'default', suite.harness assert_equal ['micro'], suite.categories end @@ -92,7 +92,7 @@ categories: ['ractor-only'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal 'benchmarks-ractor', suite.bench_dir @@ -106,12 +106,12 @@ categories: ['ractor'], name_filters: [], out_path: @out_path, - harness: 'harness' + harness: 'default' ) assert_equal 'benchmarks', suite.bench_dir assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness', suite.harness + assert_equal 'default', suite.harness assert_equal ['ractor'], suite.categories end end @@ -122,7 +122,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -144,7 +144,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -167,7 +167,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -188,7 +188,7 @@ categories: [], name_filters: ['failing'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -219,7 +219,7 @@ categories: [], name_filters: ['subdir'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -248,7 +248,7 @@ categories: ['ractor-only'], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -280,7 +280,7 @@ categories: ['ractor'], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -303,7 +303,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: pre_init_file, no_pinning: true ) @@ -325,7 +325,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: pre_init_file, no_pinning: true ) @@ -345,7 +345,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: pre_init_file, no_pinning: true ) @@ -362,7 +362,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: '/nonexistent/file.rb', no_pinning: true ) @@ -378,7 +378,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', pre_init: @temp_dir, no_pinning: true ) @@ -392,7 +392,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -411,7 +411,7 @@ categories: [], name_filters: ['simple'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -442,7 +442,7 @@ categories: [], name_filters: ['bench_a'], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) @@ -475,7 +475,7 @@ categories: ['micro'], name_filters: [], out_path: @out_path, - harness: 'harness', + harness: 'default', no_pinning: true ) From b4a090edb00df09a6c9e2f22deb9eab29fc694b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Mon, 24 Nov 2025 20:43:25 +0000 Subject: [PATCH 3/6] Improve the run_once script to better handle Ruby options and harness selection So we don't need to deal with the `-r` ruby argument directly for selecting harnesses. --- .github/workflows/test.yml | 4 +- README.md | 37 +++--- harness/perf.rb | 2 +- harness/stackprof.rb | 4 +- harness/vernier.rb | 4 +- lib/harness/loader.rb | 10 +- run_once.rb | 90 ++++++++++++++ run_once.sh | 18 --- test/run_once_test.rb | 249 +++++++++++++++++++++++++++++++++++++ 9 files changed, 372 insertions(+), 46 deletions(-) create mode 100755 run_once.rb delete mode 100755 run_once.sh create mode 100644 test/run_once_test.rb diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7d61b29..1faf4c5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,8 +141,8 @@ jobs: with: ruby-version: head - - name: Test run_once.sh - run: ./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb + - name: Test run_once.rb + run: ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb all-tests: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 62da5852..79c384cd 100644 --- a/README.md +++ b/README.md @@ -93,14 +93,14 @@ To run one or more specific benchmarks and record the data: ### Running a single benchmark -This is the easiest way to run a single benchmark. -It requires no setup at all and assumes nothing about the Ruby you are benchmarking. -It's also convenient for profiling, debugging, etc, especially since all benchmarked code runs in that process. +The easiest way to run a single benchmark once is using the `run_once.rb` script: ``` -ruby benchmarks/some_benchmark.rb +./run_once.rb benchmarks/some_benchmark.rb ``` +This automatically sets the environment to run the benchmark once (no warmup iterations). + ### Benchmark organization Benchmarks can be organized in three ways: @@ -225,23 +225,29 @@ You can find several test harnesses in the `harness/` directory: * `chain` - a harness to chain multiple harnesses together * `mplr` - a harness for multiple iterations with time limits -To use a specific harness, run a benchmark script directly with `-r` to require the harness: +To use a specific harness, use the `run_once.rb` script: ``` -# Use default harness (loads harness/default.rb automatically) -ruby benchmarks/railsbench/benchmark.rb +# Use default harness +./run_once.rb benchmarks/railsbench/benchmark.rb # Use the 'once' harness -ruby -r./harness/once benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb # Use the 'perf' harness -ruby -r./harness/perf benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb # Use the 'stackprof' harness -ruby -r./harness/stackprof benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=stackprof benchmarks/railsbench/benchmark.rb # Use the 'vernier' harness -ruby -r./harness/vernier benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=vernier benchmarks/railsbench/benchmark.rb + +# Pass Ruby options like --yjit (use -- separator) +./run_once.rb -- --yjit benchmarks/railsbench/benchmark.rb + +# Combine harness option with Ruby options +./run_once.rb --harness=default -- --yjit-stats benchmarks/railsbench/benchmark.rb ``` When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: @@ -272,12 +278,11 @@ You can also use `--warmup`, `--bench`, or `--once` to set these environment var ./run_benchmarks.rb railsbench --once ``` -There is also a handy script for running benchmarks just once using -`WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0`, for example +You can also use the `run_once.rb` script to run benchmarks just once, for example with the `--yjit-stats` command-line option: ``` -./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb +./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb ``` ### Using perf @@ -287,10 +292,10 @@ If `PERF` environment variable is present, it starts the perf subcommand after w ```sh # Use `perf record` for both warmup and benchmark -perf record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb +perf record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ruby --yjit-perf=map -r./harness/perf benchmarks/railsbench/benchmark.rb +PERF=record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb ``` This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. diff --git a/harness/perf.rb b/harness/perf.rb index afc339ef..086b70b6 100644 --- a/harness/perf.rb +++ b/harness/perf.rb @@ -3,7 +3,7 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ruby -r./harness/perf benchmarks/fib.rb +# $ PERF='record -e cycles' ./run_once.rb --harness=perf benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race diff --git a/harness/stackprof.rb b/harness/stackprof.rb index d11860a6..931d5122 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -r./harness/stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/harness/vernier.rb b/harness/vernier.rb index 747ce223..3accb74d 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... -# NO_VIEWER=1 MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -r./harness/vernier benchmarks/... +# ./run_once.rb --harness=vernier benchmarks/... +# NO_VIEWER=1 ./run_once.rb --harness=vernier benchmarks/... require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb index 5ee4d79b..19a464b5 100644 --- a/lib/harness/loader.rb +++ b/lib/harness/loader.rb @@ -1,9 +1,9 @@ -# Use harness/default.rb by default. You can change it with -r option. -# The -r option should be specified BEFORE requiring loader. +# Use harness/default.rb by default. You can change it with the --harness option in run_once.rb +# or with -r option when calling Ruby directly. # Examples: -# ruby benchmarks/railsbench/benchmark.rb # uses harness/default.rb -# ruby -r./harness/once benchmarks/railsbench/benchmark.rb # uses harness/once.rb -# ruby -r./harness/ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb +# ./run_once.rb benchmarks/railsbench/benchmark.rb # uses harness/default.rb +# ./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# ./run_once.rb --harness=ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb # Only load the default harness if no other harness has defined run_benchmark unless defined?(run_benchmark) diff --git a/run_once.rb b/run_once.rb new file mode 100755 index 00000000..9576fdd1 --- /dev/null +++ b/run_once.rb @@ -0,0 +1,90 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Script to run a single benchmark once +# Provides a clean interface for running benchmarks with different harnesses. +# Examples: +# ./run_once.rb benchmarks/railsbench/benchmark.rb +# ./run_once.rb --harness=once benchmarks/fib.rb +# ./run_once.rb --harness=stackprof benchmarks/fib.rb +# ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb +# ./run_once.rb --harness=default -- --yjit benchmarks/fib.rb + +require 'optparse' +require 'shellwords' + +# Parse options +harness = nil +ruby_args = [] +benchmark_file = nil + +parser = OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options] [--] [ruby-options] BENCHMARK_FILE" + + opts.on("--harness=HARNESS", "Harness to use (default: default, options: once, bips, perf, ractor, stackprof, vernier, warmup, stats, continuous, chain, mplr)") do |h| + harness = h + end + + opts.on("-h", "--help", "Show this help message") do + puts opts + exit + end +end + +# Parse our options - this will stop at '--' or first non-option argument +begin + parser.parse! +rescue OptionParser::InvalidOption => e + puts "Error: #{e.message}" + puts parser + exit 1 +end + +# After parsing, ARGV contains remaining args (Ruby options + benchmark file) +ARGV.each do |arg| + if arg.end_with?('.rb') && !benchmark_file + benchmark_file = arg + else + ruby_args << arg + end +end + +if !benchmark_file + puts "Error: No benchmark file specified" + puts parser + exit 1 +end + +unless File.exist?(benchmark_file) + puts "Error: Benchmark file not found: #{benchmark_file}" + exit 1 +end + +# Automatically detect ractor benchmarks +if !harness && benchmark_file.include?('benchmarks-ractor/') + harness = 'ractor' +end + +# Build the command +harness_dir = File.expand_path('harness', __dir__) +harness_args = if harness && harness != 'default' + harness_path = File.join(harness_dir, harness) + unless File.exist?("#{harness_path}.rb") + puts "Error: Harness not found: #{harness}" + puts "Available harnesses: #{Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.join(', ')}" + exit 1 + end + ['-r', harness_path] +else + [] +end + +# Set environment for running once +ENV['WARMUP_ITRS'] = '0' +ENV['MIN_BENCH_ITRS'] = '1' +ENV['MIN_BENCH_TIME'] = '0' + +# Build and execute the command +cmd = ['ruby', *ruby_args, *harness_args, benchmark_file] + +exec(*cmd) diff --git a/run_once.sh b/run_once.sh deleted file mode 100755 index bef1d8bd..00000000 --- a/run_once.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Script to run a single benchmark once -# You can pass --yjit-stats and other ruby arguments to this script. -# Automatically detects Ractor benchmarks and uses the appropriate harness. -# Examples: -# ./run_once.sh --yjit-stats benchmarks/railsbench/benchmark.rb -# ./run_once.sh benchmarks-ractor/optcarrot/benchmark.rb - -# Detect if any argument contains benchmarks-ractor/ to determine harness -HARNESS_ARGS=() -for arg in "$@"; do - if [[ "$arg" == *"benchmarks-ractor/"* ]]; then - HARNESS_ARGS=("-r" "./harness/ractor") - break - fi -done - -WARMUP_ITRS=0 MIN_BENCH_ITRS=1 MIN_BENCH_TIME=0 ruby "${HARNESS_ARGS[@]}" "$@" diff --git a/test/run_once_test.rb b/test/run_once_test.rb new file mode 100644 index 00000000..6e4acc20 --- /dev/null +++ b/test/run_once_test.rb @@ -0,0 +1,249 @@ +require_relative 'test_helper' +require 'open3' +require 'tmpdir' +require 'fileutils' + +describe 'run_once.rb' do + before do + @script_path = File.expand_path('../run_once.rb', __dir__) + @original_env = ENV.to_h + + # Create a temp directory with test benchmarks + @tmpdir = Dir.mktmpdir + @test_benchmark = File.join(@tmpdir, 'test_benchmark.rb') + File.write(@test_benchmark, <<~RUBY) + puts "Benchmark executed" + puts "WARMUP_ITRS=\#{ENV['WARMUP_ITRS']}" + puts "MIN_BENCH_ITRS=\#{ENV['MIN_BENCH_ITRS']}" + puts "MIN_BENCH_TIME=\#{ENV['MIN_BENCH_TIME']}" + RUBY + end + + after do + FileUtils.rm_rf(@tmpdir) if @tmpdir && Dir.exist?(@tmpdir) + # Restore original environment + ENV.replace(@original_env) + end + + def run_script(*args) + # Run the script and capture output + Open3.capture3('ruby', @script_path, *args) + end + + describe 'basic execution' do + it 'executes a benchmark file' do + stdout, _stderr, status = run_script(@test_benchmark) + + assert status.success?, "Script should execute successfully. stderr: #{_stderr}" + assert_match(/Benchmark executed/, stdout) + end + + it 'sets environment variables for single iteration' do + stdout, _stderr, status = run_script(@test_benchmark) + + assert status.success? + assert_match(/WARMUP_ITRS=0/, stdout) + assert_match(/MIN_BENCH_ITRS=1/, stdout) + assert_match(/MIN_BENCH_TIME=0/, stdout) + end + + it 'shows error when no benchmark file specified' do + stdout, _stderr, status = run_script + + refute status.success? + assert_match(/No benchmark file specified/, stdout) + end + + it 'shows error when benchmark file does not exist' do + stdout, _stderr, status = run_script('/nonexistent/benchmark.rb') + + refute status.success? + assert_match(/Benchmark file not found/, stdout) + end + end + + describe '--harness option' do + it 'accepts default harness' do + stdout, _stderr, status = run_script('--harness=default', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'loads custom harness when specified' do + # Create a test harness + harness_dir = File.join(File.dirname(@script_path), 'harness') + FileUtils.mkdir_p(harness_dir) + test_harness = File.join(harness_dir, 'test_harness.rb') + + begin + File.write(test_harness, <<~RUBY) + puts "TEST HARNESS LOADED" + RUBY + + stdout, _stderr, status = run_script('--harness=test_harness', @test_benchmark) + + assert status.success? + assert_match(/TEST HARNESS LOADED/, stdout) + ensure + File.delete(test_harness) if File.exist?(test_harness) + end + end + + it 'shows error for non-existent harness' do + stdout, _stderr, status = run_script('--harness=nonexistent', @test_benchmark) + + refute status.success? + assert_match(/Harness not found/, stdout) + assert_match(/Available harnesses/, stdout) + end + end + + describe 'ractor benchmark detection' do + it 'automatically uses ractor harness for ractor benchmarks' do + # Create a ractor benchmark + ractor_dir = File.join(@tmpdir, 'benchmarks-ractor', 'test') + FileUtils.mkdir_p(ractor_dir) + ractor_benchmark = File.join(ractor_dir, 'benchmark.rb') + File.write(ractor_benchmark, 'puts "Ractor benchmark"') + + _stdout, _stderr, _status = run_script(ractor_benchmark) + + # The script will try to load ractor harness + # We just verify it detected the ractor path + assert_match(/benchmarks-ractor/, ractor_benchmark) + end + end + + describe 'Ruby options pass-through' do + it 'passes Ruby options after -- separator' do + # Create a benchmark that checks for a warning flag + warning_benchmark = File.join(@tmpdir, 'warning_test.rb') + File.write(warning_benchmark, <<~RUBY) + x = 1 + x = 2 + puts "Benchmark with warnings" + RUBY + + stdout, _stderr, status = run_script('--', '-W2', warning_benchmark) + + assert status.success? + assert_match(/Benchmark with warnings/, stdout) + end + + it 'passes YJIT options after -- separator' do + yjit_benchmark = File.join(@tmpdir, 'yjit_test.rb') + File.write(yjit_benchmark, <<~RUBY) + puts "YJIT enabled" if defined?(RubyVM::YJIT) + puts "Benchmark complete" + RUBY + + stdout, _stderr, status = run_script('--', '--yjit', yjit_benchmark) + + assert status.success? + assert_match(/Benchmark complete/, stdout) + end + end + + describe '--help option' do + it 'shows help message' do + stdout, _stderr, status = run_script('--help') + + assert status.success? + assert_match(/Usage:/, stdout) + assert_match(/--harness/, stdout) + end + + it 'shows help with -h' do + stdout, _stderr, status = run_script('-h') + + assert status.success? + assert_match(/Usage:/, stdout) + end + end + + describe 'argument parsing order' do + it 'handles harness option before benchmark' do + stdout, _stderr, status = run_script('--harness=default', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'handles Ruby options after -- separator' do + stdout, _stderr, status = run_script('--', '-W0', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'handles mixed options with -- separator' do + stdout, _stderr, status = run_script('--harness=default', '--', '-W0', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + end + + describe 'script examples' do + it 'works with simple benchmark path' do + # Simulates: ./run_once.rb benchmarks/fib.rb + fib_benchmark = File.join(@tmpdir, 'fib.rb') + File.write(fib_benchmark, 'puts "Fibonacci benchmark"') + + stdout, _stderr, status = run_script(fib_benchmark) + + assert status.success? + assert_match(/Fibonacci benchmark/, stdout) + end + + it 'works with harness option' do + # Simulates: ./run_once.rb --harness=default benchmarks/fib.rb + stdout, _stderr, status = run_script('--harness=default', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + + it 'works with Ruby options after -- separator' do + # Simulates: ./run_once.rb -- --yjit benchmarks/fib.rb + stdout, _stderr, status = run_script('--', '--yjit', @test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + end + end + + describe 'edge cases' do + it 'handles benchmark files with spaces in path' do + space_dir = File.join(@tmpdir, 'dir with spaces') + FileUtils.mkdir_p(space_dir) + space_benchmark = File.join(space_dir, 'benchmark.rb') + File.write(space_benchmark, 'puts "Space test"') + + stdout, _stderr, status = run_script(space_benchmark) + + assert status.success? + assert_match(/Space test/, stdout) + end + + it 'identifies first .rb file as benchmark' do + # Even if multiple .rb files mentioned (shouldn't happen), first one wins + other_file = File.join(@tmpdir, 'other.rb') + File.write(other_file, 'puts "Wrong file"') + + stdout, _stderr, status = run_script(@test_benchmark) + + assert status.success? + assert_match(/Benchmark executed/, stdout) + refute_match(/Wrong file/, stdout) + end + + it 'rejects invalid options gracefully' do + stdout, _stderr, status = run_script('--invalid-option', @test_benchmark) + + refute status.success? + assert_match(/invalid option|Error/, stdout) + end + end +end From 66c2e95713779e4dac59266f008fcd43573e42ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 27 Nov 2025 19:30:32 +0000 Subject: [PATCH 4/6] Allow using HARNESS environment variable to select harness This allow `ruby` script direct calls to still select the harness without using any wrapper script. --- README.md | 61 ++++++++++++++++++++------------------- harness/perf.rb | 2 +- harness/stackprof.rb | 4 +-- harness/vernier.rb | 4 +-- lib/harness/loader.rb | 23 +++++++++++---- test/run_once_test.rb | 66 +++++++++++++++++++++++++++++++++++-------- 6 files changed, 108 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 79c384cd..f967dc23 100644 --- a/README.md +++ b/README.md @@ -93,13 +93,13 @@ To run one or more specific benchmarks and record the data: ### Running a single benchmark -The easiest way to run a single benchmark once is using the `run_once.rb` script: +This is the easiest way to run a single benchmark. +It requires no setup at all and assumes nothing about the Ruby you are benchmarking. +It's also convenient for profiling, debugging, etc, especially since all benchmarked code runs in that process. +```bash +ruby benchmarks/fib.rb ``` -./run_once.rb benchmarks/some_benchmark.rb -``` - -This automatically sets the environment to run the benchmark once (no warmup iterations). ### Benchmark organization @@ -225,34 +225,35 @@ You can find several test harnesses in the `harness/` directory: * `chain` - a harness to chain multiple harnesses together * `mplr` - a harness for multiple iterations with time limits -To use a specific harness, use the `run_once.rb` script: +### Selecting a harness -``` -# Use default harness -./run_once.rb benchmarks/railsbench/benchmark.rb +**Using the `HARNESS` environment variable** -# Use the 'once' harness -./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb +```bash +# Run with specific harness +HARNESS=perf ruby benchmarks/fib.rb +HARNESS=stackprof ruby benchmarks/railsbench/benchmark.rb +HARNESS=vernier ruby benchmarks/optcarrot/benchmark.rb -# Use the 'perf' harness -./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb +# Combine with Ruby options +HARNESS=perf ruby --yjit benchmarks/fib.rb +HARNESS=once ruby --yjit-stats benchmarks/railsbench/benchmark.rb +``` -# Use the 'stackprof' harness -./run_once.rb --harness=stackprof benchmarks/railsbench/benchmark.rb +**Alternative: Use `run_once.rb` with `--harness` option**: -# Use the 'vernier' harness -./run_once.rb --harness=vernier benchmarks/railsbench/benchmark.rb +```bash +./run_once.rb --harness=perf benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=stackprof benchmarks/fib.rb -# Pass Ruby options like --yjit (use -- separator) +# With Ruby options (use -- separator) ./run_once.rb -- --yjit benchmarks/railsbench/benchmark.rb - -# Combine harness option with Ruby options -./run_once.rb --harness=default -- --yjit-stats benchmarks/railsbench/benchmark.rb +./run_once.rb --harness=perf -- --yjit-stats benchmarks/fib.rb ``` When using `run_benchmarks.rb`, you can specify a harness with the `--harness` option: -``` +```bash ./run_benchmarks.rb --harness=once ./run_benchmarks.rb --harness=perf ``` @@ -278,10 +279,13 @@ You can also use `--warmup`, `--bench`, or `--once` to set these environment var ./run_benchmarks.rb railsbench --once ``` -You can also use the `run_once.rb` script to run benchmarks just once, for example -with the `--yjit-stats` command-line option: +You can also run a single benchmark directly with Ruby: -``` +```bash +# Run once with YJIT stats +ruby --yjit-stats benchmarks/railsbench/benchmark.rb + +# Or use run_once.rb script ./run_once.rb -- --yjit-stats benchmarks/railsbench/benchmark.rb ``` @@ -290,13 +294,12 @@ with the `--yjit-stats` command-line option: There is also a harness to use Linux perf. By default, it only runs a fixed number of iterations. If `PERF` environment variable is present, it starts the perf subcommand after warmup. -```sh +```bash # Use `perf record` for both warmup and benchmark -perf record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb +HARNESS=perf perf record ruby --yjit-perf=map benchmarks/railsbench/benchmark.rb # Use `perf record` only for benchmark -PERF=record ./run_once.rb --harness=perf -- --yjit-perf=map benchmarks/railsbench/benchmark.rb -``` +HARNESS=perf PERF=record ruby --yjit-perf=map benchmarks/railsbench/benchmark.rb This is the only harness that uses `run_benchmark`'s argument, `num_itrs_hint`. diff --git a/harness/perf.rb b/harness/perf.rb index 086b70b6..e13978ea 100644 --- a/harness/perf.rb +++ b/harness/perf.rb @@ -3,7 +3,7 @@ # This is a relatively minimal harness meant for use with Linux perf(1). # Example usage: # -# $ PERF='record -e cycles' ./run_once.rb --harness=perf benchmarks/fib.rb +# $ PERF='record -e cycles' HARNESS=perf ruby benchmarks/fib.rb # # When recording with perf(1), make sure the benchmark runs long enough; you # can tweak the MIN_BENCH_ITRS environment variable to lengthen the run. A race diff --git a/harness/stackprof.rb b/harness/stackprof.rb index 931d5122..8a33524e 100644 --- a/harness/stackprof.rb +++ b/harness/stackprof.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) with stackprof. # Customize stackprof options with an env var of STACKPROF_OPTS='key:value,...'. # Usage: -# STACKPROF_OPTS='mode:object' ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb -# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 ./run_once.rb --harness=stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:object' HARNESS=stackprof MIN_BENCH_TIME=0 MIN_BENCH_ITRS=1 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb +# STACKPROF_OPTS='mode:cpu,interval:10' MIN_BENCH_ITRS=10 HARNESS=stackprof MIN_BENCH_TIME=1 MIN_BENCH_ITRS=10 ruby -v -I harness-stackprof benchmarks/.../benchmark.rb require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/harness/vernier.rb b/harness/vernier.rb index 3accb74d..186da9de 100644 --- a/harness/vernier.rb +++ b/harness/vernier.rb @@ -3,8 +3,8 @@ # Profile the benchmark (ignoring initialization code) using vernier and display the profile. # Set NO_VIERWER=1 to disable automatically opening the profile in a browser. # Usage: -# ./run_once.rb --harness=vernier benchmarks/... -# NO_VIEWER=1 ./run_once.rb --harness=vernier benchmarks/... +# HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... +# NO_VIEWER=1 HARNESS=vernier MIN_BENCH_TIME=1 MIN_BENCH_ITRS=1 ruby -v -I harness-vernier benchmarks/... require_relative "../lib/harness" require_relative "../lib/harness/extra" diff --git a/lib/harness/loader.rb b/lib/harness/loader.rb index 19a464b5..5768bd59 100644 --- a/lib/harness/loader.rb +++ b/lib/harness/loader.rb @@ -1,22 +1,33 @@ -# Use harness/default.rb by default. You can change it with the --harness option in run_once.rb -# or with -r option when calling Ruby directly. +# Use harness/default.rb by default. You can change it with: +# 1. HARNESS environment variable (preferred, keeps harness close to code) +# 2. --harness option in run_once.rb +# 3. -r option when calling Ruby directly +# # Examples: -# ./run_once.rb benchmarks/railsbench/benchmark.rb # uses harness/default.rb -# ./run_once.rb --harness=once benchmarks/railsbench/benchmark.rb # uses harness/once.rb +# HARNESS=perf ruby benchmarks/railsbench/benchmark.rb # uses harness/perf.rb +# HARNESS=once ruby benchmarks/railsbench/benchmark.rb # uses harness/once.rb # ./run_once.rb --harness=ractor benchmarks/railsbench/benchmark.rb # uses harness/ractor.rb # Only load the default harness if no other harness has defined run_benchmark unless defined?(run_benchmark) + harness_name = ENV['HARNESS'] || 'default' + retries = 0 begin - require "default" + require harness_name rescue LoadError => e - if retries == 0 && e.path == "default" + if retries == 0 && e.path == harness_name retries += 1 # Add the harness directory to the load path $LOAD_PATH << File.expand_path("../../harness", __dir__) retry end + # Provide helpful error message for invalid harness + if e.path == harness_name + harness_dir = File.expand_path("../../harness", __dir__) + available = Dir.glob("#{harness_dir}/*.rb").map { |f| File.basename(f, '.rb') }.sort + raise LoadError, "Harness '#{harness_name}' not found. Available harnesses: #{available.join(', ')}" + end raise end end diff --git a/test/run_once_test.rb b/test/run_once_test.rb index 6e4acc20..1d96bc31 100644 --- a/test/run_once_test.rb +++ b/test/run_once_test.rb @@ -8,10 +8,11 @@ @script_path = File.expand_path('../run_once.rb', __dir__) @original_env = ENV.to_h - # Create a temp directory with test benchmarks @tmpdir = Dir.mktmpdir @test_benchmark = File.join(@tmpdir, 'test_benchmark.rb') + loader_path = File.expand_path('../lib/harness/loader', __dir__) File.write(@test_benchmark, <<~RUBY) + require "#{loader_path}" puts "Benchmark executed" puts "WARMUP_ITRS=\#{ENV['WARMUP_ITRS']}" puts "MIN_BENCH_ITRS=\#{ENV['MIN_BENCH_ITRS']}" @@ -21,12 +22,10 @@ after do FileUtils.rm_rf(@tmpdir) if @tmpdir && Dir.exist?(@tmpdir) - # Restore original environment ENV.replace(@original_env) end def run_script(*args) - # Run the script and capture output Open3.capture3('ruby', @script_path, *args) end @@ -71,7 +70,6 @@ def run_script(*args) end it 'loads custom harness when specified' do - # Create a test harness harness_dir = File.join(File.dirname(@script_path), 'harness') FileUtils.mkdir_p(harness_dir) test_harness = File.join(harness_dir, 'test_harness.rb') @@ -90,6 +88,26 @@ def run_script(*args) end end + it 'respects HARNESS environment variable when no --harness option provided' do + harness_dir = File.join(File.dirname(@script_path), 'harness') + FileUtils.mkdir_p(harness_dir) + test_harness = File.join(harness_dir, 'test_harness.rb') + + begin + File.write(test_harness, <<~RUBY) + puts "TEST HARNESS LOADED" + RUBY + + env = { 'HARNESS' => 'test_harness' } + stdout, _stderr, status = Open3.capture3(env, 'ruby', @script_path, @test_benchmark) + + assert status.success? + assert_match(/TEST HARNESS LOADED/, stdout) + ensure + File.delete(test_harness) if File.exist?(test_harness) + end + end + it 'shows error for non-existent harness' do stdout, _stderr, status = run_script('--harness=nonexistent', @test_benchmark) @@ -101,7 +119,6 @@ def run_script(*args) describe 'ractor benchmark detection' do it 'automatically uses ractor harness for ractor benchmarks' do - # Create a ractor benchmark ractor_dir = File.join(@tmpdir, 'benchmarks-ractor', 'test') FileUtils.mkdir_p(ractor_dir) ractor_benchmark = File.join(ractor_dir, 'benchmark.rb') @@ -109,15 +126,12 @@ def run_script(*args) _stdout, _stderr, _status = run_script(ractor_benchmark) - # The script will try to load ractor harness - # We just verify it detected the ractor path assert_match(/benchmarks-ractor/, ractor_benchmark) end end describe 'Ruby options pass-through' do it 'passes Ruby options after -- separator' do - # Create a benchmark that checks for a warning flag warning_benchmark = File.join(@tmpdir, 'warning_test.rb') File.write(warning_benchmark, <<~RUBY) x = 1 @@ -187,7 +201,6 @@ def run_script(*args) describe 'script examples' do it 'works with simple benchmark path' do - # Simulates: ./run_once.rb benchmarks/fib.rb fib_benchmark = File.join(@tmpdir, 'fib.rb') File.write(fib_benchmark, 'puts "Fibonacci benchmark"') @@ -198,7 +211,6 @@ def run_script(*args) end it 'works with harness option' do - # Simulates: ./run_once.rb --harness=default benchmarks/fib.rb stdout, _stderr, status = run_script('--harness=default', @test_benchmark) assert status.success? @@ -206,7 +218,6 @@ def run_script(*args) end it 'works with Ruby options after -- separator' do - # Simulates: ./run_once.rb -- --yjit benchmarks/fib.rb stdout, _stderr, status = run_script('--', '--yjit', @test_benchmark) assert status.success? @@ -228,7 +239,6 @@ def run_script(*args) end it 'identifies first .rb file as benchmark' do - # Even if multiple .rb files mentioned (shouldn't happen), first one wins other_file = File.join(@tmpdir, 'other.rb') File.write(other_file, 'puts "Wrong file"') @@ -246,4 +256,36 @@ def run_script(*args) assert_match(/invalid option|Error/, stdout) end end + + describe 'direct ruby execution with HARNESS env var' do + it 'uses HARNESS environment variable when running benchmark directly' do + harness_dir = File.join(File.dirname(@script_path), 'harness') + FileUtils.mkdir_p(harness_dir) + direct_harness = File.join(harness_dir, 'direct_test.rb') + + begin + File.write(direct_harness, <<~RUBY) + puts "DIRECT HARNESS RUNNING" + RUBY + + env = { 'HARNESS' => 'direct_test' } + stdout, _stderr, status = Open3.capture3(env, 'ruby', @test_benchmark) + + assert status.success?, "Direct Ruby execution should work with HARNESS env var" + assert_match(/DIRECT HARNESS RUNNING/, stdout) + assert_match(/Benchmark executed/, stdout) + ensure + File.delete(direct_harness) if File.exist?(direct_harness) + end + end + + it 'shows helpful error for invalid HARNESS value' do + env = { 'HARNESS' => 'nonexistent_harness' } + _stdout, stderr, status = Open3.capture3(env, 'ruby', @test_benchmark) + + refute status.success? + assert_match(/Harness 'nonexistent_harness' not found/, stderr) + assert_match(/Available harnesses/, stderr) + end + end end From 2de2feef81a0815492a66b581825dd4a43cf583f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 27 Nov 2025 19:45:13 +0000 Subject: [PATCH 5/6] Improve chain harness documentation and error handling --- README.md | 19 +++++++++++++++++++ harness/chain.rb | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f967dc23..11f75f20 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,25 @@ When using `run_benchmarks.rb`, you can specify a harness with the `--harness` o There is also a robust but complex CI harness in [the yjit-metrics repo](https://github.com/Shopify/yjit-metrics). +### Chain harness + +The `chain` harness allows you to combine multiple harnesses together. This is useful when you want to run a benchmark through multiple analysis tools or measurement approaches in sequence. + +Use the `HARNESS_CHAIN` environment variable to specify which harnesses to chain (comma-separated, at least 2 required): + +```bash +# Chain the 'once' and 'default' harnesses (runs benchmark once, then with default iterations) +HARNESS=chain HARNESS_CHAIN="once,default" ruby benchmarks/fib.rb + +# Profile with vernier while using ractor harness +HARNESS=chain HARNESS_CHAIN="vernier,ractor" ruby benchmarks-ractor/some_benchmark.rb + +# Using run_once.rb +HARNESS_CHAIN="once,default" ./run_once.rb --harness=chain benchmarks/fib.rb +``` + +The harnesses are executed in the order specified, with each harness wrapping the previous one. + ### Iterations and duration With the default harness, the number of iterations and duration diff --git a/harness/chain.rb b/harness/chain.rb index 62d50712..20e1fa66 100644 --- a/harness/chain.rb +++ b/harness/chain.rb @@ -34,7 +34,8 @@ def run_benchmark(n, **kwargs, &block) $current_harness = h require_relative path rescue LoadError => e - if e.path == path + expected_path = File.expand_path(h, __dir__) + if e.path == expected_path $stderr.puts "Can't find harness #{h}.rb in harness/. Exiting." exit 1 end From 01698a8d8d3f3f14e9968e972b2978f98d392141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Mon, 1 Dec 2025 17:38:24 +0000 Subject: [PATCH 6/6] Build harness arguments once during initialization --- lib/benchmark_suite.rb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 202da66e..eeda2250 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -33,6 +33,7 @@ def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pr @ractor_only = (categories == RACTOR_ONLY_CATEGORY) setup_benchmark_directories + @harness_args = build_harness_args end # Run all the benchmarks and record execution times @@ -63,6 +64,8 @@ def run(ruby:, ruby_description:) private + attr_reader :harness_args + def setup_benchmark_directories if @ractor_only @bench_dir = RACTOR_BENCHMARKS_DIR @@ -151,15 +154,6 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) ENV["RESULT_JSON_PATH"] = result_json_path # Set up the benchmarking command - # If harness is 'default', use default (no -r needed) - # Otherwise use -r to load the specific harness file with full path - harness_args = if harness == "default" - [] - else - harness_path = File.join(HARNESS_DIR, harness) - ["-r", harness_path] - end - cmd = cmd_prefix + [ *ruby, *harness_args, @@ -236,6 +230,17 @@ def setarch_prefix prefix end + # If harness is 'default', use default (no -r needed) + # Otherwise use -r to load the specific harness file with full path + def build_harness_args + if harness == "default" + [] + else + harness_path = File.join(HARNESS_DIR, harness) + ["-r", harness_path] + end + end + # Resolve the pre_init file path into a form that can be required def expand_pre_init(path) path = Pathname.new(path)