From 8e144a64b5c671372feb5155537b720fb6b58116 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 2 Dec 2025 15:29:07 +0000 Subject: [PATCH 1/6] Write a Ractor benchmark that mirrors the process structure The previous ractor knucleotide benchmark ran the whole benchmark in parallel inside multiple ractors. This is a good test that the benchmark can be parallelised but isn't directly comparable to the original Knucleotide implementation which uses processes to partition the work done by the benchmark itself. This implementation uses ractors to parallelise the work in the same way that the original uses Process.fork so it's more directly comparable. This needs to be run with the regular benchmark harness instead of the ractor harness, otherwise the ractor harness will attempt to wrap this benchmark run in multiple ractors too. --- benchmarks.yml | 2 + benchmarks/knucleotide-ractor/benchmark.rb | 71 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 benchmarks/knucleotide-ractor/benchmark.rb diff --git a/benchmarks.yml b/benchmarks.yml index 7c0297d6..58221ec3 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -91,6 +91,8 @@ graphql-native: desc: GraphQL gem parsing a large file, but using a native parser knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork +knucleotide-ractor: + desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: diff --git a/benchmarks/knucleotide-ractor/benchmark.rb b/benchmarks/knucleotide-ractor/benchmark.rb new file mode 100644 index 00000000..41d265d6 --- /dev/null +++ b/benchmarks/knucleotide-ractor/benchmark.rb @@ -0,0 +1,71 @@ +# The Computer Language Benchmarks Game +# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ +# +# k-nucleotide benchmark - Ractor implementation +# Mirrors the Process.fork version: spawns 7 ractors (one per task) + +Warning[:experimental] = false + +require_relative '../../harness/loader' + +def frequency(seq, length) + frequencies = Hash.new(0) + last_index = seq.length - length + + i = 0 + while i <= last_index + frequencies[seq.byteslice(i, length)] += 1 + i += 1 + end + + [seq.length - length + 1, frequencies] +end + +def sort_by_freq(seq, length) + n, table = frequency(seq, length) + + table.sort { |a, b| + cmp = b[1] <=> a[1] + cmp == 0 ? a[0] <=> b[0] : cmp + }.map { |seq, count| + "#{seq} #{'%.3f' % ((count * 100.0) / n)}" + }.join("\n") + "\n\n" +end + +def find_seq(seq, s) + _, table = frequency(seq, s.length) + "#{table[s] || 0}\t#{s}\n" +end + +def generate_test_sequence(size) + alu = "GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGATCACCTGAGGTCA" + + "GGAGTTCGAGACCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAATACAAAAATTAGCCGGGCGTGG" + + "TGGCGCGCGCCTGTAATCCCAGCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGGAGGCGGAGGTT" + + "GCAGTGAGCCGAGATCGCGCCACTGCACTCCAGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA" + + sequence = "" + full_copies = size / alu.length + remainder = size % alu.length + + full_copies.times { sequence << alu } + sequence << alu[0, remainder] if remainder > 0 + + sequence.upcase +end + +TEST_SEQUENCE = Ractor.make_shareable(generate_test_sequence(100_000)) + +run_benchmark(5) do + freqs = [1, 2] + nucleos = %w(GGT GGTA GGTATT GGTATTTTAATT GGTATTTTAATTTATAGT) + + ractors = freqs.map { |i| + Ractor.new(TEST_SEQUENCE, i) { |seq, len| sort_by_freq(seq, len) } + } + ractors += nucleos.map { |s| + Ractor.new(TEST_SEQUENCE, s) { |seq, nucleo| find_seq(seq, nucleo) } + } + + results = ractors.map(&:value) + results +end From 3938a74de8509c7200c737e425c086aa2facc5a0 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 2 Dec 2025 16:05:33 +0000 Subject: [PATCH 2/6] Allow pinning to be specified in the benchmarks.yml --- benchmarks.yml | 1 + lib/argument_parser.rb | 6 ++++++ lib/benchmark_runner/cli.rb | 3 ++- lib/benchmark_suite.rb | 19 ++++++++++++++----- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/benchmarks.yml b/benchmarks.yml index 58221ec3..6041618c 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -93,6 +93,7 @@ knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork knucleotide-ractor: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors + no_pinning: true lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 35de87c1..528c4c9e 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -15,6 +15,7 @@ class ArgumentParser :rss, :graph, :no_pinning, + :force_pinning, :turbo, :skip_yjit, :with_pre_init, @@ -140,6 +141,10 @@ def parse(argv) args.no_pinning = true end + opts.on("--force-pinning", "force pinning even for benchmarks marked no_pinning") do + args.force_pinning = true + end + opts.on("--turbo", "don't disable CPU turbo boost") do args.turbo = true end @@ -183,6 +188,7 @@ def default_args rss: false, graph: false, no_pinning: false, + force_pinning: false, turbo: false, skip_yjit: false, with_pre_init: nil, diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index 92206a48..639c2014 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -35,7 +35,8 @@ def run out_path: args.out_path, harness: args.harness, pre_init: args.with_pre_init, - no_pinning: args.no_pinning + no_pinning: args.no_pinning, + force_pinning: args.force_pinning ) # Benchmark with and without YJIT diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 30c4a853..ccc98459 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -19,9 +19,9 @@ class BenchmarkSuite RACTOR_CATEGORY = ["ractor"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir + attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :force_pinning, :bench_dir, :ractor_bench_dir - def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false) + def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false, force_pinning: false) @categories = categories @name_filters = name_filters @excludes = excludes @@ -29,6 +29,7 @@ def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pr @harness = harness @pre_init = pre_init ? expand_pre_init(pre_init) : nil @no_pinning = no_pinning + @force_pinning = force_pinning @ractor_only = (categories == RACTOR_ONLY_CATEGORY) setup_benchmark_directories @@ -41,13 +42,13 @@ def run(ruby:, ruby_description:) bench_failures = {} benchmark_entries = discover_benchmarks - cmd_prefix = base_cmd(ruby_description) env = benchmark_env(ruby) benchmark_entries.each_with_index do |entry, idx| puts("Running benchmark \"#{entry.name}\" (#{idx+1}/#{benchmark_entries.length})") result_json_path = File.join(out_path, "temp#{Process.pid}.json") + cmd_prefix = base_cmd(ruby_description, entry.name) result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env) if result[:success] @@ -197,13 +198,13 @@ def linux? end # Set up the base command with CPU pinning if needed - def base_cmd(ruby_description) + def base_cmd(ruby_description, benchmark_name) if linux? cmd = setarch_prefix # Pin the process to one given core to improve caching and reduce variance on CRuby # Other Rubies need to use multiple cores, e.g., for JIT threads - if ruby_description.start_with?('ruby ') && !no_pinning + if ruby_description.start_with?('ruby ') && should_pin?(benchmark_name) # The last few cores of Intel CPU may be slow E-Cores, so avoid using the last one. cpu = [(Etc.nprocessors / 2) - 1, 0].max cmd.concat(["taskset", "-c", "#{cpu}"]) @@ -215,6 +216,14 @@ def base_cmd(ruby_description) end end + def should_pin?(benchmark_name) + return false if no_pinning + return true if force_pinning + + benchmark_meta = benchmarks_metadata[benchmark_name] || {} + !benchmark_meta["no_pinning"] + end + # Generate setarch prefix for Linux def setarch_prefix # Disable address space randomization (for determinism) From da0f6df9aa9cbd705cadbab913cddec497389802 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 12:50:08 +0000 Subject: [PATCH 3/6] Don't fail tests if the user has a ruby version installed already --- lib/argument_parser.rb | 15 +++++-- test/argument_parser_test.rb | 82 ++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 528c4c9e..6e73e338 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -53,10 +53,8 @@ def parse(argv) name = name.shellsplit.first end version, *options = version.shellsplit - rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies" - unless executable = ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"].find { |path| File.executable?(path) } - abort "Cannot find '#{version}' in /opt/rubies or #{rubies_dir}" - end + executable = find_chruby_ruby(version) + abort "Cannot find '#{version}' in chruby paths" unless executable args.executables[name] = [executable, *options] end end @@ -170,6 +168,15 @@ def parse(argv) private + def find_chruby_ruby(version) + rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies" + chruby_search_paths(version, rubies_dir).find { |path| File.executable?(path) } + end + + def chruby_search_paths(version, rubies_dir) + ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"] + end + def have_yjit?(ruby) ruby_version = `#{ruby} -v --yjit 2> #{File::NULL}`.strip ruby_version.downcase.include?("yjit") diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 7008dcef..d19ae240 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -103,15 +103,11 @@ def setup_mock_ruby(path) ruby_path = File.join(tmpdir, 'opt/rubies/3.2.0/bin/ruby') setup_mock_ruby(ruby_path) - File.stub :executable?, ->(path) { - if path == "/opt/rubies/3.2.0/bin/ruby" - File.exist?(ruby_path) && File.stat(ruby_path).executable? - end - } do - parser = ArgumentParser.new + parser = ArgumentParser.new + parser.stub :chruby_search_paths, ->(version, rubies_dir) { [ruby_path] } do args = parser.parse(['--chruby=ruby-3.2.0::3.2.0']) - assert_equal '/opt/rubies/3.2.0/bin/ruby', args.executables['ruby-3.2.0'].first + assert_equal ruby_path, args.executables['ruby-3.2.0'].first end end end @@ -126,9 +122,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=my-ruby::3.3.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=my-ruby::3.3.0']) - assert_equal ruby_path, args.executables['my-ruby'].first + assert_equal ruby_path, args.executables['my-ruby'].first + end end end @@ -143,20 +141,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir - File.stub :executable?, ->(path) { - case path - when "/opt/rubies/3.2.0/bin/ruby" - File.exist?(opt_ruby) && File.stat(opt_ruby).executable? - when "#{tmpdir}/.rubies/3.2.0/bin/ruby" - File.exist?(home_ruby) && File.stat(home_ruby).executable? - else - File.method(:executable?).super_method.call(path) - end - } do - parser = ArgumentParser.new + parser = ArgumentParser.new + parser.stub :chruby_search_paths, ->(version, rd) { [opt_ruby, home_ruby] } do args = parser.parse(['--chruby=test::3.2.0']) - assert_equal '/opt/rubies/3.2.0/bin/ruby', args.executables['test'].first + assert_equal opt_ruby, args.executables['test'].first end end end @@ -171,9 +160,11 @@ def setup_mock_ruby(path) ENV['RUBIES_DIR'] = custom_rubies parser = ArgumentParser.new - args = parser.parse(['--chruby=custom::3.4.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=custom::3.4.0']) - assert_equal ruby_path, args.executables['custom'].first + assert_equal ruby_path, args.executables['custom'].first + end end end @@ -183,10 +174,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - - assert_raises(SystemExit) do - capture_io do - parser.parse(['--chruby=nonexistent::nonexistent-version-999']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + assert_raises(SystemExit) do + capture_io do + parser.parse(['--chruby=nonexistent::nonexistent-version-999']) + end end end end @@ -202,10 +194,12 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=yjit::3.2.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=yjit::3.2.0 --yjit']) - assert_equal ruby_path, args.executables['yjit'].first - assert_equal '--yjit', args.executables['yjit'].last + assert_equal ruby_path, args.executables['yjit'].first + assert_equal '--yjit', args.executables['yjit'].last + end end end @@ -219,10 +213,12 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=3.2.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=3.2.0 --yjit']) - assert args.executables.key?('3.2.0') - assert_equal ruby_path, args.executables['3.2.0'].first + assert args.executables.key?('3.2.0') + assert_equal ruby_path, args.executables['3.2.0'].first + end end end @@ -238,12 +234,14 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=ruby32::3.2.0;ruby33::3.3.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=ruby32::3.2.0;ruby33::3.3.0 --yjit']) - assert_equal 2, args.executables.size - assert_equal ruby_path_32, args.executables['ruby32'].first - assert_equal ruby_path_33, args.executables['ruby33'].first - assert_equal '--yjit', args.executables['ruby33'].last + assert_equal 2, args.executables.size + assert_equal ruby_path_32, args.executables['ruby32'].first + assert_equal ruby_path_33, args.executables['ruby33'].first + assert_equal '--yjit', args.executables['ruby33'].last + end end end end @@ -589,10 +587,12 @@ def setup_mock_ruby(path) parser = ArgumentParser.new(ruby_executable: mock_ruby) parser.stub :have_yjit?, true do - args = parser.parse(['--chruby=test::3.2.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=test::3.2.0']) - assert_equal 1, args.executables.size - assert_equal ruby_path, args.executables['test'].first + assert_equal 1, args.executables.size + assert_equal ruby_path, args.executables['test'].first + end end end end From fe5a63c1ae65a2c07cf7b764cc1c07ab58157792 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 14:35:18 +0000 Subject: [PATCH 4/6] Remove old knucleotide ractor benchmark. I don't think there is value in keeping this around now that we have a Ractor enabled version that has feature parity with the old Process based one --- benchmarks-ractor/knucleotide/benchmark.rb | 66 ---------------------- benchmarks.yml | 2 - 2 files changed, 68 deletions(-) delete mode 100644 benchmarks-ractor/knucleotide/benchmark.rb diff --git a/benchmarks-ractor/knucleotide/benchmark.rb b/benchmarks-ractor/knucleotide/benchmark.rb deleted file mode 100644 index ca5b9cd9..00000000 --- a/benchmarks-ractor/knucleotide/benchmark.rb +++ /dev/null @@ -1,66 +0,0 @@ -# The Computer Language Benchmarks Game -# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ -# -# k-nucleotide benchmark - Ractor implementation -# Mirrors the Process.fork version structure as closely as possible - -require_relative "../../harness/loader" - -def frequency(seq, length) - frequencies = Hash.new(0) - last_index = seq.length - length - - i = 0 - while i <= last_index - frequencies[seq.byteslice(i, length)] += 1 - i += 1 - end - - [seq.length - length + 1, frequencies] -end - -def sort_by_freq(seq, length) - n, table = frequency(seq, length) - - table.sort { |a, b| - cmp = b[1] <=> a[1] - cmp == 0 ? a[0] <=> b[0] : cmp - }.map! { |seq, count| - "#{seq} #{'%.3f' % ((count * 100.0) / n)}" - }.join("\n") << "\n\n" -end - -def find_seq(seq, s) - _, table = frequency(seq, s.length) - "#{table[s] || 0}\t#{s}\n" -end - -def generate_test_sequence(size) - alu = "GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGATCACCTGAGGTCA" + - "GGAGTTCGAGACCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAATACAAAAATTAGCCGGGCGTGG" + - "TGGCGCGCGCCTGTAATCCCAGCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGGAGGCGGAGGTT" + - "GCAGTGAGCCGAGATCGCGCCACTGCACTCCAGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA" - - sequence = "" - full_copies = size / alu.length - remainder = size % alu.length - - full_copies.times { sequence << alu } - sequence << alu[0, remainder] if remainder > 0 - - sequence.upcase.freeze -end - -# Make sequence shareable for Ractors -TEST_SEQUENCE = make_shareable(generate_test_sequence(100_000)) - -run_benchmark(5) do |num_ractors, ractor_args| - freqs = [1, 2] - nucleos = %w(GGT GGTA GGTATT GGTATTTTAATT GGTATTTTAATTTATAGT) - - # Sequential version - mirrors Process version but without Workers - results = [] - freqs.each { |i| results << sort_by_freq(TEST_SEQUENCE, i) } - nucleos.each { |s| results << find_seq(TEST_SEQUENCE, s) } - results -end diff --git a/benchmarks.yml b/benchmarks.yml index 6041618c..d020dc3b 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -236,8 +236,6 @@ throw: # # Ractor-only benchmarks # -ractor/knucleotide: - desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables. Counts groups in parallel using Ractors. ractor/gvl_release_acquire: desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors. ractor/json_parse_float: From 06d34e76ebfe6359543224b3cdf90f798222a014 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 14:38:28 +0000 Subject: [PATCH 5/6] Restructure the category selection For benchmarks like the knucleotide ractor benchmark, which use Ractors internally so need to be included in the ractor benchmarks, but which must not use the ractor harness New category selection is as follows: - No category: Runs all benchmarks except those with `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to the `default` harness. - `--category=ractor`: Runs benchmarks with ractor: true or `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to `harness-ractor`. - `--category=ractor-only`: Runs only benchmarks with `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to `harness-ractor`. --- benchmarks.yml | 21 ++- .../gvl_release_acquire/benchmark.rb | 0 .../json_parse_float/Gemfile | 0 .../json_parse_float/Gemfile.lock | 0 .../json_parse_float/benchmark.rb | 0 .../json_parse_string/Gemfile | 0 .../json_parse_string/Gemfile.lock | 0 .../json_parse_string/benchmark.rb | 0 burn_in.rb | 36 ++--- lib/argument_parser.rb | 3 + lib/benchmark_filter.rb | 14 +- lib/benchmark_runner/cli.rb | 1 + lib/benchmark_suite.rb | 81 ++++------- test/benchmark_filter_test.rb | 17 +++ test/benchmark_suite_test.rb | 137 +++++++++++------- 15 files changed, 183 insertions(+), 127 deletions(-) rename {benchmarks-ractor => benchmarks}/gvl_release_acquire/benchmark.rb (100%) rename {benchmarks-ractor => benchmarks}/json_parse_float/Gemfile (100%) rename {benchmarks-ractor => benchmarks}/json_parse_float/Gemfile.lock (100%) rename {benchmarks-ractor => benchmarks}/json_parse_float/benchmark.rb (100%) rename {benchmarks-ractor => benchmarks}/json_parse_string/Gemfile (100%) rename {benchmarks-ractor => benchmarks}/json_parse_string/Gemfile.lock (100%) rename {benchmarks-ractor => benchmarks}/json_parse_string/benchmark.rb (100%) diff --git a/benchmarks.yml b/benchmarks.yml index d020dc3b..5b61fd2f 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -93,7 +93,9 @@ knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork knucleotide-ractor: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors - no_pinning: true + ractor: true + ractor_only: true + default_harness: harness lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: @@ -234,11 +236,20 @@ throw: ractor: true # -# Ractor-only benchmarks +# Ractor scaling benchmarks # -ractor/gvl_release_acquire: +gvl_release_acquire: desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors. -ractor/json_parse_float: + ractor: true + ractor_only: true + default_harness: harness-ractor +json_parse_float: desc: test the performance of parsing multiple lists of json floats with ractors. -ractor/json_parse_string: + ractor: true + ractor_only: true + default_harness: harness-ractor +json_parse_string: desc: test the performance of parsing multiple lists of strings with ractors. + ractor: true + ractor_only: true + default_harness: harness-ractor diff --git a/benchmarks-ractor/gvl_release_acquire/benchmark.rb b/benchmarks/gvl_release_acquire/benchmark.rb similarity index 100% rename from benchmarks-ractor/gvl_release_acquire/benchmark.rb rename to benchmarks/gvl_release_acquire/benchmark.rb diff --git a/benchmarks-ractor/json_parse_float/Gemfile b/benchmarks/json_parse_float/Gemfile similarity index 100% rename from benchmarks-ractor/json_parse_float/Gemfile rename to benchmarks/json_parse_float/Gemfile diff --git a/benchmarks-ractor/json_parse_float/Gemfile.lock b/benchmarks/json_parse_float/Gemfile.lock similarity index 100% rename from benchmarks-ractor/json_parse_float/Gemfile.lock rename to benchmarks/json_parse_float/Gemfile.lock diff --git a/benchmarks-ractor/json_parse_float/benchmark.rb b/benchmarks/json_parse_float/benchmark.rb similarity index 100% rename from benchmarks-ractor/json_parse_float/benchmark.rb rename to benchmarks/json_parse_float/benchmark.rb diff --git a/benchmarks-ractor/json_parse_string/Gemfile b/benchmarks/json_parse_string/Gemfile similarity index 100% rename from benchmarks-ractor/json_parse_string/Gemfile rename to benchmarks/json_parse_string/Gemfile diff --git a/benchmarks-ractor/json_parse_string/Gemfile.lock b/benchmarks/json_parse_string/Gemfile.lock similarity index 100% rename from benchmarks-ractor/json_parse_string/Gemfile.lock rename to benchmarks/json_parse_string/Gemfile.lock diff --git a/benchmarks-ractor/json_parse_string/benchmark.rb b/benchmarks/json_parse_string/benchmark.rb similarity index 100% rename from benchmarks-ractor/json_parse_string/benchmark.rb rename to benchmarks/json_parse_string/benchmark.rb diff --git a/burn_in.rb b/burn_in.rb index ea305eb8..91b1ada1 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -59,14 +59,10 @@ def free_file_path(parent_dir, name_prefix) end end -def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) - # Determine the path to the benchmark script - bench_name = bench_id.sub('ractor/', '') - bench_dir, harness = if bench_name == bench_id - ['benchmarks', 'harness'] - else - ['benchmarks-ractor', 'harness-ractor'] - end +def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) + bench_dir = 'benchmarks' + entry = metadata[bench_name] || {} + harness = entry.fetch('default_harness', 'harness') script_path = File.join(bench_dir, bench_name, 'benchmark.rb') if not File.exist?(script_path) @@ -153,12 +149,12 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) return false end -def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) +def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version, metadata) error_found = false while true bench_name = bench_names.sample() - error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version) + error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) error_found ||= error if error_found @@ -201,12 +197,16 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) bench_names = [] if args.categories.include?('ractor-only') - # Only include benchmarks with ractor/ prefix (from benchmarks-ractor directory) - bench_names = metadata.keys.select { |name| name.start_with?('ractor/') } + # Include only benchmarks with ractor_only: true + metadata.each do |name, entry| + if entry['ractor_only'] + bench_names << name + end + end elsif args.categories.include?('ractor') - # Include both ractor/ prefixed benchmarks and those with ractor: true + # Include benchmarks with ractor: true or ractor_only: true metadata.each do |name, entry| - if name.start_with?('ractor/') || entry['ractor'] + if entry['ractor'] || entry['ractor_only'] bench_names << name end end @@ -221,10 +221,12 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) end end else - # Regular category filtering + # Regular category filtering - exclude ractor-only and ractor harness benchmarks metadata.each do |name, entry| category = entry.fetch('category', 'other') - if args.categories.include?(category) + is_ractor_only = entry['ractor_only'] || + (entry['ractor'] && entry['default_harness'] == 'harness-ractor') + if args.categories.include?(category) && !is_ractor_only bench_names << name end end @@ -237,7 +239,7 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) args.num_procs.times do |i| pid = Process.fork do run_time = (i < args.num_long_runs)? (3600 * 2):10 - test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version) + test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version, metadata) end end diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 6e73e338..e7a4cf4e 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -8,6 +8,7 @@ class ArgumentParser :out_path, :out_override, :harness, + :harness_explicit, :yjit_opts, :categories, :name_filters, @@ -93,6 +94,7 @@ def parse(argv) opts.on("--harness=HARNESS_DIR", "which harness to use") do |v| v = "harness-#{v}" unless v.start_with?('harness') args.harness = v + args.harness_explicit = true end opts.on("--warmup=N", "the number of warmup iterations for the default harness (default: 15)") do |n| @@ -188,6 +190,7 @@ def default_args out_path: File.expand_path("./data"), out_override: nil, harness: "harness", + harness_explicit: false, yjit_opts: "", categories: [], name_filters: [], diff --git a/lib/benchmark_filter.rb b/lib/benchmark_filter.rb index 3219d711..a9ed5586 100644 --- a/lib/benchmark_filter.rb +++ b/lib/benchmark_filter.rb @@ -18,12 +18,21 @@ def match?(name) private def matches_category?(name) - return true if @categories.empty? + if @categories.empty? + return false if ractor_harness_benchmark?(name) + return true + end benchmark_categories = get_benchmark_categories(name) @categories.intersect?(benchmark_categories) end + def ractor_harness_benchmark?(name) + benchmark_metadata = @metadata[name] || {} + benchmark_metadata['ractor_only'] || + (benchmark_metadata['ractor'] && benchmark_metadata['default_harness'] == 'harness-ractor') + end + def matches_name_filter?(name) return true if @name_filters.empty? @@ -58,7 +67,8 @@ def get_benchmark_categories(name) @category_cache[name] ||= begin benchmark_metadata = @metadata[name] || {} categories = [benchmark_metadata.fetch('category', 'other')] - categories << 'ractor' if benchmark_metadata['ractor'] + categories << 'ractor' if benchmark_metadata['ractor'] || benchmark_metadata['ractor_only'] + categories << 'ractor-only' if benchmark_metadata['ractor_only'] categories end end diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index 639c2014..d4c011c2 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -34,6 +34,7 @@ def run excludes: args.excludes, out_path: args.out_path, harness: args.harness, + harness_explicit: args.harness_explicit, pre_init: args.with_pre_init, no_pinning: args.no_pinning, force_pinning: args.force_pinning diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index ccc98459..fd12eddc 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -14,25 +14,23 @@ # BenchmarkSuite runs a collection of benchmarks and collects their results class BenchmarkSuite BENCHMARKS_DIR = "benchmarks" - RACTOR_BENCHMARKS_DIR = "benchmarks-ractor" - RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_CATEGORY = ["ractor"].freeze + RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :force_pinning, :bench_dir, :ractor_bench_dir + attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :harness_explicit, :pre_init, :no_pinning, :force_pinning, :bench_dir - def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false, force_pinning: false) + def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, harness_explicit: false, pre_init: nil, no_pinning: false, force_pinning: false) @categories = categories @name_filters = name_filters @excludes = excludes @out_path = out_path @harness = harness + @harness_explicit = harness_explicit @pre_init = pre_init ? expand_pre_init(pre_init) : nil @no_pinning = no_pinning @force_pinning = force_pinning - @ractor_only = (categories == RACTOR_ONLY_CATEGORY) - - setup_benchmark_directories + @bench_dir = BENCHMARKS_DIR end # Run all the benchmarks and record execution times @@ -49,7 +47,7 @@ def run(ruby:, ruby_description:) result_json_path = File.join(out_path, "temp#{Process.pid}.json") cmd_prefix = base_cmd(ruby_description, entry.name) - result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env) + result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, entry.name) if result[:success] bench_data[entry.name] = process_benchmark_result(result_json_path, result[:command]) @@ -63,18 +61,6 @@ def run(ruby:, ruby_description:) private - def setup_benchmark_directories - if @ractor_only - @bench_dir = RACTOR_BENCHMARKS_DIR - @ractor_bench_dir = RACTOR_BENCHMARKS_DIR - @harness = RACTOR_HARNESS - @categories = [] - else - @bench_dir = BENCHMARKS_DIR - @ractor_bench_dir = RACTOR_BENCHMARKS_DIR - end - end - def process_benchmark_result(result_json_path, command) JSON.parse(File.read(result_json_path)).tap do |json| json["command_line"] = command @@ -89,47 +75,24 @@ def discover_benchmarks end def discover_all_benchmark_entries - main_discovery = BenchmarkDiscovery.new(bench_dir) - main_entries = main_discovery.discover - - ractor_entries = if benchmark_ractor_directory? - ractor_discovery = BenchmarkDiscovery.new(ractor_bench_dir) - ractor_discovery.discover - else - [] - end - - { main: main_entries, ractor: ractor_entries } + discovery = BenchmarkDiscovery.new(bench_dir) + { main: discovery.discover } end def build_directory_map(all_entries) - combined_entries = all_entries[:main] + all_entries[:ractor] - combined_entries.each_with_object({}) do |entry, map| + all_entries[:main].each_with_object({}) do |entry, map| map[entry.name] = entry.directory end end def filter_benchmarks(all_entries, directory_map) - main_benchmarks = filter_entries( + filter_entries( all_entries[:main], categories: categories, name_filters: name_filters, excludes: excludes, directory_map: directory_map ) - - if benchmark_ractor_directory? - ractor_benchmarks = filter_entries( - all_entries[:ractor], - categories: [], - name_filters: name_filters, - excludes: excludes, - directory_map: directory_map - ) - main_benchmarks + ractor_benchmarks - else - main_benchmarks - end end def filter_entries(entries, categories:, name_filters:, excludes:, directory_map:) @@ -143,17 +106,20 @@ def filter_entries(entries, categories:, name_filters:, excludes:, directory_map entries.select { |entry| filter.match?(entry.name) } end - def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) + def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_name) # Fix for jruby/jruby#7394 in JRuby 9.4.2.0 script_path = File.expand_path(script_path) # Set up the environment for the benchmarking command ENV["RESULT_JSON_PATH"] = result_json_path + # Use per-benchmark default_harness if set, otherwise use global harness + benchmark_harness = benchmark_harness_for(benchmark_name) + # Set up the benchmarking command cmd = cmd_prefix + [ *ruby, - "-I", harness, + "-I", benchmark_harness, *pre_init, script_path, ].compact @@ -164,6 +130,14 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) result end + def benchmark_harness_for(benchmark_name) + return harness if harness_explicit + + benchmark_meta = benchmarks_metadata[benchmark_name] || {} + default = ractor_category_run? ? RACTOR_HARNESS : harness + benchmark_meta.fetch('default_harness', default) + end + def benchmark_env(ruby) # When the Ruby running this script is not the first Ruby in PATH, shell commands # like `bundle install` in a child process will not use the Ruby being benchmarked. @@ -188,10 +162,6 @@ def benchmarks_metadata @benchmarks_metadata ||= YAML.load_file('benchmarks.yml') end - def benchmark_ractor_directory? - categories == RACTOR_CATEGORY - end - # Check if running on Linux def linux? @linux ||= RbConfig::CONFIG['host_os'] =~ /linux/ @@ -219,11 +189,16 @@ def base_cmd(ruby_description, benchmark_name) def should_pin?(benchmark_name) return false if no_pinning return true if force_pinning + return false if ractor_category_run? benchmark_meta = benchmarks_metadata[benchmark_name] || {} !benchmark_meta["no_pinning"] end + def ractor_category_run? + categories == RACTOR_CATEGORY || categories == RACTOR_ONLY_CATEGORY + end + # Generate setarch prefix for Linux def setarch_prefix # Disable address space randomization (for determinism) diff --git a/test/benchmark_filter_test.rb b/test/benchmark_filter_test.rb index 5b623d10..1abdfada 100644 --- a/test/benchmark_filter_test.rb +++ b/test/benchmark_filter_test.rb @@ -47,6 +47,23 @@ assert_equal true, filter.match?('ractor_bench') end + it 'excludes ractor harness benchmarks from default runs' do + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('fib') + assert_equal true, filter.match?('ractor_bench') # ractor: true without harness-ractor runs in default + assert_equal false, filter.match?('ractor_harness_bench') # ractor: true with harness-ractor excluded + end + + it 'includes ractor harness benchmarks in ractor category' do + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('ractor_harness_bench') + assert_equal true, filter.match?('ractor_bench') + end + it 'handles regex filters' do filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], excludes: [], metadata: @metadata) diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index cd9eee3b..91443987 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -14,7 +14,6 @@ # Create mock benchmarks directory structure FileUtils.mkdir_p('benchmarks') - FileUtils.mkdir_p('benchmarks-ractor') FileUtils.mkdir_p('harness') # Create a simple benchmark file @@ -73,7 +72,7 @@ assert_equal true, suite.no_pinning end - it 'sets bench_dir to BENCHMARKS_DIR by default' do + it 'sets bench_dir to BENCHMARKS_DIR' do suite = BenchmarkSuite.new( categories: ['micro'], name_filters: [], @@ -82,37 +81,100 @@ ) assert_equal 'benchmarks', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir assert_equal 'harness', suite.harness assert_equal ['micro'], suite.categories end - it 'sets bench_dir to ractor directory and updates harness when ractor-only category is used' do + it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do suite = BenchmarkSuite.new( - categories: ['ractor-only'], + categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness' ) - assert_equal 'benchmarks-ractor', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness-ractor', suite.harness - assert_equal [], suite.categories + assert_equal 'benchmarks', suite.bench_dir + assert_equal 'harness', suite.harness + assert_equal ['ractor'], suite.categories end - it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do + it 'tracks harness_explicit flag' do + suite_explicit = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'custom-harness', + harness_explicit: true + ) + assert_equal true, suite_explicit.harness_explicit + + suite_auto = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'harness-ractor' + ) + assert_equal false, suite_auto.harness_explicit + end + end + + describe '#benchmark_harness_for' do + before do + @metadata_with_harness = { + 'simple' => { 'category' => 'micro' }, + 'custom_harness_bench' => { 'category' => 'other', 'default_harness' => 'harness' } + } + File.write('benchmarks.yml', YAML.dump(@metadata_with_harness)) + end + + it 'returns default_harness when set and harness not explicit' do + suite = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'harness-ractor', + harness_explicit: false + ) + + assert_equal 'harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'harness-ractor', suite.send(:benchmark_harness_for, 'simple') + end + + it 'ignores default_harness when harness is explicit' do + suite = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'custom-harness', + harness_explicit: true + ) + + assert_equal 'custom-harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'custom-harness', suite.send(:benchmark_harness_for, 'simple') + end + end + + describe '#ractor_category_run?' do + it 'returns true for ractor category' do suite = BenchmarkSuite.new( categories: ['ractor'], name_filters: [], out_path: @out_path, + harness: 'harness-ractor' + ) + + assert_equal true, suite.send(:ractor_category_run?) + end + + it 'returns false for other categories' do + suite = BenchmarkSuite.new( + categories: ['micro'], + name_filters: [], + out_path: @out_path, harness: 'harness' ) - assert_equal 'benchmarks', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness', suite.harness - assert_equal ['ractor'], suite.categories + assert_equal false, suite.send(:ractor_category_run?) end end @@ -232,9 +294,9 @@ assert_empty bench_failures end - it 'handles ractor-only category' do + it 'handles ractor category with ractor benchmarks' do # Create a ractor benchmark - File.write('benchmarks-ractor/ractor_test.rb', <<~RUBY) + File.write('benchmarks/ractor_test.rb', <<~RUBY) require 'json' result = { 'warmup' => [0.001], @@ -244,8 +306,14 @@ File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) RUBY + metadata = { + 'ractor_test' => { 'category' => 'other', 'ractor' => true }, + 'simple' => { 'category' => 'micro' } + } + File.write('benchmarks.yml', YAML.dump(metadata)) + suite = BenchmarkSuite.new( - categories: ['ractor-only'], + categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', @@ -257,41 +325,10 @@ bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end - # When ractor-only is specified, it should use benchmarks-ractor directory + # Should only include ractor benchmarks when ractor category specified assert_includes bench_data, 'ractor_test' + refute_includes bench_data, 'simple' assert_empty bench_failures - - # harness should be updated to harness-ractor - assert_equal 'harness-ractor', suite.harness - end - - it 'includes both regular and ractor benchmarks with ractor category' do - File.write('benchmarks-ractor/ractor_bench.rb', <<~RUBY) - require 'json' - result = { - 'warmup' => [0.001], - 'bench' => [0.001], - 'rss' => 10485760 - } - File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) - RUBY - - suite = BenchmarkSuite.new( - categories: ['ractor'], - name_filters: [], - out_path: @out_path, - harness: 'harness', - no_pinning: true - ) - - bench_data = nil - capture_io do - bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') - end - - # With ractor category, both directories should be scanned - # but we need appropriate filters - assert_instance_of Hash, bench_data end it 'expands pre_init when provided' do From dc13f62773051d290b4ec97bbae3f5878789ea16 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 22:50:33 +0000 Subject: [PATCH 6/6] Don't run ractor benchmarks on stable ruby The API has changed between 3.4.7 and HEAD. Until 4.0 is realeased we shouldn't test 3.4.7 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7d61b29..9b77dd8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ruby, head] + ruby: [head] if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3 @@ -85,7 +85,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ruby, head] + ruby: [head] if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3