diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7d61b29..9b77dd8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ruby, head] + ruby: [head] if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3 @@ -85,7 +85,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ruby, head] + ruby: [head] if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3 diff --git a/benchmarks.yml b/benchmarks.yml index 7c0297d6..5b61fd2f 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -91,6 +91,11 @@ graphql-native: desc: GraphQL gem parsing a large file, but using a native parser knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork +knucleotide-ractor: + desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors + ractor: true + ractor_only: true + default_harness: harness lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: @@ -231,13 +236,20 @@ throw: ractor: true # -# Ractor-only benchmarks +# Ractor scaling benchmarks # -ractor/knucleotide: - desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables. Counts groups in parallel using Ractors. -ractor/gvl_release_acquire: +gvl_release_acquire: desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors. -ractor/json_parse_float: + ractor: true + ractor_only: true + default_harness: harness-ractor +json_parse_float: desc: test the performance of parsing multiple lists of json floats with ractors. -ractor/json_parse_string: + ractor: true + ractor_only: true + default_harness: harness-ractor +json_parse_string: desc: test the performance of parsing multiple lists of strings with ractors. + ractor: true + ractor_only: true + default_harness: harness-ractor diff --git a/benchmarks-ractor/gvl_release_acquire/benchmark.rb b/benchmarks/gvl_release_acquire/benchmark.rb similarity index 100% rename from benchmarks-ractor/gvl_release_acquire/benchmark.rb rename to benchmarks/gvl_release_acquire/benchmark.rb diff --git a/benchmarks-ractor/json_parse_float/Gemfile b/benchmarks/json_parse_float/Gemfile similarity index 100% rename from benchmarks-ractor/json_parse_float/Gemfile rename to benchmarks/json_parse_float/Gemfile diff --git a/benchmarks-ractor/json_parse_float/Gemfile.lock b/benchmarks/json_parse_float/Gemfile.lock similarity index 100% rename from benchmarks-ractor/json_parse_float/Gemfile.lock rename to benchmarks/json_parse_float/Gemfile.lock diff --git a/benchmarks-ractor/json_parse_float/benchmark.rb b/benchmarks/json_parse_float/benchmark.rb similarity index 100% rename from benchmarks-ractor/json_parse_float/benchmark.rb rename to benchmarks/json_parse_float/benchmark.rb diff --git a/benchmarks-ractor/json_parse_string/Gemfile b/benchmarks/json_parse_string/Gemfile similarity index 100% rename from benchmarks-ractor/json_parse_string/Gemfile rename to benchmarks/json_parse_string/Gemfile diff --git a/benchmarks-ractor/json_parse_string/Gemfile.lock b/benchmarks/json_parse_string/Gemfile.lock similarity index 100% rename from benchmarks-ractor/json_parse_string/Gemfile.lock rename to benchmarks/json_parse_string/Gemfile.lock diff --git a/benchmarks-ractor/json_parse_string/benchmark.rb b/benchmarks/json_parse_string/benchmark.rb similarity index 100% rename from benchmarks-ractor/json_parse_string/benchmark.rb rename to benchmarks/json_parse_string/benchmark.rb diff --git a/benchmarks-ractor/knucleotide/benchmark.rb b/benchmarks/knucleotide-ractor/benchmark.rb similarity index 71% rename from benchmarks-ractor/knucleotide/benchmark.rb rename to benchmarks/knucleotide-ractor/benchmark.rb index ca5b9cd9..41d265d6 100644 --- a/benchmarks-ractor/knucleotide/benchmark.rb +++ b/benchmarks/knucleotide-ractor/benchmark.rb @@ -2,9 +2,11 @@ # https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ # # k-nucleotide benchmark - Ractor implementation -# Mirrors the Process.fork version structure as closely as possible +# Mirrors the Process.fork version: spawns 7 ractors (one per task) -require_relative "../../harness/loader" +Warning[:experimental] = false + +require_relative '../../harness/loader' def frequency(seq, length) frequencies = Hash.new(0) @@ -25,9 +27,9 @@ def sort_by_freq(seq, length) table.sort { |a, b| cmp = b[1] <=> a[1] cmp == 0 ? a[0] <=> b[0] : cmp - }.map! { |seq, count| + }.map { |seq, count| "#{seq} #{'%.3f' % ((count * 100.0) / n)}" - }.join("\n") << "\n\n" + }.join("\n") + "\n\n" end def find_seq(seq, s) @@ -48,19 +50,22 @@ def generate_test_sequence(size) full_copies.times { sequence << alu } sequence << alu[0, remainder] if remainder > 0 - sequence.upcase.freeze + sequence.upcase end -# Make sequence shareable for Ractors -TEST_SEQUENCE = make_shareable(generate_test_sequence(100_000)) +TEST_SEQUENCE = Ractor.make_shareable(generate_test_sequence(100_000)) -run_benchmark(5) do |num_ractors, ractor_args| +run_benchmark(5) do freqs = [1, 2] nucleos = %w(GGT GGTA GGTATT GGTATTTTAATT GGTATTTTAATTTATAGT) - # Sequential version - mirrors Process version but without Workers - results = [] - freqs.each { |i| results << sort_by_freq(TEST_SEQUENCE, i) } - nucleos.each { |s| results << find_seq(TEST_SEQUENCE, s) } + ractors = freqs.map { |i| + Ractor.new(TEST_SEQUENCE, i) { |seq, len| sort_by_freq(seq, len) } + } + ractors += nucleos.map { |s| + Ractor.new(TEST_SEQUENCE, s) { |seq, nucleo| find_seq(seq, nucleo) } + } + + results = ractors.map(&:value) results end diff --git a/burn_in.rb b/burn_in.rb index ea305eb8..91b1ada1 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -59,14 +59,10 @@ def free_file_path(parent_dir, name_prefix) end end -def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) - # Determine the path to the benchmark script - bench_name = bench_id.sub('ractor/', '') - bench_dir, harness = if bench_name == bench_id - ['benchmarks', 'harness'] - else - ['benchmarks-ractor', 'harness-ractor'] - end +def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) + bench_dir = 'benchmarks' + entry = metadata[bench_name] || {} + harness = entry.fetch('default_harness', 'harness') script_path = File.join(bench_dir, bench_name, 'benchmark.rb') if not File.exist?(script_path) @@ -153,12 +149,12 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) return false end -def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) +def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version, metadata) error_found = false while true bench_name = bench_names.sample() - error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version) + error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) error_found ||= error if error_found @@ -201,12 +197,16 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) bench_names = [] if args.categories.include?('ractor-only') - # Only include benchmarks with ractor/ prefix (from benchmarks-ractor directory) - bench_names = metadata.keys.select { |name| name.start_with?('ractor/') } + # Include only benchmarks with ractor_only: true + metadata.each do |name, entry| + if entry['ractor_only'] + bench_names << name + end + end elsif args.categories.include?('ractor') - # Include both ractor/ prefixed benchmarks and those with ractor: true + # Include benchmarks with ractor: true or ractor_only: true metadata.each do |name, entry| - if name.start_with?('ractor/') || entry['ractor'] + if entry['ractor'] || entry['ractor_only'] bench_names << name end end @@ -221,10 +221,12 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) end end else - # Regular category filtering + # Regular category filtering - exclude ractor-only and ractor harness benchmarks metadata.each do |name, entry| category = entry.fetch('category', 'other') - if args.categories.include?(category) + is_ractor_only = entry['ractor_only'] || + (entry['ractor'] && entry['default_harness'] == 'harness-ractor') + if args.categories.include?(category) && !is_ractor_only bench_names << name end end @@ -237,7 +239,7 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) args.num_procs.times do |i| pid = Process.fork do run_time = (i < args.num_long_runs)? (3600 * 2):10 - test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version) + test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version, metadata) end end diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 35de87c1..e7a4cf4e 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -8,6 +8,7 @@ class ArgumentParser :out_path, :out_override, :harness, + :harness_explicit, :yjit_opts, :categories, :name_filters, @@ -15,6 +16,7 @@ class ArgumentParser :rss, :graph, :no_pinning, + :force_pinning, :turbo, :skip_yjit, :with_pre_init, @@ -52,10 +54,8 @@ def parse(argv) name = name.shellsplit.first end version, *options = version.shellsplit - rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies" - unless executable = ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"].find { |path| File.executable?(path) } - abort "Cannot find '#{version}' in /opt/rubies or #{rubies_dir}" - end + executable = find_chruby_ruby(version) + abort "Cannot find '#{version}' in chruby paths" unless executable args.executables[name] = [executable, *options] end end @@ -94,6 +94,7 @@ def parse(argv) opts.on("--harness=HARNESS_DIR", "which harness to use") do |v| v = "harness-#{v}" unless v.start_with?('harness') args.harness = v + args.harness_explicit = true end opts.on("--warmup=N", "the number of warmup iterations for the default harness (default: 15)") do |n| @@ -140,6 +141,10 @@ def parse(argv) args.no_pinning = true end + opts.on("--force-pinning", "force pinning even for benchmarks marked no_pinning") do + args.force_pinning = true + end + opts.on("--turbo", "don't disable CPU turbo boost") do args.turbo = true end @@ -165,6 +170,15 @@ def parse(argv) private + def find_chruby_ruby(version) + rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies" + chruby_search_paths(version, rubies_dir).find { |path| File.executable?(path) } + end + + def chruby_search_paths(version, rubies_dir) + ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"] + end + def have_yjit?(ruby) ruby_version = `#{ruby} -v --yjit 2> #{File::NULL}`.strip ruby_version.downcase.include?("yjit") @@ -176,6 +190,7 @@ def default_args out_path: File.expand_path("./data"), out_override: nil, harness: "harness", + harness_explicit: false, yjit_opts: "", categories: [], name_filters: [], @@ -183,6 +198,7 @@ def default_args rss: false, graph: false, no_pinning: false, + force_pinning: false, turbo: false, skip_yjit: false, with_pre_init: nil, diff --git a/lib/benchmark_filter.rb b/lib/benchmark_filter.rb index 3219d711..a9ed5586 100644 --- a/lib/benchmark_filter.rb +++ b/lib/benchmark_filter.rb @@ -18,12 +18,21 @@ def match?(name) private def matches_category?(name) - return true if @categories.empty? + if @categories.empty? + return false if ractor_harness_benchmark?(name) + return true + end benchmark_categories = get_benchmark_categories(name) @categories.intersect?(benchmark_categories) end + def ractor_harness_benchmark?(name) + benchmark_metadata = @metadata[name] || {} + benchmark_metadata['ractor_only'] || + (benchmark_metadata['ractor'] && benchmark_metadata['default_harness'] == 'harness-ractor') + end + def matches_name_filter?(name) return true if @name_filters.empty? @@ -58,7 +67,8 @@ def get_benchmark_categories(name) @category_cache[name] ||= begin benchmark_metadata = @metadata[name] || {} categories = [benchmark_metadata.fetch('category', 'other')] - categories << 'ractor' if benchmark_metadata['ractor'] + categories << 'ractor' if benchmark_metadata['ractor'] || benchmark_metadata['ractor_only'] + categories << 'ractor-only' if benchmark_metadata['ractor_only'] categories end end diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index 92206a48..d4c011c2 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -34,8 +34,10 @@ def run excludes: args.excludes, out_path: args.out_path, harness: args.harness, + harness_explicit: args.harness_explicit, pre_init: args.with_pre_init, - no_pinning: args.no_pinning + no_pinning: args.no_pinning, + force_pinning: args.force_pinning ) # Benchmark with and without YJIT diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 30c4a853..fd12eddc 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -14,24 +14,23 @@ # BenchmarkSuite runs a collection of benchmarks and collects their results class BenchmarkSuite BENCHMARKS_DIR = "benchmarks" - RACTOR_BENCHMARKS_DIR = "benchmarks-ractor" - RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_CATEGORY = ["ractor"].freeze + RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir + attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :harness_explicit, :pre_init, :no_pinning, :force_pinning, :bench_dir - def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false) + def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, harness_explicit: false, pre_init: nil, no_pinning: false, force_pinning: false) @categories = categories @name_filters = name_filters @excludes = excludes @out_path = out_path @harness = harness + @harness_explicit = harness_explicit @pre_init = pre_init ? expand_pre_init(pre_init) : nil @no_pinning = no_pinning - @ractor_only = (categories == RACTOR_ONLY_CATEGORY) - - setup_benchmark_directories + @force_pinning = force_pinning + @bench_dir = BENCHMARKS_DIR end # Run all the benchmarks and record execution times @@ -41,14 +40,14 @@ def run(ruby:, ruby_description:) bench_failures = {} benchmark_entries = discover_benchmarks - cmd_prefix = base_cmd(ruby_description) env = benchmark_env(ruby) benchmark_entries.each_with_index do |entry, idx| puts("Running benchmark \"#{entry.name}\" (#{idx+1}/#{benchmark_entries.length})") result_json_path = File.join(out_path, "temp#{Process.pid}.json") - result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env) + cmd_prefix = base_cmd(ruby_description, entry.name) + result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, entry.name) if result[:success] bench_data[entry.name] = process_benchmark_result(result_json_path, result[:command]) @@ -62,18 +61,6 @@ def run(ruby:, ruby_description:) private - def setup_benchmark_directories - if @ractor_only - @bench_dir = RACTOR_BENCHMARKS_DIR - @ractor_bench_dir = RACTOR_BENCHMARKS_DIR - @harness = RACTOR_HARNESS - @categories = [] - else - @bench_dir = BENCHMARKS_DIR - @ractor_bench_dir = RACTOR_BENCHMARKS_DIR - end - end - def process_benchmark_result(result_json_path, command) JSON.parse(File.read(result_json_path)).tap do |json| json["command_line"] = command @@ -88,47 +75,24 @@ def discover_benchmarks end def discover_all_benchmark_entries - main_discovery = BenchmarkDiscovery.new(bench_dir) - main_entries = main_discovery.discover - - ractor_entries = if benchmark_ractor_directory? - ractor_discovery = BenchmarkDiscovery.new(ractor_bench_dir) - ractor_discovery.discover - else - [] - end - - { main: main_entries, ractor: ractor_entries } + discovery = BenchmarkDiscovery.new(bench_dir) + { main: discovery.discover } end def build_directory_map(all_entries) - combined_entries = all_entries[:main] + all_entries[:ractor] - combined_entries.each_with_object({}) do |entry, map| + all_entries[:main].each_with_object({}) do |entry, map| map[entry.name] = entry.directory end end def filter_benchmarks(all_entries, directory_map) - main_benchmarks = filter_entries( + filter_entries( all_entries[:main], categories: categories, name_filters: name_filters, excludes: excludes, directory_map: directory_map ) - - if benchmark_ractor_directory? - ractor_benchmarks = filter_entries( - all_entries[:ractor], - categories: [], - name_filters: name_filters, - excludes: excludes, - directory_map: directory_map - ) - main_benchmarks + ractor_benchmarks - else - main_benchmarks - end end def filter_entries(entries, categories:, name_filters:, excludes:, directory_map:) @@ -142,17 +106,20 @@ def filter_entries(entries, categories:, name_filters:, excludes:, directory_map entries.select { |entry| filter.match?(entry.name) } end - def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) + def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_name) # Fix for jruby/jruby#7394 in JRuby 9.4.2.0 script_path = File.expand_path(script_path) # Set up the environment for the benchmarking command ENV["RESULT_JSON_PATH"] = result_json_path + # Use per-benchmark default_harness if set, otherwise use global harness + benchmark_harness = benchmark_harness_for(benchmark_name) + # Set up the benchmarking command cmd = cmd_prefix + [ *ruby, - "-I", harness, + "-I", benchmark_harness, *pre_init, script_path, ].compact @@ -163,6 +130,14 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) result end + def benchmark_harness_for(benchmark_name) + return harness if harness_explicit + + benchmark_meta = benchmarks_metadata[benchmark_name] || {} + default = ractor_category_run? ? RACTOR_HARNESS : harness + benchmark_meta.fetch('default_harness', default) + end + def benchmark_env(ruby) # When the Ruby running this script is not the first Ruby in PATH, shell commands # like `bundle install` in a child process will not use the Ruby being benchmarked. @@ -187,23 +162,19 @@ def benchmarks_metadata @benchmarks_metadata ||= YAML.load_file('benchmarks.yml') end - def benchmark_ractor_directory? - categories == RACTOR_CATEGORY - end - # Check if running on Linux def linux? @linux ||= RbConfig::CONFIG['host_os'] =~ /linux/ end # Set up the base command with CPU pinning if needed - def base_cmd(ruby_description) + def base_cmd(ruby_description, benchmark_name) if linux? cmd = setarch_prefix # Pin the process to one given core to improve caching and reduce variance on CRuby # Other Rubies need to use multiple cores, e.g., for JIT threads - if ruby_description.start_with?('ruby ') && !no_pinning + if ruby_description.start_with?('ruby ') && should_pin?(benchmark_name) # The last few cores of Intel CPU may be slow E-Cores, so avoid using the last one. cpu = [(Etc.nprocessors / 2) - 1, 0].max cmd.concat(["taskset", "-c", "#{cpu}"]) @@ -215,6 +186,19 @@ def base_cmd(ruby_description) end end + def should_pin?(benchmark_name) + return false if no_pinning + return true if force_pinning + return false if ractor_category_run? + + benchmark_meta = benchmarks_metadata[benchmark_name] || {} + !benchmark_meta["no_pinning"] + end + + def ractor_category_run? + categories == RACTOR_CATEGORY || categories == RACTOR_ONLY_CATEGORY + end + # Generate setarch prefix for Linux def setarch_prefix # Disable address space randomization (for determinism) diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 7008dcef..d19ae240 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -103,15 +103,11 @@ def setup_mock_ruby(path) ruby_path = File.join(tmpdir, 'opt/rubies/3.2.0/bin/ruby') setup_mock_ruby(ruby_path) - File.stub :executable?, ->(path) { - if path == "/opt/rubies/3.2.0/bin/ruby" - File.exist?(ruby_path) && File.stat(ruby_path).executable? - end - } do - parser = ArgumentParser.new + parser = ArgumentParser.new + parser.stub :chruby_search_paths, ->(version, rubies_dir) { [ruby_path] } do args = parser.parse(['--chruby=ruby-3.2.0::3.2.0']) - assert_equal '/opt/rubies/3.2.0/bin/ruby', args.executables['ruby-3.2.0'].first + assert_equal ruby_path, args.executables['ruby-3.2.0'].first end end end @@ -126,9 +122,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=my-ruby::3.3.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=my-ruby::3.3.0']) - assert_equal ruby_path, args.executables['my-ruby'].first + assert_equal ruby_path, args.executables['my-ruby'].first + end end end @@ -143,20 +141,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir - File.stub :executable?, ->(path) { - case path - when "/opt/rubies/3.2.0/bin/ruby" - File.exist?(opt_ruby) && File.stat(opt_ruby).executable? - when "#{tmpdir}/.rubies/3.2.0/bin/ruby" - File.exist?(home_ruby) && File.stat(home_ruby).executable? - else - File.method(:executable?).super_method.call(path) - end - } do - parser = ArgumentParser.new + parser = ArgumentParser.new + parser.stub :chruby_search_paths, ->(version, rd) { [opt_ruby, home_ruby] } do args = parser.parse(['--chruby=test::3.2.0']) - assert_equal '/opt/rubies/3.2.0/bin/ruby', args.executables['test'].first + assert_equal opt_ruby, args.executables['test'].first end end end @@ -171,9 +160,11 @@ def setup_mock_ruby(path) ENV['RUBIES_DIR'] = custom_rubies parser = ArgumentParser.new - args = parser.parse(['--chruby=custom::3.4.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=custom::3.4.0']) - assert_equal ruby_path, args.executables['custom'].first + assert_equal ruby_path, args.executables['custom'].first + end end end @@ -183,10 +174,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - - assert_raises(SystemExit) do - capture_io do - parser.parse(['--chruby=nonexistent::nonexistent-version-999']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + assert_raises(SystemExit) do + capture_io do + parser.parse(['--chruby=nonexistent::nonexistent-version-999']) + end end end end @@ -202,10 +194,12 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=yjit::3.2.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=yjit::3.2.0 --yjit']) - assert_equal ruby_path, args.executables['yjit'].first - assert_equal '--yjit', args.executables['yjit'].last + assert_equal ruby_path, args.executables['yjit'].first + assert_equal '--yjit', args.executables['yjit'].last + end end end @@ -219,10 +213,12 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=3.2.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=3.2.0 --yjit']) - assert args.executables.key?('3.2.0') - assert_equal ruby_path, args.executables['3.2.0'].first + assert args.executables.key?('3.2.0') + assert_equal ruby_path, args.executables['3.2.0'].first + end end end @@ -238,12 +234,14 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=ruby32::3.2.0;ruby33::3.3.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=ruby32::3.2.0;ruby33::3.3.0 --yjit']) - assert_equal 2, args.executables.size - assert_equal ruby_path_32, args.executables['ruby32'].first - assert_equal ruby_path_33, args.executables['ruby33'].first - assert_equal '--yjit', args.executables['ruby33'].last + assert_equal 2, args.executables.size + assert_equal ruby_path_32, args.executables['ruby32'].first + assert_equal ruby_path_33, args.executables['ruby33'].first + assert_equal '--yjit', args.executables['ruby33'].last + end end end end @@ -589,10 +587,12 @@ def setup_mock_ruby(path) parser = ArgumentParser.new(ruby_executable: mock_ruby) parser.stub :have_yjit?, true do - args = parser.parse(['--chruby=test::3.2.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=test::3.2.0']) - assert_equal 1, args.executables.size - assert_equal ruby_path, args.executables['test'].first + assert_equal 1, args.executables.size + assert_equal ruby_path, args.executables['test'].first + end end end end diff --git a/test/benchmark_filter_test.rb b/test/benchmark_filter_test.rb index 5b623d10..1abdfada 100644 --- a/test/benchmark_filter_test.rb +++ b/test/benchmark_filter_test.rb @@ -47,6 +47,23 @@ assert_equal true, filter.match?('ractor_bench') end + it 'excludes ractor harness benchmarks from default runs' do + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('fib') + assert_equal true, filter.match?('ractor_bench') # ractor: true without harness-ractor runs in default + assert_equal false, filter.match?('ractor_harness_bench') # ractor: true with harness-ractor excluded + end + + it 'includes ractor harness benchmarks in ractor category' do + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('ractor_harness_bench') + assert_equal true, filter.match?('ractor_bench') + end + it 'handles regex filters' do filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], excludes: [], metadata: @metadata) diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index cd9eee3b..91443987 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -14,7 +14,6 @@ # Create mock benchmarks directory structure FileUtils.mkdir_p('benchmarks') - FileUtils.mkdir_p('benchmarks-ractor') FileUtils.mkdir_p('harness') # Create a simple benchmark file @@ -73,7 +72,7 @@ assert_equal true, suite.no_pinning end - it 'sets bench_dir to BENCHMARKS_DIR by default' do + it 'sets bench_dir to BENCHMARKS_DIR' do suite = BenchmarkSuite.new( categories: ['micro'], name_filters: [], @@ -82,37 +81,100 @@ ) assert_equal 'benchmarks', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir assert_equal 'harness', suite.harness assert_equal ['micro'], suite.categories end - it 'sets bench_dir to ractor directory and updates harness when ractor-only category is used' do + it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do suite = BenchmarkSuite.new( - categories: ['ractor-only'], + categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness' ) - assert_equal 'benchmarks-ractor', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness-ractor', suite.harness - assert_equal [], suite.categories + assert_equal 'benchmarks', suite.bench_dir + assert_equal 'harness', suite.harness + assert_equal ['ractor'], suite.categories end - it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do + it 'tracks harness_explicit flag' do + suite_explicit = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'custom-harness', + harness_explicit: true + ) + assert_equal true, suite_explicit.harness_explicit + + suite_auto = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'harness-ractor' + ) + assert_equal false, suite_auto.harness_explicit + end + end + + describe '#benchmark_harness_for' do + before do + @metadata_with_harness = { + 'simple' => { 'category' => 'micro' }, + 'custom_harness_bench' => { 'category' => 'other', 'default_harness' => 'harness' } + } + File.write('benchmarks.yml', YAML.dump(@metadata_with_harness)) + end + + it 'returns default_harness when set and harness not explicit' do + suite = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'harness-ractor', + harness_explicit: false + ) + + assert_equal 'harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'harness-ractor', suite.send(:benchmark_harness_for, 'simple') + end + + it 'ignores default_harness when harness is explicit' do + suite = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'custom-harness', + harness_explicit: true + ) + + assert_equal 'custom-harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'custom-harness', suite.send(:benchmark_harness_for, 'simple') + end + end + + describe '#ractor_category_run?' do + it 'returns true for ractor category' do suite = BenchmarkSuite.new( categories: ['ractor'], name_filters: [], out_path: @out_path, + harness: 'harness-ractor' + ) + + assert_equal true, suite.send(:ractor_category_run?) + end + + it 'returns false for other categories' do + suite = BenchmarkSuite.new( + categories: ['micro'], + name_filters: [], + out_path: @out_path, harness: 'harness' ) - assert_equal 'benchmarks', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness', suite.harness - assert_equal ['ractor'], suite.categories + assert_equal false, suite.send(:ractor_category_run?) end end @@ -232,9 +294,9 @@ assert_empty bench_failures end - it 'handles ractor-only category' do + it 'handles ractor category with ractor benchmarks' do # Create a ractor benchmark - File.write('benchmarks-ractor/ractor_test.rb', <<~RUBY) + File.write('benchmarks/ractor_test.rb', <<~RUBY) require 'json' result = { 'warmup' => [0.001], @@ -244,8 +306,14 @@ File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) RUBY + metadata = { + 'ractor_test' => { 'category' => 'other', 'ractor' => true }, + 'simple' => { 'category' => 'micro' } + } + File.write('benchmarks.yml', YAML.dump(metadata)) + suite = BenchmarkSuite.new( - categories: ['ractor-only'], + categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', @@ -257,41 +325,10 @@ bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end - # When ractor-only is specified, it should use benchmarks-ractor directory + # Should only include ractor benchmarks when ractor category specified assert_includes bench_data, 'ractor_test' + refute_includes bench_data, 'simple' assert_empty bench_failures - - # harness should be updated to harness-ractor - assert_equal 'harness-ractor', suite.harness - end - - it 'includes both regular and ractor benchmarks with ractor category' do - File.write('benchmarks-ractor/ractor_bench.rb', <<~RUBY) - require 'json' - result = { - 'warmup' => [0.001], - 'bench' => [0.001], - 'rss' => 10485760 - } - File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) - RUBY - - suite = BenchmarkSuite.new( - categories: ['ractor'], - name_filters: [], - out_path: @out_path, - harness: 'harness', - no_pinning: true - ) - - bench_data = nil - capture_io do - bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') - end - - # With ractor category, both directories should be scanned - # but we need appropriate filters - assert_instance_of Hash, bench_data end it 'expands pre_init when provided' do