Add benchmark SVG report in CI

.github/workflows/ci.yml

@@ -37,8 +37,43 @@ jobs:
         uses: extractions/setup-just@v1
       - name: Build extension
         run: just build-extension
-      - name: Run benchmark
-        run: just bench
+      - name: Run benchmarks and generate report
+        run: just bench heavy_work reports/benchmark_report.svg
+      - name: Post or update PR comment
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const path = 'reports/benchmark_report.svg';
+            let content = fs.readFileSync(path, 'utf8');
+            content = '<details>\n<summary>Benchmark Report</summary>\n\n' +
+                      content + '\n</details>';
+            const issue_number = context.payload.pull_request.number;
+            const header = '<!-- benchmark-report -->\n';
+            const comments = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo:  context.repo.repo,
+              issue_number
+            });
+            const existing = comments.data.find(c => c.body.startsWith(header));
+            const body = header + content;
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner:      context.repo.owner,
+                repo:       context.repo.repo,
+                comment_id: existing.id,
+                body
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner:        context.repo.owner,
+                repo:         context.repo.repo,
+                issue_number,
+                body
+              });
+            }
 
   nix:
     runs-on: ubuntu-latest

Justfile

@@ -3,8 +3,8 @@ alias t := test
 test:
     ruby -Itest test/test_tracer.rb
 
-bench name="heavy_work":
-    ruby test/benchmarks/run_benchmark.rb {{name}}
+bench name="heavy_work" write_report="console":
+    ruby test/benchmarks/run_benchmark.rb {{name}} --write-report={{write_report}}
 
 build-extension:
     cargo build --release --manifest-path gems/native-tracer/ext/native_tracer/Cargo.toml

MAINTANERS.md

@@ -39,7 +39,13 @@ the generated traces with the fixtures under `test/fixtures`.
 Benchmarks can be executed with:
 
 ```bash
-just bench
+just bench heavy_work
+```
+
+Passing a second argument writes a report instead of printing the runtime:
+
+```bash
+just bench heavy_work reports/bench.svg
 ```
 
 ## Publishing gems

test/benchmarks/README.md

@@ -5,3 +5,5 @@ These benchmarks are **not** executed in CI because they may take longer to run
 The reference traces are stored via Git LFS so the repository stays lightweight. `run_benchmark.rb` verifies the SHA-256 hash of each fixture and downloads it with `git lfs` on demand if missing.
 
 At the moment there is a single benchmark (`heavy_work`) that exercises a mixture of array and hash operations while computing prime numbers. More benchmarks will be added as we expand the suite.
+
+Use `run_benchmark.rb --write-report=console BENCHMARK` to execute a single benchmark and print the runtime. Passing a path ending with `.json` or `.svg` will run all benchmarks and write a report in the chosen format.

test/benchmarks/run_benchmark.rb

@@ -1,52 +1,112 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
 require 'json'
 require 'fileutils'
 require 'digest'
 require 'benchmark'
+require 'optparse'
 
-USAGE = "Usage: ruby run_benchmark.rb BENCHMARK_NAME"
-
-BENCHMARK = ARGV.shift || abort(USAGE)
 HASHES = {
   'heavy_work' => '912fc0347cb8a57abd94a7defd76b147f3a79e556745e45207b89529f8a59d8b'
-}
+}.freeze
 
-unless HASHES.key?(BENCHMARK)
-  abort("Unknown benchmark '#{BENCHMARK}'")
-end
-
-PROGRAM = File.join('test', 'benchmarks', 'programs', "#{BENCHMARK}.rb")
-FIXTURE = File.expand_path("fixtures/#{BENCHMARK}_trace.json", __dir__)
+PROGRAMS_DIR = File.expand_path('programs', __dir__)
+FIXTURES_DIR = File.expand_path('fixtures', __dir__)
 TMP_DIR = File.expand_path('tmp', __dir__)
-OUTPUT_DIR = File.join(TMP_DIR, BENCHMARK)
-EXPECTED_HASH = HASHES[BENCHMARK]
-
-FileUtils.mkdir_p(TMP_DIR)
-FileUtils.mkdir_p(OUTPUT_DIR)
 
-unless File.exist?(FIXTURE) && Digest::SHA256.file(FIXTURE).hexdigest == EXPECTED_HASH
-  warn "Reference trace missing or corrupt. Attempting to fetch via git lfs..."
-  system('git', 'lfs', 'pull', '--include', FIXTURE)
-end
-
-raise 'reference trace unavailable' unless File.exist?(FIXTURE)
-raise 'reference trace hash mismatch' unless Digest::SHA256.file(FIXTURE).hexdigest == EXPECTED_HASH
+WRITE_REPORT_DEFAULT = 'console'
 
-elapsed = Benchmark.realtime do
-  system('ruby', File.expand_path('../../gems/pure-ruby-tracer/lib/trace.rb', __dir__), '--out-dir', OUTPUT_DIR, PROGRAM)
-  raise 'trace failed' unless $?.success?
-end
-puts "Benchmark runtime: #{(elapsed * 1000).round} ms"
+options = { write_report: WRITE_REPORT_DEFAULT }
+OptionParser.new do |opts|
+  opts.banner = 'Usage: ruby run_benchmark.rb BENCHMARK_NAME [options]'
+  opts.on('--write-report=DEST', 'console or path to .json/.svg report') do |dest|
+    options[:write_report] = dest
+  end
+end.parse!
 
 def files_identical?(a, b)
   cmp_result = system('cmp', '-s', a, b)
   return $?.success? if !cmp_result.nil?
   File.binread(a) == File.binread(b)
 end
 
-OUTPUT_TRACE = File.join(OUTPUT_DIR, 'trace.json')
-if files_identical?(FIXTURE, OUTPUT_TRACE)
-  puts 'Trace matches reference.'
+def run_benchmark(name)
+  program = File.join('test', 'benchmarks', 'programs', "#{name}.rb")
+  fixture = File.expand_path("fixtures/#{name}_trace.json", __dir__)
+  output_dir = File.join(TMP_DIR, name)
+  expected_hash = HASHES[name]
+
+  FileUtils.mkdir_p(TMP_DIR)
+  FileUtils.mkdir_p(output_dir)
+
+  unless File.exist?(fixture) && Digest::SHA256.file(fixture).hexdigest == expected_hash
+    warn 'Reference trace missing or corrupt. Attempting to fetch via git lfs...'
+    system('git', 'lfs', 'pull', '--include', fixture)
+  end
+
+  raise 'reference trace unavailable' unless File.exist?(fixture)
+  raise 'reference trace hash mismatch' unless Digest::SHA256.file(fixture).hexdigest == expected_hash
+
+  elapsed = Benchmark.realtime do
+    system('ruby', File.expand_path('../../gems/pure-ruby-tracer/lib/trace.rb', __dir__), '--out-dir', output_dir, program)
+    raise 'trace failed' unless $?.success?
+  end
+  runtime_ms = (elapsed * 1000).round
+
+  output_trace = File.join(output_dir, 'trace.json')
+  success = files_identical?(fixture, output_trace)
+  size_bytes = File.size(output_trace)
+
+  { name: name, runtime_ms: runtime_ms, trace_size: size_bytes, success: success }
+end
+
+if options[:write_report] == 'console'
+  bench = ARGV.shift || abort('Usage: ruby run_benchmark.rb BENCHMARK_NAME [options]')
+  abort("Unknown benchmark '#{bench}'") unless HASHES.key?(bench)
+  result = run_benchmark(bench)
+  puts "Benchmark runtime: #{result[:runtime_ms]} ms"
+  if result[:success]
+    puts 'Trace matches reference.'
+  else
+    warn 'Trace differs from reference!'
+    exit 1
+  end
 else
-  warn 'Trace differs from reference!'
-  exit 1
+  benches = ARGV.empty? ? HASHES.keys.sort : ARGV
+  benches.each { |b| abort("Unknown benchmark '#{b}'") unless HASHES.key?(b) }
+  results = benches.map { |b| run_benchmark(b) }
+
+  dest = options[:write_report]
+  FileUtils.mkdir_p(File.dirname(dest))
+  case File.extname(dest)
+  when '.json'
+    data = results.map { |r| { benchmark: r[:name], runtime_ms: r[:runtime_ms], trace_bytes: r[:trace_size] } }
+    File.write(dest, JSON.pretty_generate(data))
+  when '.svg'
+    row_height = 25
+    height = 40 + row_height * results.size
+    svg = +"<svg xmlns='http://www.w3.org/2000/svg' width='500' height='#{height}'>\n"
+    svg << "  <foreignObject width='100%' height='100%'>\n"
+    svg << "    <style>table{border-collapse:collapse;font-family:sans-serif;}td,th{border:1px solid #999;padding:4px;}</style>\n"
+    svg << "    <table>\n"
+    svg << "      <thead><tr><th>Benchmark</th><th>Runtime (ms)</th><th>Trace size (bytes)</th></tr></thead>\n"
+    svg << "      <tbody>\n"
+    results.each do |r|
+      svg << "        <tr><td>#{r[:name]}</td><td>#{r[:runtime_ms]}</td><td>#{r[:trace_size]}</td></tr>\n"
+    end
+    svg << "      </tbody>\n"
+    svg << "    </table>\n"
+    svg << "  </foreignObject>\n"
+    svg << "</svg>\n"
+    File.write(dest, svg)
+  else
+    abort "Unknown report format '#{dest}'"
+  end
+
+  unless results.all? { |r| r[:success] }
+    warn 'One or more traces differ from reference!'
+    exit 1
+  end
 end
+