Use JSON output for bin/benchmark for CI

Author: lautis

.github/workflows/benchmark.yml

@@ -24,97 +24,93 @@ jobs:
 
       - name: Run benchmarks with YJIT
         run: |
-          bundle exec ruby --yjit bin/benchmark | tee benchmark_output.txt
+          bundle exec ruby --yjit bin/benchmark --json > benchmark_results.json
 
-      - name: Parse and format results
+      - name: Format results as markdown
         run: |
-          cat > parse_benchmark.rb << 'EOF'
+          cat > format_results.rb << 'EOF'
           #!/usr/bin/env ruby
+          require 'json'
           
-          input = File.read('benchmark_output.txt')
+          data = JSON.parse(File.read('benchmark_results.json'))
           
-          def parse_section(text, title)
-            lines = text.lines.map(&:chomp)
-            results = []
-            comparisons = []
-            in_comparison = false
-            in_calculating = false
-            
-            lines.each do |line|
-              # Start of calculation section
-              if line =~ /^Calculating/
-                in_calculating = true
-                next
-              end
-              
-              # Match result lines after "Calculating" section
-              # Format: "            Ruby CSV    463.779 (± 2.4%) i/s    (2.16 ms/i) - ..."
-              if in_calculating && line =~ /^\s*(.+?)\s\s+(\d+\.?\d*)\s+\(.+?\)\s+i\/s/
-                name = $1.strip
-                ips = $2
-                results << { name: name, ips: ips }
-              elsif line =~ /^Comparison:/
-                in_comparison = true
-                in_calculating = false
-              elsif in_comparison && line.strip =~ /^(.+?):\s+(.+)/
-                comparisons << "- **#{$1.strip}**: #{$2.strip}"
-              end
-            end
-            
-            return nil if results.empty?
+          output = "## 🚀 Benchmark Results\n\n"
+          output += "_Benchmarks run with YJIT enabled on Ruby #{data['ruby_version']}_\n\n"
+          
+          # Writing benchmark
+          if data['benchmarks']['writing']
+            output += "### 📝 CSV Writing Performance\n\n"
+            output += "| Library | Iterations/sec | Std Dev |\n"
+            output += "|---------|----------------|----------|\n"
             
-            # Build markdown table
-            table = "### #{title}\n\n"
-            table += "| Library | Iterations/sec |\n"
-            table += "|---------|---------------|\n"
+            writing = data['benchmarks']['writing']
+            sorted_writing = writing.sort_by { |_, v| -v['ips'] }
             
-            results.each do |result|
-              table += "| #{result[:name]} | #{result[:ips]} |\n"
+            sorted_writing.each do |name, stats|
+              output += "| #{name} | #{stats['ips'].round(1)} | ±#{stats['stddev_percentage'].round(1)}% |\n"
             end
             
-            # Add comparison info
-            if !comparisons.empty?
-              table += "\n**Comparison:**\n\n"
-              table += comparisons.join("\n") + "\n"
+            # Add comparison
+            if sorted_writing.length > 1
+              fastest = sorted_writing.first
+              output += "\n**Comparison:**\n\n"
+              sorted_writing.each do |name, stats|
+                if name == fastest[0]
+                  output += "- **#{name}**: #{stats['ips'].round(1)} i/s (fastest)\n"
+                else
+                  slowdown = fastest[1]['ips'] / stats['ips']
+                  output += "- **#{name}**: #{stats['ips'].round(1)} i/s - #{slowdown.round(2)}x slower\n"
+                end
+              end
             end
-            
-            table
+            output += "\n"
           end
           
-          output = "## 🚀 Benchmark Results\n\n"
-          output += "_Benchmarks run with YJIT enabled on Ruby #{RUBY_VERSION}_\n\n"
-          
-          # Split by === markers to get sections
-          sections = input.split(/^=== /)
-          
-          sections.each do |section|
-            if section.include?("CSV Writing Benchmark")
-              result = parse_section(section, "📝 CSV Writing Performance")
-              output += result + "\n" if result
-            elsif section.include?("CSV Reading Benchmark")
-              result = parse_section(section, "📖 CSV Reading Performance")
-              output += result + "\n" if result
+          # Reading benchmark
+          if data['benchmarks']['reading']
+            output += "### 📖 CSV Reading Performance\n\n"
+            output += "| Library | Iterations/sec | Std Dev |\n"
+            output += "|---------|----------------|----------|\n"
+            
+            reading = data['benchmarks']['reading']
+            sorted_reading = reading.sort_by { |_, v| -v['ips'] }
+            
+            sorted_reading.each do |name, stats|
+              output += "| #{name} | #{stats['ips'].round(1)} | ±#{stats['stddev_percentage'].round(1)}% |\n"
+            end
+            
+            # Add comparison
+            if sorted_reading.length > 1
+              fastest = sorted_reading.first
+              output += "\n**Comparison:**\n\n"
+              sorted_reading.each do |name, stats|
+                if name == fastest[0]
+                  output += "- **#{name}**: #{stats['ips'].round(1)} i/s (fastest)\n"
+                else
+                  slowdown = fastest[1]['ips'] / stats['ips']
+                  output += "- **#{name}**: #{stats['ips'].round(1)} i/s - #{slowdown.round(2)}x slower\n"
+                end
+              end
             end
+            output += "\n"
           end
           
           # Write to GitHub Step Summary
           if ENV['GITHUB_STEP_SUMMARY']
             File.write(ENV['GITHUB_STEP_SUMMARY'], output)
             puts "✅ Results written to GitHub Actions summary"
-          else
-            puts "⚠️  GITHUB_STEP_SUMMARY not available, printing to stdout:"
           end
           
           # Also print to stdout
-          puts "\n" + output
+          puts output
           EOF
           
-          ruby parse_benchmark.rb
+          ruby format_results.rb
 
-      - name: Upload benchmark output
+      - name: Upload benchmark results
         uses: actions/upload-artifact@v4
         if: always()
         with:
           name: benchmark-results
-          path: benchmark_output.txt
+          path: benchmark_results.json
           retention-days: 30

bin/benchmark

@@ -14,6 +14,11 @@ require 'rscsv'
 require 'csv'
 require 'osv'
 require 'stringio'
+require 'json'
+
+# Parse command line arguments
+json_output = ARGV.include?('--json')
+results = { ruby_version: RUBY_VERSION, yjit_enabled: defined?(RubyVM::YJIT), benchmarks: {} }
 
 rows = (0...1000).map do
   (0...10).map { SecureRandom.hex }
@@ -23,8 +28,8 @@ csv_string = CSV.generate do |csv|
   rows.each { |row| csv << row }
 end
 
-puts "\n=== CSV Writing Benchmark ==="
-Benchmark.ips do |x|
+puts "\n=== CSV Writing Benchmark ===" unless json_output
+write_report = Benchmark.ips do |x|
   x.report('Ruby CSV') do |times|
     times.times do
       CSV.generate do |csv|
@@ -37,11 +42,22 @@ Benchmark.ips do |x|
     times.times { Rscsv::Writer.generate_lines(rows) }
   end
 
-  x.compare!
+  x.compare! unless json_output
 end
 
-puts "\n=== CSV Reading Benchmark ==="
-Benchmark.ips do |x|
+if json_output
+  write_results = {}
+  write_report.entries.each do |entry|
+    write_results[entry.label] = {
+      ips: entry.stats.central_tendency,
+      stddev_percentage: entry.stats.error_percentage
+    }
+  end
+  results[:benchmarks][:writing] = write_results
+end
+
+puts "\n=== CSV Reading Benchmark ===" unless json_output
+read_report = Benchmark.ips do |x|
   x.report('Ruby CSV') do |times|
     times.times do
       CSV.parse(csv_string)
@@ -58,5 +74,21 @@ Benchmark.ips do |x|
     end
   end
 
-  x.compare!
+  x.compare! unless json_output
+end
+
+if json_output
+  read_results = {}
+  read_report.entries.each do |entry|
+    read_results[entry.label] = {
+      ips: entry.stats.central_tendency,
+      stddev_percentage: entry.stats.error_percentage
+    }
+  end
+  results[:benchmarks][:reading] = read_results
+end
+
+# Output JSON if requested
+if json_output
+  puts JSON.pretty_generate(results)
 end