|
| 1 | +#!/usr/bin/env ruby |
| 2 | +# ActiveModelSerializers Benchmark driver |
| 3 | +# Adapted from |
| 4 | +# https://github.com/ruby-bench/ruby-bench-suite/blob/8ad567f7e43a044ae48c36833218423bb1e2bd9d/rails/benchmarks/driver.rb |
| 5 | +require 'bundler' |
| 6 | +Bundler.setup |
| 7 | +require 'json' |
| 8 | +require 'pathname' |
| 9 | +require 'optparse' |
| 10 | +require 'digest' |
| 11 | +require 'pathname' |
| 12 | +require 'shellwords' |
| 13 | +require 'logger' |
| 14 | +require 'English' |
| 15 | + |
| 16 | +class BenchmarkDriver |
| 17 | + ROOT = Pathname File.expand_path(File.join('..', '..'), __FILE__) |
| 18 | + BASE = ENV.fetch('BASE') { ROOT.join('test', 'benchmark') } |
| 19 | + ESCAPED_BASE = Shellwords.shellescape(BASE) |
| 20 | + |
| 21 | + def self.benchmark(options) |
| 22 | + new(options).run |
| 23 | + end |
| 24 | + |
| 25 | + def self.parse_argv_and_run(argv = ARGV, options = {}) |
| 26 | + options = { |
| 27 | + repeat_count: 1, |
| 28 | + pattern: [], |
| 29 | + env: 'CACHE_ON=on' |
| 30 | + }.merge!(options) |
| 31 | + |
| 32 | + OptionParser.new do |opts| |
| 33 | + opts.banner = 'Usage: bin/bench [options]' |
| 34 | + |
| 35 | + opts.on('-r', '--repeat-count [NUM]', 'Run benchmarks [NUM] times taking the best result') do |value| |
| 36 | + options[:repeat_count] = value.to_i |
| 37 | + end |
| 38 | + |
| 39 | + opts.on('-p', '--pattern <PATTERN1,PATTERN2,PATTERN3>', 'Benchmark name pattern') do |value| |
| 40 | + options[:pattern] = value.split(',') |
| 41 | + end |
| 42 | + |
| 43 | + opts.on('-e', '--env <var1=val1,var2=val2,var3=vale>', 'ENV variables to pass in') do |value| |
| 44 | + options[:env] = value.split(',') |
| 45 | + end |
| 46 | + end.parse!(argv) |
| 47 | + |
| 48 | + benchmark(options) |
| 49 | + end |
| 50 | + |
| 51 | + attr_reader :commit_hash, :base |
| 52 | + |
| 53 | + # Based on logfmt: |
| 54 | + # https://www.brandur.org/logfmt |
| 55 | + # For more complete implementation see: |
| 56 | + # see https://github.com/arachnid-cb/logfmtr/blob/master/lib/logfmtr/base.rb |
| 57 | + # For usage see: |
| 58 | + # https://blog.codeship.com/logfmt-a-log-format-thats-easy-to-read-and-write/ |
| 59 | + # https://engineering.heroku.com/blogs/2014-09-05-hutils-explore-your-structured-data-logs/ |
| 60 | + # For Ruby parser see: |
| 61 | + # https://github.com/cyberdelia/logfmt-ruby |
| 62 | + def self.summary_logger(device = 'output.txt') |
| 63 | + require 'time' |
| 64 | + logger = Logger.new(device) |
| 65 | + logger.level = Logger::INFO |
| 66 | + logger.formatter = proc { |severity, datetime, progname, msg| |
| 67 | + msg = "'#{msg}'" |
| 68 | + "level=#{severity} time=#{datetime.utc.iso8601(6)} pid=#{Process.pid} progname=#{progname} msg=#{msg}#{$INPUT_RECORD_SEPARATOR}" |
| 69 | + } |
| 70 | + logger |
| 71 | + end |
| 72 | + |
| 73 | + def self.stdout_logger |
| 74 | + logger = Logger.new(STDOUT) |
| 75 | + logger.level = Logger::INFO |
| 76 | + logger.formatter = proc { |_, _, _, msg| "#{msg}#{$INPUT_RECORD_SEPARATOR}" } |
| 77 | + logger |
| 78 | + end |
| 79 | + |
| 80 | + def initialize(options) |
| 81 | + @writer = ENV['SUMMARIZE'] ? self.class.summary_logger : self.class.stdout_logger |
| 82 | + @repeat_count = options[:repeat_count] |
| 83 | + @pattern = options[:pattern] |
| 84 | + @commit_hash = options.fetch(:commit_hash) { `git rev-parse --short HEAD`.chomp } |
| 85 | + @base = options.fetch(:base) { ESCAPED_BASE } |
| 86 | + @env = Array(options[:env]).join(' ') |
| 87 | + @rubyopt = options[:rubyopt] # TODO: rename |
| 88 | + end |
| 89 | + |
| 90 | + def run |
| 91 | + files.each do |path| |
| 92 | + next if !@pattern.empty? && /#{@pattern.join('|')}/ !~ File.basename(path) |
| 93 | + run_single(Shellwords.shellescape(path)) |
| 94 | + end |
| 95 | + end |
| 96 | + |
| 97 | + private |
| 98 | + |
| 99 | + def files |
| 100 | + Dir[File.join(base, 'bm_*')] |
| 101 | + end |
| 102 | + |
| 103 | + def run_single(path) |
| 104 | + script = "RAILS_ENV=production #{@env} ruby #{@rubyopt} #{path}" |
| 105 | + environment = `ruby -v`.chomp.strip[/\d+\.\d+\.\d+\w+/] |
| 106 | + |
| 107 | + runs_output = measure(script) |
| 108 | + if runs_output.empty? |
| 109 | + results = { error: :no_results } |
| 110 | + return |
| 111 | + end |
| 112 | + |
| 113 | + results = {} |
| 114 | + results['commit_hash'] = commit_hash |
| 115 | + results['version'] = runs_output.first['version'] |
| 116 | + results['rails_version'] = runs_output.first['rails_version'] |
| 117 | + results['benchmark_run[environment]'] = environment |
| 118 | + results['runs'] = [] |
| 119 | + |
| 120 | + runs_output.each do |output| |
| 121 | + results['runs'] << { |
| 122 | + 'benchmark_type[category]' => output['label'], |
| 123 | + 'benchmark_run[result][iterations_per_second]' => output['iterations_per_second'].round(3), |
| 124 | + 'benchmark_run[result][total_allocated_objects_per_iteration]' => output['total_allocated_objects_per_iteration'] |
| 125 | + } |
| 126 | + end |
| 127 | + ensure |
| 128 | + results && report(results) |
| 129 | + end |
| 130 | + |
| 131 | + def report(results) |
| 132 | + @writer.info { 'Benchmark results:' } |
| 133 | + @writer.info { JSON.pretty_generate(results) } |
| 134 | + end |
| 135 | + |
| 136 | + def summarize(result) |
| 137 | + puts "#{result['label']} #{result['iterations_per_second']}/ips; #{result['total_allocated_objects_per_iteration']} objects" |
| 138 | + end |
| 139 | + |
| 140 | + # FIXME: ` provides the full output but it'll return failed output as well. |
| 141 | + def measure(script) |
| 142 | + results = Hash.new { |h, k| h[k] = [] } |
| 143 | + |
| 144 | + @repeat_count.times do |
| 145 | + output = sh(script) |
| 146 | + output.each_line do |line| |
| 147 | + next if line.nil? |
| 148 | + begin |
| 149 | + result = JSON.parse(line) |
| 150 | + rescue JSON::ParserError |
| 151 | + result = { error: line } # rubocop:disable Lint/UselessAssignment |
| 152 | + else |
| 153 | + summarize(result) |
| 154 | + results[result['label']] << result |
| 155 | + end |
| 156 | + end |
| 157 | + end |
| 158 | + |
| 159 | + results.map do |_, bm_runs| |
| 160 | + bm_runs.sort_by do |run| |
| 161 | + run['iterations_per_second'] |
| 162 | + end.last |
| 163 | + end |
| 164 | + end |
| 165 | + |
| 166 | + def sh(cmd) |
| 167 | + `#{cmd}` |
| 168 | + end |
| 169 | +end |
| 170 | + |
| 171 | +BenchmarkDriver.parse_argv_and_run if $PROGRAM_NAME == __FILE__ |
0 commit comments