|
| 1 | +# frozen_string_literal: true |
| 2 | + |
| 3 | +require_relative 'test_data' |
| 4 | + |
| 5 | +module Benchmark |
| 6 | + # monotonic system clock should be used for any time difference measurements |
| 7 | + def self.monotonic_milliseconds |
| 8 | + if defined?(Process::CLOCK_MONOTONIC) |
| 9 | + Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) / 1000.0 |
| 10 | + else |
| 11 | + Time.now.to_f * 1000.0 |
| 12 | + end |
| 13 | + end |
| 14 | + |
| 15 | + # benchmark a block, returning an array of times (to allow statistic computation) |
| 16 | + def self.measure_time(n=300, &block) |
| 17 | + values = Array.new(n) |
| 18 | + n.times do |i| |
| 19 | + t1 = monotonic_milliseconds |
| 20 | + block.call |
| 21 | + values[i] = monotonic_milliseconds - t1 |
| 22 | + end |
| 23 | + values |
| 24 | + end |
| 25 | + |
| 26 | + # Run a block in a fork and returns the data from it |
| 27 | + # the block must take a single argument and will be called with an empty hash |
| 28 | + # any data that should be communicated back to the parent process can be written to that hash |
| 29 | + def self.fork_run(&block) |
| 30 | + # fork is not supported in JRuby, for now, just run this in the same process |
| 31 | + # data collected will not be as useful, but still valid for relative comparisons over time |
| 32 | + if defined?(JRUBY_VERSION) |
| 33 | + h = {} |
| 34 | + block.call(h) |
| 35 | + return h |
| 36 | + end |
| 37 | + |
| 38 | + rd, wr = IO.pipe |
| 39 | + p1 = fork do |
| 40 | + h = {} |
| 41 | + block.call(h) |
| 42 | + wr.write(JSON.dump(h)) |
| 43 | + wr.close |
| 44 | + end |
| 45 | + Process.wait(p1) |
| 46 | + wr.close |
| 47 | + h = JSON.parse(rd.read, symbolize_names: true) |
| 48 | + rd.close |
| 49 | + return h |
| 50 | + end |
| 51 | + |
| 52 | + def self.host_os |
| 53 | + case RbConfig::CONFIG['host_os'] |
| 54 | + when /mac|darwin/ |
| 55 | + 'macos' |
| 56 | + when /linux|cygwin/ |
| 57 | + 'linux' |
| 58 | + when /mingw|mswin/ |
| 59 | + 'windows' |
| 60 | + else |
| 61 | + 'other' |
| 62 | + end |
| 63 | + end |
| 64 | + |
| 65 | + def self.initialize_report_data |
| 66 | + report_data = {'version' => '1.0'} |
| 67 | + begin |
| 68 | + report_data['commit_id'] = `git rev-parse HEAD`.strip |
| 69 | + rescue |
| 70 | + # unable to get a commit, maybe run outside of a git repo. Skip |
| 71 | + end |
| 72 | + report_data['ruby_engine'] = RUBY_ENGINE |
| 73 | + report_data['ruby_engine_version'] = RUBY_ENGINE_VERSION |
| 74 | + report_data['ruby_version'] = RUBY_VERSION |
| 75 | + |
| 76 | + report_data['cpu'] = RbConfig::CONFIG['host_cpu'] |
| 77 | + report_data['os'] = host_os |
| 78 | + report_data['execution_env'] = ENV['AWS_EXECUTION_ENV'] || ENV['EXECUTION_ENV'] || 'unknown' |
| 79 | + |
| 80 | + report_data['timestamp'] = Time.now.to_i |
| 81 | + |
| 82 | + report_data["benchmark"] = {} |
| 83 | + report_data |
| 84 | + end |
| 85 | + |
| 86 | + # abstract base class for benchmarking an SDK Gem |
| 87 | + # implementors must define the gem_name, client_klass, and operation_benchmarks methods |
| 88 | + class Gem |
| 89 | + |
| 90 | + # the name of them (eg: aws-sdk-s3) |
| 91 | + def gem_name; end |
| 92 | + |
| 93 | + # the module under Aws that contains the client (eg :S3) |
| 94 | + def client_module_name; end |
| 95 | + |
| 96 | + # return a hash with definitions for operation benchmarks to run |
| 97 | + # the key should be the name of the test (reported as the metric name) |
| 98 | + # Values should be a hash with keys: setup (proc), test (proc) and n (optional, integer) |
| 99 | + # |
| 100 | + # setup: must be a proc that takes a client. Client will be pre initialized. |
| 101 | + # Setup may initialize stubs (eg `client.stub_responses(:operation, [...])`) |
| 102 | + # Setup MUST also return a hash with the request used in the test. |
| 103 | + # This is done to avoid the cost of creating the argument in each run of the test. |
| 104 | + # |
| 105 | + # test: a proc that takes a client and request (generated from calling the setup proc) |
| 106 | + def operation_benchmarks; end |
| 107 | + |
| 108 | + # build the gem from its gemspec, then get the file size on disc |
| 109 | + # done within a temp directory to prevent accumulation of .gem artifacts |
| 110 | + def benchmark_gem_size(report_data) |
| 111 | + Dir.mktmpdir("ruby-sdk-benchmark") do |tmpdir| |
| 112 | + Dir.chdir("gems/#{gem_name}") do |
| 113 | + `gem build #{gem_name}.gemspec -o #{tmpdir}/#{gem_name}.gem` |
| 114 | + report_data['gem_size_kb'] = File.size("#{tmpdir}/#{gem_name}.gem") / 1024.0 |
| 115 | + report_data['gem_version'] = File.read("VERSION").strip |
| 116 | + end |
| 117 | + end |
| 118 | + end |
| 119 | + |
| 120 | + # benchmark requiring a gem - runs in a forked process (when supported) |
| 121 | + # to ensure state of parent process is not modified by the require |
| 122 | + # For accurate results, should be run before any SDK gems are required |
| 123 | + # in the parent process |
| 124 | + def benchmark_require(report_data) |
| 125 | + return unless gem_name |
| 126 | + |
| 127 | + report_data.merge!(Benchmark.fork_run do |out| |
| 128 | + t1 = Benchmark.monotonic_milliseconds |
| 129 | + require gem_name |
| 130 | + out[:require_time_ms] = (Benchmark.monotonic_milliseconds - t1) |
| 131 | + end) |
| 132 | + |
| 133 | + report_data.merge!(Benchmark.fork_run do |out| |
| 134 | + unless defined?(JRUBY_VERSION) |
| 135 | + r = ::MemoryProfiler.report { require gem_name } |
| 136 | + out[:require_mem_retained_kb] = r.total_retained_memsize / 1024.0 |
| 137 | + out[:require_mem_allocated_kb] = r.total_allocated_memsize / 1024.0 |
| 138 | + end |
| 139 | + end) |
| 140 | + end |
| 141 | + |
| 142 | + # benchmark creating a client - runs in a forked process (when supported) |
| 143 | + # For accurate results, should be run before the client is initialized |
| 144 | + # in the parent process to ensure cache is clean |
| 145 | + def benchmark_client(report_data) |
| 146 | + return unless client_module_name |
| 147 | + |
| 148 | + report_data.merge!(Benchmark.fork_run do |out| |
| 149 | + require gem_name |
| 150 | + client_klass = Aws.const_get(client_module_name).const_get(:Client) |
| 151 | + unless defined?(JRUBY_VERSION) |
| 152 | + r = ::MemoryProfiler.report { client_klass.new(stub_responses: true) } |
| 153 | + out[:client_mem_retained_kb] = r.total_retained_memsize / 1024.0 |
| 154 | + out[:client_mem_allocated_kb] = r.total_allocated_memsize / 1024.0 |
| 155 | + end |
| 156 | + end) |
| 157 | + end |
| 158 | + |
| 159 | + # This runs in the main process and requires service gems. |
| 160 | + # It MUST be done after ALL testing of gem loads/client creates |
| 161 | + def benchmark_operations(report_data) |
| 162 | + return unless gem_name && client_module_name && operation_benchmarks |
| 163 | + |
| 164 | + require gem_name |
| 165 | + |
| 166 | + client_klass = Aws.const_get(client_module_name).const_get(:Client) |
| 167 | + |
| 168 | + report_data[:client_init_ms] = Benchmark.measure_time(300) do |
| 169 | + client_klass.new(stub_responses: true) |
| 170 | + end |
| 171 | + |
| 172 | + values = report_data[:client_init_ms] |
| 173 | + puts "\t\t#{gem_name} client init avg: #{'%.2f' % (values.sum(0.0) / values.size)} ms" |
| 174 | + |
| 175 | + operation_benchmarks.each do |test_name, test_def| |
| 176 | + client = client_klass.new(stub_responses: true) |
| 177 | + req = test_def[:setup].call(client) |
| 178 | + |
| 179 | + |
| 180 | + # warmup (run a few iterations without measurement) |
| 181 | + 2.times { test_def[:test].call(client, req) } |
| 182 | + |
| 183 | + mem_allocated = 0 |
| 184 | + unless defined?(JRUBY_VERSION) |
| 185 | + r = ::MemoryProfiler.report { test_def[:test].call(client, req) } |
| 186 | + mem_allocated = report_data["#{test_name}_allocated_kb"] = r.total_allocated_memsize / 1024.0 |
| 187 | + end |
| 188 | + |
| 189 | + n = test_def[:n] || 300 |
| 190 | + values = Benchmark.measure_time(n) do |
| 191 | + test_def[:test].call(client, req) |
| 192 | + end |
| 193 | + report_data["#{test_name}_ms"] = values |
| 194 | + puts "\t\t#{test_name} avg: #{ '%.2f' % (values.sum(0.0) / values.size)} ms\tmem_allocated: #{'%.2f' % mem_allocated} kb" |
| 195 | + end |
| 196 | + end |
| 197 | + |
| 198 | + def self.descendants |
| 199 | + descendants = [] |
| 200 | + ObjectSpace.each_object(singleton_class) do |k| |
| 201 | + next if k.singleton_class? |
| 202 | + descendants.unshift k unless k == self |
| 203 | + end |
| 204 | + descendants |
| 205 | + end |
| 206 | + end |
| 207 | +end |
| 208 | + |
| 209 | +# require all gem benchmarks |
| 210 | +Dir[File.join(__dir__, 'gems', '*.rb')].each { |file| require file } |
| 211 | + |
0 commit comments