Skip to content

Commit 54e199c

Browse files
authored
Performance benchmark (#2857)
1 parent 23a8f80 commit 54e199c

File tree

14 files changed

+766
-0
lines changed

14 files changed

+766
-0
lines changed

.github/workflows/benchmark.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Benchmark
2+
3+
on:
4+
push:
5+
branches:
6+
- version-3
7+
8+
pull_request:
9+
branches:
10+
- version-3
11+
12+
permissions:
13+
id-token: write # required for OIDC
14+
contents: read
15+
16+
jobs:
17+
benchmark:
18+
runs-on: ubuntu-latest
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
ruby: [jruby-9.2, 2.6, 2.7, 3.2]
23+
24+
steps:
25+
- name: Setup Ruby
26+
uses: ruby/setup-ruby@v1
27+
with:
28+
ruby-version: ${{ matrix.ruby }}
29+
30+
- uses: actions/checkout@v3
31+
32+
- name: Install gems
33+
run: |
34+
bundle install --without docs repl development
35+
36+
- name: SDK Build
37+
run: bundle exec rake build
38+
39+
- name: Benchmark
40+
env:
41+
JRUBY_OPTS: -J-Xmx4g
42+
run: EXECUTION_ENV=github-action bundle exec rake benchmark
43+
44+
- name: configure aws credentials
45+
uses: aws-actions/configure-aws-credentials@v2
46+
with:
47+
role-to-assume: arn:aws:iam::469596866844:role/aws-sdk-ruby-performance-reporter
48+
role-session-name: benchmark-report
49+
aws-region: us-west-2
50+
51+
- name: Archive benchmark report
52+
run: |
53+
GH_REF=${{github.head_ref}} GH_EVENT=${{github.event_name}} bundle exec rake benchmark:archive
54+
55+
- name: Upload benchmark metrics
56+
run: |
57+
GH_REF=${{github.head_ref}} GH_EVENT=${{github.event_name}} bundle exec rake benchmark:put-metrics

Gemfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,8 @@ end
5353
group :development do
5454
gem 'rubocop', '0.81.0'
5555
end
56+
57+
group :benchmark do
58+
gem 'benchmark'
59+
gem 'memory_profiler'
60+
end

benchmark/benchmark.rb

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
# frozen_string_literal: true
2+
3+
require_relative 'test_data'
4+
5+
module Benchmark
6+
# monotonic system clock should be used for any time difference measurements
7+
def self.monotonic_milliseconds
8+
if defined?(Process::CLOCK_MONOTONIC)
9+
Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) / 1000.0
10+
else
11+
Time.now.to_f * 1000.0
12+
end
13+
end
14+
15+
# benchmark a block, returning an array of times (to allow statistic computation)
16+
def self.measure_time(n=300, &block)
17+
values = Array.new(n)
18+
n.times do |i|
19+
t1 = monotonic_milliseconds
20+
block.call
21+
values[i] = monotonic_milliseconds - t1
22+
end
23+
values
24+
end
25+
26+
# Run a block in a fork and returns the data from it
27+
# the block must take a single argument and will be called with an empty hash
28+
# any data that should be communicated back to the parent process can be written to that hash
29+
def self.fork_run(&block)
30+
# fork is not supported in JRuby, for now, just run this in the same process
31+
# data collected will not be as useful, but still valid for relative comparisons over time
32+
if defined?(JRUBY_VERSION)
33+
h = {}
34+
block.call(h)
35+
return h
36+
end
37+
38+
rd, wr = IO.pipe
39+
p1 = fork do
40+
h = {}
41+
block.call(h)
42+
wr.write(JSON.dump(h))
43+
wr.close
44+
end
45+
Process.wait(p1)
46+
wr.close
47+
h = JSON.parse(rd.read, symbolize_names: true)
48+
rd.close
49+
return h
50+
end
51+
52+
def self.host_os
53+
case RbConfig::CONFIG['host_os']
54+
when /mac|darwin/
55+
'macos'
56+
when /linux|cygwin/
57+
'linux'
58+
when /mingw|mswin/
59+
'windows'
60+
else
61+
'other'
62+
end
63+
end
64+
65+
def self.initialize_report_data
66+
report_data = {'version' => '1.0'}
67+
begin
68+
report_data['commit_id'] = `git rev-parse HEAD`.strip
69+
rescue
70+
# unable to get a commit, maybe run outside of a git repo. Skip
71+
end
72+
report_data['ruby_engine'] = RUBY_ENGINE
73+
report_data['ruby_engine_version'] = RUBY_ENGINE_VERSION
74+
report_data['ruby_version'] = RUBY_VERSION
75+
76+
report_data['cpu'] = RbConfig::CONFIG['host_cpu']
77+
report_data['os'] = host_os
78+
report_data['execution_env'] = ENV['AWS_EXECUTION_ENV'] || ENV['EXECUTION_ENV'] || 'unknown'
79+
80+
report_data['timestamp'] = Time.now.to_i
81+
82+
report_data["benchmark"] = {}
83+
report_data
84+
end
85+
86+
# abstract base class for benchmarking an SDK Gem
87+
# implementors must define the gem_name, client_klass, and operation_benchmarks methods
88+
class Gem
89+
90+
# the name of them (eg: aws-sdk-s3)
91+
def gem_name; end
92+
93+
# the module under Aws that contains the client (eg :S3)
94+
def client_module_name; end
95+
96+
# return a hash with definitions for operation benchmarks to run
97+
# the key should be the name of the test (reported as the metric name)
98+
# Values should be a hash with keys: setup (proc), test (proc) and n (optional, integer)
99+
#
100+
# setup: must be a proc that takes a client. Client will be pre initialized.
101+
# Setup may initialize stubs (eg `client.stub_responses(:operation, [...])`)
102+
# Setup MUST also return a hash with the request used in the test.
103+
# This is done to avoid the cost of creating the argument in each run of the test.
104+
#
105+
# test: a proc that takes a client and request (generated from calling the setup proc)
106+
def operation_benchmarks; end
107+
108+
# build the gem from its gemspec, then get the file size on disc
109+
# done within a temp directory to prevent accumulation of .gem artifacts
110+
def benchmark_gem_size(report_data)
111+
Dir.mktmpdir("ruby-sdk-benchmark") do |tmpdir|
112+
Dir.chdir("gems/#{gem_name}") do
113+
`gem build #{gem_name}.gemspec -o #{tmpdir}/#{gem_name}.gem`
114+
report_data['gem_size_kb'] = File.size("#{tmpdir}/#{gem_name}.gem") / 1024.0
115+
report_data['gem_version'] = File.read("VERSION").strip
116+
end
117+
end
118+
end
119+
120+
# benchmark requiring a gem - runs in a forked process (when supported)
121+
# to ensure state of parent process is not modified by the require
122+
# For accurate results, should be run before any SDK gems are required
123+
# in the parent process
124+
def benchmark_require(report_data)
125+
return unless gem_name
126+
127+
report_data.merge!(Benchmark.fork_run do |out|
128+
t1 = Benchmark.monotonic_milliseconds
129+
require gem_name
130+
out[:require_time_ms] = (Benchmark.monotonic_milliseconds - t1)
131+
end)
132+
133+
report_data.merge!(Benchmark.fork_run do |out|
134+
unless defined?(JRUBY_VERSION)
135+
r = ::MemoryProfiler.report { require gem_name }
136+
out[:require_mem_retained_kb] = r.total_retained_memsize / 1024.0
137+
out[:require_mem_allocated_kb] = r.total_allocated_memsize / 1024.0
138+
end
139+
end)
140+
end
141+
142+
# benchmark creating a client - runs in a forked process (when supported)
143+
# For accurate results, should be run before the client is initialized
144+
# in the parent process to ensure cache is clean
145+
def benchmark_client(report_data)
146+
return unless client_module_name
147+
148+
report_data.merge!(Benchmark.fork_run do |out|
149+
require gem_name
150+
client_klass = Aws.const_get(client_module_name).const_get(:Client)
151+
unless defined?(JRUBY_VERSION)
152+
r = ::MemoryProfiler.report { client_klass.new(stub_responses: true) }
153+
out[:client_mem_retained_kb] = r.total_retained_memsize / 1024.0
154+
out[:client_mem_allocated_kb] = r.total_allocated_memsize / 1024.0
155+
end
156+
end)
157+
end
158+
159+
# This runs in the main process and requires service gems.
160+
# It MUST be done after ALL testing of gem loads/client creates
161+
def benchmark_operations(report_data)
162+
return unless gem_name && client_module_name && operation_benchmarks
163+
164+
require gem_name
165+
166+
client_klass = Aws.const_get(client_module_name).const_get(:Client)
167+
168+
report_data[:client_init_ms] = Benchmark.measure_time(300) do
169+
client_klass.new(stub_responses: true)
170+
end
171+
172+
values = report_data[:client_init_ms]
173+
puts "\t\t#{gem_name} client init avg: #{'%.2f' % (values.sum(0.0) / values.size)} ms"
174+
175+
operation_benchmarks.each do |test_name, test_def|
176+
client = client_klass.new(stub_responses: true)
177+
req = test_def[:setup].call(client)
178+
179+
180+
# warmup (run a few iterations without measurement)
181+
2.times { test_def[:test].call(client, req) }
182+
183+
mem_allocated = 0
184+
unless defined?(JRUBY_VERSION)
185+
r = ::MemoryProfiler.report { test_def[:test].call(client, req) }
186+
mem_allocated = report_data["#{test_name}_allocated_kb"] = r.total_allocated_memsize / 1024.0
187+
end
188+
189+
n = test_def[:n] || 300
190+
values = Benchmark.measure_time(n) do
191+
test_def[:test].call(client, req)
192+
end
193+
report_data["#{test_name}_ms"] = values
194+
puts "\t\t#{test_name} avg: #{ '%.2f' % (values.sum(0.0) / values.size)} ms\tmem_allocated: #{'%.2f' % mem_allocated} kb"
195+
end
196+
end
197+
198+
def self.descendants
199+
descendants = []
200+
ObjectSpace.each_object(singleton_class) do |k|
201+
next if k.singleton_class?
202+
descendants.unshift k unless k == self
203+
end
204+
descendants
205+
end
206+
end
207+
end
208+
209+
# require all gem benchmarks
210+
Dir[File.join(__dir__, 'gems', '*.rb')].each { |file| require file }
211+

benchmark/gems/cloudwatch.rb

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# frozen_string_literal: true
2+
3+
module Benchmark
4+
module Gems
5+
class CloudWatch < Benchmark::Gem
6+
7+
def gem_name
8+
'aws-sdk-cloudwatch'
9+
end
10+
11+
def client_module_name
12+
:CloudWatch
13+
end
14+
15+
def operation_benchmarks
16+
{
17+
put_metric_data_small: {
18+
setup: proc do |client|
19+
{namespace: 'namespace', metric_data: [{metric_name: "metric", timestamp: Time.now, value: 1.0, unit: "Seconds"}]}
20+
end,
21+
test: proc do |client, req|
22+
client.put_metric_data(req)
23+
end
24+
},
25+
put_metric_data_large: {
26+
setup: proc do |client|
27+
{namespace: 'namespace', metric_data:
28+
(0...10).map do
29+
{metric_name: "metric", timestamp: Time.now, values: (0...150).to_a, unit: "Seconds"}
30+
end}
31+
end,
32+
test: proc do |client, req|
33+
client.put_metric_data(req)
34+
end
35+
},
36+
}
37+
end
38+
39+
end
40+
end
41+
end

benchmark/gems/cloudwatchlogs.rb

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# frozen_string_literal: true
2+
3+
module Benchmark
4+
module Gems
5+
class CloudWatchLogs < Benchmark::Gem
6+
7+
def gem_name
8+
'aws-sdk-cloudwatchlogs'
9+
end
10+
11+
def client_module_name
12+
:CloudWatchLogs
13+
end
14+
15+
def operation_benchmarks
16+
{
17+
put_log_events_small: {
18+
setup: proc do |client|
19+
{log_group_name: 'log_group', log_stream_name: 'log_stream', log_events: (0...5).map { |i| { timestamp: Time.now.to_i, message: "test log event #{i}"}}}
20+
end,
21+
test: proc do |client, req|
22+
client.put_log_events(req)
23+
end
24+
},
25+
put_log_events_large: {
26+
setup: proc do |client|
27+
{log_group_name: 'log_group', log_stream_name: 'log_stream', log_events: (0...5000).map { |i| { timestamp: Time.now.to_i, message: "test log event #{i} - #{TestData.random_value(i)}"}}}
28+
end,
29+
test: proc do |client, req|
30+
client.put_log_events(req)
31+
end
32+
},
33+
get_log_events_small: {
34+
setup: proc do |client|
35+
client.stub_responses(:get_log_events, [{events: (0...5).map { |i| { timestamp: Time.now.to_i, message: "test log event #{i} - #{TestData.random_value(i)}"}}}])
36+
{log_group_name: 'log_group', log_stream_name: 'log_stream'}
37+
end,
38+
test: proc do |client, req|
39+
client.get_log_events(req)
40+
end
41+
},
42+
get_log_events_large: {
43+
setup: proc do |client|
44+
client.stub_responses(:get_log_events, [{events: (0...5000).map { |i| { timestamp: Time.now.to_i, message: "test log event #{i} - #{TestData.random_value(i)}"}}}])
45+
{log_group_name: 'log_group', log_stream_name: 'log_stream'}
46+
end,
47+
test: proc do |client, req|
48+
client.get_log_events(req)
49+
end
50+
},
51+
filter_log_events_large: {
52+
setup: proc do |client|
53+
client.stub_responses(:filter_log_events, [{events: (0...5000).map { |i| { timestamp: Time.now.to_i, message: "test log event #{i} - #{TestData.random_value(i)}"}}}])
54+
{log_group_name: 'log_group', log_stream_names: ['log_stream'], start_time: 1, end_time: 1000}
55+
end,
56+
test: proc do |client, req|
57+
client.filter_log_events(req)
58+
end
59+
}
60+
}
61+
end
62+
end
63+
end
64+
end

0 commit comments

Comments
 (0)