Skip to content

Commit 5817a0b

Browse files
supercaracalsupercaracal
authored
perf: command builder shouldn't be called multiple times (#406)
1 parent ff2969d commit 5817a0b

File tree

11 files changed

+147
-41
lines changed

11 files changed

+147
-41
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
.bundle
22
*.gem
3+
*.json
34
Gemfile.lock

‎Gemfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@ gem 'rubocop-minitest', require: false
1515
gem 'rubocop-performance', require: false
1616
gem 'rubocop-rake', require: false
1717
gem 'stackprof', platform: :mri
18+
gem 'vernier', platform: :mri if RUBY_ENGINE == 'ruby' && Integer(RUBY_VERSION.split('.').first) > 2

‎lib/redis_client/cluster.rb‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,30 +62,37 @@ def blocking_call_v(timeout, command, &block)
6262
end
6363

6464
def scan(*args, **kwargs, &block)
65-
raise ArgumentError, 'block required' unless block
65+
return to_enum(__callee__, *args, **kwargs) unless block_given?
6666

67+
command = @command_builder.generate(['SCAN', ZERO_CURSOR_FOR_SCAN] + args, kwargs)
6768
seed = Random.new_seed
68-
cursor = ZERO_CURSOR_FOR_SCAN
6969
loop do
70-
cursor, keys = router.scan('SCAN', cursor, *args, seed: seed, **kwargs)
70+
cursor, keys = router.scan(command, seed: seed)
71+
command[1] = cursor
7172
keys.each(&block)
7273
break if cursor == ZERO_CURSOR_FOR_SCAN
7374
end
7475
end
7576

7677
def sscan(key, *args, **kwargs, &block)
77-
node = router.assign_node(['SSCAN', key])
78-
router.try_delegate(node, :sscan, key, *args, **kwargs, &block)
78+
return to_enum(__callee__, key, *args, **kwargs) unless block_given?
79+
80+
command = @command_builder.generate(['SSCAN', key, ZERO_CURSOR_FOR_SCAN] + args, kwargs)
81+
router.scan_single_key(command, arity: 1, &block)
7982
end
8083

8184
def hscan(key, *args, **kwargs, &block)
82-
node = router.assign_node(['HSCAN', key])
83-
router.try_delegate(node, :hscan, key, *args, **kwargs, &block)
85+
return to_enum(__callee__, key, *args, **kwargs) unless block_given?
86+
87+
command = @command_builder.generate(['HSCAN', key, ZERO_CURSOR_FOR_SCAN] + args, kwargs)
88+
router.scan_single_key(command, arity: 2, &block)
8489
end
8590

8691
def zscan(key, *args, **kwargs, &block)
87-
node = router.assign_node(['ZSCAN', key])
88-
router.try_delegate(node, :zscan, key, *args, **kwargs, &block)
92+
return to_enum(__callee__, key, *args, **kwargs) unless block_given?
93+
94+
command = @command_builder.generate(['ZSCAN', key, ZERO_CURSOR_FOR_SCAN] + args, kwargs)
95+
router.scan_single_key(command, arity: 2, &block)
8996
end
9097

9198
def pipelined(exception: true)

‎lib/redis_client/cluster/noop_command_builder.rb‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# frozen_string_literal: true
2+
3+
class RedisClient
4+
class Cluster
5+
module NoopCommandBuilder
6+
module_function
7+
8+
def generate(args, _kwargs = nil)
9+
args
10+
end
11+
end
12+
end
13+
end

‎lib/redis_client/cluster/pipeline.rb‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
require 'redis_client'
44
require 'redis_client/cluster/errors'
5+
require 'redis_client/cluster/noop_command_builder'
56
require 'redis_client/connection_mixin'
67
require 'redis_client/middlewares'
78
require 'redis_client/pooled'
@@ -229,7 +230,7 @@ def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Met
229230

230231
def append_pipeline(node_key)
231232
@pipelines ||= {}
232-
@pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(@command_builder)
233+
@pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(::RedisClient::Cluster::NoopCommandBuilder)
233234
@pipelines[node_key].add_outer_index(@size)
234235
@size += 1
235236
@pipelines[node_key]

‎lib/redis_client/cluster/router.rb‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,6 @@ def try_send(node, method, command, args, retry_count: 3, &block)
104104
end
105105
end
106106

107-
def try_delegate(node, method, *args, retry_count: 3, **kwargs, &block)
108-
handle_redirection(node, nil, retry_count: retry_count) do |on_node|
109-
on_node.public_send(method, *args, **kwargs, &block)
110-
end
111-
end
112-
113107
def handle_redirection(node, command, retry_count:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
114108
yield node
115109
rescue ::RedisClient::CircuitBreaker::OpenCircuitError
@@ -153,9 +147,7 @@ def handle_redirection(node, command, retry_count:) # rubocop:disable Metrics/Ab
153147
raise
154148
end
155149

156-
def scan(*command, seed: nil, **kwargs) # rubocop:disable Metrics/AbcSize
157-
command = @command_builder.generate(command, kwargs)
158-
150+
def scan(command, seed: nil) # rubocop:disable Metrics/AbcSize
159151
command[1] = ZERO_CURSOR_FOR_SCAN if command.size == 1
160152
input_cursor = Integer(command[1])
161153

@@ -180,6 +172,16 @@ def scan(*command, seed: nil, **kwargs) # rubocop:disable Metrics/AbcSize
180172
raise
181173
end
182174

175+
def scan_single_key(command, arity:, &block)
176+
node = assign_node(command)
177+
loop do
178+
cursor, values = handle_redirection(node, nil, retry_count: 3) { |n| n.call_v(command) }
179+
command[2] = cursor
180+
arity < 2 ? values.each(&block) : values.each_slice(arity, &block)
181+
break if cursor == ZERO_CURSOR_FOR_SCAN
182+
end
183+
end
184+
183185
def assign_node(command)
184186
handle_node_reload_error do
185187
node_key = find_node_key(command)

‎lib/redis_client/cluster/transaction.rb‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
require 'redis_client'
44
require 'redis_client/cluster/errors'
5+
require 'redis_client/cluster/noop_command_builder'
56
require 'redis_client/cluster/pipeline'
67

78
class RedisClient
@@ -18,7 +19,7 @@ def initialize(router, command_builder, node: nil, slot: nil, asking: false)
1819
@router = router
1920
@command_builder = command_builder
2021
@retryable = true
21-
@pipeline = ::RedisClient::Pipeline.new(@command_builder)
22+
@pipeline = ::RedisClient::Pipeline.new(::RedisClient::Cluster::NoopCommandBuilder)
2223
@pending_commands = []
2324
@node = node
2425
prepare_tx unless @node.nil?

‎lib/redis_client/cluster_config.rb‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
require 'redis_client/cluster'
66
require 'redis_client/cluster/errors'
77
require 'redis_client/cluster/node_key'
8+
require 'redis_client/cluster/noop_command_builder'
89
require 'redis_client/command_builder'
910

1011
class RedisClient
@@ -64,7 +65,7 @@ def initialize( # rubocop:disable Metrics/ParameterLists
6465
end
6566

6667
def inspect
67-
"#<#{self.class.name} #{startup_nodes.values}>"
68+
"#<#{self.class.name} #{startup_nodes.values.map { |v| v.reject { |k| k == :command_builder } }}>"
6869
end
6970

7071
def read_timeout
@@ -187,6 +188,7 @@ def build_startup_nodes(configs)
187188
def augment_client_config(config)
188189
config = @client_config.merge(config)
189190
config = config.merge(host: @fixed_hostname) unless @fixed_hostname.empty?
191+
config[:command_builder] = ::RedisClient::Cluster::NoopCommandBuilder # prevent twice call
190192
config
191193
end
192194
end

‎test/prof_stack2.rb‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# frozen_string_literal: true
2+
3+
require 'vernier'
4+
require 'redis_cluster_client'
5+
require 'testing_constants'
6+
7+
module ProfStack2
8+
SIZE = 40
9+
ATTEMPTS = 1000
10+
11+
module_function
12+
13+
def run
14+
client = make_client
15+
prepare(client)
16+
17+
case mode = ENV.fetch('PROFILE_MODE', :single).to_sym
18+
when :single
19+
execute(client, mode) do |client|
20+
ATTEMPTS.times { |i| client.call('GET', "key#{i}") }
21+
end
22+
when :transaction
23+
execute(client, mode) do |client|
24+
ATTEMPTS.times do |i|
25+
client.multi do |tx|
26+
SIZE.times do |j|
27+
n = SIZE * i + j
28+
tx.call('SET', "{group:#{i}}:key:#{n}", n)
29+
end
30+
end
31+
end
32+
end
33+
when :pipeline
34+
execute(client, mode) do |client|
35+
ATTEMPTS.times do |i|
36+
client.pipelined do |pi|
37+
SIZE.times do |j|
38+
n = SIZE * i + j
39+
pi.call('GET', "key#{n}")
40+
end
41+
end
42+
end
43+
end
44+
end
45+
end
46+
47+
def execute(client, mode)
48+
Vernier.profile(out: "vernier_#{mode}.json") do
49+
yield(client)
50+
end
51+
end
52+
53+
def make_client
54+
::RedisClient.cluster(
55+
nodes: TEST_NODE_URIS,
56+
replica: true,
57+
replica_affinity: :random,
58+
fixed_hostname: TEST_FIXED_HOSTNAME,
59+
# concurrency: { model: :on_demand, size: 6 },
60+
# concurrency: { model: :pooled, size: 6 },
61+
concurrency: { model: :none },
62+
**TEST_GENERIC_OPTIONS
63+
).new_client
64+
end
65+
66+
def prepare(client)
67+
ATTEMPTS.times do |i|
68+
client.pipelined do |pi|
69+
SIZE.times do |j|
70+
n = SIZE * i + j
71+
pi.call('SET', "key#{n}", "val#{n}")
72+
end
73+
end
74+
end
75+
end
76+
end
77+
78+
ProfStack2.run

‎test/redis_client/test_cluster.rb‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,6 @@ def test_blocking_call
9292
end
9393

9494
def test_scan
95-
assert_raises(ArgumentError) { @client.scan }
96-
9795
10.times { |i| @client.call('SET', "key#{i}", i) }
9896
wait_for_replication
9997
want = (0..9).map { |i| "key#{i}" }
@@ -117,9 +115,9 @@ def test_hscan
117115
10.times do |i|
118116
10.times { |j| @client.call('HSET', "key#{i}", "field#{j}", j) }
119117
wait_for_replication
120-
want = (0..9).map { |j| "field#{j}" }
118+
want = (0..9).map { |j| ["field#{j}", j.to_s] }
121119
got = []
122-
@client.hscan("key#{i}", 'COUNT', '5') { |field| got << field }
120+
@client.hscan("key#{i}", 'COUNT', '5') { |pair| got << pair }
123121
assert_equal(want, got.sort)
124122
end
125123
end
@@ -128,9 +126,9 @@ def test_zscan
128126
10.times do |i|
129127
10.times { |j| @client.call('ZADD', "key#{i}", j, "member#{j}") }
130128
wait_for_replication
131-
want = (0..9).map { |j| "member#{j}" }
129+
want = (0..9).map { |j| ["member#{j}", j.to_s] }
132130
got = []
133-
@client.zscan("key#{i}", 'COUNT', '5') { |member| got << member }
131+
@client.zscan("key#{i}", 'COUNT', '5') { |pair| got << pair }
134132
assert_equal(want, got.sort)
135133
end
136134
end

0 commit comments

Comments
 (0)