Skip to content

Commit ca7840c

Browse files
supercaracalsupercaracal
authored
fix: the replica affinity option is not working (#109)
1 parent 1b41b3b commit ca7840c

File tree

7 files changed

+190
-15
lines changed

7 files changed

+190
-15
lines changed

‎.github/workflows/test.yaml‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -209,13 +209,13 @@ jobs:
209209
with:
210210
ruby-version: '3.1'
211211
bundler-cache: true
212-
- name: Pull Docker images
213-
run: docker pull redis:$REDIS_VERSION
214212
- name: Get IP address of host
215213
id: host_ip_addr
216214
run: |
217215
host_ip_addr=$(ip a | grep eth0 | grep inet | awk '{print $2}' | cut -d'/' -f1)
218216
echo "::set-output name=host_ip_addr::$host_ip_addr"
217+
- name: Pull Docker images
218+
run: docker pull redis:$REDIS_VERSION
219219
- name: Run containers
220220
run: docker compose -f $DOCKER_COMPOSE_FILE up -d
221221
env:
@@ -266,8 +266,10 @@ jobs:
266266
runs-on: ubuntu-latest
267267
env:
268268
REDIS_VERSION: '7.0.1'
269-
DOCKER_COMPOSE_FILE: 'docker-compose.yaml'
270-
REDIS_CLIENT_MAX_THREADS: '37'
269+
DOCKER_COMPOSE_FILE: 'docker-compose.latency.yaml'
270+
REDIS_REPLICA_SIZE: '2'
271+
REDIS_CLIENT_MAX_THREADS: '5'
272+
DELAY_TIME: '2ms'
271273
steps:
272274
- name: Check out code
273275
uses: actions/checkout@v3
@@ -284,7 +286,31 @@ jobs:
284286
run: bundle exec rake wait
285287
- name: Print containers
286288
run: docker compose -f $DOCKER_COMPOSE_FILE ps
289+
- name: Rebuild cluster for balancing of replicas
290+
run: bundle exec rake build_cluster_for_bench
291+
env:
292+
DEBUG: "1"
293+
- name: Print topology
294+
run: |
295+
for i in {1..9}
296+
do
297+
echo "node$i: $(docker compose -f $DOCKER_COMPOSE_FILE exec node$i redis-cli cluster nodes | grep myself)"
298+
done
299+
- name: Ping nodes
300+
run: |
301+
for i in {1..9}
302+
do
303+
node_addr="$(docker compose -f $DOCKER_COMPOSE_FILE exec node$i redis-cli cluster nodes | grep myself | awk '{print $2}' | cut -d'@' -f1 | cut -d':' -f1)"
304+
echo "node$i:"
305+
ping -c 5 $node_addr
306+
done
287307
- name: Run minitest
288308
run: bundle exec rake bench
309+
- name: Reset qdisc
310+
run: |
311+
for i in {5..9..2}
312+
do
313+
docker compose -f $DOCKER_COMPOSE_FILE exec node$i tc qdisc del dev eth0 root netem
314+
done
289315
- name: Stop containers
290316
run: docker compose -f $DOCKER_COMPOSE_FILE down

‎Rakefile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,11 @@ task :build_cluster, %i[addr1 addr2] do |_, args|
6262
timeout: 30.0
6363
).rebuild
6464
end
65+
66+
desc 'Build cluster for benchmark'
67+
task :build_cluster_for_bench do
68+
$LOAD_PATH.unshift(File.expand_path('test', __dir__))
69+
require 'cluster_controller'
70+
nodes = (6379..6387).map { |port| "redis://127.0.0.1:#{port}" }
71+
::ClusterController.new(nodes, shard_size: 3, replica_size: 2, timeout: 30.0).rebuild
72+
end

‎docker-compose.latency.yaml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
---
2+
# 3 shards plus with each 2 replicas simutlated network delay
3+
services:
4+
node1: &node
5+
image: "redis:${REDIS_VERSION:-7}"
6+
command: >
7+
redis-server
8+
--maxmemory 64mb
9+
--maxmemory-policy allkeys-lru
10+
--appendonly yes
11+
--cluster-enabled yes
12+
--cluster-config-file nodes.conf
13+
--cluster-node-timeout 5000
14+
restart: "${RESTART_POLICY:-always}"
15+
healthcheck:
16+
test: ["CMD", "redis-cli", "ping"]
17+
interval: "7s"
18+
timeout: "5s"
19+
retries: 10
20+
ports:
21+
- "6379:6379"
22+
node2:
23+
<<: *node
24+
ports:
25+
- "6380:6379"
26+
node3:
27+
<<: *node
28+
ports:
29+
- "6381:6379"
30+
node4:
31+
<<: *node
32+
ports:
33+
- "6382:6379"
34+
node5: &far_node
35+
<<: *node
36+
command: >
37+
bash -c "apt-get update > /dev/null
38+
&& apt-get install --no-install-recommends --no-install-suggests -y iproute2 iputils-ping > /dev/null
39+
&& rm -rf /var/lib/apt/lists/*
40+
&& tc qdisc add dev eth0 root netem delay ${DELAY_TIME:-20ms}
41+
&& redis-server
42+
--maxmemory 64mb
43+
--maxmemory-policy allkeys-lru
44+
--appendonly yes
45+
--cluster-enabled yes
46+
--cluster-config-file nodes.conf
47+
--cluster-node-timeout 5000"
48+
cap_add:
49+
- NET_ADMIN
50+
ports:
51+
- "6383:6379"
52+
node6:
53+
<<: *node
54+
ports:
55+
- "6384:6379"
56+
node7:
57+
<<: *far_node
58+
ports:
59+
- "6385:6379"
60+
node8:
61+
<<: *node
62+
ports:
63+
- "6386:6379"
64+
node9:
65+
<<: *far_node
66+
ports:
67+
- "6387:6379"
68+
clustering:
69+
image: "redis:${REDIS_VERSION:-7}"
70+
command: >
71+
bash -c "apt-get update > /dev/null
72+
&& apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null
73+
&& rm -rf /var/lib/apt/lists/*
74+
&& yes yes | redis-cli --cluster create
75+
$$(dig node1 +short):6379
76+
$$(dig node2 +short):6379
77+
$$(dig node3 +short):6379
78+
$$(dig node4 +short):6379
79+
$$(dig node5 +short):6379
80+
$$(dig node6 +short):6379
81+
$$(dig node7 +short):6379
82+
$$(dig node8 +short):6379
83+
$$(dig node9 +short):6379
84+
--cluster-replicas 2"
85+
depends_on:
86+
node1:
87+
condition: service_healthy
88+
node2:
89+
condition: service_healthy
90+
node3:
91+
condition: service_healthy
92+
node4:
93+
condition: service_healthy
94+
node5:
95+
condition: service_healthy
96+
node6:
97+
condition: service_healthy
98+
node7:
99+
condition: service_healthy
100+
node8:
101+
condition: service_healthy
102+
node9:
103+
condition: service_healthy

‎lib/redis_client/cluster_config.rb‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,17 @@ def initialize( # rubocop:disable Metrics/ParameterLists
4242
@mutex = Mutex.new
4343
end
4444

45+
def dup
46+
self.class.new(
47+
nodes: @node_configs,
48+
replica: @replica,
49+
replica_affinity: @replica_affinity,
50+
fixed_hostname: @fixed_hostname,
51+
client_implementation: @client_implementation,
52+
**@client_config
53+
)
54+
end
55+
4556
def inspect
4657
"#<#{self.class.name} #{per_node_key.values}>"
4758
end
@@ -79,10 +90,6 @@ def add_node(host, port)
7990
@mutex.synchronize { @node_configs << { host: host, port: port } }
8091
end
8192

82-
def dup
83-
self.class.new(nodes: @node_configs, replica: @replica, fixed_hostname: @fixed_hostname, **@client_config)
84-
end
85-
8693
private
8794

8895
def build_node_configs(addrs)

‎test/bench_command.rb‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def new_test_client
9494
end
9595
end
9696

97-
class ScaleRead < BenchmarkWrapper
97+
class ScaleReadRandom < BenchmarkWrapper
9898
include Mixin
9999

100100
private
@@ -103,6 +103,24 @@ def new_test_client
103103
config = ::RedisClient::ClusterConfig.new(
104104
nodes: TEST_NODE_URIS,
105105
replica: true,
106+
replica_affinity: :random,
107+
fixed_hostname: TEST_FIXED_HOSTNAME,
108+
**TEST_GENERIC_OPTIONS
109+
)
110+
::RedisClient::Cluster.new(config)
111+
end
112+
end
113+
114+
class ScaleReadLatency < BenchmarkWrapper
115+
include Mixin
116+
117+
private
118+
119+
def new_test_client
120+
config = ::RedisClient::ClusterConfig.new(
121+
nodes: TEST_NODE_URIS,
122+
replica: true,
123+
replica_affinity: :latency,
106124
fixed_hostname: TEST_FIXED_HOSTNAME,
107125
**TEST_GENERIC_OPTIONS
108126
)

‎test/redis_client/test_cluster.rb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def test_call
3838
assert(@client.call('PING') { |r| r == 'PONG' })
3939

4040
assert_equal(2, @client.call('HSET', 'hash', { foo: 1, bar: 2 }))
41+
wait_for_replication
4142
assert_equal(%w[1 2], @client.call('HMGET', 'hash', %w[foo bar]))
4243
end
4344

@@ -53,6 +54,7 @@ def test_call_once
5354
assert(@client.call_once('PING') { |r| r == 'PONG' })
5455

5556
assert_equal(2, @client.call_once('HSET', 'hash', { foo: 1, bar: 2 }))
57+
wait_for_replication
5658
assert_equal(%w[1 2], @client.call_once('HMGET', 'hash', %w[foo bar]))
5759
end
5860

‎test/redis_client/test_cluster_config.rb‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,23 @@
55

66
class RedisClient
77
class TestClusterConfig < TestingWrapper
8+
def test_dup
9+
orig = ::RedisClient::ClusterConfig.new
10+
copy = orig.dup
11+
refute_equal(orig.object_id, copy.object_id)
12+
13+
::RedisClient::ClusterConfig.instance_method(:initialize).parameters.each do |type, name|
14+
case type
15+
when :key
16+
want = orig.instance_variable_get("@#{name}".to_sym)
17+
got = copy.instance_variable_get("@#{name}".to_sym)
18+
next if got.nil?
19+
20+
assert_equal(want, got, "Case: #{type}=#{name}")
21+
end
22+
end
23+
end
24+
825
def test_inspect
926
want = '#<RedisClient::ClusterConfig [{:host=>"127.0.0.1", :port=>6379}]>'
1027
got = ::RedisClient::ClusterConfig.new.inspect
@@ -100,12 +117,6 @@ def test_add_node
100117
assert_equal([{ host: '127.0.0.1', port: 6379 }, { host: '127.0.0.2', port: 6380 }], config.instance_variable_get(:@node_configs))
101118
end
102119

103-
def test_dup
104-
orig = ::RedisClient::ClusterConfig.new
105-
copy = orig.dup
106-
refute_equal(orig.object_id, copy.object_id)
107-
end
108-
109120
def test_command_builder
110121
assert_equal(::RedisClient::CommandBuilder, ::RedisClient::ClusterConfig.new.command_builder)
111122
end

0 commit comments

Comments
 (0)