Skip to content

Commit 3318b38

Browse files
supercaracalsupercaracal
authored
ci: add memory profiling cases with massive cluster (#195)
1 parent da34f07 commit 3318b38

File tree

7 files changed

+426
-3
lines changed

7 files changed

+426
-3
lines changed

‎.github/workflows/test.yaml

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
runs-on: ubuntu-latest
2525
strategy:
2626
fail-fast: false
27-
max-parallel: 16
27+
max-parallel: 8
2828
matrix:
2929
include:
3030
- {redis: '7.0', ruby: '3.2'}
@@ -188,7 +188,7 @@ jobs:
188188
- name: Print cpu info
189189
run: grep 'model name' /proc/cpuinfo
190190
- name: Print memory info
191-
run: free
191+
run: free -w
192192
- name: Print disk info
193193
run: df -h
194194
- name: Run minitest
@@ -237,3 +237,62 @@ jobs:
237237
MEMORY_PROFILE_MODE: ${{ matrix.mode }}
238238
- name: Stop containers
239239
run: docker compose -f $DOCKER_COMPOSE_FILE down
240+
massive:
241+
name: Massive Cluster
242+
timeout-minutes: 10
243+
runs-on: ubuntu-latest
244+
env:
245+
REDIS_VERSION: '7.0'
246+
DOCKER_COMPOSE_FILE: 'compose.large.yaml'
247+
REDIS_SHARD_SIZE: '10'
248+
REDIS_REPLICA_SIZE: '2'
249+
steps:
250+
- name: Check out code
251+
uses: actions/checkout@v3
252+
- name: Set up Ruby
253+
uses: ruby/setup-ruby@v1
254+
with:
255+
ruby-version: '3.2'
256+
bundler-cache: true
257+
- name: Print user limits
258+
run: ulimit -a
259+
- name: Print kernel params
260+
run: |
261+
sysctl fs.file-max
262+
sysctl vm.swappiness
263+
sysctl vm.overcommit_memory
264+
sysctl net.ipv4.tcp_sack
265+
sysctl net.ipv4.tcp_timestamps
266+
sysctl net.ipv4.tcp_window_scaling
267+
sysctl net.ipv4.tcp_congestion_control
268+
sysctl net.ipv4.tcp_syncookies
269+
sysctl net.ipv4.tcp_tw_reuse
270+
sysctl net.ipv4.tcp_max_syn_backlog
271+
sysctl net.core.somaxconn
272+
sysctl net.core.rmem_max
273+
sysctl net.core.wmem_max
274+
- name: Tune kernel params for redis
275+
run: |
276+
# https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/
277+
sudo sysctl -w net.ipv4.tcp_tw_reuse=1 # reuse sockets quickly
278+
sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 # backlog setting
279+
sudo sysctl -w net.core.somaxconn=1024 # up the number of connections per port
280+
sudo sysctl -w vm.overcommit_memory=1
281+
- name: Pull Docker images
282+
run: docker pull redis:$REDIS_VERSION
283+
- name: Run containers
284+
run: docker compose -f $DOCKER_COMPOSE_FILE up -d
285+
- name: Print memory info
286+
run: free -w
287+
- name: Wait for Redis cluster to be ready
288+
run: bundle exec rake wait
289+
env:
290+
DEBUG: '1'
291+
- name: Print containers
292+
run: docker compose -f $DOCKER_COMPOSE_FILE ps
293+
- name: Run memory profiler
294+
run: bundle exec rake prof
295+
env:
296+
MEMORY_PROFILE_MODE: pipelining_in_moderation
297+
- name: Stop containers
298+
run: docker compose -f $DOCKER_COMPOSE_FILE down

‎Rakefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ task :wait do
5252
require 'cluster_controller'
5353
::ClusterController.new(
5454
TEST_NODE_URIS,
55+
shard_size: TEST_SHARD_SIZE,
5556
replica_size: TEST_REPLICA_SIZE,
5657
**TEST_GENERIC_OPTIONS
5758
).wait_for_cluster_to_be_ready

‎bin/render_compose

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env ruby
2+
# frozen_string_literal: true
3+
4+
require 'erb'
5+
6+
path = File.join(File.expand_path('..', __dir__), 'compose.yaml.erb')
7+
content = File.read(path)
8+
template = ERB.new(content, trim_mode: '<>')
9+
10+
# rubocop:disable Lint/UselessAssignment
11+
shards = ENV.fetch('REDIS_SHARD_SIZE', '10').to_i
12+
n = shards + shards * ENV.fetch('REDIS_REPLICA_SIZE', '2').to_i
13+
port = 6379
14+
# rubocop:enable Lint/UselessAssignment
15+
16+
template.run

‎compose.large.yaml

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
---
2+
# bin/render_compose > compose.large.yaml
3+
#
4+
# https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/
5+
# https://github.com/redis/redis/blob/unstable/redis.conf
6+
services:
7+
node001: &node
8+
image: "redis:${REDIS_VERSION:-7}"
9+
command: >
10+
redis-server
11+
--maxmemory 32mb
12+
--maxmemory-policy allkeys-lru
13+
--maxmemory-clients 32%
14+
--maxclients 256
15+
--tcp-backlog 1024
16+
--timeout 0
17+
--tcp-keepalive 300
18+
--save ""
19+
--rdb-del-sync-files no
20+
--replica-serve-stale-data yes
21+
--replica-read-only yes
22+
--repl-backlog-size 2mb
23+
--repl-backlog-ttl 0
24+
--repl-disable-tcp-nodelay yes
25+
--repl-diskless-sync yes
26+
--repl-diskless-sync-delay 0
27+
--repl-diskless-sync-max-replicas 0
28+
--repl-diskless-load on-empty-db
29+
--repl-ping-replica-period 60
30+
--repl-timeout 300
31+
--min-replicas-to-write 0
32+
--min-replicas-max-lag 0
33+
--shutdown-timeout 0
34+
--shutdown-on-sigint nosave force now
35+
--shutdown-on-sigterm nosave force now
36+
--client-output-buffer-limit replica 0 0 0
37+
--client-query-buffer-limit 4mb
38+
--appendonly no
39+
--appendfsync no
40+
--cluster-enabled yes
41+
--cluster-config-file nodes.conf
42+
--cluster-node-timeout 300000
43+
--cluster-replica-validity-factor 5
44+
restart: "${RESTART_POLICY:-always}"
45+
healthcheck:
46+
test: ["CMD", "redis-cli", "ping"]
47+
interval: "7s"
48+
timeout: "5s"
49+
retries: 10
50+
ports:
51+
- "6379:6379"
52+
node002:
53+
<<: *node
54+
ports:
55+
- "6380:6379"
56+
node003:
57+
<<: *node
58+
ports:
59+
- "6381:6379"
60+
node004:
61+
<<: *node
62+
ports:
63+
- "6382:6379"
64+
node005:
65+
<<: *node
66+
ports:
67+
- "6383:6379"
68+
node006:
69+
<<: *node
70+
ports:
71+
- "6384:6379"
72+
node007:
73+
<<: *node
74+
ports:
75+
- "6385:6379"
76+
node008:
77+
<<: *node
78+
ports:
79+
- "6386:6379"
80+
node009:
81+
<<: *node
82+
ports:
83+
- "6387:6379"
84+
node010:
85+
<<: *node
86+
ports:
87+
- "6388:6379"
88+
node011:
89+
<<: *node
90+
ports:
91+
- "6389:6379"
92+
node012:
93+
<<: *node
94+
ports:
95+
- "6390:6379"
96+
node013:
97+
<<: *node
98+
ports:
99+
- "6391:6379"
100+
node014:
101+
<<: *node
102+
ports:
103+
- "6392:6379"
104+
node015:
105+
<<: *node
106+
ports:
107+
- "6393:6379"
108+
node016:
109+
<<: *node
110+
ports:
111+
- "6394:6379"
112+
node017:
113+
<<: *node
114+
ports:
115+
- "6395:6379"
116+
node018:
117+
<<: *node
118+
ports:
119+
- "6396:6379"
120+
node019:
121+
<<: *node
122+
ports:
123+
- "6397:6379"
124+
node020:
125+
<<: *node
126+
ports:
127+
- "6398:6379"
128+
node021:
129+
<<: *node
130+
ports:
131+
- "6399:6379"
132+
node022:
133+
<<: *node
134+
ports:
135+
- "6400:6379"
136+
node023:
137+
<<: *node
138+
ports:
139+
- "6401:6379"
140+
node024:
141+
<<: *node
142+
ports:
143+
- "6402:6379"
144+
node025:
145+
<<: *node
146+
ports:
147+
- "6403:6379"
148+
node026:
149+
<<: *node
150+
ports:
151+
- "6404:6379"
152+
node027:
153+
<<: *node
154+
ports:
155+
- "6405:6379"
156+
node028:
157+
<<: *node
158+
ports:
159+
- "6406:6379"
160+
node029:
161+
<<: *node
162+
ports:
163+
- "6407:6379"
164+
node030:
165+
<<: *node
166+
ports:
167+
- "6408:6379"
168+
clustering:
169+
image: "redis:${REDIS_VERSION:-7}"
170+
command: >
171+
bash -c "apt-get update > /dev/null
172+
&& apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null
173+
&& rm -rf /var/lib/apt/lists/*
174+
&& yes yes | redis-cli --cluster create
175+
$$(dig node001 +short):6379
176+
$$(dig node002 +short):6379
177+
$$(dig node003 +short):6379
178+
$$(dig node004 +short):6379
179+
$$(dig node005 +short):6379
180+
$$(dig node006 +short):6379
181+
$$(dig node007 +short):6379
182+
$$(dig node008 +short):6379
183+
$$(dig node009 +short):6379
184+
$$(dig node010 +short):6379
185+
$$(dig node011 +short):6379
186+
$$(dig node012 +short):6379
187+
$$(dig node013 +short):6379
188+
$$(dig node014 +short):6379
189+
$$(dig node015 +short):6379
190+
$$(dig node016 +short):6379
191+
$$(dig node017 +short):6379
192+
$$(dig node018 +short):6379
193+
$$(dig node019 +short):6379
194+
$$(dig node020 +short):6379
195+
$$(dig node021 +short):6379
196+
$$(dig node022 +short):6379
197+
$$(dig node023 +short):6379
198+
$$(dig node024 +short):6379
199+
$$(dig node025 +short):6379
200+
$$(dig node026 +short):6379
201+
$$(dig node027 +short):6379
202+
$$(dig node028 +short):6379
203+
$$(dig node029 +short):6379
204+
$$(dig node030 +short):6379
205+
--cluster-replicas 2"
206+
depends_on:
207+
node001:
208+
condition: service_healthy
209+
node002:
210+
condition: service_healthy
211+
node003:
212+
condition: service_healthy
213+
node004:
214+
condition: service_healthy
215+
node005:
216+
condition: service_healthy
217+
node006:
218+
condition: service_healthy
219+
node007:
220+
condition: service_healthy
221+
node008:
222+
condition: service_healthy
223+
node009:
224+
condition: service_healthy
225+
node010:
226+
condition: service_healthy
227+
node011:
228+
condition: service_healthy
229+
node012:
230+
condition: service_healthy
231+
node013:
232+
condition: service_healthy
233+
node014:
234+
condition: service_healthy
235+
node015:
236+
condition: service_healthy
237+
node016:
238+
condition: service_healthy
239+
node017:
240+
condition: service_healthy
241+
node018:
242+
condition: service_healthy
243+
node019:
244+
condition: service_healthy
245+
node020:
246+
condition: service_healthy
247+
node021:
248+
condition: service_healthy
249+
node022:
250+
condition: service_healthy
251+
node023:
252+
condition: service_healthy
253+
node024:
254+
condition: service_healthy
255+
node025:
256+
condition: service_healthy
257+
node026:
258+
condition: service_healthy
259+
node027:
260+
condition: service_healthy
261+
node028:
262+
condition: service_healthy
263+
node029:
264+
condition: service_healthy
265+
node030:
266+
condition: service_healthy

0 commit comments

Comments
 (0)