Skip to content

Commit 973e6c0

Browse files
wangyeleiwangyelei
authored
chore: sync redis acl after hscale (#2315)
1 parent 70c3632 commit 973e6c0

File tree

9 files changed

+191
-3
lines changed

9 files changed

+191
-3
lines changed

‎addons/redis/redis-cluster-scripts/redis-cluster-common.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,8 @@ check_redis_server_ready() {
159159
if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
160160
check_ready_cmd="redis-cli -h $host -p $port -a $REDIS_DEFAULT_PASSWORD ping"
161161
fi
162-
set_xtrace_when_ut_mode_false
163162
output=$($check_ready_cmd)
163+
set_xtrace_when_ut_mode_false
164164
status=$?
165165
if [ $status -ne 0 ] || [ "$output" != "PONG" ] ; then
166166
echo "Failed to execute the check ready command: $check_ready_cmd" >&2
@@ -253,8 +253,8 @@ get_cluster_info() {
253253
if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
254254
command="redis-cli -h $cluster_node -p $cluster_node_port -a $REDIS_DEFAULT_PASSWORD cluster info"
255255
fi
256-
set_xtrace_when_ut_mode_false
257256
cluster_info=$($command)
257+
set_xtrace_when_ut_mode_false
258258
status=$?
259259
if [ $status -ne 0 ]; then
260260
echo "Failed to execute the get cluster info command" >&2
@@ -272,8 +272,8 @@ get_cluster_nodes_info() {
272272
if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
273273
command="redis-cli -h $cluster_node -p $cluster_node_port -a $REDIS_DEFAULT_PASSWORD cluster nodes"
274274
fi
275-
set_xtrace_when_ut_mode_false
276275
cluster_nodes_info=$($command)
276+
set_xtrace_when_ut_mode_false
277277
status=$?
278278
if [ $status -ne 0 ]; then
279279
echo "Failed to execute the get cluster nodes info command" >&2

‎addons/redis/redis-cluster-scripts/redis-cluster-manage.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -890,6 +890,69 @@ scale_out_redis_cluster_shard() {
890890
return 0
891891
}
892892

893+
sync_acl_for_redis_cluster_shard() {
894+
echo "Sync ACL rules for redis cluster shard..."
895+
set +ex
896+
redis_base_cmd="redis-cli -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD"
897+
if [ -z "$REDIS_DEFAULT_PASSWORD" ]; then
898+
redis_base_cmd="redis-cli -p $SERVICE_PORT"
899+
fi
900+
is_ok=false
901+
acl_list=""
902+
# 1. get acl list from other pods
903+
for pod_name in $(echo "$KB_CLUSTER_POD_NAME_LIST" | tr ',' ' '); do
904+
pod_ip=$(parse_host_ip_from_built_in_envs "$pod_name" "$KB_CLUSTER_POD_NAME_LIST" "$KB_CLUSTER_POD_IP_LIST")
905+
if is_empty "$pod_ip"; then
906+
echo "Failed to get the host ip of the pod $pod_name"
907+
continue
908+
fi
909+
910+
cluster_info=$(get_cluster_info_with_retry "$pod_ip" "$SERVICE_PORT")
911+
status=$?
912+
if [ $status -ne 0 ]; then
913+
continue
914+
fi
915+
cluster_state=$(echo "$cluster_info" | awk -F: '/cluster_state/{print $2}' | tr -d '[:space:]')
916+
if is_empty "$cluster_state" || equals "$cluster_state" "ok"; then
917+
acl_list=$($redis_base_cmd -h "$pod_ip" ACL LIST)
918+
is_ok=true
919+
break
920+
fi
921+
done
922+
923+
if [ "$is_ok" = false ]; then
924+
echo "Failed to get ACL LIST from other shard pods" >&2
925+
exit 1
926+
fi
927+
928+
if [ -z "$acl_list" ]; then
929+
echo "No ACL rules found in other pods, skip synchronization" >&2
930+
return
931+
fi
932+
# 2. apply acl list to current shard pods
933+
set -e
934+
while IFS= read -r user_rule; do
935+
[[ -z "$user_rule" ]] && continue
936+
937+
if [[ "$user_rule" =~ ^user[[:space:]]+([^[:space:]]+) ]]; then
938+
username="${BASH_REMATCH[1]}"
939+
else
940+
# skip invalid user rule
941+
continue
942+
fi
943+
944+
if [[ "$username" == "default" ]]; then
945+
continue
946+
fi
947+
rule_part="${user_rule#user $username }"
948+
for pod_fqdn in $(echo "$CURRENT_SHARD_POD_FQDN_LIST" | tr ',' '\n'); do
949+
$redis_base_cmd -h $pod_fqdn ACL SETUSER "$username" $rule_part >&2
950+
$redis_base_cmd -h $pod_fqdn ACL save >&2
951+
done
952+
done <<< "$acl_list"
953+
set_xtrace_when_ut_mode_false
954+
}
955+
893956
scale_in_redis_cluster_shard() {
894957
# check KB_CLUSTER_COMPONENT_IS_SCALING_IN env
895958
if is_empty "$KB_CLUSTER_COMPONENT_IS_SCALING_IN"; then
@@ -971,6 +1034,7 @@ initialize_or_scale_out_redis_cluster() {
9711034
return 1
9721035
fi
9731036
else
1037+
sync_acl_for_redis_cluster_shard
9741038
echo "Redis Cluster already initialized, scaling out the shard..."
9751039
if scale_out_redis_cluster_shard; then
9761040
echo "Redis Cluster scale out shard successfully"

‎addons/redis/redis-cluster-scripts/sync-acl.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
3+
redis_base_cmd="redis-cli -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD"
4+
if [ -z "$REDIS_DEFAULT_PASSWORD" ]; then
5+
redis_base_cmd="redis-cli -p $SERVICE_PORT"
6+
fi
7+
8+
is_ok=false
9+
acl_list=""
10+
# 1. get acl list from other pods
11+
for pod_fqdn in $(echo "$CURRENT_SHARD_POD_FQDN_LIST" | tr ',' '\n'); do
12+
if [[ "$pod_fqdn" == "$KB_JOIN_MEMBER_POD_FQDN" ]]; then
13+
continue
14+
fi
15+
acl_list=$($redis_base_cmd -h "$pod_fqdn" ACL LIST)
16+
if [ $? -eq 0 ]; then
17+
is_ok=true
18+
break
19+
fi
20+
done
21+
22+
if [ "$is_ok" = false ]; then
23+
echo "Failed to get ACL LIST from other pods" >&2
24+
exit 1
25+
fi
26+
27+
if [ -z "$acl_list" ]; then
28+
echo "No ACL rules found in other pods, skip synchronization" >&2
29+
exit 0
30+
fi
31+
32+
set -e
33+
# 2. apply acl list to current pod
34+
while IFS= read -r user_rule; do
35+
[[ -z "$user_rule" ]] && continue
36+
37+
if [[ "$user_rule" =~ ^user[[:space:]]+([^[:space:]]+) ]]; then
38+
username="${BASH_REMATCH[1]}"
39+
else
40+
# skip invalid user rule
41+
continue
42+
fi
43+
44+
if [[ "$username" == "default" ]]; then
45+
continue
46+
fi
47+
rule_part="${user_rule#user $username }"
48+
$redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL SETUSER "$username" $rule_part >&2
49+
done <<< "$acl_list"
50+
51+
$redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL save >&2

‎addons/redis/scripts-ut-spec/redis_cluster_manage_spec.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1772,6 +1772,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
17721772
return 0
17731773
}
17741774

1775+
sync_acl_for_redis_cluster_shard() {
1776+
return 0
1777+
}
1778+
17751779
setup() {
17761780
export KB_CLUSTER_POD_IP_LIST="172.42.0.1,172.42.0.2,172.42.0.3,172.42.0.4,172.42.0.5,172.42.0.6"
17771781
export KB_CLUSTER_POD_NAME_LIST="redis-shard-98x-0,redis-shard-98x-1,redis-shard-7hy-0,redis-shard-7hy-1,redis-shard-jwl-0,redis-shard-jwl-1"

‎addons/redis/scripts/sync-acl.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
3+
redis_base_cmd="redis-cli -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD"
4+
if [ -z "$REDIS_DEFAULT_PASSWORD" ]; then
5+
redis_base_cmd="redis-cli -p $SERVICE_PORT"
6+
fi
7+
8+
is_ok=false
9+
acl_list=""
10+
# 1. get acl list from other pods
11+
for pod_fqdn in $(echo "$REDIS_POD_FQDN_LIST" | tr ',' '\n'); do
12+
if [[ "$pod_fqdn" == "$KB_JOIN_MEMBER_POD_FQDN" ]]; then
13+
continue
14+
fi
15+
acl_list=$($redis_base_cmd -h "$pod_fqdn" ACL LIST)
16+
if [ $? -eq 0 ]; then
17+
is_ok=true
18+
break
19+
fi
20+
done
21+
22+
if [ "$is_ok" = false ]; then
23+
echo "Failed to get ACL LIST from other pods" >&2
24+
exit 1
25+
fi
26+
27+
if [ -z "$acl_list" ]; then
28+
echo "No ACL rules found in other pods, skip synchronization" >&2
29+
exit 0
30+
fi
31+
32+
set -e
33+
# 2. apply acl list to current pod
34+
while IFS= read -r user_rule; do
35+
[[ -z "$user_rule" ]] && continue
36+
37+
if [[ "$user_rule" =~ ^user[[:space:]]+([^[:space:]]+) ]]; then
38+
username="${BASH_REMATCH[1]}"
39+
else
40+
# skip invalid user rule
41+
continue
42+
fi
43+
44+
if [[ "$username" == "default" ]]; then
45+
continue
46+
fi
47+
rule_part="${user_rule#user $username }"
48+
$redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL SETUSER "$username" $rule_part >&2
49+
done <<< "$acl_list"
50+
51+
$redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL save >&2

‎addons/redis/templates/cmpd-redis-cluster.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,14 @@ spec:
504504
- /bin/bash
505505
- -c
506506
- /scripts/{{ $redisClusterSwitchoverScripts }} > /tmp/switchover.log 2>&1
507+
memberJoin:
508+
exec:
509+
container: redis-cluster
510+
command:
511+
- /bin/bash
512+
- -c
513+
- /scripts/sync-acl.sh
514+
targetPodSelector: Any
507515
runtime:
508516
initContainers:
509517
- name: init-dbctl

‎addons/redis/templates/cmpd-redis.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,14 @@ spec:
377377
- /bin/bash
378378
- -c
379379
- /scripts/redis-switchover.sh > /tmp/switchover.log 2>&1
380+
memberJoin:
381+
exec:
382+
container: redis
383+
command:
384+
- /bin/bash
385+
- -c
386+
- /scripts/sync-acl.sh
387+
targetPodSelector: Any
380388
runtime:
381389
initContainers:
382390
- name: init-dbctl

‎addons/redis/templates/cmpv-redis-cluster.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ spec:
3434
preTerminate: {{ $redisRepository }}:{{ .imageTag }}
3535
memberLeave: {{ $redisRepository }}:{{ .imageTag }}
3636
metrics: {{ include "metrics.repository" $ }}:0.1.2-beta.1
37+
memberJoin: {{ $redisRepository }}:{{ .imageTag }}
3738
init-dbctl: {{ $.Values.dbctlImage.registry | default ( $.Values.image.registry | default "docker.io" ) }}/{{ $.Values.dbctlImage.repository }}:{{ $.Values.dbctlImage.tag }}
3839
{{- end }}
3940
{{- end }}

‎addons/redis/templates/cmpv-redis.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,6 @@ spec:
3333
postProvision: {{ $redisRepository }}:{{ .imageTag }}
3434
accountProvision: {{ $redisRepository }}:{{ .imageTag }}
3535
switchover: {{ $redisRepository }}:{{ .imageTag }}
36+
memberJoin: {{ $redisRepository }}:{{ .imageTag }}
3637
{{- end }}
3738
{{- end }}

0 commit comments

Comments
 (0)