chore: sync redis acl after hscale

addons/redis/redis-cluster-scripts/redis-cluster-common.sh

@@ -159,8 +159,8 @@ check_redis_server_ready() {
   if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
     check_ready_cmd="redis-cli -h $host -p $port -a $REDIS_DEFAULT_PASSWORD ping"
   fi
-  set_xtrace_when_ut_mode_false
   output=$($check_ready_cmd)
+  set_xtrace_when_ut_mode_false
   status=$?
   if [ $status -ne 0 ] || [ "$output" != "PONG" ] ; then
     echo "Failed to execute the check ready command: $check_ready_cmd" >&2
@@ -253,8 +253,8 @@ get_cluster_info() {
   if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
     command="redis-cli -h $cluster_node -p $cluster_node_port -a $REDIS_DEFAULT_PASSWORD cluster info"
   fi
-  set_xtrace_when_ut_mode_false
   cluster_info=$($command)
+  set_xtrace_when_ut_mode_false
   status=$?
   if [ $status -ne 0 ]; then
     echo "Failed to execute the get cluster info command" >&2
@@ -272,8 +272,8 @@ get_cluster_nodes_info() {
   if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
     command="redis-cli -h $cluster_node -p $cluster_node_port -a $REDIS_DEFAULT_PASSWORD cluster nodes"
   fi
-  set_xtrace_when_ut_mode_false
   cluster_nodes_info=$($command)
+  set_xtrace_when_ut_mode_false
   status=$?
   if [ $status -ne 0 ]; then
     echo "Failed to execute the get cluster nodes info command" >&2

addons/redis/redis-cluster-scripts/redis-cluster-manage.sh

@@ -890,6 +890,69 @@ scale_out_redis_cluster_shard() {
   return 0
 }
 
+sync_acl_for_redis_cluster_shard() {
+  echo "Sync ACL rules for redis cluster shard..."
+  set +ex
+  redis_base_cmd="redis-cli -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD"
+  if [ -z "$REDIS_DEFAULT_PASSWORD" ]; then
+     redis_base_cmd="redis-cli -p $SERVICE_PORT"
+  fi
+  is_ok=false
+  acl_list=""
+  # 1. get acl list from other pods
+  for pod_name in $(echo "$KB_CLUSTER_POD_NAME_LIST" | tr ',' ' '); do
+    pod_ip=$(parse_host_ip_from_built_in_envs "$pod_name" "$KB_CLUSTER_POD_NAME_LIST" "$KB_CLUSTER_POD_IP_LIST")
+    if is_empty "$pod_ip"; then
+      echo "Failed to get the host ip of the pod $pod_name"
+      continue
+    fi
+
+    cluster_info=$(get_cluster_info_with_retry "$pod_ip" "$SERVICE_PORT")
+    status=$?
+    if [ $status -ne 0 ]; then
+      continue
+    fi
+    cluster_state=$(echo "$cluster_info" | awk -F: '/cluster_state/{print $2}' | tr -d '[:space:]')
+    if is_empty "$cluster_state" || equals "$cluster_state" "ok"; then
+       acl_list=$($redis_base_cmd -h "$pod_ip" ACL LIST)
+       is_ok=true
+       break
+    fi
+  done
+
+  if [ "$is_ok" = false ]; then
+      echo "Failed to get ACL LIST from other shard pods" >&2
+      exit 1
+  fi
+
+  if [ -z "$acl_list" ]; then
+      echo "No ACL rules found in other pods, skip synchronization" >&2
+      return
+  fi
+  # 2. apply acl list to current shard pods
+  set -e
+  while IFS= read -r user_rule; do
+      [[ -z "$user_rule" ]] && continue
+
+      if [[ "$user_rule" =~ ^user[[:space:]]+([^[:space:]]+) ]]; then
+          username="${BASH_REMATCH[1]}"
+      else
+        # skip invalid user rule
+        continue
+      fi
+
+      if [[ "$username" == "default" ]]; then
+          continue
+      fi
+      rule_part="${user_rule#user $username }"
+      for pod_fqdn in $(echo "$CURRENT_SHARD_POD_FQDN_LIST" | tr ',' '\n'); do
+         $redis_base_cmd -h $pod_fqdn ACL SETUSER "$username" $rule_part >&2
+         $redis_base_cmd -h $pod_fqdn ACL save >&2
+      done
+  done <<< "$acl_list"
+  set_xtrace_when_ut_mode_false
+}
+
 scale_in_redis_cluster_shard() {
   # check KB_CLUSTER_COMPONENT_IS_SCALING_IN env
   if is_empty "$KB_CLUSTER_COMPONENT_IS_SCALING_IN"; then
@@ -971,6 +1034,7 @@ initialize_or_scale_out_redis_cluster() {
       return 1
     fi
   else
+    sync_acl_for_redis_cluster_shard
     echo "Redis Cluster already initialized, scaling out the shard..."
     if scale_out_redis_cluster_shard; then
       echo "Redis Cluster scale out shard successfully"

addons/redis/redis-cluster-scripts/sync-acl.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+redis_base_cmd="redis-cli -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD"
+if [ -z "$REDIS_DEFAULT_PASSWORD" ]; then
+   redis_base_cmd="redis-cli -p $SERVICE_PORT"
+fi
+
+is_ok=false
+acl_list=""
+# 1. get acl list from other pods
+for pod_fqdn in $(echo "$CURRENT_SHARD_POD_FQDN_LIST" | tr ',' '\n'); do
+    if [[ "$pod_fqdn" == "$KB_JOIN_MEMBER_POD_FQDN" ]]; then
+        continue
+    fi
+    acl_list=$($redis_base_cmd -h "$pod_fqdn" ACL LIST)
+    if [ $? -eq 0 ]; then
+        is_ok=true
+        break
+    fi
+done
+
+if [ "$is_ok" = false ]; then
+    echo "Failed to get ACL LIST from other pods" >&2
+    exit 1
+fi
+
+if [ -z "$acl_list" ]; then
+    echo "No ACL rules found in other pods, skip synchronization" >&2
+    exit 0
+fi
+
+set -e
+# 2. apply acl list to current pod
+while IFS= read -r user_rule; do
+    [[ -z "$user_rule" ]] && continue
+
+    if [[ "$user_rule" =~ ^user[[:space:]]+([^[:space:]]+) ]]; then
+        username="${BASH_REMATCH[1]}"
+    else
+      # skip invalid user rule
+      continue
+    fi
+
+    if [[ "$username" == "default" ]]; then
+        continue
+    fi
+    rule_part="${user_rule#user $username }"
+    $redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL SETUSER "$username" $rule_part >&2
+done <<< "$acl_list"
+
+$redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL save >&2

addons/redis/scripts-ut-spec/redis_cluster_manage_spec.sh

@@ -1772,6 +1772,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         return 0
       }
 
+      sync_acl_for_redis_cluster_shard() {
+        return 0
+      }
+
       setup() {
         export KB_CLUSTER_POD_IP_LIST="172.42.0.1,172.42.0.2,172.42.0.3,172.42.0.4,172.42.0.5,172.42.0.6"
         export KB_CLUSTER_POD_NAME_LIST="redis-shard-98x-0,redis-shard-98x-1,redis-shard-7hy-0,redis-shard-7hy-1,redis-shard-jwl-0,redis-shard-jwl-1"

addons/redis/scripts/sync-acl.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+redis_base_cmd="redis-cli -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD"
+if [ -z "$REDIS_DEFAULT_PASSWORD" ]; then
+   redis_base_cmd="redis-cli -p $SERVICE_PORT"
+fi
+
+is_ok=false
+acl_list=""
+# 1. get acl list from other pods
+for pod_fqdn in $(echo "$REDIS_POD_FQDN_LIST" | tr ',' '\n'); do
+    if [[ "$pod_fqdn" == "$KB_JOIN_MEMBER_POD_FQDN" ]]; then
+        continue
+    fi
+    acl_list=$($redis_base_cmd -h "$pod_fqdn" ACL LIST)
+    if [ $? -eq 0 ]; then
+        is_ok=true
+        break
+    fi
+done
+
+if [ "$is_ok" = false ]; then
+    echo "Failed to get ACL LIST from other pods" >&2
+    exit 1
+fi
+
+if [ -z "$acl_list" ]; then
+    echo "No ACL rules found in other pods, skip synchronization" >&2
+    exit 0
+fi
+
+set -e
+# 2. apply acl list to current pod
+while IFS= read -r user_rule; do
+    [[ -z "$user_rule" ]] && continue
+
+    if [[ "$user_rule" =~ ^user[[:space:]]+([^[:space:]]+) ]]; then
+        username="${BASH_REMATCH[1]}"
+    else
+      # skip invalid user rule
+      continue
+    fi
+
+    if [[ "$username" == "default" ]]; then
+        continue
+    fi
+    rule_part="${user_rule#user $username }"
+    $redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL SETUSER "$username" $rule_part >&2
+done <<< "$acl_list"
+
+$redis_base_cmd -h $KB_JOIN_MEMBER_POD_FQDN ACL save >&2

addons/redis/templates/cmpd-redis-cluster.yaml

@@ -504,6 +504,14 @@ spec:
           - /bin/bash
           - -c
           - /scripts/{{ $redisClusterSwitchoverScripts }}  > /tmp/switchover.log 2>&1
+    memberJoin:
+      exec:
+        container: redis-cluster
+        command:
+          - /bin/bash
+          - -c
+          - /scripts/sync-acl.sh
+        targetPodSelector: Any
   runtime:
     initContainers:
       - name: init-dbctl

addons/redis/templates/cmpd-redis.yaml

@@ -377,6 +377,14 @@ spec:
           - /bin/bash
           - -c
           - /scripts/redis-switchover.sh > /tmp/switchover.log 2>&1
+    memberJoin:
+      exec:
+        container: redis
+        command:
+          - /bin/bash
+          - -c
+          - /scripts/sync-acl.sh
+        targetPodSelector: Any
   runtime:
     initContainers:
       - name: init-dbctl

addons/redis/templates/cmpv-redis-cluster.yaml

@@ -34,6 +34,7 @@ spec:
       preTerminate: {{ $redisRepository }}:{{ .imageTag }}
       memberLeave: {{ $redisRepository }}:{{ .imageTag }}
       metrics: {{ include "metrics.repository" $ }}:0.1.2-beta.1
+      memberJoin: {{ $redisRepository }}:{{ .imageTag }}
       init-dbctl: {{ $.Values.dbctlImage.registry | default ( $.Values.image.registry | default "docker.io" ) }}/{{ $.Values.dbctlImage.repository }}:{{ $.Values.dbctlImage.tag }}
   {{- end }}
   {{- end }}

addons/redis/templates/cmpv-redis.yaml

@@ -33,5 +33,6 @@ spec:
       postProvision: {{ $redisRepository }}:{{ .imageTag }}
       accountProvision: {{ $redisRepository }}:{{ .imageTag }}
       switchover: {{ $redisRepository }}:{{ .imageTag }}
+      memberJoin: {{ $redisRepository }}:{{ .imageTag }}
   {{- end }}
   {{- end }}