fix: reset sentinel config on reconcile (#1533)

Signed-off-by: yangw <[email protected]>

Author: drivebyer

internal/controller/common/redis/heal.go

@@ -22,6 +22,9 @@ import (
 
 type Healer interface {
 	SentinelMonitor(ctx context.Context, rs *rsvb2.RedisSentinel, master string) error
+	// SentinelSet set the config for specific master
+	// reference: https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#reconfiguring-sentinel-at-runtime
+	SentinelSet(ctx context.Context, rs *rsvb2.RedisSentinel, master string) error
 	SentinelReset(ctx context.Context, rs *rsvb2.RedisSentinel) error
 
 	// UpdatePodRoleLabel connect to all redis pods and update pod role label `redis-role` to `master` or `slave` according to their role.
@@ -84,6 +87,35 @@ func (h *healer) UpdateRedisRoleLabel(ctx context.Context, ns string, labels map
 	return nil
 }
 
+func (h *healer) SentinelSet(ctx context.Context, rs *rsvb2.RedisSentinel, master string) error {
+	pods, err := h.getSentinelPods(ctx, rs)
+	if err != nil {
+		return err
+	}
+	sentinelPass, err := NewChecker(h.k8s).GetPassword(ctx, rs.Namespace, rs.Spec.KubernetesConfig.ExistingPasswordSecret)
+	if err != nil {
+		return err
+	}
+	for _, pod := range pods.Items {
+		connInfo := createConnectionInfo(ctx, pod, sentinelPass, rs.Spec.TLS, h.k8s, rs.Namespace, "26379")
+
+		for k, v := range map[string]string{
+			"down-after-milliseconds": rs.Spec.RedisSentinelConfig.DownAfterMilliseconds,
+			"parallel-syncs":          rs.Spec.RedisSentinelConfig.ParallelSyncs,
+			"failover-timeout":        rs.Spec.RedisSentinelConfig.FailoverTimeout,
+		} {
+			if v == "" {
+				continue
+			}
+			err = h.redis.Connect(connInfo).SentinelSet(ctx, rs.Spec.RedisSentinelConfig.MasterGroupName, k, v)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // SentinelReset range all sentinel execute `sentinel reset *`
 func (h *healer) SentinelReset(ctx context.Context, rs *rsvb2.RedisSentinel) error {
 	pods, err := h.getSentinelPods(ctx, rs)

internal/controller/redissentinel/redissentinel_controller.go

@@ -115,7 +115,6 @@ func (r *RedisSentinelReconciler) reconcileSentinel(ctx context.Context, instanc
 	if err := k8sutils.CreateRedisSentinel(ctx, r.K8sClient, instance, r.K8sClient, r.Client); err != nil {
 		return intctrlutil.RequeueE(ctx, err, "")
 	}
-
 	if instance.Spec.RedisSentinelConfig == nil {
 		return intctrlutil.Reconciled()
 	}
@@ -138,14 +137,15 @@ func (r *RedisSentinelReconciler) reconcileSentinel(ctx context.Context, instanc
 			monitorAddr = master.Status.PodIP
 		}
 	}
-
 	if err := r.Healer.SentinelMonitor(ctx, instance, monitorAddr); err != nil {
 		return intctrlutil.RequeueE(ctx, err, "")
 	}
+	if err := r.Healer.SentinelSet(ctx, instance, monitorAddr); err != nil {
+		return intctrlutil.RequeueE(ctx, err, "")
+	}
 	if err := r.Healer.SentinelReset(ctx, instance); err != nil {
 		return intctrlutil.RequeueE(ctx, err, "")
 	}
-
 	return intctrlutil.Reconciled()
 }
 

internal/service/redis/client.go

@@ -54,6 +54,7 @@ type Service interface {
 	IsMaster(ctx context.Context) (bool, error)
 	GetAttachedReplicaCount(ctx context.Context) (int, error)
 	SentinelMonitor(ctx context.Context, master *ConnectionInfo, masterGroupName, quorum string) error
+	SentinelSet(ctx context.Context, masterGroupName, key, value string) error
 	SentinelReset(ctx context.Context, masterGroupName string) error
 	GetClusterInfo(ctx context.Context) (*ClusterStatus, error)
 }
@@ -81,6 +82,24 @@ func (s *service) createClient() *rediscli.Client {
 	return rediscli.NewClient(opts)
 }
 
+func (c *service) SentinelSet(ctx context.Context, masterGroupName, key, value string) error {
+	client := c.createClient()
+	if client == nil {
+		return nil
+	}
+	defer client.Close()
+
+	cmd := rediscli.NewStringCmd(ctx, "SENTINEL", "SET", masterGroupName, key, value)
+	err := client.Process(ctx, cmd)
+	if err != nil {
+		return err
+	}
+	if err = cmd.Err(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *service) SentinelReset(ctx context.Context, masterGroupName string) error {
 	client := c.createClient()
 	if client == nil {