From e5053b9e22586d6032e956b42af69b7e3afc912f Mon Sep 17 00:00:00 2001 From: chris-gooch Date: Mon, 8 Sep 2025 15:52:56 +0100 Subject: [PATCH 1/3] Add ability to scale up single-node clusters Signed-off-by: chris-gooch --- .../rediscluster/rediscluster_controller.go | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/internal/controller/rediscluster/rediscluster_controller.go b/internal/controller/rediscluster/rediscluster_controller.go index 7a5263d8f3..fdcb9ad7d1 100644 --- a/internal/controller/rediscluster/rediscluster_controller.go +++ b/internal/controller/rediscluster/rediscluster_controller.go @@ -117,10 +117,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu k8sutils.ReshardRedisCluster(ctx, r.K8sClient, instance, shardIdx, shardMoveNodeIdx, true) monitoring.RedisClusterReshardTotal.WithLabelValues(instance.Namespace, instance.Name).Inc() } - logger.Info("Redis cluster is downscaled... Rebalancing the cluster") // Step 3 Rebalance the cluster - k8sutils.RebalanceRedisCluster(ctx, r.K8sClient, instance) - logger.Info("Redis cluster is downscaled... Rebalancing the cluster is done") + if leaderReplicas > 1 { + logger.Info("Redis cluster is downscaled... Rebalancing the cluster") + k8sutils.RebalanceRedisCluster(ctx, r.K8sClient, instance) + logger.Info("Redis cluster is downscaled... Rebalancing the cluster is done") + } else { + logger.Info("Redis cluster is downscaled... Skipping rebalance for single-node cluster") + } monitoring.RedisClusterRebalanceTotal.WithLabelValues(instance.Namespace, instance.Name).Inc() return intctrlutil.RequeueAfter(ctx, time.Second*10, "") } else { @@ -208,22 +212,32 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu logger.Info("Creating redis cluster by executing cluster creation commands") leaderCount := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, instance, "leader") if leaderCount != leaderReplicas { - logger.Info("Not all leader are part of the cluster...", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas) - if leaderCount <= 2 { - k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) - } else { - if leaderCount < leaderReplicas { - // Scale up the cluster - // Step 2 : Add Redis Node + logger.Info("Not all leaders are part of the cluster...", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas) + if leaderCount < leaderReplicas { + // Check if the single node has slots assigned and is a functioning cluster + // This helps distingush between scaling up vs initial multi-node creation + if slotsAssigned, err := r.Checker.CheckClusterSlotsAssigned(ctx, instance); err != nil { + logger.Error(err, "Failed to check cluster slots, creating multi-node cluster") + k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) + } else if slotsAssigned { + // This is a functioing single-node cluster being scaled up + logger.Info("Scaling up existing single-node cluster", "Current.Leaders", leaderCount, "Desired.Leaders", leaderReplicas) k8sutils.AddRedisNodeToCluster(ctx, r.K8sClient, instance) monitoring.RedisClusterAddingNodeAttempt.WithLabelValues(instance.Namespace, instance.Name).Inc() - // Step 3 Rebalance the cluster using the empty masters + // Rebalance the cluster using the empty masters + k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) + } else { + // Multi-node cluster scaling up + logger.Info("Adding node to existing multi-node cluster", "Current.Leaders", leaderCount, "Desired.Leaders", leaderReplicas) + k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) + monitoring.RedisClusterAddingNodeAttempt.WithLabelValues(instance.Namespace, instance.Name).Inc() + // Rebalance the cluster using empty masters k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) } } } else { if followerReplicas > 0 { - logger.Info("All leader are part of the cluster, adding follower/replicas", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas, "Follower.Replicas", followerReplicas) + logger.Info("All leaders are part of the cluster, adding follower/replicas", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas, "Follower.Replicas", followerReplicas) k8sutils.ExecuteRedisReplicationCommand(ctx, r.K8sClient, instance) } else { logger.Info("no follower/replicas configured, skipping replication configuration", "Leaders.Count", leaderCount, "Leader.Size", leaderReplicas, "Follower.Replicas", followerReplicas) From e43c64757c61e6242173f23c5ec8732488960a31 Mon Sep 17 00:00:00 2001 From: chris-gooch Date: Mon, 8 Sep 2025 16:46:44 +0100 Subject: [PATCH 2/3] Add ability to scale up single-node clusters Signed-off-by: chris-gooch --- .../rediscluster/rediscluster_controller.go | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/internal/controller/rediscluster/rediscluster_controller.go b/internal/controller/rediscluster/rediscluster_controller.go index fdcb9ad7d1..1126f4da8f 100644 --- a/internal/controller/rediscluster/rediscluster_controller.go +++ b/internal/controller/rediscluster/rediscluster_controller.go @@ -214,18 +214,25 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if leaderCount != leaderReplicas { logger.Info("Not all leaders are part of the cluster...", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas) if leaderCount < leaderReplicas { - // Check if the single node has slots assigned and is a functioning cluster - // This helps distingush between scaling up vs initial multi-node creation - if slotsAssigned, err := r.Checker.CheckClusterSlotsAssigned(ctx, instance); err != nil { - logger.Error(err, "Failed to check cluster slots, creating multi-node cluster") - k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) - } else if slotsAssigned { - // This is a functioing single-node cluster being scaled up - logger.Info("Scaling up existing single-node cluster", "Current.Leaders", leaderCount, "Desired.Leaders", leaderReplicas) - k8sutils.AddRedisNodeToCluster(ctx, r.K8sClient, instance) - monitoring.RedisClusterAddingNodeAttempt.WithLabelValues(instance.Namespace, instance.Name).Inc() - // Rebalance the cluster using the empty masters - k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) + // Check if we are expanding an existing single-node cluster or creating a new multi-node cluster + if leaderCount == 1 && leaderReplicas > 1 { + // Check if the single node has slots assigned and is a functioning cluster + // This helps distingush between scaling up vs initial multi-node creation + if slotsAssigned, err := r.Checker.CheckClusterSlotsAssigned(ctx, instance); err != nil { + logger.Error(err, "Failed to check cluster slots, creating multi-node cluster") + k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) + } else if slotsAssigned { + // This is a functioing single-node cluster being scaled up + logger.Info("Scaling up existing single-node cluster", "Current.Leaders", leaderCount, "Desired.Leaders", leaderReplicas) + k8sutils.AddRedisNodeToCluster(ctx, r.K8sClient, instance) + monitoring.RedisClusterAddingNodeAttempt.WithLabelValues(instance.Namespace, instance.Name).Inc() + // Rebalance the cluster using the empty masters + k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) + } else { + // Single node exists but has no slots - this is likely initial multi-node creation + logger.Info("Creating multi-node cluster") + k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) + } } else { // Multi-node cluster scaling up logger.Info("Adding node to existing multi-node cluster", "Current.Leaders", leaderCount, "Desired.Leaders", leaderReplicas) From 556ae644d45d7006063b54ffe5e6dfd0889d650f Mon Sep 17 00:00:00 2001 From: chris-gooch Date: Mon, 8 Sep 2025 16:48:31 +0100 Subject: [PATCH 3/3] Add ability to scale up single-node clusters Signed-off-by: chris-gooch --- internal/controller/rediscluster/rediscluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/rediscluster/rediscluster_controller.go b/internal/controller/rediscluster/rediscluster_controller.go index 1126f4da8f..3f9bdb25fe 100644 --- a/internal/controller/rediscluster/rediscluster_controller.go +++ b/internal/controller/rediscluster/rediscluster_controller.go @@ -230,7 +230,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu k8sutils.RebalanceRedisClusterEmptyMasters(ctx, r.K8sClient, instance) } else { // Single node exists but has no slots - this is likely initial multi-node creation - logger.Info("Creating multi-node cluster") + logger.Info("Creating multi-node cluster", "Current.Leaders", leaderCount, "Desired.Leaders", leaderReplicas) k8sutils.ExecuteRedisClusterCommand(ctx, r.K8sClient, instance) } } else {