Skip to content

fix: support cluster creation/scale with NodePort and TLS enabled #1568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 23, 2025
Merged

fix: support cluster creation/scale with NodePort and TLS enabled #1568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eb90b5a
fix: support cluster creation/scale with NodePort and TLS enabled
drivebyer Oct 23, 2025
7be5d05
fix test
drivebyer Oct 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/samber/lo v1.52.0
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
Expand Down
11 changes: 8 additions & 3 deletions internal/agent/bootstrap/redis/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ func GenerateConfig() error {
externalConfigFile, _ = util.CoalesceEnv("EXTERNAL_CONFIG_FILE", "/etc/redis/external.conf.d/redis-additional.conf")
redisMajorVersion, _ = util.CoalesceEnv("REDIS_MAJOR_VERSION", "v7")
redisPort, _ = util.CoalesceEnv("REDIS_PORT", "6379")
nodeport, _ = util.CoalesceEnv("NODEPORT", "false")
tlsMode, _ = util.CoalesceEnv("TLS_MODE", "false")
)

if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" {
Expand Down Expand Up @@ -70,7 +72,7 @@ func GenerateConfig() error {
fmt.Println("Setting up redis in standalone mode")
}

if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" {
if tlsMode == "true" {
redisTLSCert, _ := util.CoalesceEnv("REDIS_TLS_CERT", "")
redisTLSCertKey, _ := util.CoalesceEnv("REDIS_TLS_CERT_KEY", "")
redisTLSCAKey, _ := util.CoalesceEnv("REDIS_TLS_CA_KEY", "")
Expand All @@ -83,7 +85,7 @@ func GenerateConfig() error {

if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" {
cfg.Append("tls-cluster", "yes")
if redisMajorVersion == "v7" {
if redisMajorVersion == "v7" && nodeport == "false" {
cfg.Append("cluster-preferred-endpoint-type", "hostname")
}
}
Expand All @@ -108,7 +110,7 @@ func GenerateConfig() error {
fmt.Println("Running without persistence mode")
}

if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" {
if tlsMode == "true" {
cfg.Append("port", "0")
cfg.Append("tls-port", redisPort)
} else {
Expand All @@ -126,6 +128,9 @@ func GenerateConfig() error {

if clusterAnnouncePort != "" {
cfg.Append("cluster-announce-port", clusterAnnouncePort)
if tlsMode == "true" {
cfg.Append("cluster-announce-tls-port", clusterAnnouncePort)
}
}
if clusterAnnounceBusPort != "" {
cfg.Append("cluster-announce-bus-port", clusterAnnounceBusPort)
Expand Down
96 changes: 15 additions & 81 deletions internal/k8sutils/cluster-scaling.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package k8sutils
import (
"context"
"fmt"
"net"
"strconv"
"strings"

Expand Down Expand Up @@ -33,13 +34,7 @@ func ReshardRedisCluster(ctx context.Context, client kubernetes.Interface, cr *r
Namespace: cr.Namespace,
}
cmd = []string{"redis-cli", "--cluster", "reshard"}

if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
} else {
cmd = append(cmd, getRedisServerAddress(ctx, client, transferPOD, *cr.Spec.Port))
}

cmd = append(cmd, getEndpoint(ctx, client, cr, transferPOD))
if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key)
if err != nil {
Expand Down Expand Up @@ -75,7 +70,6 @@ func ReshardRedisCluster(ctx context.Context, client kubernetes.Interface, cr *r

cmd = append(cmd, "--cluster-yes")

log.FromContext(ctx).V(1).Info("redis cluster reshard command is", "Command", cmd)
log.FromContext(ctx).Info(fmt.Sprintf("transferring %s slots from shard %d to shard %d", slots, shardIdx, transferNodeIdx))
executeCommand(ctx, client, cr, cmd, transferNodeName)
log.FromContext(ctx).Info(fmt.Sprintf("transferring %s slots from shard %d to shard %d completed", slots, shardIdx, transferNodeIdx))
Expand Down Expand Up @@ -142,15 +136,8 @@ func RebalanceRedisClusterEmptyMasters(ctx context.Context, client kubernetes.In
Namespace: cr.Namespace,
}
cmd = []string{"redis-cli", "--cluster", "rebalance"}

if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
} else {
cmd = append(cmd, getRedisServerAddress(ctx, client, pod, *cr.Spec.Port))
}

cmd = append(cmd, getEndpoint(ctx, client, cr, pod))
cmd = append(cmd, "--cluster-use-empty-masters")

if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key)
if err != nil {
Expand All @@ -162,7 +149,6 @@ func RebalanceRedisClusterEmptyMasters(ctx context.Context, client kubernetes.In

cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...)

log.FromContext(ctx).V(1).Info("Redis cluster rebalance command is", "Command", cmd)
executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-1")
}

Expand Down Expand Up @@ -196,13 +182,7 @@ func RebalanceRedisCluster(ctx context.Context, client kubernetes.Interface, cr
Namespace: cr.Namespace,
}
cmd = []string{"redis-cli", "--cluster", "rebalance"}

if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
} else {
cmd = append(cmd, getRedisServerAddress(ctx, client, pod, *cr.Spec.Port))
}

cmd = append(cmd, getEndpoint(ctx, client, cr, pod))
if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key)
if err != nil {
Expand All @@ -214,15 +194,13 @@ func RebalanceRedisCluster(ctx context.Context, client kubernetes.Interface, cr

cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...)

log.FromContext(ctx).V(1).Info("Redis cluster rebalance command is", "Command", cmd)
executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-1")
}

// Add redis cluster node would add a node to the existing redis cluster using redis-cli
func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) {
var cmd []string
cmd := []string{"redis-cli", "--cluster", "add-node"}
activeRedisNode := CheckRedisNodeCount(ctx, client, cr, "leader")

newPod := RedisDetails{
PodName: cr.Name + "-leader-" + strconv.Itoa(int(activeRedisNode)),
Namespace: cr.Namespace,
Expand All @@ -231,17 +209,8 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr
PodName: cr.Name + "-leader-0",
Namespace: cr.Namespace,
}

cmd = []string{"redis-cli", "--cluster", "add-node"}

if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
} else {
cmd = append(cmd, getRedisServerAddress(ctx, client, newPod, *cr.Spec.Port))
cmd = append(cmd, getRedisServerAddress(ctx, client, existingPod, *cr.Spec.Port))
}

cmd = append(cmd, getEndpoint(ctx, client, cr, newPod))
cmd = append(cmd, getEndpoint(ctx, client, cr, existingPod))
if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key)
if err != nil {
Expand All @@ -253,7 +222,6 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr

cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...)

log.FromContext(ctx).V(1).Info("Redis cluster add-node command is", "Command", cmd)
executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0")
}

Expand Down Expand Up @@ -305,47 +273,25 @@ func RemoveRedisFollowerNodesFromCluster(ctx context.Context, client kubernetes.
followerNodeIDs := getAttachedFollowerNodeIDs(ctx, redisClient, lastLeaderPodNodeID)

cmd = append(cmd, "--cluster", "del-node")
if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
} else {
cmd = append(cmd, getRedisServerAddress(ctx, client, existingPod, *cr.Spec.Port))
}

cmd = append(cmd, getEndpoint(ctx, client, cr, existingPod))
for _, followerNodeID := range followerNodeIDs {
cmd = append(cmd, followerNodeID)
log.FromContext(ctx).V(1).Info("Redis cluster follower remove command is", "Command", cmd)
executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0")
cmd = cmd[:len(cmd)-1]
}
}

// Remove redis cluster node would remove last node to the existing redis cluster using redis-cli
func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster, removePod RedisDetails) {
var cmd []string
redisClient := configureRedisClient(ctx, client, cr, cr.Name+"-leader-0")
defer redisClient.Close()
// currentRedisCount := CheckRedisNodeCount(ctx, client, cr, "leader")

existingPod := RedisDetails{
PodName: cr.Name + "-leader-0",
Namespace: cr.Namespace,
}
//removePod := RedisDetails{
// PodName: cr.Name + "-leader-" + strconv.Itoa(int(currentRedisCount)-1),
// Namespace: cr.Namespace,
//}

cmd = []string{"redis-cli", "--cluster", "del-node"}

if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
} else {
cmd = append(cmd, getRedisServerAddress(ctx, client, existingPod, *cr.Spec.Port))
}

removePodNodeID := getRedisNodeID(ctx, client, cr, removePod)
cmd = append(cmd, removePodNodeID)

cmd := []string{"redis-cli", "--cluster", "del-node"}
cmd = append(cmd, getEndpoint(ctx, client, cr, existingPod))
cmd = append(cmd, getRedisNodeID(ctx, client, cr, removePod))
if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key)
if err != nil {
Expand All @@ -354,13 +300,7 @@ func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface
cmd = append(cmd, "-a")
cmd = append(cmd, pass)
}

cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...)

log.FromContext(ctx).V(1).Info("Redis cluster leader remove command is", "Command", cmd)
if getRedisClusterSlots(ctx, redisClient, removePodNodeID) != "0" {
log.FromContext(ctx).V(1).Info("Skipping execution remove leader not empty", "cmd", cmd)
}
executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0")
}

Expand Down Expand Up @@ -399,17 +339,11 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, cr *rcvb2
PodName: slavePodName,
Namespace: cr.Namespace,
}

cmd = []string{"redis-cli", "-h"}

if *cr.Spec.ClusterVersion == "v7" {
cmd = append(cmd, getRedisHostname(pod, cr, "leader"))
} else {
cmd = append(cmd, getRedisServerIP(ctx, client, pod))
host, port, err := net.SplitHostPort(getEndpoint(ctx, client, cr, pod))
if err != nil {
return err
}
cmd = append(cmd, "-p")
cmd = append(cmd, strconv.Itoa(*cr.Spec.Port))

cmd = []string{"redis-cli", "-h", host, "-p", port}
if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key)
if err != nil {
Expand Down
7 changes: 2 additions & 5 deletions internal/k8sutils/redis-cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,10 +368,7 @@ func (service RedisClusterService) CreateRedisClusterService(ctx context.Context
log.FromContext(ctx).Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole)
return err
}
additionalServiceType := cr.Spec.KubernetesConfig.GetServiceType()
if additionalServiceType == "NodePort" {
// If NodePort is enabled, we need to create a service for every redis pod.
// Then use --cluster-announce-ip --cluster-announce-port --cluster-announce-bus-port to make cluster.
if cr.Spec.KubernetesConfig.GetServiceType() == "NodePort" {
err = service.createOrUpdateClusterNodePortService(ctx, cr, cl)
if err != nil {
log.FromContext(ctx).Error(err, "Cannot create nodeport service for Redis", "Setup.Type", service.RedisServiceRole)
Expand All @@ -383,7 +380,7 @@ func (service RedisClusterService) CreateRedisClusterService(ctx context.Context
additionalExtraPorts = append(additionalExtraPorts, busPort)
}
if cr.Spec.KubernetesConfig.ShouldCreateAdditionalService() {
err = CreateOrUpdateService(ctx, cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, additionalServiceType, *cr.Spec.Port, cl, additionalExtraPorts...)
err = CreateOrUpdateService(ctx, cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, cr.Spec.KubernetesConfig.GetServiceType(), *cr.Spec.Port, cl, additionalExtraPorts...)
if err != nil {
log.FromContext(ctx).Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole)
return err
Expand Down
Loading