diff --git a/go.mod b/go.mod index 31840ef769..5c2e5396e2 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/samber/lo v1.52.0 github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index 38a0919fe0..962b5a29eb 100644 --- a/go.sum +++ b/go.sum @@ -203,6 +203,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= +github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= diff --git a/internal/agent/bootstrap/redis/config.go b/internal/agent/bootstrap/redis/config.go index 5de7bf23de..f2ef107985 100644 --- a/internal/agent/bootstrap/redis/config.go +++ b/internal/agent/bootstrap/redis/config.go @@ -38,6 +38,8 @@ func GenerateConfig() error { externalConfigFile, _ = util.CoalesceEnv("EXTERNAL_CONFIG_FILE", "/etc/redis/external.conf.d/redis-additional.conf") redisMajorVersion, _ = util.CoalesceEnv("REDIS_MAJOR_VERSION", "v7") redisPort, _ = util.CoalesceEnv("REDIS_PORT", "6379") + nodeport, _ = util.CoalesceEnv("NODEPORT", "false") + tlsMode, _ = util.CoalesceEnv("TLS_MODE", "false") ) if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" { @@ -70,7 +72,7 @@ func GenerateConfig() error { fmt.Println("Setting up redis in standalone mode") } - if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" { + if tlsMode == "true" { redisTLSCert, _ := util.CoalesceEnv("REDIS_TLS_CERT", "") redisTLSCertKey, _ := util.CoalesceEnv("REDIS_TLS_CERT_KEY", "") redisTLSCAKey, _ := util.CoalesceEnv("REDIS_TLS_CA_KEY", "") @@ -83,7 +85,7 @@ func GenerateConfig() error { if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" { cfg.Append("tls-cluster", "yes") - if redisMajorVersion == "v7" { + if redisMajorVersion == "v7" && nodeport == "false" { cfg.Append("cluster-preferred-endpoint-type", "hostname") } } @@ -108,7 +110,7 @@ func GenerateConfig() error { fmt.Println("Running without persistence mode") } - if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" { + if tlsMode == "true" { cfg.Append("port", "0") cfg.Append("tls-port", redisPort) } else { @@ -126,6 +128,9 @@ func GenerateConfig() error { if clusterAnnouncePort != "" { cfg.Append("cluster-announce-port", clusterAnnouncePort) + if tlsMode == "true" { + cfg.Append("cluster-announce-tls-port", clusterAnnouncePort) + } } if clusterAnnounceBusPort != "" { cfg.Append("cluster-announce-bus-port", clusterAnnounceBusPort) diff --git a/internal/k8sutils/cluster-scaling.go b/internal/k8sutils/cluster-scaling.go index 4db01beae5..c7fc8ca75d 100644 --- a/internal/k8sutils/cluster-scaling.go +++ b/internal/k8sutils/cluster-scaling.go @@ -3,6 +3,7 @@ package k8sutils import ( "context" "fmt" + "net" "strconv" "strings" @@ -33,13 +34,7 @@ func ReshardRedisCluster(ctx context.Context, client kubernetes.Interface, cr *r Namespace: cr.Namespace, } cmd = []string{"redis-cli", "--cluster", "reshard"} - - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, transferPOD, *cr.Spec.Port)) - } - + cmd = append(cmd, getEndpoint(ctx, client, cr, transferPOD)) if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { @@ -75,7 +70,6 @@ func ReshardRedisCluster(ctx context.Context, client kubernetes.Interface, cr *r cmd = append(cmd, "--cluster-yes") - log.FromContext(ctx).V(1).Info("redis cluster reshard command is", "Command", cmd) log.FromContext(ctx).Info(fmt.Sprintf("transferring %s slots from shard %d to shard %d", slots, shardIdx, transferNodeIdx)) executeCommand(ctx, client, cr, cmd, transferNodeName) log.FromContext(ctx).Info(fmt.Sprintf("transferring %s slots from shard %d to shard %d completed", slots, shardIdx, transferNodeIdx)) @@ -142,15 +136,8 @@ func RebalanceRedisClusterEmptyMasters(ctx context.Context, client kubernetes.In Namespace: cr.Namespace, } cmd = []string{"redis-cli", "--cluster", "rebalance"} - - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, pod, *cr.Spec.Port)) - } - + cmd = append(cmd, getEndpoint(ctx, client, cr, pod)) cmd = append(cmd, "--cluster-use-empty-masters") - if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { @@ -162,7 +149,6 @@ func RebalanceRedisClusterEmptyMasters(ctx context.Context, client kubernetes.In cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) - log.FromContext(ctx).V(1).Info("Redis cluster rebalance command is", "Command", cmd) executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-1") } @@ -196,13 +182,7 @@ func RebalanceRedisCluster(ctx context.Context, client kubernetes.Interface, cr Namespace: cr.Namespace, } cmd = []string{"redis-cli", "--cluster", "rebalance"} - - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, pod, *cr.Spec.Port)) - } - + cmd = append(cmd, getEndpoint(ctx, client, cr, pod)) if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { @@ -214,15 +194,13 @@ func RebalanceRedisCluster(ctx context.Context, client kubernetes.Interface, cr cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) - log.FromContext(ctx).V(1).Info("Redis cluster rebalance command is", "Command", cmd) executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-1") } // Add redis cluster node would add a node to the existing redis cluster using redis-cli func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster) { - var cmd []string + cmd := []string{"redis-cli", "--cluster", "add-node"} activeRedisNode := CheckRedisNodeCount(ctx, client, cr, "leader") - newPod := RedisDetails{ PodName: cr.Name + "-leader-" + strconv.Itoa(int(activeRedisNode)), Namespace: cr.Namespace, @@ -231,17 +209,8 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr PodName: cr.Name + "-leader-0", Namespace: cr.Namespace, } - - cmd = []string{"redis-cli", "--cluster", "add-node"} - - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, newPod, *cr.Spec.Port)) - cmd = append(cmd, getRedisServerAddress(ctx, client, existingPod, *cr.Spec.Port)) - } - + cmd = append(cmd, getEndpoint(ctx, client, cr, newPod)) + cmd = append(cmd, getEndpoint(ctx, client, cr, existingPod)) if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { @@ -253,7 +222,6 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, cr cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) - log.FromContext(ctx).V(1).Info("Redis cluster add-node command is", "Command", cmd) executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0") } @@ -305,15 +273,9 @@ func RemoveRedisFollowerNodesFromCluster(ctx context.Context, client kubernetes. followerNodeIDs := getAttachedFollowerNodeIDs(ctx, redisClient, lastLeaderPodNodeID) cmd = append(cmd, "--cluster", "del-node") - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, existingPod, *cr.Spec.Port)) - } - + cmd = append(cmd, getEndpoint(ctx, client, cr, existingPod)) for _, followerNodeID := range followerNodeIDs { cmd = append(cmd, followerNodeID) - log.FromContext(ctx).V(1).Info("Redis cluster follower remove command is", "Command", cmd) executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0") cmd = cmd[:len(cmd)-1] } @@ -321,31 +283,15 @@ func RemoveRedisFollowerNodesFromCluster(ctx context.Context, client kubernetes. // Remove redis cluster node would remove last node to the existing redis cluster using redis-cli func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster, removePod RedisDetails) { - var cmd []string redisClient := configureRedisClient(ctx, client, cr, cr.Name+"-leader-0") defer redisClient.Close() - // currentRedisCount := CheckRedisNodeCount(ctx, client, cr, "leader") - existingPod := RedisDetails{ PodName: cr.Name + "-leader-0", Namespace: cr.Namespace, } - //removePod := RedisDetails{ - // PodName: cr.Name + "-leader-" + strconv.Itoa(int(currentRedisCount)-1), - // Namespace: cr.Namespace, - //} - - cmd = []string{"redis-cli", "--cluster", "del-node"} - - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port)) - } else { - cmd = append(cmd, getRedisServerAddress(ctx, client, existingPod, *cr.Spec.Port)) - } - - removePodNodeID := getRedisNodeID(ctx, client, cr, removePod) - cmd = append(cmd, removePodNodeID) - + cmd := []string{"redis-cli", "--cluster", "del-node"} + cmd = append(cmd, getEndpoint(ctx, client, cr, existingPod)) + cmd = append(cmd, getRedisNodeID(ctx, client, cr, removePod)) if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { @@ -354,13 +300,7 @@ func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface cmd = append(cmd, "-a") cmd = append(cmd, pass) } - cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) - - log.FromContext(ctx).V(1).Info("Redis cluster leader remove command is", "Command", cmd) - if getRedisClusterSlots(ctx, redisClient, removePodNodeID) != "0" { - log.FromContext(ctx).V(1).Info("Skipping execution remove leader not empty", "cmd", cmd) - } executeCommand(ctx, client, cr, cmd, cr.Name+"-leader-0") } @@ -399,17 +339,11 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, cr *rcvb2 PodName: slavePodName, Namespace: cr.Namespace, } - - cmd = []string{"redis-cli", "-h"} - - if *cr.Spec.ClusterVersion == "v7" { - cmd = append(cmd, getRedisHostname(pod, cr, "leader")) - } else { - cmd = append(cmd, getRedisServerIP(ctx, client, pod)) + host, port, err := net.SplitHostPort(getEndpoint(ctx, client, cr, pod)) + if err != nil { + return err } - cmd = append(cmd, "-p") - cmd = append(cmd, strconv.Itoa(*cr.Spec.Port)) - + cmd = []string{"redis-cli", "-h", host, "-p", port} if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { diff --git a/internal/k8sutils/redis-cluster.go b/internal/k8sutils/redis-cluster.go index 9d2f27569f..3a79ee6ae5 100644 --- a/internal/k8sutils/redis-cluster.go +++ b/internal/k8sutils/redis-cluster.go @@ -368,10 +368,7 @@ func (service RedisClusterService) CreateRedisClusterService(ctx context.Context log.FromContext(ctx).Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err } - additionalServiceType := cr.Spec.KubernetesConfig.GetServiceType() - if additionalServiceType == "NodePort" { - // If NodePort is enabled, we need to create a service for every redis pod. - // Then use --cluster-announce-ip --cluster-announce-port --cluster-announce-bus-port to make cluster. + if cr.Spec.KubernetesConfig.GetServiceType() == "NodePort" { err = service.createOrUpdateClusterNodePortService(ctx, cr, cl) if err != nil { log.FromContext(ctx).Error(err, "Cannot create nodeport service for Redis", "Setup.Type", service.RedisServiceRole) @@ -383,7 +380,7 @@ func (service RedisClusterService) CreateRedisClusterService(ctx context.Context additionalExtraPorts = append(additionalExtraPorts, busPort) } if cr.Spec.KubernetesConfig.ShouldCreateAdditionalService() { - err = CreateOrUpdateService(ctx, cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, additionalServiceType, *cr.Spec.Port, cl, additionalExtraPorts...) + err = CreateOrUpdateService(ctx, cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, cr.Spec.KubernetesConfig.GetServiceType(), *cr.Spec.Port, cl, additionalExtraPorts...) if err != nil { log.FromContext(ctx).Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/internal/k8sutils/redis.go b/internal/k8sutils/redis.go index 09f8ec87cd..905d236a4c 100644 --- a/internal/k8sutils/redis.go +++ b/internal/k8sutils/redis.go @@ -10,10 +10,12 @@ import ( "strconv" "strings" - common "github.com/OT-CONTAINER-KIT/redis-operator/api/common/v1beta2" + commonapi "github.com/OT-CONTAINER-KIT/redis-operator/api/common/v1beta2" rcvb2 "github.com/OT-CONTAINER-KIT/redis-operator/api/rediscluster/v1beta2" rrvb2 "github.com/OT-CONTAINER-KIT/redis-operator/api/redisreplication/v1beta2" + common "github.com/OT-CONTAINER-KIT/redis-operator/internal/controller/common" redis "github.com/redis/go-redis/v9" + "github.com/samber/lo" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -28,6 +30,14 @@ type RedisDetails struct { Namespace string } +func (rd *RedisDetails) FQDN() string { + return fmt.Sprintf("%s.%s.%s.svc", rd.PodName, common.GetHeadlessServiceNameFromPodName(rd.PodName), rd.Namespace) +} + +func (rd *RedisDetails) String() string { + return fmt.Sprintf("%s.%s", rd.PodName, rd.Namespace) +} + // getRedisServerIP will return the IP of redis service func getRedisServerIP(ctx context.Context, client kubernetes.Interface, redisInfo RedisDetails) string { log.FromContext(ctx).V(1).Info("Fetching Redis pod", "namespace", redisInfo.Namespace, "podName", redisInfo.PodName) @@ -68,10 +78,48 @@ func getRedisServerAddress(ctx context.Context, client kubernetes.Interface, rd return fmt.Sprintf(format, ip, port) } -// getRedisHostname will return the complete FQDN for redis -func getRedisHostname(redisInfo RedisDetails, cr *rcvb2.RedisCluster, role string) string { - fqdn := fmt.Sprintf("%s.%s-%s-headless.%s.svc", redisInfo.PodName, cr.Name, role, cr.Namespace) - return fqdn +func getEndpoint(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster, rd RedisDetails) string { + var ( + host string + port int + ) + port = *cr.Spec.Port + if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { + host = rd.FQDN() + } else { + host = getRedisServerIP(ctx, client, rd) + if host == "" { + return "" + } + // if ip is IPv6, wrap it in brackets + if net.ParseIP(host).To4() == nil { + host = "[" + host + "]" + } + } + if cr.Spec.KubernetesConfig.GetServiceType() == "NodePort" { + svc, err := getService(ctx, client, cr.Namespace, rd.PodName) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to get service for redis pod", "Pod", rd.PodName) + return "" + } + if svc.Spec.Type != corev1.ServiceTypeNodePort { + log.FromContext(ctx).Error(errors.New("service type mismatch"), "Expected NodePort service type", "Pod", rd.PodName, "ActualType", svc.Spec.Type) + return "" + } + svcPort, ok := lo.Find(svc.Spec.Ports, func(item corev1.ServicePort) bool { + return item.Name == "redis-client" + }) + if ok { + port = int(svcPort.NodePort) + } + pod, err := client.CoreV1().Pods(rd.Namespace).Get(ctx, rd.PodName, metav1.GetOptions{}) + if err != nil { + log.FromContext(ctx).Error(err, "") + return "" + } + host = pod.Status.HostIP + } + return host + ":" + strconv.Itoa(port) } // CreateSingleLeaderRedisCommand will create command for single leader cluster creation @@ -151,20 +199,14 @@ func CreateMultipleLeaderRedisCommand(ctx context.Context, client kubernetes.Int Command: []string{"redis-cli", "--cluster", "create"}, } replicas := cr.Spec.GetReplicaCounts("leader") - for podCount := 0; podCount < int(replicas); podCount++ { - podName := cr.Name + "-leader-" + strconv.Itoa(podCount) - var address string - if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { - address = getRedisHostname(RedisDetails{PodName: podName, Namespace: cr.Namespace}, cr, "leader") + fmt.Sprintf(":%d", *cr.Spec.Port) - } else { - address = getRedisServerAddress(ctx, client, RedisDetails{PodName: podName, Namespace: cr.Namespace}, *cr.Spec.Port) + rd := RedisDetails{ + PodName: cr.Name + "-leader-" + strconv.Itoa(podCount), + Namespace: cr.Namespace, } - cmd.AddFlag(address) + cmd.AddFlag(getEndpoint(ctx, client, cr, rd)) } cmd.AddFlag("--cluster-yes") - - log.FromContext(ctx).V(1).Info("Redis cluster creation command", "CommandBase", []string{"redis-cli", "cluster", "create"}, "Replicas", replicas) return cmd } @@ -212,18 +254,16 @@ func ExecuteRedisClusterCommand(ctx context.Context, client kubernetes.Interface cmd.AddFlag(pass) } cmd.AddFlag(getRedisTLSArgs(cr.Spec.TLS, cr.Name+"-leader-0")...) - log.FromContext(ctx).V(1).Info("Redis cluster creation command is", "Command", cmd) executeCommand(ctx, client, cr, cmd.Args(), cr.Name+"-leader-0") } -func getRedisTLSArgs(tlsConfig *common.TLSConfig, clientHost string) []string { +func getRedisTLSArgs(tlsConfig *commonapi.TLSConfig, clientHost string) []string { cmd := []string{} if tlsConfig != nil { cmd = append(cmd, "--tls") cmd = append(cmd, "--cacert") cmd = append(cmd, "/tls/ca.crt") - cmd = append(cmd, "-h") - cmd = append(cmd, clientHost) + cmd = append(cmd, "--insecure") } return cmd } @@ -231,18 +271,9 @@ func getRedisTLSArgs(tlsConfig *common.TLSConfig, clientHost string) []string { // createRedisReplicationCommand will create redis replication creation command func createRedisReplicationCommand(ctx context.Context, client kubernetes.Interface, cr *rcvb2.RedisCluster, leaderPod RedisDetails, followerPod RedisDetails) []string { cmd := []string{"redis-cli", "--cluster", "add-node"} - var followerAddress, leaderAddress string - - if cr.Spec.ClusterVersion != nil && *cr.Spec.ClusterVersion == "v7" { - followerAddress = getRedisHostname(followerPod, cr, "follower") + fmt.Sprintf(":%d", *cr.Spec.Port) - leaderAddress = getRedisHostname(leaderPod, cr, "leader") + fmt.Sprintf(":%d", *cr.Spec.Port) - } else { - followerAddress = getRedisServerAddress(ctx, client, followerPod, *cr.Spec.Port) - leaderAddress = getRedisServerAddress(ctx, client, leaderPod, *cr.Spec.Port) - } - - cmd = append(cmd, followerAddress, leaderAddress, "--cluster-slave") - + cmd = append(cmd, getEndpoint(ctx, client, cr, followerPod)) + cmd = append(cmd, getEndpoint(ctx, client, cr, leaderPod)) + cmd = append(cmd, "--cluster-slave") if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil { pass, err := getRedisPassword(ctx, client, cr.Namespace, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Name, *cr.Spec.KubernetesConfig.ExistingPasswordSecret.Key) if err != nil { @@ -251,13 +282,7 @@ func createRedisReplicationCommand(ctx context.Context, client kubernetes.Interf cmd = append(cmd, "-a", pass) } } - cmd = append(cmd, getRedisTLSArgs(cr.Spec.TLS, leaderPod.PodName)...) - - log.FromContext(ctx).V(1).Info("Generated Redis replication command", - "FollowerAddress", followerAddress, "LeaderAddress", leaderAddress, - "Command", cmd) - return cmd } @@ -286,7 +311,7 @@ func ExecuteRedisReplicationCommand(ctx context.Context, client kubernetes.Inter Namespace: cr.Namespace, } podIP = getRedisServerIP(ctx, client, followerPod) - if !checkRedisNodePresence(ctx, cr, nodes, podIP) { + if !checkRedisNodePresence(ctx, nodes, podIP) { log.FromContext(ctx).V(1).Info("Adding node to cluster.", "Node.IP", podIP, "Follower.Pod", followerPod) cmd := createRedisReplicationCommand(ctx, client, cr, leaderPod, followerPod) redisClient := configureRedisClient(ctx, client, cr, followerPod.PodName) @@ -571,7 +596,7 @@ func getContainerID(ctx context.Context, client kubernetes.Interface, cr *rcvb2. } // checkRedisNodePresence will check if the redis node exist in cluster or not -func checkRedisNodePresence(ctx context.Context, cr *rcvb2.RedisCluster, nodeList []clusterNodesResponse, nodeName string) bool { +func checkRedisNodePresence(ctx context.Context, nodeList []clusterNodesResponse, nodeName string) bool { log.FromContext(ctx).V(1).Info("Checking if Node is in cluster", "Node", nodeName) for _, node := range nodeList { s := strings.Split(node[1], ":") diff --git a/internal/k8sutils/redis_test.go b/internal/k8sutils/redis_test.go index ffe2e16996..b8e10a092a 100644 --- a/internal/k8sutils/redis_test.go +++ b/internal/k8sutils/redis_test.go @@ -22,7 +22,6 @@ import ( ) func TestCheckRedisNodePresence(t *testing.T) { - cr := &rcvb2.RedisCluster{} output := "205dd1780dda981f9320c9d47d069b3c0ceaa358 172.17.0.24:6379@16379 slave b65312dcf5537b8826c344783f078096fdb7f27c 0 1654197347000 1 connected\nfaa21623054227826e93dd71314cce3706491dac 172.17.0.28:6379@16379 slave d54557b21bc5a5aa947ce58b7dbadc5d39bdd551 0 1654197347000 2 connected\nb65312dcf5537b8826c344783f078096fdb7f27c 172.17.0.25:6379@16379 master - 0 1654197346000 1 connected 0-5460\nd54557b21bc5a5aa947ce58b7dbadc5d39bdd551 172.17.0.29:6379@16379 myself,master - 0 1654197347000 2 connected 5461-10922\nc9fa05269c4e662295bf34eb93f1315f962493ba 172.17.0.3:6379@16379 master - 0 1654197348006 3 connected 10923-16383" csvOutput := csv.NewReader(strings.NewReader(output)) csvOutput.Comma = ' ' @@ -47,7 +46,7 @@ func TestCheckRedisNodePresence(t *testing.T) { for _, tt := range tests { testname := fmt.Sprintf("%s,%s", tt.nodes, tt.ip) t.Run(testname, func(t *testing.T) { - ans := checkRedisNodePresence(context.TODO(), cr, tt.nodes, tt.ip) + ans := checkRedisNodePresence(context.TODO(), tt.nodes, tt.ip) if ans != tt.want { t.Errorf("got %t, want %t", ans, tt.want) } @@ -303,39 +302,6 @@ func TestGetRedisServerAddress(t *testing.T) { } } -func TestGetRedisHostname(t *testing.T) { - tests := []struct { - name string - redisInfo RedisDetails - redisCluster *rcvb2.RedisCluster - role string - expected string - }{ - { - name: "standard configuration", - redisInfo: RedisDetails{ - PodName: "redis-pod", - Namespace: "default", - }, - redisCluster: &rcvb2.RedisCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mycluster", - Namespace: "default", - }, - }, - role: "master", - expected: "redis-pod.mycluster-master-headless.default.svc", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fqdn := getRedisHostname(tt.redisInfo, tt.redisCluster, tt.role) - assert.Equal(t, tt.expected, fqdn, "FQDN should match the expected output") - }) - } -} - func TestCreateSingleLeaderRedisCommand(t *testing.T) { cr := &rcvb2.RedisCluster{} invocation := CreateSingleLeaderRedisCommand(context.TODO(), cr) @@ -423,7 +389,7 @@ func TestGetRedisTLSArgs(t *testing.T) { name: "with TLS configuration", tlsConfig: &common.TLSConfig{}, clientHost: "redis-host", - expected: []string{"--tls", "--cacert", "/tls/ca.crt", "-h", "redis-host"}, + expected: []string{"--tls", "--cacert", "/tls/ca.crt", "--insecure"}, }, { name: "without TLS configuration", diff --git a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml deleted file mode 100644 index 3473eb09ae..0000000000 --- a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json -apiVersion: chainsaw.kyverno.io/v1alpha1 -kind: Test -metadata: - name: nodeport-redis-cluster -spec: - steps: - - name: Setup redis cluster use nodeport - try: - - apply: - file: cluster.yaml - - assert: - file: ready-cluster.yaml - - - name: Try saving a key With Password - try: - - script: - timeout: 30s - content: | - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- redis-cli -c -p 6379 set foo-0 bar-0 - check: - (contains($stdout, 'OK')): true diff --git a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/cluster.yaml b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/cluster.yaml deleted file mode 100644 index 634018552e..0000000000 --- a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/cluster.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -apiVersion: redis.redis.opstreelabs.in/v1beta2 -kind: RedisCluster -metadata: - name: redis-cluster-v1beta2 -spec: - clusterSize: 3 - clusterVersion: v6 - persistenceEnabled: true - podSecurityContext: - runAsUser: 1000 - fsGroup: 1000 - kubernetesConfig: - service: - serviceType: NodePort - image: quay.io/opstree/redis:v6.2.14 - imagePullPolicy: Always - resources: - requests: - cpu: 101m - memory: 128Mi - limits: - cpu: 101m - memory: 128Mi - redisExporter: - enabled: true - image: quay.io/opstree/redis-exporter:v1.44.0 - imagePullPolicy: Always - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 100m - memory: 128Mi - storage: - volumeClaimTemplate: - spec: - # storageClassName: standard - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi - nodeConfVolume: true - nodeConfVolumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml deleted file mode 100644 index a0708b5a5e..0000000000 --- a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: redis.redis.opstreelabs.in/v1beta2 -kind: RedisCluster -metadata: - name: redis-cluster-v1beta2 -status: - readyFollowerReplicas: 3 - readyLeaderReplicas: 3 - state: Ready - reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/cluster.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/cluster.yaml index 2432c2495e..a03c48e845 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/cluster.yaml @@ -22,6 +22,7 @@ spec: fsGroup: 1000 kubernetesConfig: service: + serviceType: NodePort includeBusPort: true headless: includeBusPort: true diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml index 867a4cd42c..6f6bc90bc1 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/ready-svc.yaml @@ -31,9 +31,6 @@ spec: app: redis-cluster-v1beta2-leader redis_setup_type: cluster role: leader - type: ClusterIP -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -71,9 +68,6 @@ spec: app: redis-cluster-v1beta2-leader redis_setup_type: cluster role: leader - type: ClusterIP -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -95,7 +89,6 @@ metadata: kind: RedisCluster name: redis-cluster-v1beta2 spec: - clusterIP: None ports: - name: redis-client port: 6379 @@ -109,9 +102,6 @@ spec: app: redis-cluster-v1beta2-leader redis_setup_type: cluster role: leader - type: ClusterIP -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -149,9 +139,6 @@ spec: app: redis-cluster-v1beta2-follower redis_setup_type: cluster role: follower - type: ClusterIP -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -185,9 +172,6 @@ spec: app: redis-cluster-v1beta2-follower redis_setup_type: cluster role: follower - type: ClusterIP -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -209,7 +193,6 @@ metadata: kind: RedisCluster name: redis-cluster-v1beta2 spec: - clusterIP: None ports: - name: redis-client port: 6379 @@ -223,9 +206,6 @@ spec: app: redis-cluster-v1beta2-follower redis_setup_type: cluster role: follower - type: ClusterIP -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -250,4 +230,3 @@ spec: cluster: redis-cluster-v1beta2 redis-role: master redis_setup_type: cluster - type: ClusterIP