Skip to content

Commit 7e7b872

Browse files
drivebyerdrivebyer
authored
fix: cluster TLS configuration for init-config container (#1489)
* fix: cluster TLS configuration for init-config container - Refactored Redis cluster configuration generation to improve IP handling and code structure - Replace exec commands with Go-native implementations using fqdn.FqdnHostname() for hostname resolution - Add proper IP address handling in nodes.conf file updates using regex patterns - Improve TLS configuration with cluster-preferred-endpoint-type for Redis v7 - Update E2E tests to support TLS verification in cluster mode This change is only applicable when the GenerateConfigInInitContainer feature gate is enabled. Signed-off-by: yangw <[email protected]>
1 parent 9280c02 commit 7e7b872

File tree

8 files changed

+234
-133
lines changed

8 files changed

+234
-133
lines changed

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ require (
2626

2727
require (
2828
emperror.dev/errors v0.8.0 // indirect
29+
github.com/Showmax/go-fqdn v1.0.0 // indirect
2930
github.com/beorn7/perks v1.0.1 // indirect
3031
github.com/blang/semver/v4 v4.0.0 // indirect
3132
github.com/cespare/xxhash/v2 v2.3.0 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
55
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
66
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
77
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
8+
github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM=
9+
github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko=
810
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
911
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
1012
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=

‎internal/agent/bootstrap/redis/config.go‎

Lines changed: 127 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package bootstrap
22

33
import (
4+
"bufio"
5+
"bytes"
46
"fmt"
57
"log"
68
"os"
7-
"os/exec"
89
"path/filepath"
10+
"regexp"
911
"strings"
1012

1113
agentutil "github.com/OT-CONTAINER-KIT/redis-operator/internal/agent/util"
1214
"github.com/OT-CONTAINER-KIT/redis-operator/internal/consts"
1315
"github.com/OT-CONTAINER-KIT/redis-operator/internal/util"
16+
"github.com/Showmax/go-fqdn"
1417
)
1518

1619
// defaultRedisConfig from https://github.com/OT-CONTAINER-KIT/redis/blob/master/redis.conf
@@ -34,156 +37,127 @@ func GenerateConfig() error {
3437
nodeConfDir, _ = util.CoalesceEnv("NODE_CONF_DIR", "/node-conf")
3538
externalConfigFile, _ = util.CoalesceEnv("EXTERNAL_CONFIG_FILE", "/etc/redis/external.conf.d/redis-additional.conf")
3639
redisMajorVersion, _ = util.CoalesceEnv("REDIS_MAJOR_VERSION", "v7")
40+
redisPort, _ = util.CoalesceEnv("REDIS_PORT", "6379")
3741
)
3842

39-
// set_redis_password - configure Redis password
40-
{
41-
if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" {
42-
cfg.Append("masterauth", val)
43-
cfg.Append("requirepass", val)
44-
cfg.Append("protected-mode", "yes")
45-
} else {
46-
fmt.Println("Redis is running without password which is not recommended")
47-
cfg.Append("protected-mode", "no")
48-
}
43+
if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" {
44+
cfg.Append("masterauth", val)
45+
cfg.Append("requirepass", val)
46+
cfg.Append("protected-mode", "yes")
47+
} else {
48+
fmt.Println("Redis is running without password which is not recommended")
49+
cfg.Append("protected-mode", "no")
4950
}
5051

51-
// redis_mode_setup - configure Redis mode (cluster or standalone)
52-
{
53-
if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" {
54-
cfg.Append("cluster-enabled", "yes")
55-
cfg.Append("cluster-node-timeout", "5000")
56-
cfg.Append("cluster-require-full-coverage", "no")
57-
cfg.Append("cluster-migration-barrier", "1")
58-
cfg.Append("cluster-config-file", fmt.Sprintf("%s/nodes.conf", nodeConfDir))
59-
60-
// Get Pod IP
61-
cmd := exec.Command("hostname", "-i")
62-
output, err := cmd.Output()
52+
if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" {
53+
nodeConfPath := filepath.Join(nodeConfDir, "nodes.conf")
54+
55+
cfg.Append("cluster-enabled", "yes")
56+
cfg.Append("cluster-node-timeout", "5000")
57+
cfg.Append("cluster-require-full-coverage", "no")
58+
cfg.Append("cluster-migration-barrier", "1")
59+
cfg.Append("cluster-config-file", nodeConfPath)
60+
61+
if ip, err := util.GetLocalIP(); err != nil {
62+
log.Printf("Warning: Failed to get local IP: %v", err)
63+
} else {
64+
_, err = updateMyselfIP(nodeConfPath, strings.TrimSpace(ip))
6365
if err != nil {
64-
log.Printf("Warning: Failed to get pod IP: %v", err)
65-
} else {
66-
podIP := strings.TrimSpace(string(output))
67-
68-
// Update IP in nodes.conf file
69-
nodesConfPath := filepath.Join(nodeConfDir, "nodes.conf")
70-
if _, err := os.Stat(nodesConfPath); err == nil {
71-
cmd := exec.Command("sed", "-i", fmt.Sprintf("/myself/ s/[0-9]\\{1,3\\}\\.[0-9]\\{1,3\\}\\.[0-9]\\{1,3\\}\\.[0-9]\\{1,3\\}/%s/", podIP), nodesConfPath)
72-
if err := cmd.Run(); err != nil {
73-
log.Printf("Warning: Failed to update nodes.conf: %v", err)
74-
}
75-
}
66+
log.Printf("Warning: Failed to update nodes.conf: %v", err)
7667
}
77-
} else {
78-
fmt.Println("Setting up redis in standalone mode")
7968
}
69+
} else {
70+
fmt.Println("Setting up redis in standalone mode")
8071
}
8172

82-
// tls_setup - configure TLS
83-
{
84-
if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" {
85-
redisTLSCert, _ := util.CoalesceEnv("REDIS_TLS_CERT", "")
86-
redisTLSCertKey, _ := util.CoalesceEnv("REDIS_TLS_CERT_KEY", "")
87-
redisTLSCAKey, _ := util.CoalesceEnv("REDIS_TLS_CA_KEY", "")
88-
89-
cfg.Append("tls-cert-file", redisTLSCert)
90-
cfg.Append("tls-key-file", redisTLSCertKey)
91-
cfg.Append("tls-ca-cert-file", redisTLSCAKey)
92-
cfg.Append("tls-auth-clients", "optional")
93-
cfg.Append("tls-replication", "yes")
94-
95-
if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" {
96-
cfg.Append("tls-cluster", "yes")
97-
98-
if redisMajorVersion == "v7" {
99-
cfg.Append("cluster-preferred-endpoint-type", "hostname")
100-
}
73+
if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" {
74+
redisTLSCert, _ := util.CoalesceEnv("REDIS_TLS_CERT", "")
75+
redisTLSCertKey, _ := util.CoalesceEnv("REDIS_TLS_CERT_KEY", "")
76+
redisTLSCAKey, _ := util.CoalesceEnv("REDIS_TLS_CA_KEY", "")
77+
78+
cfg.Append("tls-cert-file", redisTLSCert)
79+
cfg.Append("tls-key-file", redisTLSCertKey)
80+
cfg.Append("tls-ca-cert-file", redisTLSCAKey)
81+
cfg.Append("tls-auth-clients", "optional")
82+
cfg.Append("tls-replication", "yes")
83+
84+
if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" {
85+
cfg.Append("tls-cluster", "yes")
86+
if redisMajorVersion == "v7" {
87+
cfg.Append("cluster-preferred-endpoint-type", "hostname")
10188
}
102-
} else {
103-
fmt.Println("Running without TLS mode")
10489
}
90+
} else {
91+
fmt.Println("Running without TLS mode")
10592
}
10693

107-
// acl_setup - configure ACL
108-
{
109-
if aclMode, ok := util.CoalesceEnv("ACL_MODE", ""); ok && aclMode == "true" {
110-
cfg.Append("aclfile", "/etc/redis/user.acl")
111-
} else {
112-
fmt.Println("ACL_MODE is not true, skipping ACL file modification")
113-
}
94+
if aclMode, ok := util.CoalesceEnv("ACL_MODE", ""); ok && aclMode == "true" {
95+
cfg.Append("aclfile", "/etc/redis/user.acl")
96+
} else {
97+
fmt.Println("ACL_MODE is not true, skipping ACL file modification")
11498
}
11599

116-
// persistence_setup - configure persistence
117-
{
118-
if persistenceEnabled == "true" {
119-
cfg.Append("save", "900 1")
120-
cfg.Append("save", "300 10")
121-
cfg.Append("save", "60 10000")
122-
cfg.Append("Appendonly", "yes")
123-
cfg.Append("Appendfilename", "\"Appendonly.aof\"")
124-
cfg.Append("dir", dataDir)
125-
} else {
126-
fmt.Println("Running without persistence mode")
127-
}
100+
if persistenceEnabled == "true" {
101+
cfg.Append("save", "900 1")
102+
cfg.Append("save", "300 10")
103+
cfg.Append("save", "60 10000")
104+
cfg.Append("Appendonly", "yes")
105+
cfg.Append("Appendfilename", "\"Appendonly.aof\"")
106+
cfg.Append("dir", dataDir)
107+
} else {
108+
fmt.Println("Running without persistence mode")
128109
}
129110

130-
// port_setup - configure ports
131-
{
132-
redisPort, _ := util.CoalesceEnv("REDIS_PORT", "6379")
133-
134-
if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" {
135-
cfg.Append("port", "0")
136-
cfg.Append("tls-port", redisPort)
137-
} else {
138-
cfg.Append("port", redisPort)
139-
}
111+
if tlsMode, ok := util.CoalesceEnv("TLS_MODE", ""); ok && tlsMode == "true" {
112+
cfg.Append("port", "0")
113+
cfg.Append("tls-port", redisPort)
114+
} else {
115+
cfg.Append("port", redisPort)
116+
}
140117

141-
if nodePort, ok := util.CoalesceEnv("NODEPORT", ""); ok && nodePort == "true" {
142-
podHostname, _ := os.Hostname()
143-
announcePortVar := "announce_port_" + strings.ReplaceAll(podHostname, "-", "_")
144-
announceBusPortVar := "announce_bus_port_" + strings.ReplaceAll(podHostname, "-", "_")
118+
if nodePort, ok := util.CoalesceEnv("NODEPORT", ""); ok && nodePort == "true" {
119+
podHostname, _ := os.Hostname()
120+
announcePortVar := "announce_port_" + strings.ReplaceAll(podHostname, "-", "_")
121+
announceBusPortVar := "announce_bus_port_" + strings.ReplaceAll(podHostname, "-", "_")
145122

146-
// Get environment variables
147-
clusterAnnouncePort := os.Getenv(announcePortVar)
148-
clusterAnnounceBusPort := os.Getenv(announceBusPortVar)
123+
// Get environment variables
124+
clusterAnnouncePort := os.Getenv(announcePortVar)
125+
clusterAnnounceBusPort := os.Getenv(announceBusPortVar)
149126

150-
if clusterAnnouncePort != "" {
151-
cfg.Append("cluster-announce-port", clusterAnnouncePort)
152-
}
153-
if clusterAnnounceBusPort != "" {
154-
cfg.Append("cluster-announce-bus-port", clusterAnnounceBusPort)
155-
}
127+
if clusterAnnouncePort != "" {
128+
cfg.Append("cluster-announce-port", clusterAnnouncePort)
129+
}
130+
if clusterAnnounceBusPort != "" {
131+
cfg.Append("cluster-announce-bus-port", clusterAnnounceBusPort)
156132
}
157133
}
158134

159-
// external_config - include external config file
160-
{
161-
if _, err := os.Stat(externalConfigFile); err == nil {
162-
cfg.Append("include", externalConfigFile)
163-
}
135+
if _, err := os.Stat(externalConfigFile); err == nil {
136+
cfg.Append("include", externalConfigFile)
164137
}
165138

166139
// Add cluster announcement IP and hostname for cluster mode
167140
if setupMode, ok := util.CoalesceEnv("SETUP_MODE", ""); ok && setupMode == "cluster" {
168-
// Get Pod hostname and IP
169-
podHostname, err := os.Hostname()
170-
if err == nil {
171-
var clusterAnnounceIP string
141+
var err error
142+
var clusterAnnounceIP string
143+
if nodePort, ok := util.CoalesceEnv("NODEPORT", ""); ok && nodePort == "true" {
144+
clusterAnnounceIP = os.Getenv("HOST_IP")
145+
} else {
146+
clusterAnnounceIP, err = util.GetLocalIP()
147+
if err != nil {
148+
log.Printf("Warning: Failed to get local IP: %v", err)
149+
}
150+
}
151+
if clusterAnnounceIP != "" {
152+
cfg.Append("cluster-announce-ip", clusterAnnounceIP)
153+
}
172154

173-
if nodePort, ok := util.CoalesceEnv("NODEPORT", ""); ok && nodePort == "true" {
174-
clusterAnnounceIP = os.Getenv("HOST_IP")
155+
if redisMajorVersion == "v7" {
156+
fqdnName, err := fqdn.FqdnHostname()
157+
if err != nil {
158+
log.Printf("Warning: Failed to get FQDN: %v", err)
175159
} else {
176-
cmd := exec.Command("hostname", "-i")
177-
output, err := cmd.Output()
178-
if err == nil {
179-
clusterAnnounceIP = strings.TrimSpace(string(output))
180-
}
181-
}
182-
if clusterAnnounceIP != "" {
183-
cfg.Append("cluster-announce-ip", clusterAnnounceIP)
184-
}
185-
if redisMajorVersion == "v7" {
186-
cfg.Append("cluster-announce-hostname", podHostname)
160+
cfg.Append("cluster-announce-hostname", fqdnName)
187161
}
188162
}
189163
}
@@ -192,6 +166,36 @@ func GenerateConfig() error {
192166
cfg.Append("maxmemory", maxMemory)
193167
}
194168

195-
// Commit configuration to file
196169
return cfg.Commit()
197170
}
171+
172+
func updateMyselfIP(nodesConfPath, newIP string) (updated []byte, err error) {
173+
raw, err := os.ReadFile(nodesConfPath)
174+
if err != nil {
175+
return nil, err
176+
}
177+
ipRe := regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`)
178+
var out bytes.Buffer
179+
scanner := bufio.NewScanner(bytes.NewReader(raw))
180+
changed := false
181+
182+
for scanner.Scan() {
183+
line := scanner.Text()
184+
if bytes.Contains([]byte(line), []byte("myself")) {
185+
replaced := ipRe.ReplaceAllString(line, newIP)
186+
if replaced != line {
187+
changed = true
188+
line = replaced
189+
}
190+
}
191+
out.WriteString(line)
192+
out.WriteByte('\n')
193+
}
194+
if err := scanner.Err(); err != nil {
195+
return nil, err
196+
}
197+
if changed {
198+
return out.Bytes(), os.WriteFile(nodesConfPath, out.Bytes(), 0o644)
199+
}
200+
return nil, nil
201+
}

‎internal/agent/bootstrap/redis/config_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package bootstrap
2+
3+
import (
4+
"os"
5+
"strings"
6+
"testing"
7+
)
8+
9+
func Test_updateMyselfIP(t *testing.T) {
10+
testData := `7a6b5f4f99496c97f4e32c30c077aa95cab92664 10.244.0.246:0@16379,,tls-port=6379,shard-id=a03445a0d3f6d405af261041e0cb77a8a176f42b slave b66f2fa597eeda567cf05f3701419be9a3b2f50e 0 1756463509000 1 connected
11+
93ad60e9ce21430683a3534d2c96ab1b8077cfe8 10.244.0.237:0@16379,,tls-port=6379,shard-id=2f177491b895051f91e91e554a2a9da2cd167aeb master - 0 1756463509685 2 connected 5461-10922
12+
b66f2fa597eeda567cf05f3701419be9a3b2f50e 10.244.0.222:0@16379,,tls-port=6379,shard-id=a03445a0d3f6d405af261041e0cb77a8a176f42b myself,master - 0 0 1 connected 0-5460
13+
88456cac1830f3e00f6ab681fb819b4b1d7ad36b 10.244.0.252:0@16379,,tls-port=6379,shard-id=b61110535f09b9c0703517f79da79118fee8d1a4 slave 580e234a8dcd74717c37d01ed8097929c64536ff 0 1756463509691 3 connected
14+
580e234a8dcd74717c37d01ed8097929c64536ff 10.244.0.240:0@16379,,tls-port=6379,shard-id=b61110535f09b9c0703517f79da79118fee8d1a4 master - 0 1756463509583 3 connected 10923-16383
15+
c0fc3c21460fec045775d2dcde220fb26ca668c1 10.244.0.249:0@16379,,tls-port=6379,shard-id=2f177491b895051f91e91e554a2a9da2cd167aeb slave 93ad60e9ce21430683a3534d2c96ab1b8077cfe8 0 1756463509583 2 connected
16+
vars currentEpoch 3 lastVoteEpoch 0
17+
`
18+
19+
tmpFile := "/tmp/test_nodes.conf"
20+
err := os.WriteFile(tmpFile, []byte(testData), 0o644)
21+
if err != nil {
22+
t.Fatalf("Failed to create test file: %v", err)
23+
}
24+
defer os.Remove(tmpFile)
25+
26+
newIP := "10.244.0.9"
27+
updated, err := updateMyselfIP(tmpFile, newIP)
28+
if err != nil {
29+
t.Errorf("updateMyselfIP() error = %v", err)
30+
}
31+
32+
if updated == nil {
33+
t.Errorf("Expected changes to be made, but updated is nil")
34+
return
35+
}
36+
expectedContent := strings.ReplaceAll(testData, "10.244.0.222", newIP)
37+
updatedContent := string(updated)
38+
39+
if updatedContent != expectedContent {
40+
t.Errorf("Expected updated content to match:\nExpected:\n%s\nGot:\n%s", expectedContent, updatedContent)
41+
}
42+
43+
t.Logf("Successfully updated nodes.conf with new IP %s", newIP)
44+
}

‎internal/k8sutils/redis.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,14 +110,17 @@ func repairDisconnectedMasters(ctx context.Context, client kubernetes.Interface,
110110
if !nodeFailedOrDisconnected(node) {
111111
continue
112112
}
113-
podName, err := getMasterHostFromClusterNode(node)
113+
host, err := getMasterHostFromClusterNode(node)
114114
if err != nil {
115115
lastError = err
116116
log.FromContext(ctx).V(1).Error(err, "Failed to get pod name from cluster node. Continuing with other nodes.", "Node", node)
117117
continue
118118
}
119119
ip := getRedisServerIP(ctx, client, RedisDetails{
120-
PodName: podName,
120+
// host may be FQDN like redis-cluster-leader-0.redis-cluster-leader-headless.default.svc.cluster.local
121+
// or it may be like redis-cluster-leader-0
122+
// we need to adapt
123+
PodName: strings.Split(host, ".")[0],
121124
Namespace: cr.Namespace,
122125
})
123126
err = redisClient.ClusterMeet(ctx, ip, strconv.Itoa(*cr.Spec.Port)).Err()

0 commit comments

Comments
 (0)