Merge pull request kubernetes#85745 from aojea/kubeadm_etcd_ipv6

kubeadm: default etcd address based on the IP family

Author: k8s-ci-robot

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -162,7 +162,12 @@ func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint
 		}
 	} else {
 		// Default to etcd static pod on localhost
-		defaultArguments["etcd-servers"] = fmt.Sprintf("https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort)
+		// localhost IP family should be the same that the AdvertiseAddress
+		etcdLocalhostAddress := "127.0.0.1"
+		if utilsnet.IsIPv6String(localAPIEndpoint.AdvertiseAddress) {
+			etcdLocalhostAddress = "::1"
+		}
+		defaultArguments["etcd-servers"] = fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort)))
 		defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName)
 		defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)
 		defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)
@@ -175,7 +180,7 @@ func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint
 		}
 	}
 
-	// TODO: The following code should be remvoved after dual-stack is GA.
+	// TODO: The following code should be removed after dual-stack is GA.
 	// Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value.
 	if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present {
 		defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled)

cmd/kubeadm/app/phases/controlplane/manifests_test.go

@@ -266,7 +266,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--requestheader-allowed-names=front-proxy-client",
 				"--authorization-mode=Node,RBAC",
 				"--advertise-address=2001:db8::1",
-				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
+				fmt.Sprintf("--etcd-servers=https://[::1]:%d", kubeadmconstants.EtcdListenClientPort),
 				"--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
 				"--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
 				"--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
@@ -278,7 +278,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
 				Etcd: kubeadmapi.Etcd{
 					External: &kubeadmapi.ExternalEtcd{
-						Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"},
+						Endpoints: []string{"https://[2001:abcd:bcda::1]:2379", "https://[2001:abcd:bcda::2]:2379"},
 						CAFile:    "fuz",
 						CertFile:  "fiz",
 						KeyFile:   "faz",
@@ -311,7 +311,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--requestheader-allowed-names=front-proxy-client",
 				"--authorization-mode=Node,RBAC",
 				"--advertise-address=2001:db8::1",
-				"--etcd-servers=https://8.6.4.1:2379,https://8.6.4.2:2379",
+				"--etcd-servers=https://[2001:abcd:bcda::1]:2379,https://[2001:abcd:bcda::2]:2379",
 				"--etcd-cafile=fuz",
 				"--etcd-certfile=fiz",
 				"--etcd-keyfile=faz",
@@ -323,7 +323,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
 				Etcd: kubeadmapi.Etcd{
 					External: &kubeadmapi.ExternalEtcd{
-						Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"},
+						Endpoints: []string{"http://[::1]:2379", "http://[::1]:2380"},
 					},
 				},
 				CertificatesDir: testCertsDir,
@@ -353,7 +353,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--requestheader-allowed-names=front-proxy-client",
 				"--authorization-mode=Node,RBAC",
 				"--advertise-address=2001:db8::1",
-				"--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:2380",
+				"--etcd-servers=http://[::1]:2379,http://[::1]:2380",
 			},
 		},
 		{

cmd/kubeadm/app/phases/etcd/BUILD

@@ -35,6 +35,7 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
     ],
 )
 

cmd/kubeadm/app/phases/etcd/local.go

@@ -18,7 +18,9 @@ package etcd
 
 import (
 	"fmt"
+	"net"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -33,6 +35,7 @@ import (
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
+	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -176,7 +179,8 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A
 		etcdVolumeName:  staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
 		certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
 	}
-	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd)
+	// probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress
+	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress))
 	return staticpodutil.ComponentPod(v1.Container{
 		Name:            kubeadmconstants.Etcd,
 		Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
@@ -193,9 +197,14 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A
 
 // getEtcdCommand builds the right etcd command from the given config object
 func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string {
+	// localhost IP family should be the same that the AdvertiseAddress
+	etcdLocalhostAddress := "127.0.0.1"
+	if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) {
+		etcdLocalhostAddress = "::1"
+	}
 	defaultArguments := map[string]string{
 		"name":                        nodeName,
-		"listen-client-urls":          fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP("127.0.0.1"), etcdutil.GetClientURL(endpoint)),
+		"listen-client-urls":          fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)),
 		"advertise-client-urls":       etcdutil.GetClientURL(endpoint),
 		"listen-peer-urls":            etcdutil.GetPeerURL(endpoint),
 		"initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint),
@@ -209,7 +218,7 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A
 		"peer-trusted-ca-file":        filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
 		"peer-client-cert-auth":       "true",
 		"snapshot-count":              "10000",
-		"listen-metrics-urls":         fmt.Sprintf("http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort),
+		"listen-metrics-urls":         fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))),
 	}
 
 	if len(initialCluster) == 0 {

cmd/kubeadm/app/phases/etcd/local_test.go

@@ -266,8 +266,8 @@ func TestGetEtcdCommand(t *testing.T) {
 			expected: []string{
 				"etcd",
 				"--name=foo",
-				fmt.Sprintf("--listen-client-urls=https://127.0.0.1:%d,https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort),
-				fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort),
+				fmt.Sprintf("--listen-client-urls=https://[::1]:%d,https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort),
+				fmt.Sprintf("--listen-metrics-urls=http://[::1]:%d", kubeadmconstants.EtcdMetricsPort),
 				fmt.Sprintf("--advertise-client-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort),
 				fmt.Sprintf("--listen-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort),
 				fmt.Sprintf("--initial-advertise-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort),

cmd/kubeadm/app/util/staticpod/utils.go

@@ -269,8 +269,11 @@ func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
 // GetEtcdProbeEndpoint takes a kubeadm Etcd configuration object and attempts to parse
 // the first URL in the listen-metrics-urls argument, returning an etcd probe hostname,
 // port and scheme
-func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd) (string, int, v1.URIScheme) {
+func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd, isIPv6 bool) (string, int, v1.URIScheme) {
 	localhost := "127.0.0.1"
+	if isIPv6 {
+		localhost = "::1"
+	}
 	if cfg.Local == nil || cfg.Local.ExtraArgs == nil {
 		return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
 	}

cmd/kubeadm/app/util/staticpod/utils_test.go

@@ -65,6 +65,13 @@ func TestGetAPIServerProbeAddress(t *testing.T) {
 			},
 			expected: "10.10.10.10",
 		},
+		{
+			desc: "filled in ipv6 AdvertiseAddress endpoint returns it",
+			endpoint: &kubeadmapi.APIEndpoint{
+				AdvertiseAddress: "2001:abcd:bcda::1",
+			},
+			expected: "2001:abcd:bcda::1",
+		},
 	}
 
 	for _, test := range tests {
@@ -103,6 +110,17 @@ func TestGetControllerManagerProbeAddress(t *testing.T) {
 			},
 			expected: "10.10.10.10",
 		},
+		{
+			desc: "setting controller manager extra ipv6 address arg to something acknowledges it",
+			cfg: &kubeadmapi.ClusterConfiguration{
+				ControllerManager: kubeadmapi.ControlPlaneComponent{
+					ExtraArgs: map[string]string{
+						kubeControllerManagerAddressArg: "2001:abcd:bcda::1",
+					},
+				},
+			},
+			expected: "2001:abcd:bcda::1",
+		},
 	}
 
 	for _, test := range tests {
@@ -119,6 +137,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) {
 	var tests = []struct {
 		name             string
 		cfg              *kubeadmapi.Etcd
+		isIPv6           bool
 		expectedHostname string
 		expectedPort     int
 		expectedScheme   v1.URIScheme
@@ -131,6 +150,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) {
 						"listen-metrics-urls": "https://1.2.3.4:1234,https://4.3.2.1:2381"},
 				},
 			},
+			isIPv6:           false,
 			expectedHostname: "1.2.3.4",
 			expectedPort:     1234,
 			expectedScheme:   v1.URISchemeHTTPS,
@@ -143,6 +163,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) {
 						"listen-metrics-urls": "http://1.2.3.4:1234"},
 				},
 			},
+			isIPv6:           false,
 			expectedHostname: "1.2.3.4",
 			expectedPort:     1234,
 			expectedScheme:   v1.URISchemeHTTP,
@@ -155,6 +176,7 @@ func TestGetEtcdProbeEndpoint(t *testing.T) {
 						"listen-metrics-urls": "1.2.3.4"},
 				},
 			},
+			isIPv6:           false,
 			expectedHostname: "127.0.0.1",
 			expectedPort:     kubeadmconstants.EtcdMetricsPort,
 			expectedScheme:   v1.URISchemeHTTP,
@@ -167,23 +189,87 @@ func TestGetEtcdProbeEndpoint(t *testing.T) {
 						"listen-metrics-urls": "https://1.2.3.4"},
 				},
 			},
+			isIPv6:           false,
 			expectedHostname: "1.2.3.4",
 			expectedPort:     kubeadmconstants.EtcdMetricsPort,
 			expectedScheme:   v1.URISchemeHTTPS,
 		},
+		{
+			name: "etcd probe URL from two IPv6 URLs",
+			cfg: &kubeadmapi.Etcd{
+				Local: &kubeadmapi.LocalEtcd{
+					ExtraArgs: map[string]string{
+						"listen-metrics-urls": "https://[2001:abcd:bcda::1]:1234,https://[2001:abcd:bcda::2]:2381"},
+				},
+			},
+			isIPv6:           true,
+			expectedHostname: "2001:abcd:bcda::1",
+			expectedPort:     1234,
+			expectedScheme:   v1.URISchemeHTTPS,
+		},
+		{
+			name: "etcd probe localhost IPv6 URL with HTTP scheme",
+			cfg: &kubeadmapi.Etcd{
+				Local: &kubeadmapi.LocalEtcd{
+					ExtraArgs: map[string]string{
+						"listen-metrics-urls": "http://[::1]:1234"},
+				},
+			},
+			isIPv6:           true,
+			expectedHostname: "::1",
+			expectedPort:     1234,
+			expectedScheme:   v1.URISchemeHTTP,
+		},
+		{
+			name: "etcd probe IPv6 URL with HTTP scheme",
+			cfg: &kubeadmapi.Etcd{
+				Local: &kubeadmapi.LocalEtcd{
+					ExtraArgs: map[string]string{
+						"listen-metrics-urls": "http://[2001:abcd:bcda::1]:1234"},
+				},
+			},
+			isIPv6:           true,
+			expectedHostname: "2001:abcd:bcda::1",
+			expectedPort:     1234,
+			expectedScheme:   v1.URISchemeHTTP,
+		},
+		{
+			name: "etcd probe IPv6 URL without port",
+			cfg: &kubeadmapi.Etcd{
+				Local: &kubeadmapi.LocalEtcd{
+					ExtraArgs: map[string]string{
+						"listen-metrics-urls": "https://[2001:abcd:bcda::1]"},
+				},
+			},
+			isIPv6:           true,
+			expectedHostname: "2001:abcd:bcda::1",
+			expectedPort:     kubeadmconstants.EtcdMetricsPort,
+			expectedScheme:   v1.URISchemeHTTPS,
+		},
 		{
 			name: "etcd probe URL from defaults",
 			cfg: &kubeadmapi.Etcd{
 				Local: &kubeadmapi.LocalEtcd{},
 			},
+			isIPv6:           false,
 			expectedHostname: "127.0.0.1",
 			expectedPort:     kubeadmconstants.EtcdMetricsPort,
 			expectedScheme:   v1.URISchemeHTTP,
 		},
+		{
+			name: "etcd probe URL from defaults if IPv6",
+			cfg: &kubeadmapi.Etcd{
+				Local: &kubeadmapi.LocalEtcd{},
+			},
+			isIPv6:           true,
+			expectedHostname: "::1",
+			expectedPort:     kubeadmconstants.EtcdMetricsPort,
+			expectedScheme:   v1.URISchemeHTTP,
+		},
 	}
 	for _, rt := range tests {
 		t.Run(rt.name, func(t *testing.T) {
-			hostname, port, scheme := GetEtcdProbeEndpoint(rt.cfg)
+			hostname, port, scheme := GetEtcdProbeEndpoint(rt.cfg, rt.isIPv6)
 			if hostname != rt.expectedHostname {
 				t.Errorf("%q test case failed:\n\texpected hostname: %s\n\tgot: %s",
 					rt.name, rt.expectedHostname, hostname)