Merge pull request kubernetes#85609 from Nordix/kubeadm-dualstack-node-cidr-mask-size

Add support for new dual-stack flags for kubernetes-controller-manager in kubeadm

Author: k8s-ci-robot

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -19,6 +19,7 @@ go_test(
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
+        "//pkg/features:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/github.com/lithammer/dedent:go_default_library",

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -260,11 +260,13 @@ func isValidAuthzMode(authzMode string) bool {
 // the available bits to maximize the number used for nodes, but still have the node
 // CIDR be a multiple of eight.
 //
-func calcNodeCidrSize(podSubnet string) string {
+func calcNodeCidrSize(podSubnet string) (string, bool) {
 	maskSize := "24"
+	isIPv6 := false
 	if ip, podCidr, err := net.ParseCIDR(podSubnet); err == nil {
 		if utilsnet.IsIPv6(ip) {
 			var nodeCidrSize int
+			isIPv6 = true
 			podNetSize, totalBits := podCidr.Mask.Size()
 			switch {
 			case podNetSize == 112:
@@ -280,7 +282,7 @@ func calcNodeCidrSize(podSubnet string) string {
 			maskSize = strconv.Itoa(nodeCidrSize)
 		}
 	}
-	return maskSize
+	return maskSize, isIPv6
 }
 
 // getControllerManagerCommand builds the right controller manager command from the given config object and version
@@ -324,12 +326,25 @@ func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration) []string
 
 	// TODO: The following code should be remvoved after dual-stack is GA.
 	// Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value.
-	if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present {
+	enabled, present := cfg.FeatureGates[features.IPv6DualStack]
+	if present {
 		defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled)
-	} else if cfg.Networking.PodSubnet != "" {
-		// TODO(Arvinderpal): Needs to be fixed once PR #73977 lands. Should be a list of maskSizes.
-		maskSize := calcNodeCidrSize(cfg.Networking.PodSubnet)
-		defaultArguments["node-cidr-mask-size"] = maskSize
+	}
+	if cfg.Networking.PodSubnet != "" {
+		if enabled {
+			// any errors will be caught during validation
+			subnets := strings.Split(cfg.Networking.PodSubnet, ",")
+			for _, podSubnet := range subnets {
+				if maskSize, isIPv6 := calcNodeCidrSize(podSubnet); isIPv6 {
+					defaultArguments["node-cidr-mask-size-ipv6"] = maskSize
+				} else {
+					defaultArguments["node-cidr-mask-size-ipv4"] = maskSize
+				}
+			}
+		} else {
+			maskSize, _ := calcNodeCidrSize(cfg.Networking.PodSubnet)
+			defaultArguments["node-cidr-mask-size"] = maskSize
+		}
 	}
 
 	command := []string{"kube-controller-manager"}

cmd/kubeadm/app/phases/controlplane/manifests_test.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 const (
@@ -719,6 +720,73 @@ func TestGetControllerManagerCommand(t *testing.T) {
 				"--service-cluster-ip-range=fd03::/112",
 			},
 		},
+		{
+			name: "dual-stack networking for " + cpVersion,
+			cfg: &kubeadmapi.ClusterConfiguration{
+				Networking: kubeadmapi.Networking{
+					PodSubnet:     "2001:db8::/64,10.1.0.0/16",
+					ServiceSubnet: "fd03::/112,192.168.0.0/16",
+				},
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: cpVersion,
+				FeatureGates:      map[string]bool{string(features.IPv6DualStack): true},
+			},
+			expected: []string{
+				"kube-controller-manager",
+				"--bind-address=127.0.0.1",
+				"--leader-elect=true",
+				"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+				"--root-ca-file=" + testCertsDir + "/ca.crt",
+				"--service-account-private-key-file=" + testCertsDir + "/sa.key",
+				"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
+				"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
+				"--use-service-account-credentials=true",
+				"--controllers=*,bootstrapsigner,tokencleaner",
+				"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+				"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+				"--client-ca-file=" + testCertsDir + "/ca.crt",
+				"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
+				"--feature-gates=IPv6DualStack=true",
+				"--allocate-node-cidrs=true",
+				"--cluster-cidr=2001:db8::/64,10.1.0.0/16",
+				"--node-cidr-mask-size-ipv4=24",
+				"--node-cidr-mask-size-ipv6=80",
+				"--service-cluster-ip-range=fd03::/112,192.168.0.0/16",
+			},
+		},
+		{
+			name: "dual-stack networking custom extra-args for " + cpVersion,
+			cfg: &kubeadmapi.ClusterConfiguration{
+				Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16,2001:db8::/64"},
+				ControllerManager: kubeadmapi.ControlPlaneComponent{
+					ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "20", "node-cidr-mask-size-ipv6": "96"},
+				},
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: cpVersion,
+				FeatureGates:      map[string]bool{string(features.IPv6DualStack): true},
+			},
+			expected: []string{
+				"kube-controller-manager",
+				"--bind-address=127.0.0.1",
+				"--leader-elect=true",
+				"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+				"--root-ca-file=" + testCertsDir + "/ca.crt",
+				"--service-account-private-key-file=" + testCertsDir + "/sa.key",
+				"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
+				"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
+				"--use-service-account-credentials=true",
+				"--controllers=*,bootstrapsigner,tokencleaner",
+				"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+				"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+				"--client-ca-file=" + testCertsDir + "/ca.crt",
+				"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
+				"--feature-gates=IPv6DualStack=true",
+				"--allocate-node-cidrs=true",
+				"--cluster-cidr=10.0.1.15/16,2001:db8::/64",
+				"--node-cidr-mask-size-ipv4=20",
+				"--node-cidr-mask-size-ipv6=96",
+			},
+		},
 	}
 
 	for _, rt := range tests {
@@ -738,75 +806,92 @@ func TestCalcNodeCidrSize(t *testing.T) {
 		name           string
 		podSubnet      string
 		expectedPrefix string
+		expectedIPv6   bool
 	}{
 		{
 			name:           "Malformed pod subnet",
 			podSubnet:      "10.10.10/160",
 			expectedPrefix: "24",
+			expectedIPv6:   false,
 		},
 		{
 			name:           "V4: Always uses 24",
 			podSubnet:      "10.10.10.10/16",
 			expectedPrefix: "24",
+			expectedIPv6:   false,
 		},
 		{
 			name:           "V6: Use pod subnet size, when not enough space",
 			podSubnet:      "2001:db8::/128",
 			expectedPrefix: "128",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: Use pod subnet size, when not enough space",
 			podSubnet:      "2001:db8::/113",
 			expectedPrefix: "113",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: Special case with 256 nodes",
 			podSubnet:      "2001:db8::/112",
 			expectedPrefix: "120",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: Using /120 for node CIDR",
 			podSubnet:      "2001:db8::/104",
 			expectedPrefix: "120",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: Using /112 for node CIDR",
 			podSubnet:      "2001:db8::/103",
 			expectedPrefix: "112",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: Using /112 for node CIDR",
 			podSubnet:      "2001:db8::/96",
 			expectedPrefix: "112",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: Using /104 for node CIDR",
 			podSubnet:      "2001:db8::/95",
 			expectedPrefix: "104",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: For /64 pod net, use /80",
 			podSubnet:      "2001:db8::/64",
 			expectedPrefix: "80",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: For /48 pod net, use /64",
 			podSubnet:      "2001:db8::/48",
 			expectedPrefix: "64",
+			expectedIPv6:   true,
 		},
 		{
 			name:           "V6: For /32 pod net, use /48",
 			podSubnet:      "2001:db8::/32",
 			expectedPrefix: "48",
+			expectedIPv6:   true,
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			actualPrefix := calcNodeCidrSize(test.podSubnet)
+			actualPrefix, actualIPv6 := calcNodeCidrSize(test.podSubnet)
 			if actualPrefix != test.expectedPrefix {
 				t.Errorf("Case [%s]\nCalc of node CIDR size for pod subnet %q failed: Expected %q, saw %q",
 					test.name, test.podSubnet, test.expectedPrefix, actualPrefix)
 			}
+			if actualIPv6 != test.expectedIPv6 {
+				t.Errorf("Case [%s]\nCalc of node CIDR size for pod subnet %q failed: Expected isIPv6=%v, saw isIPv6=%v",
+					test.name, test.podSubnet, test.expectedIPv6, actualIPv6)
+			}
 		})
 	}