kubeadm: make the scheduler and KCM connect to local endpoint

Pinning the kube-controller-manager and kube-scheduler kubeconfig files
to point to the control-plane-endpoint can be problematic during
immutable upgrades if one of these components ends up contacting an N-1
kube-apiserver:
https://kubernetes.io/docs/setup/release/version-skew-policy/#kube-controller-manager-kube-scheduler-and-cloud-controller-manager
For example, the components can send a request for a non-existing API
version.

Instead of using the CPE for these components, use the LocalAPIEndpoint.
This guarantees that the components would talk to the local
kube-apiserver, which should be the same version, unless the user
explicitly patched manifests.

Author: neolit123

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -362,32 +362,37 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu
 }
 
 func getKubeConfigSpecsBase(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) {
-	apiServer, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
+	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
 	if err != nil {
 		return nil, err
 	}
+	localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	return map[string]*kubeConfigSpec{
 		kubeadmconstants.AdminKubeConfigFileName: {
-			APIServer:  apiServer,
+			APIServer:  controlPlaneEndpoint,
 			ClientName: "kubernetes-admin",
 			ClientCertAuth: &clientCertAuth{
 				Organizations: []string{kubeadmconstants.SystemPrivilegedGroup},
 			},
 		},
 		kubeadmconstants.KubeletKubeConfigFileName: {
-			APIServer:  apiServer,
+			APIServer:  controlPlaneEndpoint,
 			ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
 			ClientCertAuth: &clientCertAuth{
 				Organizations: []string{kubeadmconstants.NodesGroup},
 			},
 		},
 		kubeadmconstants.ControllerManagerKubeConfigFileName: {
-			APIServer:      apiServer,
+			APIServer:      localAPIEndpoint,
 			ClientName:     kubeadmconstants.ControllerManagerUser,
 			ClientCertAuth: &clientCertAuth{},
 		},
 		kubeadmconstants.SchedulerKubeConfigFileName: {
-			APIServer:      apiServer,
+			APIServer:      localAPIEndpoint,
 			ClientName:     kubeadmconstants.SchedulerUser,
 			ClientCertAuth: &clientCertAuth{},
 		},

cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go

@@ -167,8 +167,22 @@ func TestGetKubeConfigSpecs(t *testing.T) {
 				if err != nil {
 					t.Error(err)
 				}
-				if spec.APIServer != controlPlaneEndpoint {
-					t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer endpoint into spec for %s", assertion.kubeConfigFile)
+				localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
+				if err != nil {
+					t.Error(err)
+				}
+
+				switch assertion.kubeConfigFile {
+				case kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.KubeletKubeConfigFileName:
+					if spec.APIServer != controlPlaneEndpoint {
+						t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s",
+							assertion.kubeConfigFile, controlPlaneEndpoint, spec.APIServer)
+					}
+				case kubeadmconstants.ControllerManagerKubeConfigFileName, kubeadmconstants.SchedulerKubeConfigFileName:
+					if spec.APIServer != localAPIEndpoint {
+						t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s",
+							assertion.kubeConfigFile, localAPIEndpoint, spec.APIServer)
+					}
 				}
 
 				// Asserts CA certs and CA keys loaded into specs

cmd/kubeadm/app/util/endpoint.go

@@ -34,22 +34,10 @@ import (
 // - if the controlPlaneEndpoint is defined but without a port number, use the controlPlaneEndpoint + localEndpoint.BindPort is used.
 // - Otherwise, in case the controlPlaneEndpoint is not defined, use the localEndpoint.AdvertiseAddress + the localEndpoint.BindPort.
 func GetControlPlaneEndpoint(controlPlaneEndpoint string, localEndpoint *kubeadmapi.APIEndpoint) (string, error) {
-	// parse the bind port
-	bindPortString := strconv.Itoa(int(localEndpoint.BindPort))
-	if _, err := ParsePort(bindPortString); err != nil {
-		return "", errors.Wrapf(err, "invalid value %q given for api.bindPort", localEndpoint.BindPort)
-	}
-
-	// parse the AdvertiseAddress
-	var ip = net.ParseIP(localEndpoint.AdvertiseAddress)
-	if ip == nil {
-		return "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", localEndpoint.AdvertiseAddress)
-	}
-
-	// set the control-plane url using localEndpoint.AdvertiseAddress + the localEndpoint.BindPort
-	controlPlaneURL := &url.URL{
-		Scheme: "https",
-		Host:   net.JoinHostPort(ip.String(), bindPortString),
+	// get the URL of the local endpoint
+	localAPIEndpoint, err := GetLocalAPIEndpoint(localEndpoint)
+	if err != nil {
+		return "", err
 	}
 
 	// if the controlplane endpoint is defined
@@ -62,22 +50,32 @@ func GetControlPlaneEndpoint(controlPlaneEndpoint string, localEndpoint *kubeadm
 		}
 
 		// if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport
+		localEndpointPort := strconv.Itoa(int(localEndpoint.BindPort))
 		if port != "" {
-			if port != bindPortString {
+			if port != localEndpointPort {
 				fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address")
 			}
 		} else {
-			port = bindPortString
+			port = localEndpointPort
 		}
 
 		// overrides the control-plane url using the controlPlaneAddress (and eventually the bindport)
-		controlPlaneURL = &url.URL{
-			Scheme: "https",
-			Host:   net.JoinHostPort(host, port),
-		}
+		return formatURL(host, port).String(), nil
 	}
 
-	return controlPlaneURL.String(), nil
+	return localAPIEndpoint, nil
+}
+
+// GetLocalAPIEndpoint parses an APIEndpoint and returns it as a string,
+// or returns and error in case it cannot be parsed.
+func GetLocalAPIEndpoint(localEndpoint *kubeadmapi.APIEndpoint) (string, error) {
+	// get the URL of the local endpoint
+	localEndpointIP, localEndpointPort, err := parseAPIEndpoint(localEndpoint)
+	if err != nil {
+		return "", err
+	}
+	url := formatURL(localEndpointIP.String(), localEndpointPort)
+	return url.String(), nil
 }
 
 // ParseHostPort parses a network address of the form "host:port", "ipv4:port", "[ipv6]:port" into host and port;
@@ -123,3 +121,29 @@ func ParsePort(port string) (int, error) {
 
 	return 0, errors.New("port must be a valid number between 1 and 65535, inclusive")
 }
+
+// parseAPIEndpoint parses an APIEndpoint and returns the AdvertiseAddress as net.IP and the BindPort as string.
+// If the BindPort or AdvertiseAddress are invalid it returns an error.
+func parseAPIEndpoint(localEndpoint *kubeadmapi.APIEndpoint) (net.IP, string, error) {
+	// parse the bind port
+	bindPortString := strconv.Itoa(int(localEndpoint.BindPort))
+	if _, err := ParsePort(bindPortString); err != nil {
+		return nil, "", errors.Wrapf(err, "invalid value %q given for api.bindPort", localEndpoint.BindPort)
+	}
+
+	// parse the AdvertiseAddress
+	var ip = net.ParseIP(localEndpoint.AdvertiseAddress)
+	if ip == nil {
+		return nil, "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", localEndpoint.AdvertiseAddress)
+	}
+
+	return ip, bindPortString, nil
+}
+
+// formatURL takes a host and a port string and creates a net.URL using https scheme
+func formatURL(host, port string) *url.URL {
+	return &url.URL{
+		Scheme: "https",
+		Host:   net.JoinHostPort(host, port),
+	}
+}