Merge pull request kubernetes#125582 from chrischdi/pr-kubeadm-kep-4471

kubeadm: implement ControlPlaneKubeletLocalMode

Author: k8s-ci-robot

cmd/kubeadm/app/cmd/join.go

@@ -219,6 +219,8 @@ func newCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
 	joinRunner.AppendPhase(phases.NewControlPlanePreparePhase())
 	joinRunner.AppendPhase(phases.NewCheckEtcdPhase())
 	joinRunner.AppendPhase(phases.NewKubeletStartPhase())
+	joinRunner.AppendPhase(phases.NewEtcdJoinPhase())
+	joinRunner.AppendPhase(phases.NewKubeletWaitBootstrapPhase())
 	joinRunner.AppendPhase(phases.NewControlPlaneJoinPhase())
 	joinRunner.AppendPhase(phases.NewWaitControlPlanePhase())
 

cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
 	markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane"
 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
@@ -35,6 +36,11 @@ var controlPlaneJoinExample = cmdutil.Examples(`
 	kubeadm join phase control-plane-join all
 `)
 
+var etcdJoinExample = cmdutil.Examples(`
+	# Joins etcd for a control plane instance
+	kubeadm join phase control-plane-join-etcd all
+`)
+
 func getControlPlaneJoinPhaseFlags(name string) []string {
 	flags := []string{
 		options.CfgPath,
@@ -73,13 +79,37 @@ func NewControlPlaneJoinPhase() workflow.Phase {
 	}
 }
 
+// NewEtcdJoinPhase creates a kubeadm workflow phase that implements joining etcd
+func NewEtcdJoinPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:          "etcd-join",
+		Short:         fmt.Sprintf("[EXPERIMENTAL] Join etcd for control plane nodes (only used when feature gate %s is enabled)", features.ControlPlaneKubeletLocalMode),
+		Run:           runEtcdPhase,
+		Example:       etcdJoinExample,
+		InheritFlags:  getControlPlaneJoinPhaseFlags("etcd"),
+		ArgsValidator: cobra.NoArgs,
+		// TODO: unhide this phase once ControlPlaneKubeletLocalMode goes GA:
+		// https://github.com/kubernetes/enhancements/issues/4471
+		Hidden: true,
+		// Only run this phase as if `ControlPlaneKubeletLocalMode` is activated.
+		RunIf: func(c workflow.RunData) (bool, error) {
+			return checkFeatureState(c, features.ControlPlaneKubeletLocalMode, true)
+		},
+	}
+}
+
 func newEtcdLocalSubphase() workflow.Phase {
 	return workflow.Phase{
 		Name:          "etcd",
 		Short:         "Add a new local etcd member",
 		Run:           runEtcdPhase,
 		InheritFlags:  getControlPlaneJoinPhaseFlags("etcd"),
 		ArgsValidator: cobra.NoArgs,
+		// Only run this phase as subphase of control-plane-join phase if
+		// `ControlPlaneKubeletLocalMode` is deactivated.
+		RunIf: func(c workflow.RunData) (bool, error) {
+			return checkFeatureState(c, features.ControlPlaneKubeletLocalMode, false)
+		},
 	}
 }
 

cmd/kubeadm/app/cmd/phases/join/data.go

@@ -20,11 +20,14 @@ package phases
 import (
 	"io"
 
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 )
 
 // JoinData is the interface to use for join phases.
@@ -44,3 +47,17 @@ type JoinData interface {
 	ManifestDir() string
 	CertificateWriteDir() string
 }
+
+func checkFeatureState(c workflow.RunData, featureGate string, state bool) (bool, error) {
+	data, ok := c.(JoinData)
+	if !ok {
+		return false, errors.New("control-plane-join phase invoked with an invalid data struct")
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return false, err
+	}
+
+	return state == features.Enabled(cfg.FeatureGates, featureGate), nil
+}

cmd/kubeadm/app/cmd/phases/join/kubelet.go

@@ -40,8 +40,10 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
@@ -85,6 +87,27 @@ func NewKubeletStartPhase() workflow.Phase {
 	}
 }
 
+// NewKubeletWaitBootstrapPhase creates a kubeadm workflow phase that start kubelet on a node.
+func NewKubeletWaitBootstrapPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "kubelet-wait-bootstrap",
+		Short: "[EXPERIMENTAL] Wait for the kubelet to bootstrap itself (only used when feature gate ControlPlaneKubeletLocalMode is enabled)",
+		Run:   runKubeletWaitBootstrapPhase,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.NodeCRISocket,
+			options.DryRun,
+		},
+		// TODO: unhide this phase once ControlPlaneKubeletLocalMode goes GA:
+		// https://github.com/kubernetes/enhancements/issues/4471
+		Hidden: true,
+		// Only run this phase as if `ControlPlaneKubeletLocalMode` is activated.
+		RunIf: func(c workflow.RunData) (bool, error) {
+			return checkFeatureState(c, features.ControlPlaneKubeletLocalMode, true)
+		},
+	}
+}
+
 func getKubeletStartJoinData(c workflow.RunData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) {
 	data, ok := c.(JoinData)
 	if !ok {
@@ -117,8 +140,36 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 	}
 	bootstrapKubeConfigFile := filepath.Join(data.KubeConfigDir(), kubeadmconstants.KubeletBootstrapKubeConfigFileName)
 
-	// Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk
-	defer os.Remove(bootstrapKubeConfigFile)
+	// Do not delete the bootstrapKubeConfigFile at the end of this function when
+	// using ControlPlaneKubeletLocalMode. The KubeletWaitBootstrapPhase will delete
+	// it when the feature is enabled.
+	if !features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
+		// Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk
+		defer func() {
+			_ = os.Remove(bootstrapKubeConfigFile)
+		}()
+	}
+
+	// Create the bootstrap client before we possibly overwrite the server address
+	// for ControlPlaneKubeletLocalMode.
+	bootstrapClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg)
+	if err != nil {
+		return errors.Errorf("could not create client from bootstrap kubeconfig")
+	}
+
+	if features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
+		// Set the server url to LocalAPIEndpoint if the feature gate is enabled so the config
+		// which gets passed to the kubelet forces it to talk to the local kube-apiserver.
+		if cfg.ControlPlane != nil {
+			for c, conf := range tlsBootstrapCfg.Clusters {
+				conf.Server, err = kubeadmutil.GetLocalAPIEndpoint(&cfg.ControlPlane.LocalAPIEndpoint)
+				if err != nil {
+					return errors.Wrapf(err, "could not get LocalAPIEndpoint when %s is enabled", features.ControlPlaneKubeletLocalMode)
+				}
+				tlsBootstrapCfg.Clusters[c] = conf
+			}
+		}
+	}
 
 	// Write the bootstrap kubelet config file or the TLS-Bootstrapped kubelet config file down to disk
 	klog.V(1).Infof("[kubelet-start] writing bootstrap kubelet config file at %s", bootstrapKubeConfigFile)
@@ -142,11 +193,6 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 		}
 	}
 
-	bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile)
-	if err != nil {
-		return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile)
-	}
-
 	// Obtain the name of this Node.
 	nodeName, _, err := kubeletphase.GetNodeNameAndHostname(&cfg.NodeRegistration)
 	if err != nil {
@@ -205,6 +251,36 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 	fmt.Println("[kubelet-start] Starting the kubelet")
 	kubeletphase.TryStartKubelet()
 
+	// Run the same code as KubeletWaitBootstrapPhase would do if the ControlPlaneKubeletLocalMode feature gate is disabled.
+	if !features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
+		if err := runKubeletWaitBootstrapPhase(c); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// runKubeletWaitBootstrapPhase waits for the kubelet to finish its TLS bootstrap process.
+// This process is executed by the kubelet and completes with the node joining the cluster
+// with a dedicates set of credentials as required by the node authorizer.
+func runKubeletWaitBootstrapPhase(c workflow.RunData) (returnErr error) {
+	data, ok := c.(JoinData)
+	if !ok {
+		return errors.New("kubelet-start phase invoked with an invalid data struct")
+	}
+	cfg := data.Cfg()
+	initCfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	bootstrapKubeConfigFile := filepath.Join(data.KubeConfigDir(), kubeadmconstants.KubeletBootstrapKubeConfigFileName)
+	// Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk
+	defer func() {
+		_ = os.Remove(bootstrapKubeConfigFile)
+	}()
+
 	// Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf
 	// Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process
 	// times out, display a somewhat user-friendly message.

cmd/kubeadm/app/features/features.go

@@ -38,6 +38,8 @@ const (
 	EtcdLearnerMode = "EtcdLearnerMode"
 	// WaitForAllControlPlaneComponents is expected to be alpha in v1.30
 	WaitForAllControlPlaneComponents = "WaitForAllControlPlaneComponents"
+	// ControlPlaneKubeletLocalMode is expected to be in alpha in v1.31, beta in v1.32
+	ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode"
 )
 
 // InitFeatureGates are the default feature gates for the init command
@@ -53,6 +55,7 @@ var InitFeatureGates = FeatureList{
 	},
 	EtcdLearnerMode:                  {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
 	WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
+	ControlPlaneKubeletLocalMode:     {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
 }
 
 // Feature represents a feature being gated