Merge pull request kubernetes#128354 from SataQiu/join-control-plane-e2e

kubeadm: support joining control plane nodes in dryrun mode without a real initialized control plane

Author: k8s-ci-robot

cmd/kubeadm/app/cmd/join.go

@@ -607,6 +607,7 @@ func (j *joinData) Client() (clientset.Interface, error) {
 			WithWriter(os.Stdout).
 			AppendReactor(dryRun.GetClusterInfoReactor()).
 			AppendReactor(dryRun.GetKubeadmConfigReactor()).
+			AppendReactor(dryRun.GetKubeadmCertsReactor()).
 			AppendReactor(dryRun.GetKubeProxyConfigReactor()).
 			AppendReactor(dryRun.GetKubeletConfigReactor())
 

cmd/kubeadm/app/cmd/phases/join/checketcd.go

@@ -56,6 +56,11 @@ func runCheckEtcdPhase(c workflow.RunData) error {
 		return nil
 	}
 
+	if data.DryRun() {
+		fmt.Println("[dryrun] Would check that the etcd cluster is healthy")
+		return nil
+	}
+
 	fmt.Println("[check-etcd] Checking that the etcd cluster is healthy")
 
 	// Checks that the etcd cluster is healthy

cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go

@@ -292,6 +292,11 @@ func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error {
 	fmt.Println("[kubeconfig] Generating kubeconfig files")
 	fmt.Printf("[kubeconfig] Using kubeconfig folder %q\n", data.KubeConfigDir())
 
+	// If we're dry-running, load certs from tmp dir, and defer to restore to the path originally specified by the user
+	certsDir := cfg.CertificatesDir
+	cfg.CertificatesDir = data.CertificateWriteDir()
+	defer func() { cfg.CertificatesDir = certsDir }()
+
 	// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
 	// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
 	// following steps of the join --control-plane workflow
@@ -303,6 +308,14 @@ func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error {
 }
 
 func bootstrapClient(data JoinData) (clientset.Interface, error) {
+	if data.DryRun() {
+		client, err := data.Client()
+		if err != nil {
+			return nil, err
+		}
+		return client, nil
+	}
+
 	tlsBootstrapCfg, err := data.TLSBootstrapCfg()
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to access the cluster")

cmd/kubeadm/app/phases/etcd/local.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/images"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/users"
@@ -139,19 +140,20 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm
 // for an additional etcd member that is joining an existing local/stacked etcd cluster.
 // Other members of the etcd cluster will be notified of the joining node in beforehand as well.
 func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool, certificatesDir string) error {
-	// creates an etcd client that connects to all the local/stacked etcd members
-	klog.V(1).Info("creating etcd client that connects to etcd pods")
-	etcdClient, err := etcdutil.NewFromCluster(client, certificatesDir)
-	if err != nil {
-		return err
-	}
-
 	etcdPeerAddress := etcdutil.GetPeerURL(endpoint)
 
 	var cluster []etcdutil.Member
+	var etcdClient *etcdutil.Client
+	var err error
 	if isDryRun {
 		fmt.Printf("[etcd] Would add etcd member: %s\n", etcdPeerAddress)
 	} else {
+		// Creates an etcd client that connects to all the local/stacked etcd members.
+		klog.V(1).Info("creating etcd client that connects to etcd pods")
+		etcdClient, err = etcdutil.NewFromCluster(client, certificatesDir)
+		if err != nil {
+			return err
+		}
 		klog.V(1).Infof("[etcd] Adding etcd member: %s", etcdPeerAddress)
 		if features.Enabled(cfg.FeatureGates, features.EtcdLearnerMode) {
 			cluster, err = etcdClient.AddMemberAsLearner(nodeName, etcdPeerAddress)
@@ -323,5 +325,11 @@ func prepareAndWriteEtcdStaticPod(manifestDir string, patchesDir string, cfg *ku
 		return err
 	}
 
+	// If dry-running, print the static etcd pod manifest file.
+	if isDryRun {
+		realPath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir)
+		outputPath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, kubeadmconstants.GetStaticPodDirectory())
+		return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{dryrunutil.NewFileToPrint(realPath, outputPath)}, os.Stdout)
+	}
 	return nil
 }

cmd/kubeadm/app/util/apiclient/dryrun.go

@@ -415,6 +415,24 @@ func (d *DryRun) GetKubeadmConfigReactor() *testing.SimpleReactor {
 	}
 }
 
+// GetKubeadmCertsReactor returns a reactor that handles the GET action of the "kubeadm-certs" Secret.
+func (d *DryRun) GetKubeadmCertsReactor() *testing.SimpleReactor {
+	return &testing.SimpleReactor{
+		Verb:     "get",
+		Resource: "secrets",
+		Reaction: func(action testing.Action) (bool, runtime.Object, error) {
+			a := action.(testing.GetAction)
+			if a.GetName() != constants.KubeadmCertsSecret || a.GetNamespace() != metav1.NamespaceSystem {
+				return false, nil, nil
+			}
+
+			obj := getKubeadmCertsSecret()
+			d.LogObject(obj, action.GetResource().GroupVersion())
+			return true, obj, nil
+		},
+	}
+}
+
 // GetKubeletConfigReactor returns a reactor that handles the GET action of the "kubelet-config"
 // ConfigMap.
 func (d *DryRun) GetKubeletConfigReactor() *testing.SimpleReactor {
@@ -513,6 +531,17 @@ func getConfigMap(namespace, name string, data map[string]string) *corev1.Config
 	}
 }
 
+// getSecret returns a fake Secret object.
+func getSecret(namespace, name string, data map[string][]byte) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Data: data,
+	}
+}
+
 // getClusterInfoConfigMap returns a fake "cluster-info" ConfigMap.
 func getClusterInfoConfigMap() *corev1.ConfigMap {
 	kubeconfig := dedent.Dedent(`apiVersion: v1
@@ -546,6 +575,7 @@ controllerManager:
   extraArgs:
   - name: cluster-signing-duration
     value: 24h
+controlPlaneEndpoint: 192.168.0.101:6443
 dns: {}
 encryptionAlgorithm: RSA-2048
 etcd:
@@ -568,6 +598,23 @@ scheduler: {}
 	return getConfigMap(metav1.NamespaceSystem, constants.KubeadmConfigConfigMap, data)
 }
 
+// getKubeadmCertsSecret returns a fake "kubeadm-certs" Secret.
+func getKubeadmCertsSecret() *corev1.Secret {
+	// The cert data is empty because the actual content is not relevant for the dryrun test.
+	data := map[string][]byte{
+		constants.CACertName:                                   {},
+		constants.CAKeyName:                                    {},
+		constants.FrontProxyCACertName:                         {},
+		constants.FrontProxyCAKeyName:                          {},
+		constants.ServiceAccountPrivateKeyName:                 {},
+		constants.ServiceAccountPublicKeyName:                  {},
+		strings.ReplaceAll(constants.EtcdCACertName, "/", "-"): {},
+		strings.ReplaceAll(constants.EtcdCAKeyName, "/", "-"):  {},
+	}
+
+	return getSecret(metav1.NamespaceSystem, constants.KubeadmCertsSecret, data)
+}
+
 // getKubeletConfigMap returns a fake "kubelet-config" ConfigMap.
 func getKubeletConfigMap() *corev1.ConfigMap {
 	configData := `apiVersion: kubelet.config.k8s.io/v1beta1