Merge pull request kubernetes#126776 from neolit123/1.31-improve-dry-run-logic

kubeadm: refactor the dry-run logic

Author: k8s-ci-robot

cmd/kubeadm/app/cmd/init.go

@@ -28,6 +28,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
@@ -349,10 +350,7 @@ func newInitData(cmd *cobra.Command, args []string, initOptions *initOptions, ou
 	// if dry running creates a temporary folder for saving kubeadm generated files
 	dryRunDir := ""
 	if initOptions.dryRun || cfg.DryRun {
-		// the KUBEADM_INIT_DRYRUN_DIR environment variable allows overriding the dry-run temporary
-		// directory from the command line. This makes it possible to run "kubeadm init" integration
-		// tests without root.
-		if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm(os.Getenv("KUBEADM_INIT_DRYRUN_DIR"), "kubeadm-init-dryrun"); err != nil {
+		if dryRunDir, err = kubeadmconstants.GetDryRunDir(kubeadmconstants.EnvVarInitDryRunDir, "kubeadm-init-dryrun", klog.Warningf); err != nil {
 			return nil, errors.Wrap(err, "couldn't create a temporary directory")
 		}
 	}
@@ -502,12 +500,16 @@ func (d *initData) OutputWriter() io.Writer {
 
 // getDryRunClient creates a fake client that answers some GET calls in order to be able to do the full init flow in dry-run mode.
 func getDryRunClient(d *initData) (clientset.Interface, error) {
-	svcSubnetCIDR, err := kubeadmconstants.GetKubernetesServiceCIDR(d.cfg.Networking.ServiceSubnet)
-	if err != nil {
-		return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", d.cfg.Networking.ServiceSubnet)
+	dryRun := apiclient.NewDryRun()
+	if err := dryRun.WithKubeConfigFile(d.KubeConfigPath()); err != nil {
+		return nil, err
 	}
-	dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String())
-	return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil
+	dryRun.WithDefaultMarshalFunction().
+		WithWriter(os.Stdout).
+		PrependReactor(dryRun.GetNodeReactor()).
+		PrependReactor(dryRun.PatchNodeReactor())
+
+	return dryRun.FakeClient(), nil
 }
 
 // Client returns a Kubernetes client to be used by kubeadm.

cmd/kubeadm/app/cmd/join.go

@@ -45,6 +45,8 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
+	"k8s.io/kubernetes/cmd/kubeadm/app/discovery/token"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
@@ -464,7 +466,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 	// if dry running, creates a temporary folder to save kubeadm generated files
 	dryRunDir := ""
 	if opt.dryRun || cfg.DryRun {
-		if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-join-dryrun"); err != nil {
+		if dryRunDir, err = kubeadmconstants.GetDryRunDir(kubeadmconstants.EnvVarJoinDryRunDir, "kubeadm-join-dryrun", klog.Warningf); err != nil {
 			return nil, errors.Wrap(err, "couldn't create a temporary directory on dryrun")
 		}
 	}
@@ -535,8 +537,19 @@ func (j *joinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) {
 	if j.tlsBootstrapCfg != nil {
 		return j.tlsBootstrapCfg, nil
 	}
+
+	var (
+		client clientset.Interface
+		err    error
+	)
+	if j.dryRun {
+		client, err = j.Client()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create a client for TLS bootstrap")
+		}
+	}
 	klog.V(1).Infoln("[preflight] Discovering cluster-info")
-	tlsBootstrapCfg, err := discovery.For(j.cfg)
+	tlsBootstrapCfg, err := discovery.For(client, j.cfg)
 	j.tlsBootstrapCfg = tlsBootstrapCfg
 	return tlsBootstrapCfg, err
 }
@@ -550,19 +563,58 @@ func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) {
 		return nil, err
 	}
 	klog.V(1).Infoln("[preflight] Fetching init configuration")
-	initCfg, err := fetchInitConfigurationFromJoinConfiguration(j.cfg, j.tlsBootstrapCfg)
+	var client clientset.Interface
+	if j.dryRun {
+		var err error
+		client, err = j.Client()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get dry-run client for fetching InitConfiguration")
+		}
+	}
+	initCfg, err := fetchInitConfigurationFromJoinConfiguration(j.cfg, client, j.tlsBootstrapCfg)
 	j.initCfg = initCfg
 	return initCfg, err
 }
 
 // Client returns the Client for accessing the cluster with the identity defined in admin.conf.
 func (j *joinData) Client() (clientset.Interface, error) {
-	if j.client != nil {
+	pathAdmin := filepath.Join(j.KubeConfigDir(), kubeadmconstants.AdminKubeConfigFileName)
+
+	if j.dryRun {
+		dryRun := apiclient.NewDryRun()
+		// For the dynamic dry-run client use this kubeconfig only if it exists.
+		// That would happen presumably after TLS bootstrap.
+		if _, err := os.Stat(pathAdmin); err == nil {
+			if err := dryRun.WithKubeConfigFile(pathAdmin); err != nil {
+				return nil, err
+			}
+		} else if j.tlsBootstrapCfg != nil {
+			if err := dryRun.WithKubeConfig(j.tlsBootstrapCfg); err != nil {
+				return nil, err
+			}
+		} else if j.cfg.Discovery.BootstrapToken != nil {
+			insecureConfig := token.BuildInsecureBootstrapKubeConfig(j.cfg.Discovery.BootstrapToken.APIServerEndpoint)
+			resetConfig, err := clientcmd.NewDefaultClientConfig(*insecureConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create API client configuration from kubeconfig")
+			}
+			if err := dryRun.WithRestConfig(resetConfig); err != nil {
+				return nil, err
+			}
+		}
+
+		dryRun.WithDefaultMarshalFunction().
+			WithWriter(os.Stdout).
+			AppendReactor(dryRun.GetClusterInfoReactor()).
+			AppendReactor(dryRun.GetKubeadmConfigReactor()).
+			AppendReactor(dryRun.GetKubeProxyConfigReactor()).
+			AppendReactor(dryRun.GetKubeletConfigReactor())
+
+		j.client = dryRun.FakeClient()
 		return j.client, nil
 	}
-	path := filepath.Join(j.KubeConfigDir(), kubeadmconstants.AdminKubeConfigFileName)
 
-	client, err := kubeconfigutil.ClientSetFromFile(path)
+	client, err := kubeconfigutil.ClientSetFromFile(pathAdmin)
 	if err != nil {
 		return nil, errors.Wrap(err, "[preflight] couldn't create Kubernetes client")
 	}
@@ -593,10 +645,18 @@ func (j *joinData) PatchesDir() string {
 }
 
 // fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery
-func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
-	// Retrieves the kubeadm configuration
+func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, client clientset.Interface, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
+	var err error
+
 	klog.V(1).Infoln("[preflight] Retrieving KubeConfig objects")
-	initConfiguration, err := fetchInitConfiguration(tlsBootstrapCfg)
+	if client == nil {
+		// creates a client to access the cluster using the bootstrap token identity
+		client, err = kubeconfigutil.ToClientSet(tlsBootstrapCfg)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to access the cluster")
+		}
+	}
+	initConfiguration, err := fetchInitConfiguration(client)
 	if err != nil {
 		return nil, err
 	}
@@ -618,15 +678,8 @@ func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfigurati
 }
 
 // fetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap
-func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
-	// creates a client to access the cluster using the bootstrap token identity
-	tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg)
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to access the cluster")
-	}
-
-	// Fetches the init configuration
-	initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, nil, "preflight", true, false)
+func fetchInitConfiguration(client clientset.Interface) (*kubeadmapi.InitConfiguration, error) {
+	initConfiguration, err := configutil.FetchInitConfigurationFromCluster(client, nil, "preflight", true, false)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
 	}

cmd/kubeadm/app/cmd/phases/join/kubelet.go

@@ -30,6 +30,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog/v2"
@@ -150,11 +151,18 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 		}()
 	}
 
-	// Create the bootstrap client before we possibly overwrite the server address
-	// for ControlPlaneKubeletLocalMode.
-	bootstrapClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg)
-	if err != nil {
-		return errors.Errorf("could not create client from bootstrap kubeconfig")
+	var client clientset.Interface
+	// If dry-use the client from joinData, else create a new bootstrap client
+	if data.DryRun() {
+		client, err = data.Client()
+		if err != nil {
+			return err
+		}
+	} else {
+		client, err = kubeconfigutil.ToClientSet(tlsBootstrapCfg)
+		if err != nil {
+			return errors.Errorf("could not create client from bootstrap kubeconfig")
+		}
 	}
 
 	if features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
@@ -204,7 +212,7 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 	// A new Node with the same name as an existing control-plane Node can cause undefined
 	// behavior and ultimately control-plane failure.
 	klog.V(1).Infof("[kubelet-start] Checking for an existing Node in the cluster with name %q and status %q", nodeName, v1.NodeReady)
-	node, err := bootstrapClient.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
 	if err != nil && !apierrors.IsNotFound(err) {
 		return errors.Wrapf(err, "cannot get Node %q", nodeName)
 	}

cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go

@@ -75,7 +75,7 @@ func runCleanupNode(c workflow.RunData) error {
 				klog.Warningln("[reset] Please ensure kubelet is stopped manually")
 			}
 		} else {
-			fmt.Println("[reset] Would stop the kubelet service")
+			fmt.Println("[dryrun] Would stop the kubelet service")
 		}
 	}
 
@@ -96,7 +96,7 @@ func runCleanupNode(c workflow.RunData) error {
 			dirsToClean = append(dirsToClean, kubeletRunDirectory)
 		}
 	} else {
-		fmt.Printf("[reset] Would unmount mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
+		fmt.Printf("[dryrun] Would unmount mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
 	}
 
 	if !r.DryRun() {
@@ -105,7 +105,7 @@ func runCleanupNode(c workflow.RunData) error {
 			klog.Warningf("[reset] Failed to remove containers: %v\n", err)
 		}
 	} else {
-		fmt.Println("[reset] Would remove Kubernetes-managed containers")
+		fmt.Println("[dryrun] Would remove Kubernetes-managed containers")
 	}
 
 	// Remove contents from the config and pki directories
@@ -115,7 +115,7 @@ func runCleanupNode(c workflow.RunData) error {
 
 	dirsToClean = append(dirsToClean, certsDir)
 	if r.CleanupTmpDir() {
-		tempDir := path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)
+		tempDir := path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDir)
 		dirsToClean = append(dirsToClean, tempDir)
 	}
 	resetConfigDir(kubeadmconstants.KubernetesDir, dirsToClean, r.DryRun())
@@ -127,7 +127,7 @@ func runCleanupNode(c workflow.RunData) error {
 				klog.Warningf("[reset] Failed to remove users and groups: %v\n", err)
 			}
 		} else {
-			fmt.Println("[reset] Would remove users and groups created for rootless control-plane")
+			fmt.Println("[dryrun] Would remove users and groups created for rootless control-plane")
 		}
 	}
 
@@ -156,7 +156,7 @@ func resetConfigDir(configPathDir string, dirsToClean []string, isDryRun bool) {
 			}
 		}
 	} else {
-		fmt.Printf("[reset] Would delete contents of directories: %v\n", dirsToClean)
+		fmt.Printf("[dryrun] Would delete contents of directories: %v\n", dirsToClean)
 	}
 
 	filesToClean := []string{
@@ -176,7 +176,7 @@ func resetConfigDir(configPathDir string, dirsToClean []string, isDryRun bool) {
 			}
 		}
 	} else {
-		fmt.Printf("[reset] Would delete files: %v\n", filesToClean)
+		fmt.Printf("[dryrun] Would delete files: %v\n", filesToClean)
 	}
 }
 

cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go

@@ -185,7 +185,7 @@ func TestConfigDirCleaner(t *testing.T) {
 			dirsToClean := []string{
 				filepath.Join(tmpDir, test.resetDir),
 				filepath.Join(tmpDir, kubeadmconstants.ManifestsSubDirName),
-				filepath.Join(tmpDir, kubeadmconstants.TempDirForKubeadm),
+				filepath.Join(tmpDir, kubeadmconstants.TempDir),
 			}
 			resetConfigDir(tmpDir, dirsToClean, false)
 

cmd/kubeadm/app/cmd/phases/reset/removeetcdmember.go

@@ -74,8 +74,8 @@ func runRemoveETCDMemberPhase(c workflow.RunData) error {
 					}
 				}
 			} else {
-				fmt.Println("[reset] Would remove the etcd member on this node from the etcd cluster")
-				fmt.Printf("[reset] Would delete contents of the etcd data directory: %v\n", etcdDataDir)
+				fmt.Println("[dryrun] Would remove the etcd member on this node from the etcd cluster")
+				fmt.Printf("[dryrun] Would delete contents of the etcd data directory: %v\n", etcdDataDir)
 			}
 		}
 		// This could happen if the phase `cleanup-node` is run before the `remove-etcd-member`.

cmd/kubeadm/app/cmd/reset.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"path"
 
 	"github.com/lithammer/dedent"
@@ -39,7 +40,9 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
 )
 
@@ -104,7 +107,10 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W
 		return nil, err
 	}
 
-	var initCfg *kubeadmapi.InitConfiguration
+	var (
+		initCfg *kubeadmapi.InitConfiguration
+		client  clientset.Interface
+	)
 
 	// Either use the config file if specified, or convert public kubeadm API to the internal ResetConfiguration and validates cfg.
 	resetCfg, err := configutil.LoadOrDefaultResetConfiguration(opts.cfgPath, opts.externalcfg, configutil.LoadOrDefaultConfigurationOptions{
@@ -115,7 +121,21 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W
 		return nil, err
 	}
 
-	client, err := cmdutil.GetClientSet(opts.kubeconfigPath, false)
+	dryRunFlag := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, resetCfg.DryRun, opts.externalcfg.DryRun).(bool)
+	if dryRunFlag {
+		dryRun := apiclient.NewDryRun().WithDefaultMarshalFunction().WithWriter(os.Stdout)
+		dryRun.AppendReactor(dryRun.GetKubeadmConfigReactor()).
+			AppendReactor(dryRun.GetKubeletConfigReactor()).
+			AppendReactor(dryRun.GetKubeProxyConfigReactor())
+		client = dryRun.FakeClient()
+		_, err = os.Stat(opts.kubeconfigPath)
+		if err == nil {
+			err = dryRun.WithKubeConfigFile(opts.kubeconfigPath)
+		}
+	} else {
+		client, err = kubeconfigutil.ClientSetFromFile(opts.kubeconfigPath)
+	}
+
 	if err == nil {
 		klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", opts.kubeconfigPath)
 		initCfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
@@ -162,7 +182,7 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W
 		outputWriter:          out,
 		cfg:                   initCfg,
 		resetCfg:              resetCfg,
-		dryRun:                cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, resetCfg.DryRun, opts.externalcfg.DryRun).(bool),
+		dryRun:                dryRunFlag,
 		forceReset:            cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.ForceReset, resetCfg.Force, opts.externalcfg.Force).(bool),
 		cleanupTmpDir:         cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.CleanupTmpDir, resetCfg.CleanupTmpDir, opts.externalcfg.CleanupTmpDir).(bool),
 	}, nil
@@ -184,7 +204,7 @@ func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) {
 	)
 	flagSet.BoolVar(
 		&resetOptions.externalcfg.CleanupTmpDir, options.CleanupTmpDir, resetOptions.externalcfg.CleanupTmpDir,
-		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)),
+		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDir)),
 	)
 	options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath)
 	options.AddConfigFlag(flagSet, &resetOptions.cfgPath)

cmd/kubeadm/app/cmd/reset_test.go

@@ -219,6 +219,9 @@ func TestNewResetData(t *testing.T) {
 			resetOptions := newResetOptions()
 			cmd := newCmdReset(nil, nil, resetOptions)
 
+			// make sure all cases use dry-run as we are not constructing a kubeconfig
+			tc.flags[options.DryRun] = "true"
+
 			// sets cmd flags (that will be reflected on the reset options)
 			for f, v := range tc.flags {
 				cmd.Flags().Set(f, v)