kubeadm: refactor the dry-run logic

The current dryrun client implemnetation is suboptimal
and sparse. It has the following problems:

- When an object CREATE or UPDATE reaches the default dryrun client
the operation is a NO-OP, which means subsequent GET calls must
fully emulate the object that exists in the store.
- There are multiple implmentations of a DryRunGetter interface
such the one in init_dryrun.go but there are no implementations
for reset, upgrade, join.
- There is a specific DryRunGetter that is backed by a real
client in clientbacked_dryrun.go, but this is used for upgrade
and does not work in conjuction with a fake client.

This commit does the following changes:

- Removes all existing *dryrun*.go implementations.
- Add a new DryRun implementation in dryrun.go that implements
3 clients - fake clientset, real clientset, real dynamic client.
- The DryRun object uses the method chaining pattern.
- Allows the user opt-in into real clients only if needed, by passing
a real kubeconfig. By default only constructs a fake client.
- The default reactor chain for the fake client, always logs the
object action, then for GET or LIST actions attempts to use the
real dynamic client to get the object. If a real object does not
exist it attempts to get the object from the fake object store.
- The user can prepend or append reactors to the chain.
- All known needed reactors for operations during init, join,
reset, upgrade are added as methods of the DryRun struct.
- Adds detailed unit test for the DryRun struct and its methods
including reactors.

Additional changes:
- Use the new DryRun implementation in all command workflows -
init, join, reset, upgrade.
- Ensure that --dry-run works even if there is no active cluster
by returning faked objects. For join, a faked cluster-info
with a fake bootstrap token and CA are used.

Author: neolit123

cmd/kubeadm/app/cmd/init.go

@@ -28,6 +28,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
@@ -349,10 +350,7 @@ func newInitData(cmd *cobra.Command, args []string, initOptions *initOptions, ou
 	// if dry running creates a temporary folder for saving kubeadm generated files
 	dryRunDir := ""
 	if initOptions.dryRun || cfg.DryRun {
-		// the KUBEADM_INIT_DRYRUN_DIR environment variable allows overriding the dry-run temporary
-		// directory from the command line. This makes it possible to run "kubeadm init" integration
-		// tests without root.
-		if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm(os.Getenv("KUBEADM_INIT_DRYRUN_DIR"), "kubeadm-init-dryrun"); err != nil {
+		if dryRunDir, err = kubeadmconstants.GetDryRunDir(kubeadmconstants.EnvVarInitDryRunDir, "kubeadm-init-dryrun", klog.Warningf); err != nil {
 			return nil, errors.Wrap(err, "couldn't create a temporary directory")
 		}
 	}
@@ -502,12 +500,16 @@ func (d *initData) OutputWriter() io.Writer {
 
 // getDryRunClient creates a fake client that answers some GET calls in order to be able to do the full init flow in dry-run mode.
 func getDryRunClient(d *initData) (clientset.Interface, error) {
-	svcSubnetCIDR, err := kubeadmconstants.GetKubernetesServiceCIDR(d.cfg.Networking.ServiceSubnet)
-	if err != nil {
-		return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", d.cfg.Networking.ServiceSubnet)
+	dryRun := apiclient.NewDryRun()
+	if err := dryRun.WithKubeConfigFile(d.KubeConfigPath()); err != nil {
+		return nil, err
 	}
-	dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String())
-	return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil
+	dryRun.WithDefaultMarshalFunction().
+		WithWriter(os.Stdout).
+		PrependReactor(dryRun.GetNodeReactor()).
+		PrependReactor(dryRun.PatchNodeReactor())
+
+	return dryRun.FakeClient(), nil
 }
 
 // Client returns a Kubernetes client to be used by kubeadm.

cmd/kubeadm/app/cmd/join.go

@@ -45,6 +45,8 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
+	"k8s.io/kubernetes/cmd/kubeadm/app/discovery/token"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
@@ -464,7 +466,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 	// if dry running, creates a temporary folder to save kubeadm generated files
 	dryRunDir := ""
 	if opt.dryRun || cfg.DryRun {
-		if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-join-dryrun"); err != nil {
+		if dryRunDir, err = kubeadmconstants.GetDryRunDir(kubeadmconstants.EnvVarJoinDryRunDir, "kubeadm-join-dryrun", klog.Warningf); err != nil {
 			return nil, errors.Wrap(err, "couldn't create a temporary directory on dryrun")
 		}
 	}
@@ -535,8 +537,19 @@ func (j *joinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) {
 	if j.tlsBootstrapCfg != nil {
 		return j.tlsBootstrapCfg, nil
 	}
+
+	var (
+		client clientset.Interface
+		err    error
+	)
+	if j.dryRun {
+		client, err = j.Client()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create a client for TLS bootstrap")
+		}
+	}
 	klog.V(1).Infoln("[preflight] Discovering cluster-info")
-	tlsBootstrapCfg, err := discovery.For(j.cfg)
+	tlsBootstrapCfg, err := discovery.For(client, j.cfg)
 	j.tlsBootstrapCfg = tlsBootstrapCfg
 	return tlsBootstrapCfg, err
 }
@@ -550,19 +563,58 @@ func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) {
 		return nil, err
 	}
 	klog.V(1).Infoln("[preflight] Fetching init configuration")
-	initCfg, err := fetchInitConfigurationFromJoinConfiguration(j.cfg, j.tlsBootstrapCfg)
+	var client clientset.Interface
+	if j.dryRun {
+		var err error
+		client, err = j.Client()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get dry-run client for fetching InitConfiguration")
+		}
+	}
+	initCfg, err := fetchInitConfigurationFromJoinConfiguration(j.cfg, client, j.tlsBootstrapCfg)
 	j.initCfg = initCfg
 	return initCfg, err
 }
 
 // Client returns the Client for accessing the cluster with the identity defined in admin.conf.
 func (j *joinData) Client() (clientset.Interface, error) {
-	if j.client != nil {
+	pathAdmin := filepath.Join(j.KubeConfigDir(), kubeadmconstants.AdminKubeConfigFileName)
+
+	if j.dryRun {
+		dryRun := apiclient.NewDryRun()
+		// For the dynamic dry-run client use this kubeconfig only if it exists.
+		// That would happen presumably after TLS bootstrap.
+		if _, err := os.Stat(pathAdmin); err == nil {
+			if err := dryRun.WithKubeConfigFile(pathAdmin); err != nil {
+				return nil, err
+			}
+		} else if j.tlsBootstrapCfg != nil {
+			if err := dryRun.WithKubeConfig(j.tlsBootstrapCfg); err != nil {
+				return nil, err
+			}
+		} else if j.cfg.Discovery.BootstrapToken != nil {
+			insecureConfig := token.BuildInsecureBootstrapKubeConfig(j.cfg.Discovery.BootstrapToken.APIServerEndpoint)
+			resetConfig, err := clientcmd.NewDefaultClientConfig(*insecureConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to create API client configuration from kubeconfig")
+			}
+			if err := dryRun.WithRestConfig(resetConfig); err != nil {
+				return nil, err
+			}
+		}
+
+		dryRun.WithDefaultMarshalFunction().
+			WithWriter(os.Stdout).
+			AppendReactor(dryRun.GetClusterInfoReactor()).
+			AppendReactor(dryRun.GetKubeadmConfigReactor()).
+			AppendReactor(dryRun.GetKubeProxyConfigReactor()).
+			AppendReactor(dryRun.GetKubeletConfigReactor())
+
+		j.client = dryRun.FakeClient()
 		return j.client, nil
 	}
-	path := filepath.Join(j.KubeConfigDir(), kubeadmconstants.AdminKubeConfigFileName)
 
-	client, err := kubeconfigutil.ClientSetFromFile(path)
+	client, err := kubeconfigutil.ClientSetFromFile(pathAdmin)
 	if err != nil {
 		return nil, errors.Wrap(err, "[preflight] couldn't create Kubernetes client")
 	}
@@ -593,10 +645,18 @@ func (j *joinData) PatchesDir() string {
 }
 
 // fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery
-func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
-	// Retrieves the kubeadm configuration
+func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, client clientset.Interface, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
+	var err error
+
 	klog.V(1).Infoln("[preflight] Retrieving KubeConfig objects")
-	initConfiguration, err := fetchInitConfiguration(tlsBootstrapCfg)
+	if client == nil {
+		// creates a client to access the cluster using the bootstrap token identity
+		client, err = kubeconfigutil.ToClientSet(tlsBootstrapCfg)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to access the cluster")
+		}
+	}
+	initConfiguration, err := fetchInitConfiguration(client)
 	if err != nil {
 		return nil, err
 	}
@@ -618,15 +678,8 @@ func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfigurati
 }
 
 // fetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap
-func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
-	// creates a client to access the cluster using the bootstrap token identity
-	tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg)
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to access the cluster")
-	}
-
-	// Fetches the init configuration
-	initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, nil, "preflight", true, false)
+func fetchInitConfiguration(client clientset.Interface) (*kubeadmapi.InitConfiguration, error) {
+	initConfiguration, err := configutil.FetchInitConfigurationFromCluster(client, nil, "preflight", true, false)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
 	}

cmd/kubeadm/app/cmd/phases/join/kubelet.go

@@ -30,6 +30,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog/v2"
@@ -150,11 +151,18 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 		}()
 	}
 
-	// Create the bootstrap client before we possibly overwrite the server address
-	// for ControlPlaneKubeletLocalMode.
-	bootstrapClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg)
-	if err != nil {
-		return errors.Errorf("could not create client from bootstrap kubeconfig")
+	var client clientset.Interface
+	// If dry-use the client from joinData, else create a new bootstrap client
+	if data.DryRun() {
+		client, err = data.Client()
+		if err != nil {
+			return err
+		}
+	} else {
+		client, err = kubeconfigutil.ToClientSet(tlsBootstrapCfg)
+		if err != nil {
+			return errors.Errorf("could not create client from bootstrap kubeconfig")
+		}
 	}
 
 	if features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
@@ -204,7 +212,7 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) {
 	// A new Node with the same name as an existing control-plane Node can cause undefined
 	// behavior and ultimately control-plane failure.
 	klog.V(1).Infof("[kubelet-start] Checking for an existing Node in the cluster with name %q and status %q", nodeName, v1.NodeReady)
-	node, err := bootstrapClient.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
 	if err != nil && !apierrors.IsNotFound(err) {
 		return errors.Wrapf(err, "cannot get Node %q", nodeName)
 	}

cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go

@@ -75,7 +75,7 @@ func runCleanupNode(c workflow.RunData) error {
 				klog.Warningln("[reset] Please ensure kubelet is stopped manually")
 			}
 		} else {
-			fmt.Println("[reset] Would stop the kubelet service")
+			fmt.Println("[dryrun] Would stop the kubelet service")
 		}
 	}
 
@@ -96,7 +96,7 @@ func runCleanupNode(c workflow.RunData) error {
 			dirsToClean = append(dirsToClean, kubeletRunDirectory)
 		}
 	} else {
-		fmt.Printf("[reset] Would unmount mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
+		fmt.Printf("[dryrun] Would unmount mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
 	}
 
 	if !r.DryRun() {
@@ -105,7 +105,7 @@ func runCleanupNode(c workflow.RunData) error {
 			klog.Warningf("[reset] Failed to remove containers: %v\n", err)
 		}
 	} else {
-		fmt.Println("[reset] Would remove Kubernetes-managed containers")
+		fmt.Println("[dryrun] Would remove Kubernetes-managed containers")
 	}
 
 	// Remove contents from the config and pki directories
@@ -115,7 +115,7 @@ func runCleanupNode(c workflow.RunData) error {
 
 	dirsToClean = append(dirsToClean, certsDir)
 	if r.CleanupTmpDir() {
-		tempDir := path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)
+		tempDir := path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDir)
 		dirsToClean = append(dirsToClean, tempDir)
 	}
 	resetConfigDir(kubeadmconstants.KubernetesDir, dirsToClean, r.DryRun())
@@ -127,7 +127,7 @@ func runCleanupNode(c workflow.RunData) error {
 				klog.Warningf("[reset] Failed to remove users and groups: %v\n", err)
 			}
 		} else {
-			fmt.Println("[reset] Would remove users and groups created for rootless control-plane")
+			fmt.Println("[dryrun] Would remove users and groups created for rootless control-plane")
 		}
 	}
 
@@ -156,7 +156,7 @@ func resetConfigDir(configPathDir string, dirsToClean []string, isDryRun bool) {
 			}
 		}
 	} else {
-		fmt.Printf("[reset] Would delete contents of directories: %v\n", dirsToClean)
+		fmt.Printf("[dryrun] Would delete contents of directories: %v\n", dirsToClean)
 	}
 
 	filesToClean := []string{
@@ -176,7 +176,7 @@ func resetConfigDir(configPathDir string, dirsToClean []string, isDryRun bool) {
 			}
 		}
 	} else {
-		fmt.Printf("[reset] Would delete files: %v\n", filesToClean)
+		fmt.Printf("[dryrun] Would delete files: %v\n", filesToClean)
 	}
 }
 

cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go

@@ -185,7 +185,7 @@ func TestConfigDirCleaner(t *testing.T) {
 			dirsToClean := []string{
 				filepath.Join(tmpDir, test.resetDir),
 				filepath.Join(tmpDir, kubeadmconstants.ManifestsSubDirName),
-				filepath.Join(tmpDir, kubeadmconstants.TempDirForKubeadm),
+				filepath.Join(tmpDir, kubeadmconstants.TempDir),
 			}
 			resetConfigDir(tmpDir, dirsToClean, false)
 

cmd/kubeadm/app/cmd/phases/reset/removeetcdmember.go

@@ -74,8 +74,8 @@ func runRemoveETCDMemberPhase(c workflow.RunData) error {
 					}
 				}
 			} else {
-				fmt.Println("[reset] Would remove the etcd member on this node from the etcd cluster")
-				fmt.Printf("[reset] Would delete contents of the etcd data directory: %v\n", etcdDataDir)
+				fmt.Println("[dryrun] Would remove the etcd member on this node from the etcd cluster")
+				fmt.Printf("[dryrun] Would delete contents of the etcd data directory: %v\n", etcdDataDir)
 			}
 		}
 		// This could happen if the phase `cleanup-node` is run before the `remove-etcd-member`.

cmd/kubeadm/app/cmd/reset.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"path"
 
 	"github.com/lithammer/dedent"
@@ -39,7 +40,9 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
 )
 
@@ -104,7 +107,10 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W
 		return nil, err
 	}
 
-	var initCfg *kubeadmapi.InitConfiguration
+	var (
+		initCfg *kubeadmapi.InitConfiguration
+		client  clientset.Interface
+	)
 
 	// Either use the config file if specified, or convert public kubeadm API to the internal ResetConfiguration and validates cfg.
 	resetCfg, err := configutil.LoadOrDefaultResetConfiguration(opts.cfgPath, opts.externalcfg, configutil.LoadOrDefaultConfigurationOptions{
@@ -115,7 +121,21 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W
 		return nil, err
 	}
 
-	client, err := cmdutil.GetClientSet(opts.kubeconfigPath, false)
+	dryRunFlag := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, resetCfg.DryRun, opts.externalcfg.DryRun).(bool)
+	if dryRunFlag {
+		dryRun := apiclient.NewDryRun().WithDefaultMarshalFunction().WithWriter(os.Stdout)
+		dryRun.AppendReactor(dryRun.GetKubeadmConfigReactor()).
+			AppendReactor(dryRun.GetKubeletConfigReactor()).
+			AppendReactor(dryRun.GetKubeProxyConfigReactor())
+		client = dryRun.FakeClient()
+		_, err = os.Stat(opts.kubeconfigPath)
+		if err == nil {
+			err = dryRun.WithKubeConfigFile(opts.kubeconfigPath)
+		}
+	} else {
+		client, err = kubeconfigutil.ClientSetFromFile(opts.kubeconfigPath)
+	}
+
 	if err == nil {
 		klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", opts.kubeconfigPath)
 		initCfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
@@ -162,7 +182,7 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W
 		outputWriter:          out,
 		cfg:                   initCfg,
 		resetCfg:              resetCfg,
-		dryRun:                cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, resetCfg.DryRun, opts.externalcfg.DryRun).(bool),
+		dryRun:                dryRunFlag,
 		forceReset:            cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.ForceReset, resetCfg.Force, opts.externalcfg.Force).(bool),
 		cleanupTmpDir:         cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.CleanupTmpDir, resetCfg.CleanupTmpDir, opts.externalcfg.CleanupTmpDir).(bool),
 	}, nil
@@ -184,7 +204,7 @@ func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) {
 	)
 	flagSet.BoolVar(
 		&resetOptions.externalcfg.CleanupTmpDir, options.CleanupTmpDir, resetOptions.externalcfg.CleanupTmpDir,
-		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)),
+		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDir)),
 	)
 	options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath)
 	options.AddConfigFlag(flagSet, &resetOptions.cfgPath)

cmd/kubeadm/app/cmd/reset_test.go

@@ -219,6 +219,9 @@ func TestNewResetData(t *testing.T) {
 			resetOptions := newResetOptions()
 			cmd := newCmdReset(nil, nil, resetOptions)
 
+			// make sure all cases use dry-run as we are not constructing a kubeconfig
+			tc.flags[options.DryRun] = "true"
+
 			// sets cmd flags (that will be reflected on the reset options)
 			for f, v := range tc.flags {
 				cmd.Flags().Set(f, v)