kubeadm: support dryrunning upgrade wihout a real cluster

Make the following changes:
- When dryrunning if the given kubeconfig does not exist
create a DryRun object without a real client. This means only
a fake client will be used for all actions.
- Skip the preflight check if manifests exist during dryrun.
Print "would ..." instead.
- Add new reactors that handle objects during upgrade.
- Add unit tests for new reactors.
- Print message on "upgrade node" that this is not a CP node
if the apiserver manifest is missing.
- Add a new function GetNodeName() that uses 3 different methods
for fetching the node name. Solves a long standing issue where
we only used the cert in kubelet.conf for determining node name.
- Various other minor fixes.

Author: neolit123

cmd/kubeadm/app/cmd/phases/upgrade/apply/preflight.go

@@ -80,7 +80,7 @@ func runPreflight(c workflow.RunData) error {
 
 	// Run healthchecks against the cluster.
 	klog.V(1).Infoln("[upgrade/preflight] Verifying the cluster health")
-	if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrors, printer); err != nil {
+	if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrors, data.DryRun(), printer); err != nil {
 		return err
 	}
 

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -223,13 +223,13 @@ func newApplyData(cmd *cobra.Command, args []string, applyFlags *applyFlags) (*a
 		return nil, cmdutil.TypeMismatchErr("printConfig", "bool")
 	}
 
-	client, err := getClient(applyFlags.kubeConfigPath, *dryRun)
+	printer := &output.TextPrinter{}
+
+	client, err := getClient(applyFlags.kubeConfigPath, *dryRun, printer)
 	if err != nil {
 		return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", applyFlags.kubeConfigPath)
 	}
 
-	printer := &output.TextPrinter{}
-
 	// Fetches the cluster configuration.
 	klog.V(1).Infoln("[upgrade] retrieving configuration from cluster")
 	initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", false, false)

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -71,7 +72,7 @@ func enforceRequirements(flagSet *pflag.FlagSet, flags *applyPlanFlags, args []s
 		}
 	}
 
-	client, err := getClient(flags.kubeConfigPath, *isDryRun)
+	client, err := getClient(flags.kubeConfigPath, *isDryRun, printer)
 	if err != nil {
 		return nil, nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath)
 	}
@@ -137,7 +138,7 @@ func enforceRequirements(flagSet *pflag.FlagSet, flags *applyPlanFlags, args []s
 	}
 
 	// Run healthchecks against the cluster
-	if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrorsSet, printer); err != nil {
+	if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrorsSet, dryRun, printer); err != nil {
 		return nil, nil, nil, nil, errors.Wrap(err, "[upgrade/health] FATAL")
 	}
 
@@ -189,32 +190,57 @@ func runPreflightChecks(client clientset.Interface, ignorePreflightErrors sets.S
 }
 
 // getClient gets a real or fake client depending on whether the user is dry-running or not
-func getClient(file string, dryRun bool) (clientset.Interface, error) {
+func getClient(file string, dryRun bool, printer output.Printer) (clientset.Interface, error) {
 	if dryRun {
+		// Default the server version to the kubeadm version.
+		serverVersion := constants.CurrentKubernetesVersion.Info()
+
 		dryRun := apiclient.NewDryRun()
-		if err := dryRun.WithKubeConfigFile(file); err != nil {
-			return nil, err
-		}
 		dryRun.WithDefaultMarshalFunction().
 			WithWriter(os.Stdout).
 			PrependReactor(dryRun.HealthCheckJobReactor()).
 			PrependReactor(dryRun.PatchNodeReactor())
 
-		// In order for fakeclient.Discovery().ServerVersion() to return the backing API Server's
-		// real version; we have to do some clever API machinery tricks. First, we get the real
-		// API Server's version.
-		realServerVersion, err := dryRun.Client().Discovery().ServerVersion()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to get server version")
+		// If the kubeconfig exists, construct a real client from it and get the real serverVersion.
+		if _, err := os.Stat(file); err == nil {
+			_, _ = printer.Printf("[dryrun] Creating a real client from %q\n", file)
+			if err := dryRun.WithKubeConfigFile(file); err != nil {
+				return nil, err
+			}
+			serverVersion, err = dryRun.Client().Discovery().ServerVersion()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to get server version")
+			}
+		} else if os.IsNotExist(err) {
+			// If the file (supposedly admin.conf) does not exist, add more reactors.
+			// Knowing the node name is required by the ListPodsReactor. For that we try to use
+			// the kubelet.conf client, if it exists. If not, it falls back to hostname.
+			_, _ = printer.Printf("[dryrun] Dryrunning without a real client\n")
+			kubeconfigPath := filepath.Join(constants.KubernetesDir, constants.KubeletKubeConfigFileName)
+			nodeName, err := configutil.GetNodeName(kubeconfigPath)
+			if err != nil {
+				return nil, err
+			}
+			dryRun.PrependReactor(dryRun.GetKubeadmConfigReactor()).
+				PrependReactor(dryRun.GetKubeletConfigReactor()).
+				PrependReactor(dryRun.GetKubeProxyConfigReactor()).
+				PrependReactor(dryRun.GetNodeReactor()).
+				PrependReactor(dryRun.ListPodsReactor(nodeName)).
+				PrependReactor(dryRun.GetCoreDNSConfigReactor()).
+				PrependReactor(dryRun.ListDeploymentsReactor())
+		} else {
+			// Throw an error if the file exists but there was a different stat error.
+			return nil, errors.Wrapf(err, "could not create a client from %q", file)
 		}
+
 		// Obtain the FakeDiscovery object for this fake client.
 		fakeClient := dryRun.FakeClient()
 		fakeClientDiscovery, ok := fakeClient.Discovery().(*fakediscovery.FakeDiscovery)
 		if !ok {
 			return nil, errors.New("could not set fake discovery's server version")
 		}
-		// Lastly, set the right server version to be used.
-		fakeClientDiscovery.FakedServerVersion = realServerVersion
+		// Set the right server version for it.
+		fakeClientDiscovery.FakedServerVersion = serverVersion
 
 		return fakeClient, nil
 	}

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -17,6 +17,7 @@ limitations under the License.
 package upgrade
 
 import (
+	"fmt"
 	"io"
 	"os"
 
@@ -26,6 +27,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
@@ -37,6 +39,7 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
 )
 
 // nodeOptions defines all the options exposed via flags by kubeadm upgrade node.
@@ -84,7 +87,15 @@ func newCmdNode(out io.Writer) *cobra.Command {
 				return err
 			}
 
-			return nodeRunner.Run(args)
+			if err := nodeRunner.Run(args); err != nil {
+				return err
+			}
+			if nodeOptions.dryRun {
+				fmt.Println("[upgrade/successful] Finished dryrunning successfully!")
+				return nil
+			}
+
+			return nil
 		},
 		Args: cobra.NoArgs,
 	}
@@ -150,6 +161,7 @@ func newNodeData(cmd *cobra.Command, nodeOptions *nodeOptions, out io.Writer) (*
 	isControlPlaneNode := true
 	filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory())
 	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		klog.V(1).Infof("assuming this is not a control plane node because %q is missing", filepath)
 		isControlPlaneNode = false
 	}
 	if len(nodeOptions.kubeConfigPath) == 0 {
@@ -171,7 +183,9 @@ func newNodeData(cmd *cobra.Command, nodeOptions *nodeOptions, out io.Writer) (*
 	if !ok {
 		return nil, cmdutil.TypeMismatchErr("dryRun", "bool")
 	}
-	client, err := getClient(nodeOptions.kubeConfigPath, *dryRun)
+
+	printer := &output.TextPrinter{}
+	client, err := getClient(nodeOptions.kubeConfigPath, *dryRun, printer)
 	if err != nil {
 		return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", nodeOptions.kubeConfigPath)
 	}

cmd/kubeadm/app/phases/upgrade/health.go

@@ -46,16 +46,18 @@ const createJobHealthCheckPrefix = "upgrade-health-check"
 
 // healthCheck is a helper struct for easily performing healthchecks against the cluster and printing the output
 type healthCheck struct {
-	name   string
-	client clientset.Interface
-	cfg    *kubeadmapi.ClusterConfiguration
+	name    string
+	client  clientset.Interface
+	cfg     *kubeadmapi.ClusterConfiguration
+	dryRun  bool
+	printer output.Printer
 	// f is invoked with a k8s client and a kubeadm ClusterConfiguration passed to it. Should return an optional error
-	f func(clientset.Interface, *kubeadmapi.ClusterConfiguration) error
+	f func(clientset.Interface, *kubeadmapi.ClusterConfiguration, bool, output.Printer) error
 }
 
 // Check is part of the preflight.Checker interface
 func (c *healthCheck) Check() (warnings, errors []error) {
-	if err := c.f(c.client, c.cfg); err != nil {
+	if err := c.f(c.client, c.cfg, c.dryRun, c.printer); err != nil {
 		return nil, []error{err}
 	}
 	return nil, nil
@@ -70,32 +72,38 @@ func (c *healthCheck) Name() string {
 // - the cluster can accept a workload
 // - all control-plane Nodes are Ready
 // - (if static pod-hosted) that all required Static Pod manifests exist on disk
-func CheckClusterHealth(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration, ignoreChecksErrors sets.Set[string], printer output.Printer) error {
+func CheckClusterHealth(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration, ignoreChecksErrors sets.Set[string], dryRun bool, printer output.Printer) error {
 	_, _ = printer.Println("[upgrade] Running cluster health checks")
 
 	healthChecks := []preflight.Checker{
 		&healthCheck{
-			name:   "CreateJob",
-			client: client,
-			cfg:    cfg,
-			f:      createJob,
+			name:    "CreateJob",
+			client:  client,
+			cfg:     cfg,
+			f:       createJob,
+			dryRun:  dryRun,
+			printer: printer,
 		},
 		&healthCheck{
-			name:   "ControlPlaneNodesReady",
-			client: client,
-			f:      controlPlaneNodesReady,
+			name:    "ControlPlaneNodesReady",
+			client:  client,
+			f:       controlPlaneNodesReady,
+			dryRun:  dryRun,
+			printer: printer,
 		},
 		&healthCheck{
-			name: "StaticPodManifest",
-			f:    staticPodManifestHealth,
+			name:    "StaticPodManifest",
+			f:       staticPodManifestHealth,
+			dryRun:  dryRun,
+			printer: printer,
 		},
 	}
 
 	return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors)
 }
 
 // createJob is a check that verifies that a Job can be created in the cluster
-func createJob(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) error {
+func createJob(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration, _ bool, _ output.Printer) error {
 	const (
 		fieldSelector = "spec.unschedulable=false"
 		ns            = metav1.NamespaceSystem
@@ -213,7 +221,7 @@ func createJob(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration)
 }
 
 // controlPlaneNodesReady checks whether all control-plane Nodes in the cluster are in the Running state
-func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error {
+func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration, _ bool, _ output.Printer) error {
 	selectorControlPlane := labels.SelectorFromSet(map[string]string{
 		constants.LabelNodeRoleControlPlane: "",
 	})
@@ -232,18 +240,22 @@ func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterCon
 }
 
 // staticPodManifestHealth makes sure the required static pods are presents
-func staticPodManifestHealth(_ clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error {
+func staticPodManifestHealth(_ clientset.Interface, _ *kubeadmapi.ClusterConfiguration, dryRun bool, printer output.Printer) error {
 	var nonExistentManifests []string
 	for _, component := range constants.ControlPlaneComponents {
 		manifestFile := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
+		if dryRun {
+			_, _ = printer.Printf("[dryrun] would check if %s exists\n", manifestFile)
+			continue
+		}
 		if _, err := os.Stat(manifestFile); os.IsNotExist(err) {
 			nonExistentManifests = append(nonExistentManifests, manifestFile)
 		}
 	}
 	if len(nonExistentManifests) == 0 {
 		return nil
 	}
-	return errors.Errorf("The control plane seems to be Static Pod-hosted, but some of the manifests don't seem to exist on disk. This probably means you're running 'kubeadm upgrade' on a remote machine, which is not supported for a Static Pod-hosted cluster. Manifest files not found: %v", nonExistentManifests)
+	return errors.Errorf("manifest files not found: %v", nonExistentManifests)
 }
 
 // getNotReadyNodes returns a string slice of nodes in the cluster that are NotReady

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -113,11 +113,6 @@ func (w *fakeWaiter) WaitForPodsWithLabel(kvLabel string) error {
 	return w.errsToReturn[waitForPodsWithLabel]
 }
 
-// WaitForPodToDisappear just returns a dummy nil, to indicate that the program should just proceed
-func (w *fakeWaiter) WaitForPodToDisappear(podName string) error {
-	return nil
-}
-
 // SetTimeout is a no-op; we don't use it in this implementation
 func (w *fakeWaiter) SetTimeout(_ time.Duration) {}