KCP: block upgrade to versions with old registry, improve registry

handling

Signed-off-by: Stefan Büringer [email protected]

Author: sbueringer

bootstrap/kubeadm/api/v1beta1/kubeadm_types.go

@@ -125,9 +125,16 @@ type ClusterConfiguration struct {
 	CertificatesDir string `json:"certificatesDir,omitempty"`
 
 	// ImageRepository sets the container registry to pull images from.
-	// If empty, `registry.k8s.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`)
-	// `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `registry.k8s.io`
-	// will be used for all the other images.
+	// * If not set, the default registry of kubeadm will be used, i.e.
+	//   * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0
+	//   * k8s.gcr.io (old registry): all older versions
+	//   Please note that when imageRepository is not set we don't allow upgrades to
+	//   versions >= v1.22.0 which use the old registry (k8s.gcr.io). Please use
+	//   a newer patch version with the new registry instead (i.e. >= v1.22.17,
+	//   >= v1.23.15, >= v1.24.9, >= v1.25.0).
+	// * If the version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`)
+	//  `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components
+	//   and for kube-proxy, while `registry.k8s.io` will be used for all the other images.
 	// +optional
 	ImageRepository string `json:"imageRepository,omitempty"`
 

bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml

@@ -60,6 +60,11 @@ type KubeadmControlPlaneSpec struct {
 	Replicas *int32 `json:"replicas,omitempty"`
 
 	// Version defines the desired Kubernetes version.
+	// Please note that if kubeadmConfigSpec.ClusterConfiguration.imageRepository is not set
+	// we don't allow upgrades to versions >= v1.22.0 for which kubeadm uses the old registry (k8s.gcr.io).
+	// Please use a newer patch version with the new registry instead. The default registries of kubeadm are:
+	//   * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0
+	//   * k8s.gcr.io (old registry): all older versions
 	Version string `json:"version"`
 
 	// MachineTemplate contains information about how machines

bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	"sigs.k8s.io/cluster-api/internal/util/kubeadm"
 	"sigs.k8s.io/cluster-api/util/container"
 	"sigs.k8s.io/cluster-api/util/version"
 )
@@ -601,7 +602,10 @@ func (in *KubeadmControlPlane) validateVersion(previousVersion string) (allErrs
 		return allErrs
 	}
 
-	// Since upgrades to the next minor version are allowed, irrespective of the patch version.
+	// Validate that the update is upgrading at most one minor version.
+	// Note: Skipping a minor version is not allowed.
+	// Note: Checking against this ceilVersion allows upgrading to the next minor
+	// version irrespective of the patch version.
 	ceilVersion := semver.Version{
 		Major: fromVersion.Major,
 		Minor: fromVersion.Minor + 2,
@@ -616,6 +620,31 @@ func (in *KubeadmControlPlane) validateVersion(previousVersion string) (allErrs
 		)
 	}
 
+	// The Kubernetes ecosystem has been requested to move users to the new registry due to cost issues.
+	// This validation enforces the move to the new registry by forcing users to upgrade to kubeadm versions
+	// with the new registry.
+	// NOTE: This only affects users relying on the community maintained registry.
+	// NOTE: Pinning to the upstream registry is not recommended because it could lead to issues
+	// given how the migration has been implemented in kubeadm.
+	//
+	// Block if imageRepository is not set (i.e. the default registry should be used),
+	if (in.Spec.KubeadmConfigSpec.ClusterConfiguration == nil ||
+		in.Spec.KubeadmConfigSpec.ClusterConfiguration.ImageRepository == "") &&
+		// the version changed (i.e. we have an upgrade),
+		toVersion.NE(fromVersion) &&
+		// the version is >= v1.22.0 and < v1.26.0
+		toVersion.GTE(kubeadm.MinKubernetesVersionImageRegistryMigration) &&
+		toVersion.LT(kubeadm.NextKubernetesVersionImageRegistryMigration) &&
+		// and the default registry of the new Kubernetes/kubeadm version is the old default registry.
+		kubeadm.GetDefaultRegistry(toVersion) == kubeadm.OldDefaultImageRepository {
+		allErrs = append(allErrs,
+			field.Forbidden(
+				field.NewPath("spec", "version"),
+				"cannot upgrade to a Kubernetes/kubeadm version which is using the old default registry. Please use a newer Kubernetes patch release which is using the new default registry (>= v1.22.17, >= v1.23.15, >= v1.24.9)",
+			),
+		)
+	}
+
 	return allErrs
 }
 

controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go

@@ -451,14 +451,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
 	kubernetesVersion := before.DeepCopy()
 	kubernetesVersion.Spec.KubeadmConfigSpec.ClusterConfiguration.KubernetesVersion = "some kubernetes version"
 
-	prevKCPWithVersion := func(version string) *KubeadmControlPlane {
-		prev := before.DeepCopy()
-		prev.Spec.Version = version
-		return prev
-	}
-	skipMinorControlPlaneVersion := prevKCPWithVersion("v1.18.1")
-	emptyControlPlaneVersion := prevKCPWithVersion("")
-
 	controlPlaneEndpoint := before.DeepCopy()
 	controlPlaneEndpoint.Spec.KubeadmConfigSpec.ClusterConfiguration.ControlPlaneEndpoint = "some control plane endpoint"
 
@@ -611,13 +603,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
 		DataDir: "/data",
 	}
 
-	disallowedUpgrade118Prev := prevKCPWithVersion("v1.18.8")
-	disallowedUpgrade119Version := before.DeepCopy()
-	disallowedUpgrade119Version.Spec.Version = "v1.19.0"
-
-	disallowedUpgrade120AlphaVersion := before.DeepCopy()
-	disallowedUpgrade120AlphaVersion.Spec.Version = "v1.20.0-alpha.0.734_ba502ee555924a"
-
 	updateNTPServers := before.DeepCopy()
 	updateNTPServers.Spec.KubeadmConfigSpec.NTP.Servers = []string{"new-server"}
 
@@ -925,36 +910,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
 			before:    withoutClusterConfiguration,
 			kcp:       afterEtcdLocalDirAddition,
 		},
-		{
-			name:      "should fail when skipping control plane minor versions",
-			expectErr: true,
-			before:    before,
-			kcp:       skipMinorControlPlaneVersion,
-		},
-		{
-			name:      "should fail when no control plane version is passed",
-			expectErr: true,
-			before:    before,
-			kcp:       emptyControlPlaneVersion,
-		},
-		{
-			name:      "should pass if control plane version is the same",
-			expectErr: false,
-			before:    before,
-			kcp:       before.DeepCopy(),
-		},
-		{
-			name:      "should return error when trying to upgrade to v1.19.0",
-			expectErr: true,
-			before:    disallowedUpgrade118Prev,
-			kcp:       disallowedUpgrade119Version,
-		},
-		{
-			name:      "should return error when trying to upgrade two minor versions",
-			expectErr: true,
-			before:    disallowedUpgrade118Prev,
-			kcp:       disallowedUpgrade120AlphaVersion,
-		},
 		{
 			name:      "should not return an error when maxSurge value is updated to 0",
 			expectErr: false,
@@ -1051,6 +1006,162 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
 	}
 }
 
+func TestValidateVersion(t *testing.T) {
+	tests := []struct {
+		name                 string
+		clusterConfiguration *bootstrapv1.ClusterConfiguration
+		oldVersion           string
+		newVersion           string
+		expectErr            bool
+	}{
+		// Basic validation of old and new version.
+		{
+			name:       "error when old version is empty",
+			oldVersion: "",
+			newVersion: "v1.16.6",
+			expectErr:  true,
+		},
+		{
+			name:       "error when old version is invalid",
+			oldVersion: "invalid-version",
+			newVersion: "v1.18.1",
+			expectErr:  true,
+		},
+		{
+			name:       "error when new version is empty",
+			oldVersion: "v1.16.6",
+			newVersion: "",
+			expectErr:  true,
+		},
+		{
+			name:       "error when new version is invalid",
+			oldVersion: "v1.18.1",
+			newVersion: "invalid-version",
+			expectErr:  true,
+		},
+		// Validation that we block upgrade to v1.19.0.
+		// Note: Upgrading to v1.19.0 is not supported, because of issues in v1.19.0,
+		// see: https://github.com/kubernetes-sigs/cluster-api/issues/3564
+		{
+			name:       "error when upgrading to v1.19.0",
+			oldVersion: "v1.18.8",
+			newVersion: "v1.19.0",
+			expectErr:  true,
+		},
+		{
+			name:       "pass when both versions are v1.19.0",
+			oldVersion: "v1.19.0",
+			newVersion: "v1.19.0",
+			expectErr:  false,
+		},
+		// Validation for skip-level upgrades.
+		{
+			name:       "error when upgrading two minor versions",
+			oldVersion: "v1.18.8",
+			newVersion: "v1.20.0-alpha.0.734_ba502ee555924a",
+			expectErr:  true,
+		},
+		{
+			name:       "pass when upgrading one minor version",
+			oldVersion: "v1.20.1",
+			newVersion: "v1.21.18",
+			expectErr:  false,
+		},
+		// Validation for usage of the old registry.
+		// Notes:
+		// * kubeadm versions < v1.22 are always using the old registry.
+		// * kubeadm versions >= v1.25.0 are always using the new registry.
+		// * kubeadm versions in between are using the new registry
+		//   starting with certain patch versions.
+		// This test validates that we don't block upgrades for < v1.22.0 and >= v1.25.0
+		// and block upgrades to kubeadm versions in between with the old registry.
+		{
+			name: "pass when imageRepository is set",
+			clusterConfiguration: &bootstrapv1.ClusterConfiguration{
+				ImageRepository: "k8s.gcr.io",
+			},
+			oldVersion: "v1.21.1",
+			newVersion: "v1.22.16",
+			expectErr:  false,
+		},
+		{
+			name:       "pass when version didn't change",
+			oldVersion: "v1.22.16",
+			newVersion: "v1.22.16",
+			expectErr:  false,
+		},
+		{
+			name:       "pass when new version is < v1.22.0",
+			oldVersion: "v1.20.10",
+			newVersion: "v1.21.5",
+			expectErr:  false,
+		},
+		{
+			name:       "error when new version is using old registry (v1.22.0 <= version <= v1.22.16)",
+			oldVersion: "v1.21.1",
+			newVersion: "v1.22.16", // last patch release using old registry
+			expectErr:  true,
+		},
+		{
+			name:       "pass when new version is using new registry (>= v1.22.17)",
+			oldVersion: "v1.21.1",
+			newVersion: "v1.22.17", // first patch release using new registry
+			expectErr:  false,
+		},
+		{
+			name:       "error when new version is using old registry (v1.23.0 <= version <= v1.23.14)",
+			oldVersion: "v1.22.17",
+			newVersion: "v1.23.14", // last patch release using old registry
+			expectErr:  true,
+		},
+		{
+			name:       "pass when new version is using new registry (>= v1.23.15)",
+			oldVersion: "v1.22.17",
+			newVersion: "v1.23.15", // first patch release using new registry
+			expectErr:  false,
+		},
+		{
+			name:       "error when new version is using old registry (v1.24.0 <= version <= v1.24.8)",
+			oldVersion: "v1.23.1",
+			newVersion: "v1.24.8", // last patch release using old registry
+			expectErr:  true,
+		},
+		{
+			name:       "pass when new version is using new registry (>= v1.24.9)",
+			oldVersion: "v1.23.1",
+			newVersion: "v1.24.9", // first patch release using new registry
+			expectErr:  false,
+		},
+		{
+			name:       "pass when new version is using new registry (>= v1.25.0)",
+			oldVersion: "v1.24.8",
+			newVersion: "v1.25.0", // uses new registry
+			expectErr:  false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			kcp := KubeadmControlPlane{
+				Spec: KubeadmControlPlaneSpec{
+					KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{
+						ClusterConfiguration: tt.clusterConfiguration,
+					},
+					Version: tt.newVersion,
+				},
+			}
+
+			allErrs := kcp.validateVersion(tt.oldVersion)
+			if tt.expectErr {
+				g.Expect(allErrs).ToNot(HaveLen(0))
+			} else {
+				g.Expect(allErrs).To(HaveLen(0))
+			}
+		})
+	}
+}
 func TestKubeadmControlPlaneValidateUpdateAfterDefaulting(t *testing.T) {
 	before := &KubeadmControlPlane{
 		ObjectMeta: metav1.ObjectMeta{