Cluster K8s Upgrade (#105)

simcod · vknabel · web-flow · commit c18b5ef70e88 · 2025-11-05T16:35:17.000+01:00
Co-authored-by: Valentin Knabel &lt;dev@vknabel.com&gt;
diff --git a/.gitignore b/.gitignore
@@ -6,7 +6,7 @@
 *.dylib
 bin/*
 test/external-crds
-test/e2e/frmwrk/artifacts
+test/e2e/frmwrk/data/clusterctl-templates/cluster-template-*.yaml
 _artifacts
 Dockerfile.cross
 .release
diff --git a/Makefile b/Makefile
@@ -141,10 +141,14 @@ E2E_DEFAULT_FLAVOR ?= "calico"
 # Can be something like: basic && !move
 E2E_LABEL_FILTER ?= ""
 
+E2E_TEMPLATES := test/e2e/frmwrk/data/clusterctl-templates
 .PHONY: test-e2e
 test-e2e: manifests generate fmt vet ginkgo
 	rm -rf $(ARTIFACTS)
 
+	$(KUSTOMIZE) build $(E2E_TEMPLATES)/upgrade --load-restrictor LoadRestrictionsNone > $(E2E_TEMPLATES)/cluster-template-upgrade.yaml
+	$(KUSTOMIZE) build $(E2E_TEMPLATES)/upgrade-workerless --load-restrictor LoadRestrictionsNone > $(E2E_TEMPLATES)/cluster-template-upgrade-workerless.yaml
+	
 	@METAL_API_URL=$(E2E_METAL_API_URL) \
 	METAL_API_HMAC=$(E2E_METAL_API_HMAC) \
 	METAL_API_HMAC_AUTH_TYPE=$(E2E_METAL_API_HMAC_AUTH_TYPE) \
@@ -162,6 +166,7 @@ test-e2e: manifests generate fmt vet ginkgo
 	FIREWALL_NETWORKS=$(E2E_FIREWALL_NETWORKS) \
 	ARTIFACTS=$(ARTIFACTS) \
 	E2E_DEFAULT_FLAVOR=$(E2E_DEFAULT_FLAVOR) \
+	KUBETEST_CONFIGURATION="$(shell git rev-parse --show-toplevel)/test/e2e/frmwrk/data/kubetest/conformance.yaml" \
 	$(GINKGO) -vv -r --junit-report="junit.e2e_suite.xml" --output-dir="$(ARTIFACTS)" --label-filter="$(E2E_LABEL_FILTER)" -timeout 60m ./test/e2e/frmwrk
 
 .PHONY: lint
diff --git a/go.mod b/go.mod
@@ -3,7 +3,6 @@ module github.com/metal-stack/cluster-api-provider-metal-stack
 go 1.23.0
 
 require (
-	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46
 	github.com/go-logr/logr v1.4.2
 	github.com/metal-stack/metal-go v0.40.4
 	github.com/metal-stack/metal-lib v0.20.2
@@ -47,6 +46,7 @@ require (
 	github.com/docker/docker v28.0.2+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
 	github.com/fatih/color v1.18.0 // indirect
diff --git a/test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go b/test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go
diff --git a/test/e2e/frmwrk/config/capi-e2e-config.yaml b/test/e2e/frmwrk/config/capi-e2e-config.yaml
@@ -124,6 +124,8 @@ providers:
       - sourcePath: "../../../../config/clusterctl-templates/cluster-template.yaml"
       - sourcePath: "../../../../config/clusterctl-templates/cluster-template-calico.yaml"
       - sourcePath: "../../../../config/clusterctl-templates/cluster-template-pre-v1.33.yaml"
+      - sourcePath: "../data/clusterctl-templates/cluster-template-upgrade.yaml"
+      - sourcePath: "../data/clusterctl-templates/cluster-template-upgrade-workerless.yaml"
     replacements:
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
@@ -133,7 +135,7 @@ variables:
   # NAMESPACE:
   KUBERNETES_VERSION: "v1.34.1"
   EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"
-  # METAL_PARTITION: 
+  # METAL_PARTITION:
   # METAL_PROJECT_ID:
   # METAL_NODE_NETWORK_ID:
   # CONTROL_PLANE_IP:
@@ -145,6 +147,15 @@ variables:
   WORKER_MACHINE_SIZE: "c1-medium-x86"
   CONTROL_PLANE_MACHINE_COUNT: "1"
   WORKER_MACHINE_COUNT: "1"
+  # K8s Upgrade variables
+  KUBERNETES_VERSION_UPGRADE_FROM: "v1.33.5"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.34.1"
+  CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "k8s-upgrade-controlplane"
+  WORKERS_MACHINE_TEMPLATE_UPGRADE_TO: "k8s-upgrade-worker"
+  KUBERNETES_IMAGE_UPGRADE_TO: "capms-ubuntu-1.34.1"
+  # TODO: remove once merged into upstream
+  # https://github.com/kubernetes-sigs/cluster-api/pull/12948
+  KUBETEST_CONFIGURATION: "test/e2e/frmwrk/data/kubetest/conformance.yaml" # must be absolute, actual path provided by Makefile
 
 intervals:
   default/wait-controllers: ["4m", "10s"]
diff --git a/test/e2e/frmwrk/data/clusterctl-templates/bases/cluster-template-mt-control-plane-upgrade-to.yaml b/test/e2e/frmwrk/data/clusterctl-templates/bases/cluster-template-mt-control-plane-upgrade-to.yaml
@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: MetalStackMachineTemplate
+metadata:
+  name: k8s-upgrade-controlplane
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      size: ${CONTROL_PLANE_MACHINE_SIZE}
+      image: ${KUBERNETES_IMAGE_UPGRADE_TO}
diff --git a/test/e2e/frmwrk/data/clusterctl-templates/bases/cluster-template-mt-worker-upgrade-to.yaml b/test/e2e/frmwrk/data/clusterctl-templates/bases/cluster-template-mt-worker-upgrade-to.yaml
@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: MetalStackMachineTemplate
+metadata:
+  name: k8s-upgrade-worker
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      size: ${WORKER_MACHINE_SIZE}
+      image: ${KUBERNETES_IMAGE_UPGRADE_TO}
diff --git a/test/e2e/frmwrk/data/clusterctl-templates/upgrade-workerless/kustomization.yaml b/test/e2e/frmwrk/data/clusterctl-templates/upgrade-workerless/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+  - ../../../../../../config/clusterctl-templates/cluster-template-calico.yaml
+  - ../bases/cluster-template-mt-control-plane-upgrade-to.yaml
diff --git a/test/e2e/frmwrk/data/clusterctl-templates/upgrade/kustomization.yaml b/test/e2e/frmwrk/data/clusterctl-templates/upgrade/kustomization.yaml
@@ -0,0 +1,4 @@
+resources:
+  - ../../../../../../config/clusterctl-templates/cluster-template-calico.yaml
+  - ../bases/cluster-template-mt-control-plane-upgrade-to.yaml
+  - ../bases/cluster-template-mt-worker-upgrade-to.yaml
diff --git a/test/e2e/frmwrk/data/kubetest/conformance.yaml b/test/e2e/frmwrk/data/kubetest/conformance.yaml
@@ -0,0 +1,8 @@
+ginkgo.focus: \[Conformance\]
+ginkgo.skip: \[Serial\]|HostPort.*\[Conformance\].*
+disable-log-dump: true
+ginkgo.progress: true
+ginkgo.slowSpecThreshold: 120.0
+ginkgo.flakeAttempts: 3
+ginkgo.trace: true
+ginkgo.v: true
diff --git a/test/e2e/frmwrk/shared_cluster.go b/test/e2e/frmwrk/shared_cluster.go
@@ -135,6 +135,7 @@ func (e2e *E2ECluster) SetupMetalStackPreconditions(ctx context.Context) {
 }
 
 func (e2e *E2ECluster) Teardown(ctx context.Context) {
+	e2e.teardownAddons(ctx)
 	e2e.teardownCluster(ctx)
 	e2e.teardownControlPlaneIP(ctx)
 	e2e.teardownFirewall(ctx)
@@ -285,6 +286,8 @@ func (e2e *E2ECluster) setupControlPlaneIP(ctx context.Context) {
 
 	e2e.Refs.ControlPlaneIP = ip.Payload
 	e2e.ControlPlaneIP = *e2e.Refs.ControlPlaneIP.Ipaddress
+
+	GinkgoWriter.Printf("Control Plane IP allocated: %s\n", *e2e.Refs.ControlPlaneIP.Ipaddress)
 }
 
 func (e2e *E2ECluster) teardownControlPlaneIP(ctx context.Context) {
@@ -315,7 +318,6 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
 		Flavor:                   e2e.E2EContext.Environment.Flavor,
 		LogFolder:                path.Join(e2e.E2EContext.Environment.artifactsPath, "clusters", e2e.ClusterName),
 		ClusterctlVariables:      e2e.Variables(),
-		InfrastructureProvider:   "metal-stack:v0.6.2",
 	})
 
 	By("Apply cluster template")
@@ -339,54 +341,75 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
 	Expect(e2e.Refs.Cluster).NotTo(BeNil(), "failed to get cluster")
 }
 
-func (e2e *E2ECluster) teardownCluster(ctx context.Context) {
-	if e2e.Refs.Cluster == nil {
-		return
-	}
-	Expect(e2e.Refs.Cluster).NotTo(BeNil(), "cluster not created yet")
+func (e2e *E2ECluster) teardownAddons(ctx context.Context) {
+	By("Teardown Addons")
 
-	resources := framework.GetCAPIResources(ctx, framework.GetCAPIResourcesInput{
-		Lister:    e2e.E2EContext.Environment.Bootstrap.GetClient(),
-		Namespace: e2e.NamespaceName,
-		IncludeTypes: []metav1.TypeMeta{
-			{
-				Kind:       "HelmReleaseProxy",
-				APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
-			},
-			{
-				Kind:       "HelmChartProxy",
-				APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
-			},
-			{
-				Kind:       "ClusterResourceSetBinding",
-				APIVersion: "addons.cluster.x-k8s.io/v1beta1",
-			},
-			{
-				Kind:       "ClusterResourceSet",
-				APIVersion: "addons.cluster.x-k8s.io/v1beta1",
-			},
+	includeTypes := []metav1.TypeMeta{
+		{
+			Kind:       "HelmChartProxy",
+			APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
 		},
-	})
+		{
+			Kind:       "HelmReleaseProxy",
+			APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
+		},
+		{
+			Kind:       "ClusterResourceSet",
+			APIVersion: "addons.cluster.x-k8s.io/v1beta1",
+		},
+		{
+			Kind:       "ClusterResourceSetBinding",
+			APIVersion: "addons.cluster.x-k8s.io/v1beta1",
+		},
+	}
+
+	resources := []*unstructured.Unstructured{}
+	for _, typ := range includeTypes {
+		typeList := new(unstructured.UnstructuredList)
+		typeList.SetAPIVersion(typ.APIVersion)
+		typeList.SetKind(typ.Kind)
+
+		if err := e2e.E2EContext.Environment.Bootstrap.GetClient().List(ctx, typeList, client.InNamespace(e2e.NamespaceName)); err != nil {
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+			if apierrors.IsForbidden(err) {
+				fmt.Printf("Warning: failed to list %s resources due to a rbac issue: %v", typeList.GroupVersionKind(), err)
+				continue
+			}
+			Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err))
+		}
+		for i := range typeList.Items {
+			obj := typeList.Items[i]
+			resources = append(resources, &obj)
+		}
+	}
 
 	for _, r := range resources {
+		By(fmt.Sprintf("Deleting resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
 		err := e2e.E2EContext.Environment.Bootstrap.GetClient().Delete(ctx, r)
 		Expect(err).To(Or(
 			Not(HaveOccurred()),
 			Satisfy(apierrors.IsNotFound)),
 			fmt.Sprintf("failed to delete resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind),
 		)
-	}
-
-	for _, r := range resources {
 		Eventually(func() bool {
+			By(fmt.Sprintf("Waiting for resource %s/%s of kind %s to be deleted", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
 			err := e2e.E2EContext.Environment.Bootstrap.GetClient().Get(ctx, client.ObjectKeyFromObject(r), r)
 			return apierrors.IsNotFound(err)
 		}, e2e.E2EContext.E2EConfig.GetIntervals("default", "wait-delete-resource")...).Should(BeTrue(),
 			fmt.Sprintf("resource %s/%s of kind %s still exists", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind),
 		)
 	}
+}
+
+func (e2e *E2ECluster) teardownCluster(ctx context.Context) {
+	if e2e.Refs.Cluster == nil {
+		return
+	}
+	Expect(e2e.Refs.Cluster).NotTo(BeNil(), "cluster not created yet")
 
-	deleteClusterAndWait(ctx, framework.DeleteClusterAndWaitInput{
+	capi_e2e_DeleteClusterAndWait(ctx, framework.DeleteClusterAndWaitInput{
 		ClusterProxy:         e2e.E2EContext.Environment.Bootstrap,
 		ClusterctlConfigPath: e2e.E2EContext.Environment.ClusterctlConfigPath,
 		ArtifactFolder:       e2e.E2EContext.Environment.artifactsPath,
@@ -428,9 +451,9 @@ func (ec *E2ECluster) Dump(ctx context.Context) {
 	})
 }
 
-// deleteClusterAndWait deletes a cluster object and waits for it to be gone.
+// capi_e2e_DeleteClusterAndWait deletes a cluster object and waits for it to be gone.
 // TODO: remove once cluster expectation has been fixed in framework
-func deleteClusterAndWait(ctx context.Context, input framework.DeleteClusterAndWaitInput, intervals ...any) {
+func capi_e2e_DeleteClusterAndWait(ctx context.Context, input framework.DeleteClusterAndWaitInput, intervals ...any) {
 	var (
 		retryableOperationInterval = 3 * time.Second
 		// retryableOperationTimeout requires a higher value especially for self-hosted upgrades.
diff --git a/test/e2e/frmwrk/shared_context.go b/test/e2e/frmwrk/shared_context.go
@@ -276,10 +276,22 @@ func (ee *E2EContext) TeardownMetalStackProject(ctx context.Context) {
 			})
 
 			for _, r := range resources {
+				By(fmt.Sprintf("Deleting resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
 				err := ee.Environment.Bootstrap.GetClient().Delete(ctx, r)
 				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to delete resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
 			}
 
+			for _, r := range resources {
+				By(fmt.Sprintf("Waiting for resource %s/%s of kind %s to be deleted", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
+				Eventually(ctx, func(g Gomega) {
+					err := ee.Environment.Bootstrap.GetClient().Get(ctx, client.ObjectKey{
+						Namespace: r.GetNamespace(),
+						Name:      r.GetName(),
+					}, r)
+					g.Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred(), fmt.Sprintf("failed to get resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
+				}, "5m", "10s").WithContext(ctx).Should(Succeed(), fmt.Sprintf("timed out waiting for resource %s/%s of kind %s to be deleted", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
+			}
+
 			framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
 				Deleter: ee.Environment.Bootstrap.GetClient(),
 				Name:    ns.Name,

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+resources:`
	`2`	`+ - ../../../../../../config/clusterctl-templates/cluster-template-calico.yaml`
	`3`	`+ - ../bases/cluster-template-mt-control-plane-upgrade-to.yaml`