Enable k8s upgrade in slef hosted test

lentzi90 · lentzi90 · commit 4d500ed35091 · 2023-10-24T08:49:51.000+03:00
This enables the k8s upgrade that is part of the self hosted test. It
was previously disabled since the dynamic installation of kubernetes
components on the nodes made it hard to change the version in the middle
of a test.

Also updated the README tables for what versions are tested.
diff --git a/Makefile b/Makefile
@@ -165,7 +165,8 @@ e2e-templates: $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \
 		 cluster-template-multi-network.yaml \
 		 cluster-template-without-lb.yaml \
 		 cluster-template.yaml \
-		 cluster-template-flatcar.yaml)
+		 cluster-template-flatcar.yaml \
+		 cluster-template-k8s-upgrade.yaml)
 # Currently no templates that require CI artifacts
 # $(addprefix $(E2E_TEMPLATES_DIR)/, add-templates-here.yaml) \
 
diff --git a/README.md b/README.md
@@ -48,25 +48,28 @@ This provider's versions are compatible with the following versions of Cluster A
 
 This provider's versions are able to install and manage the following versions of Kubernetes:
 
-|                                    | v1.22 | v1.23 | v1.24 | v1.25 | v1.26 | v1.27 | v1.28 |
-|------------------------------------| ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| OpenStack Provider v1alpha5 (v0.6) | +     | ✓     | ✓     | ✓     | +     | +     | +     |
-| OpenStack Provider v1alpha6 (v0.7) | +     | ✓     | ✓     | ✓     | ✓     | ✓     | +     |
-| OpenStack Provider v1alpha7        | +     | +     | +     | +     | ✓     | ✓     | ✓     |
+|                                    | v1.25 | v1.26 | v1.27 | v1.28 |
+|------------------------------------| ----- | ----- | ----- | ----- |
+| OpenStack Provider v1alpha5 (v0.6) | ✓     | +     | +     | +     |
+| OpenStack Provider v1alpha6 (v0.7) | ✓     | ✓     | ✓     | +     |
+| OpenStack Provider v1alpha7        | +     | ✓     | ✓     | ★     |
 
 This provider's versions are able to install Kubernetes to the following versions of OpenStack:
 
 |                                    | Queens | Rocky | Stein | Train | Ussuri | Victoria | Wallaby | Xena | Yoga |
 |------------------------------------| ------ | ----- | ----- | ----- | ------ | -------- | ------- | ---- | ---- |
-| OpenStack Provider v1alpha5 (v0.6) | +      | +     | +     | +     | +      | ✓        | ✓       | ✓    | ✓    |
-| OpenStack Provider v1alpha6 (v0.7) | +      | +     | +     | +     | +      | ✓        | ✓       | ✓    | ✓    |
-| OpenStack Provider v1alpha7        |        | +     | +     | +     | +      | ✓        | ✓       | ✓    | ✓    |
+| OpenStack Provider v1alpha5 (v0.6) | +      | +     | +     | +     | +      | ✓        | ✓       | ✓    | ★    |
+| OpenStack Provider v1alpha6 (v0.7) | +      | +     | +     | +     | +      | ✓        | ✓       | ✓    | ★    |
+| OpenStack Provider v1alpha7        |        | +     | +     | +     | +      | ✓        | ✓       | ✓    | ★    |
 
 Test status:
 
-- `✓` tested
+- `★` currently testing
+- `✓` previously tested
 - `+` should work, but we weren't able to test it
 
+Older versions may also work but we have not verified.
+
 Each version of Cluster API for OpenStack will attempt to support two Kubernetes versions.
 
 **NOTE:** As the versioning for this project is tied to the versioning of Cluster API, future modifications to this
diff --git a/hack/ci/cloud-init/controller.yaml.tpl b/hack/ci/cloud-init/controller.yaml.tpl
@@ -62,6 +62,7 @@
     # https://docs.openstack.org/glance/latest/admin/quotas.html
     IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2022-12-05/amphora-x64-haproxy.qcow2,"
     IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.27.2.img,"
     IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.28.2.img,"
     IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.28.2.img"
 
diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml
@@ -165,6 +165,12 @@ variables:
   # used to ensure we deploy to the correct management cluster
   KUBE_CONTEXT: "kind-capo-e2e"
   KUBERNETES_VERSION: "v1.28.2"
+  KUBERNETES_VERSION_UPGRADE_FROM: "v1.27.2"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.28.2"
+  ETCD_VERSION_UPGRADE_TO: "3.5.9-0"
+  COREDNS_VERSION_UPGRADE_TO: "v1.10.1"
+  CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "upgrade-to-control-plane"
+  WORKERS_MACHINE_TEMPLATE_UPGRADE_TO: "upgrade-to-md-0"
   CNI: "../../data/cni/calico.yaml"
   CCM: "../../data/ccm/cloud-controller-manager.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
@@ -179,6 +185,7 @@ variables:
   OPENSTACK_FAILURE_DOMAIN: "testaz1"
   OPENSTACK_FAILURE_DOMAIN_ALT: "testaz2"
   OPENSTACK_IMAGE_NAME: "ubuntu-2204-kube-v1.28.2"
+  OPENSTACK_IMAGE_NAME_UPGRADE_FROM: "ubuntu-2204-kube-v1.27.2"
   OPENSTACK_NODE_MACHINE_FLAVOR: "m1.small"
   OPENSTACK_SSH_KEY_NAME: "cluster-api-provider-openstack-sigs-k8s-io"
   OPENSTACK_EXTERNAL_NETWORK_ID: ""
diff --git a/test/e2e/data/kustomize/common-patches/ccm/kustomization.yaml b/test/e2e/data/kustomize/common-patches/ccm/kustomization.yaml
@@ -6,9 +6,6 @@ kind: Component
 resources:
 - ccm.yaml
 
-patchesStrategicMerge:
-- patch-ccm.yaml
-
 patches:
 - target:
     kind: OpenStackCluster
@@ -18,3 +15,4 @@ patches:
     kind: KubeadmControlPlane
     name: \${CLUSTER_NAME}-control-plane
   path: patch-ccm-cloud-config.yaml
+- path: patch-ccm.yaml
diff --git a/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml b/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml
@@ -6,11 +6,9 @@ kind: Component
 resources:
 - cni.yaml
 
-patchesStrategicMerge:
-- patch-cni.yaml
-
 patches:
 - target:
     kind: OpenStackCluster
     name: \${CLUSTER_NAME}
   path: patch-cluster.yaml
+- path: patch-cni.yaml
diff --git a/test/e2e/data/kustomize/k8s-upgrade/kcp-patch.yaml b/test/e2e/data/kustomize/k8s-upgrade/kcp-patch.yaml
@@ -0,0 +1,10 @@
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+      kind: OpenStackMachineTemplate
+      name: ${CLUSTER_NAME}-upgrade-from-control-plane
diff --git a/test/e2e/data/kustomize/k8s-upgrade/kustomization.yaml b/test/e2e/data/kustomize/k8s-upgrade/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+resources:
+- ../default
+- upgrade-from-template.yaml
+- upgrade-to-template.yaml
+
+patches:
+- path: kcp-patch.yaml
+- path: md-patch.yaml
diff --git a/test/e2e/data/kustomize/k8s-upgrade/md-patch.yaml b/test/e2e/data/kustomize/k8s-upgrade/md-patch.yaml
@@ -0,0 +1,11 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+        kind: OpenStackMachineTemplate
+        name: ${CLUSTER_NAME}-upgrade-from-md-0
diff --git a/test/e2e/data/kustomize/k8s-upgrade/upgrade-from-template.yaml b/test/e2e/data/kustomize/k8s-upgrade/upgrade-from-template.yaml
@@ -0,0 +1,34 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-upgrade-from-control-plane
+spec:
+  template:
+    spec:
+      cloudName: ${OPENSTACK_CLOUD}
+      flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR}
+      identityRef:
+        kind: Secret
+        name: ${CLUSTER_NAME}-cloud-config
+      image: ${OPENSTACK_IMAGE_NAME_UPGRADE_FROM}
+      sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
+      tags:
+      - control-plane
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-upgrade-from-md-0
+spec:
+  template:
+    spec:
+      cloudName: ${OPENSTACK_CLOUD}
+      flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR}
+      identityRef:
+        kind: Secret
+        name: ${CLUSTER_NAME}-cloud-config
+      image: ${OPENSTACK_IMAGE_NAME_UPGRADE_FROM}
+      sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
+      tags:
+      - machine
diff --git a/test/e2e/data/kustomize/k8s-upgrade/upgrade-to-template.yaml b/test/e2e/data/kustomize/k8s-upgrade/upgrade-to-template.yaml
@@ -0,0 +1,43 @@
+# This is just a copy of the normal cluster template OSMTs.
+# It is needed because we need a static name that we can put in the e2e config.
+# The normal templates has the cluster name as part of the OSMT name.
+# See CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO and WORKERS_MACHINE_TEMPLATE_UPGRADE_TO
+# in the e2e config.
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackMachineTemplate
+metadata:
+  name: upgrade-to-control-plane
+  labels:
+    clusterctl.cluster.x-k8s.io/move: ""
+spec:
+  template:
+    spec:
+      cloudName: ${OPENSTACK_CLOUD}
+      flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR}
+      identityRef:
+        kind: Secret
+        name: ${CLUSTER_NAME}-cloud-config
+      image: ${OPENSTACK_IMAGE_NAME}
+      sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
+      tags:
+      - control-plane
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackMachineTemplate
+metadata:
+  name: upgrade-to-md-0
+  labels:
+    clusterctl.cluster.x-k8s.io/move: ""
+spec:
+  template:
+    spec:
+      cloudName: ${OPENSTACK_CLOUD}
+      flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR}
+      identityRef:
+        kind: Secret
+        name: ${CLUSTER_NAME}-cloud-config
+      image: ${OPENSTACK_IMAGE_NAME}
+      sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
+      tags:
+      - machine
diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go
@@ -53,6 +53,7 @@ const (
 	FlavorMDRemediation        = "md-remediation"
 	FlavorKCPRemediation       = "kcp-remediation"
 	FlavorFlatcar              = "flatcar"
+	FlavorKubernetesUpgrade    = "k8s-upgrade"
 )
 
 // DefaultScheme returns the default scheme to use for testing.
diff --git a/test/e2e/suites/e2e/self_hosted_test.go b/test/e2e/suites/e2e/self_hosted_test.go
@@ -30,23 +30,16 @@ import (
 
 var _ = Describe("When testing Cluster API provider Openstack working on [self-hosted] clusters", func() {
 	ctx := context.TODO()
-	shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false)
 	shared.SetEnvVar("DOWNLOAD_E2E_IMAGE", "true", false)
-	// TODO(lentzi90): Since we currently rely on USE_CI_ARTIFACTS to get kubernetes set up,
-	// we are stuck with one version only and cannot do the k8s upgrade that is part of the
-	// SelfHostedSpec. The reason for this is that the script used to install kubernetes tools comes
-	// with the KubeadmControlPlane and that does not change when upgrading the k8s version
-	// (only the machineTemplate changes).
-	// Until we have a way to work around this, the k8s upgrade test is disabled.
 	capie2e.SelfHostedSpec(ctx, func() capie2e.SelfHostedSpecInput {
 		return capie2e.SelfHostedSpecInput{
 			E2EConfig:             e2eCtx.E2EConfig,
 			ClusterctlConfigPath:  e2eCtx.Environment.ClusterctlConfigPath,
 			BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
 			ArtifactFolder:        e2eCtx.Settings.ArtifactFolder,
-			SkipUpgrade:           true,
+			SkipUpgrade:           false,
 			SkipCleanup:           false,
-			Flavor:                shared.FlavorDefault,
+			Flavor:                shared.FlavorKubernetesUpgrade,
 		}
 	})
 })
