Add clusterctl upgrade tests

These tests make use of the CAPI e2e framework. The test spec creates a
secondary management cluster with older versions of the controllers. A
workload cluster is created to test the functionality of the old
controllers before they are upgraded. Then clusterctl upgrade is used to
upgrade them and the workload cluster is scaled to check that things are
working also after the upgrade.

Author: lentzi90

.gitignore

@@ -173,3 +173,6 @@ sshuttle.pid
 
 # Book
 docs/book/book/
+
+# venv
+.venv

Makefile

@@ -120,7 +120,7 @@ LDFLAGS := $(shell source ./hack/version.sh; version::ldflags)
 ## --------------------------------------
 
 # The number of ginkgo tests to run concurrently
-E2E_GINKGO_PARALLEL=2
+E2E_GINKGO_PARALLEL ?= 2
 
 E2E_ARGS ?=
 
@@ -143,6 +143,7 @@ E2E_KUSTOMIZE_DIR=test/e2e/data/kustomize
 .PHONY: e2e-templates
 e2e-templates: ## Generate cluster templates for e2e tests
 e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \
+		 cluster-template-v1alpha5.yaml \
 		 cluster-template-external-cloud-provider.yaml \
 		 cluster-template-multi-az.yaml \
 		 cluster-template-multi-network.yaml \

test/e2e/data/e2e_conf.yaml

@@ -35,6 +35,7 @@ providers:
   - name: v1.2.4
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/core-components.yaml"
     type: url
+    contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1/metadata.yaml"
     replacements:
@@ -48,6 +49,7 @@ providers:
   - name: v1.2.4
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/bootstrap-components.yaml"
     type: url
+    contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1/metadata.yaml"
     replacements:
@@ -61,6 +63,7 @@ providers:
   - name: v1.2.4
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/control-plane-components.yaml"
     type: url
+    contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1/metadata.yaml"
     replacements:
@@ -71,17 +74,36 @@ providers:
 - name: openstack
   type: InfrastructureProvider
   versions:
+  # This is only for clusterctl upgrade tests
+  - name: v0.6.3
+    value: "https://github.com/kubernetes-sigs/cluster-api-provider-openstack/releases/download/v0.6.3/infrastructure-components.yaml"
+    type: url
+    contract: v1beta1
+    files:
+    - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
+    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
+    replacements:
+    - old: "imagePullPolicy: Always"
+      new: "imagePullPolicy: IfNotPresent"
+    - old: "--v=2"
+      new: "--v=4"
+    - old: "--leader-elect"
+      new: "--leader-elect=false\n        - --sync-period=1m"
   - name: v0.6.99
     value: ../../../config/default
-    contract: v1beta1
+    # This is the upcoming version.
+    # Specify no contract so that upgrade tests that start from a specific contract won't pick it up.
+    # contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
     - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
     - sourcePath: "./infrastructure-openstack/cluster-template-external-cloud-provider.yaml"
     - sourcePath: "./infrastructure-openstack/cluster-template-without-lb.yaml"
     replacements:
+    # TODO: We should use e2e here instead of main, but we need a way to get it into the workload cluster
+    # for upgrade tests for that to work.
     - old: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:dev
-      new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
+      new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:main
     - old: "imagePullPolicy: Always"
       new: "imagePullPolicy: IfNotPresent"
     - old: "--v=2"
@@ -116,6 +138,7 @@ variables:
   OPENSTACK_VOLUME_TYPE_ALT: "test-volume-type"
   CONFORMANCE_WORKER_MACHINE_COUNT: "5"
   CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
+  INIT_WITH_KUBERNETES_VERSION: "v1.25.0"
 
 intervals:
   conformance/wait-control-plane: ["30m", "10s"]

test/e2e/data/kustomize/v1alpha5/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+resources:
+- ../../../../../kustomize/v1alpha5/default
+
+components:
+- ../common-patches

test/e2e/shared/defaults.go

@@ -48,6 +48,7 @@ const (
 	FlavorExternalCloudProvider = "external-cloud-provider-ci-artifacts"
 	FlavorMultiNetwork          = "multi-network-ci-artifacts"
 	FlavorMultiAZ               = "multi-az-ci-artifacts"
+	FlavorV1alpha5              = "v1alpha5-ci-artifacts"
 )
 
 // DefaultScheme returns the default scheme to use for testing.

test/e2e/suites/e2e/clusterctl_upgrade_test.go

@@ -0,0 +1,53 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+
+	"sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared"
+)
+
+// Note: There isn't really anything fixing this to the v0.6 release. The test will
+// simply pick the latest release with the correct contract from the E2EConfig, as seen here
+// https://github.com/kubernetes-sigs/cluster-api/blob/3abb9089485f51d46054096c17ccb8b59f0f7331/test/e2e/clusterctl_upgrade.go#L265
+// When new minor releases are added (with the same contract) we will need to work on this
+// if we want to continue testing v0.6.
+var _ = Describe("When testing clusterctl upgrades (v0.6=>current) [clusterctl-upgrade]", func() {
+	ctx := context.TODO()
+	shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false)
+
+	capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
+		return capi_e2e.ClusterctlUpgradeSpecInput{
+			E2EConfig:                 e2eCtx.E2EConfig,
+			ClusterctlConfigPath:      e2eCtx.Environment.ClusterctlConfigPath,
+			BootstrapClusterProxy:     e2eCtx.Environment.BootstrapClusterProxy,
+			ArtifactFolder:            e2eCtx.Settings.ArtifactFolder,
+			SkipCleanup:               false,
+			InitWithBinary:            "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/clusterctl-{OS}-{ARCH}",
+			InitWithProvidersContract: "v1beta1",
+			MgmtFlavor:                shared.FlavorDefault,
+			WorkloadFlavor:            shared.FlavorV1alpha5,
+		}
+	})
+})