Skip to content

Commit 3430748

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1371 from Nordix/lentzi90/e2e-upgrade
🌱 Add e2e clusterctl upgrade tests
2 parents 9f670e4 + 40f1a33 commit 3430748

File tree

10 files changed

+178
-3
lines changed

10 files changed

+178
-3
lines changed

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ E2E_KUSTOMIZE_DIR=test/e2e/data/kustomize
151151
.PHONY: e2e-templates
152152
e2e-templates: ## Generate cluster templates for e2e tests
153153
e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \
154+
cluster-template-v1alpha5.yaml \
154155
cluster-template-external-cloud-provider.yaml \
155156
cluster-template-multi-az.yaml \
156157
cluster-template-multi-network.yaml \

‎scripts/ci-e2e.sh‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
# limitations under the License.
1616

1717
################################################################################
18-
# usage: ci-conformance.sh
19-
# This program runs the clusterctl conformance e2e tests.
18+
# usage: ci-e2e.sh
19+
# This program runs the e2e tests.
2020
################################################################################
2121

2222
set -x
@@ -84,6 +84,20 @@ fi
8484

8585
"hack/ci/create_devstack.sh"
8686

87+
# Upload image for e2e clusterctl upgrade tests
88+
source "${REPO_ROOT}/hack/ci/${RESOURCE_TYPE}.sh"
89+
CONTAINER_ARCHIVE="${ARTIFACTS}/capo-e2e-image.tar"
90+
SSH_KEY="$(get_ssh_private_key_file)"
91+
SSH_ARGS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no"
92+
CONTROLLER_IP=${CONTROLLER_IP:-"10.0.3.15"}
93+
94+
make e2e-image
95+
docker save -o "${CONTAINER_ARCHIVE}" gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
96+
scp -i "${SSH_KEY}" ${SSH_ARGS} "${CONTAINER_ARCHIVE}" "cloud@${CONTROLLER_IP}:capo-e2e-image.tar"
97+
ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chown root:root capo-e2e-image.tar
98+
ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chmod u=rw,g=r,o=r capo-e2e-image.tar
99+
ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo mv capo-e2e-image.tar /var/www/html/capo-e2e-image.tar
100+
87101
export OPENSTACK_CLOUD_YAML_FILE
88102
OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
89103
make test-e2e

‎test/e2e/data/e2e_conf.yaml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ providers:
3535
- name: v1.2.4
3636
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/core-components.yaml"
3737
type: url
38+
contract: v1beta1
3839
files:
3940
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
4041
replacements:
@@ -48,6 +49,7 @@ providers:
4849
- name: v1.2.4
4950
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/bootstrap-components.yaml"
5051
type: url
52+
contract: v1beta1
5153
files:
5254
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
5355
replacements:
@@ -61,6 +63,7 @@ providers:
6163
- name: v1.2.4
6264
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/control-plane-components.yaml"
6365
type: url
66+
contract: v1beta1
6467
files:
6568
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
6669
replacements:
@@ -71,9 +74,26 @@ providers:
7174
- name: openstack
7275
type: InfrastructureProvider
7376
versions:
77+
# This is only for clusterctl upgrade tests
78+
- name: v0.6.3
79+
value: "https://github.com/kubernetes-sigs/cluster-api-provider-openstack/releases/download/v0.6.3/infrastructure-components.yaml"
80+
type: url
81+
contract: v1beta1
82+
files:
83+
- sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
84+
- sourcePath: "./infrastructure-openstack/cluster-template.yaml"
85+
replacements:
86+
- old: "imagePullPolicy: Always"
87+
new: "imagePullPolicy: IfNotPresent"
88+
- old: "--v=2"
89+
new: "--v=4"
90+
- old: "--leader-elect"
91+
new: "--leader-elect=false\n - --sync-period=1m"
7492
- name: v0.6.99
7593
value: ../../../config/default
76-
contract: v1beta1
94+
# This is the upcoming version.
95+
# Specify no contract so that upgrade tests that start from a specific contract won't pick it up.
96+
# contract: v1beta1
7797
files:
7898
- sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
7999
- sourcePath: "./infrastructure-openstack/cluster-template.yaml"
@@ -116,6 +136,7 @@ variables:
116136
OPENSTACK_VOLUME_TYPE_ALT: "test-volume-type"
117137
CONFORMANCE_WORKER_MACHINE_COUNT: "5"
118138
CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
139+
INIT_WITH_KUBERNETES_VERSION: "v1.25.0"
119140

120141
intervals:
121142
conformance/wait-control-plane: ["30m", "10s"]

‎test/e2e/data/kustomize/default/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ resources:
44

55
components:
66
- ../common-patches
7+
- ../upgrade-patches

‎test/e2e/data/kustomize/upgrade-patches/ci-hack-kcp.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
# Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7457
3+
# There is a small but important difference between these two:
4+
# path: /a/b/c
5+
# *creates* the c array, overwriting anything that was there before
6+
# path: /a/b/c/-
7+
# *adds* to the c array and does not work if the array is missing
8+
#
9+
# We add to the postKubeadmCommands (instead of pre*) since we need the CI artifacts
10+
# script to run first. Without this, the container images are not imported properly.
11+
- op: add
12+
path: /spec/kubeadmConfigSpec/postKubeadmCommands
13+
value:
14+
- /usr/local/bin/ci-artifacts-openstack.sh
15+
- op: add
16+
path: /spec/kubeadmConfigSpec/files/-
17+
value:
18+
content: |
19+
#!/bin/bash
20+
DOWNLOAD_E2E_IMAGE=${DOWNLOAD_E2E_IMAGE:=false}
21+
if [ ! "${DOWNLOAD_E2E_IMAGE}" = true ]; then
22+
echo "Not downloading E2E image, exiting"
23+
exit 0
24+
fi
25+
# Download the locally built CAPO controller image
26+
echo "Downloading ${E2E_IMAGE_URL}"
27+
wget "${E2E_IMAGE_URL}" -O "/tmp/capo-controller-manager.tar"
28+
sudo ctr -n k8s.io images import "/tmp/capo-controller-manager.tar" || echo "* ignoring expected 'ctr images import' result"
29+
owner: root:root
30+
path: /usr/local/bin/ci-artifacts-openstack.sh
31+
permissions: "0750"

‎test/e2e/data/kustomize/upgrade-patches/ci-hack-kct.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
# Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7457
3+
# There is a small but important difference between these two:
4+
# path: /a/b/c
5+
# *creates* the c array, overwriting anything that was there before
6+
# path: /a/b/c/-
7+
# *adds* to the c array and does not work if the array is missing
8+
#
9+
# We add to the postKubeadmCommands (instead of pre*) since we need the CI artifacts
10+
# script to run first. Without this, the container images are not imported properly.
11+
- op: add
12+
path: /spec/template/spec/postKubeadmCommands
13+
value:
14+
- /usr/local/bin/ci-artifacts-openstack.sh
15+
- op: add
16+
path: /spec/template/spec/files/-
17+
value:
18+
content: |
19+
#!/bin/bash
20+
DOWNLOAD_E2E_IMAGE=${DOWNLOAD_E2E_IMAGE:=false}
21+
if [ ! "${DOWNLOAD_E2E_IMAGE}" = true ]; then
22+
echo "Not downloading E2E image, exiting"
23+
exit 0
24+
fi
25+
# Download the locally built CAPO controller image
26+
echo "Downloading ${E2E_IMAGE_URL}"
27+
wget "${E2E_IMAGE_URL}" -O "/tmp/capo-controller-manager.tar"
28+
sudo ctr -n k8s.io images import "/tmp/capo-controller-manager.tar" || echo "* ignoring expected 'ctr images import' result"
29+
owner: root:root
30+
path: /usr/local/bin/ci-artifacts-openstack.sh
31+
permissions: "0750"

‎test/e2e/data/kustomize/upgrade-patches/kustomization.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Modifications to release templates for clusterctl upgrade scenarios
2+
---
3+
apiVersion: kustomize.config.k8s.io/v1alpha1
4+
kind: Component
5+
6+
patches:
7+
- target:
8+
kind: KubeadmControlPlane
9+
name: \${CLUSTER_NAME}-control-plane
10+
path: ci-hack-kcp.yaml
11+
- target:
12+
kind: KubeadmConfigTemplate
13+
name: \${CLUSTER_NAME}-md-0
14+
path: ci-hack-kct.yaml

‎test/e2e/data/kustomize/v1alpha5/kustomization.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
resources:
3+
- ../../../../../kustomize/v1alpha5/default
4+
5+
components:
6+
- ../common-patches

‎test/e2e/shared/defaults.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ const (
4848
FlavorExternalCloudProvider = "external-cloud-provider-ci-artifacts"
4949
FlavorMultiNetwork = "multi-network-ci-artifacts"
5050
FlavorMultiAZ = "multi-az-ci-artifacts"
51+
FlavorV1alpha5 = "v1alpha5-ci-artifacts"
5152
)
5253

5354
// DefaultScheme returns the default scheme to use for testing.

‎test/e2e/suites/e2e/clusterctl_upgrade_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
/*
5+
Copyright 2021 The Kubernetes Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package e2e
21+
22+
import (
23+
"context"
24+
25+
. "github.com/onsi/ginkgo"
26+
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
27+
28+
"sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared"
29+
)
30+
31+
// Note: There isn't really anything fixing this to the v0.6 release. The test will
32+
// simply pick the latest release with the correct contract from the E2EConfig, as seen here
33+
// https://github.com/kubernetes-sigs/cluster-api/blob/3abb9089485f51d46054096c17ccb8b59f0f7331/test/e2e/clusterctl_upgrade.go#L265
34+
// When new minor releases are added (with the same contract) we will need to work on this
35+
// if we want to continue testing v0.6.
36+
var _ = Describe("When testing clusterctl upgrades (v0.6=>current) [clusterctl-upgrade]", func() {
37+
ctx := context.TODO()
38+
shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false)
39+
shared.SetEnvVar("DOWNLOAD_E2E_IMAGE", "true", false)
40+
shared.SetEnvVar("E2E_IMAGE_URL", "http://10.0.3.15/capo-e2e-image.tar", false)
41+
42+
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
43+
return capi_e2e.ClusterctlUpgradeSpecInput{
44+
E2EConfig: e2eCtx.E2EConfig,
45+
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
46+
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
47+
ArtifactFolder: e2eCtx.Settings.ArtifactFolder,
48+
SkipCleanup: false,
49+
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/clusterctl-{OS}-{ARCH}",
50+
InitWithProvidersContract: "v1beta1",
51+
MgmtFlavor: shared.FlavorDefault,
52+
WorkloadFlavor: shared.FlavorV1alpha5,
53+
}
54+
})
55+
})

0 commit comments

Comments
 (0)