Skip to content

Commit 0909161

Browse files
nojnhuhnojnhuh
committed
Add clusterctl upgrade tests with AKS workload clusters
1 parent 87d3aa5 commit 0909161

File tree

7 files changed

+547
-0
lines changed

7 files changed

+547
-0
lines changed

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,7 @@ generate-e2e-templates: $(KUSTOMIZE) ## Generate Azure infrastructure templates
531531
$(KUSTOMIZE) build $(AZURE_TEMPLATES)/v1beta1/cluster-template-node-drain --load-restrictor LoadRestrictionsNone > $(AZURE_TEMPLATES)/v1beta1/cluster-template-node-drain.yaml
532532
$(KUSTOMIZE) build $(AZURE_TEMPLATES)/v1beta1/cluster-template-upgrades --load-restrictor LoadRestrictionsNone > $(AZURE_TEMPLATES)/v1beta1/cluster-template-upgrades.yaml
533533
$(KUSTOMIZE) build $(AZURE_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(AZURE_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in.yaml
534+
$(KUSTOMIZE) build $(AZURE_TEMPLATES)/v1beta1/cluster-template-aks --load-restrictor LoadRestrictionsNone > $(AZURE_TEMPLATES)/v1beta1/cluster-template-aks.yaml
534535

535536
.PHONY: generate-addons
536537
generate-addons: fetch-calico-manifests ## Generate metric-server, calico, calico-ipv6, azure cni v1 addons.

‎test/e2e/aks_machinepools.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,3 +137,32 @@ func AKSMachinePoolSpec(ctx context.Context, inputGetter func() AKSMachinePoolSp
137137

138138
wg.Wait()
139139
}
140+
141+
type AKSMachinePoolPostUpgradeSpecInput struct {
142+
MgmtCluster framework.ClusterProxy
143+
ClusterName string
144+
ClusterNamespace string
145+
}
146+
147+
func AKSMachinePoolPostUpgradeSpec(ctx context.Context, inputGetter func() AKSMachinePoolPostUpgradeSpecInput) {
148+
input := inputGetter()
149+
150+
cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
151+
Getter: input.MgmtCluster.GetClient(),
152+
Name: input.ClusterName,
153+
Namespace: input.ClusterNamespace,
154+
})
155+
mps := framework.GetMachinePoolsByCluster(ctx, framework.GetMachinePoolsByClusterInput{
156+
Lister: input.MgmtCluster.GetClient(),
157+
ClusterName: input.ClusterName,
158+
Namespace: input.ClusterNamespace,
159+
})
160+
AKSMachinePoolSpec(ctx, func() AKSMachinePoolSpecInput {
161+
return AKSMachinePoolSpecInput{
162+
MgmtCluster: input.MgmtCluster,
163+
Cluster: cluster,
164+
MachinePools: mps,
165+
WaitIntervals: e2eConfig.GetIntervals("default", "wait-machine-pool-nodes"),
166+
}
167+
})
168+
}

‎test/e2e/capi_test.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
. "github.com/onsi/ginkgo/v2"
3131
. "github.com/onsi/gomega"
3232
"k8s.io/utils/ptr"
33+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3334
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
3435
"sigs.k8s.io/cluster-api/test/framework"
3536
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -181,6 +182,8 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
181182

182183
if os.Getenv("USE_LOCAL_KIND_REGISTRY") != "true" {
183184
Context("API Version Upgrade", func() {
185+
var aksKubernetesVersion string
186+
184187
BeforeEach(func() {
185188
// Unset resource group and vnet env variables, since the upgrade test creates 2 clusters,
186189
// and will result in both the clusters using the same vnet and resource group.
@@ -199,6 +202,9 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
199202
identity, err := identityClient.Get(ctx, identityRG, identityName, nil)
200203
Expect(err).NotTo(HaveOccurred())
201204
Expect(os.Setenv("AZURE_CLIENT_ID_CLOUD_PROVIDER", *identity.Properties.ClientID)).To(Succeed())
205+
206+
aksKubernetesVersion, err = GetAKSKubernetesVersion(ctx, e2eConfig, AKSKubernetesVersion)
207+
Expect(err).NotTo(HaveOccurred())
202208
})
203209

204210
Context("upgrade from an old version of v1beta1 to current, and scale workload clusters created in the old version", func() {
@@ -250,6 +256,82 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
250256
}
251257
})
252258
})
259+
260+
Context("upgrade from an old version of v1beta1 to current, and scale AKS workload clusters created in the old version", func() {
261+
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
262+
return capi_e2e.ClusterctlUpgradeSpecInput{
263+
E2EConfig: e2eConfig,
264+
ClusterctlConfigPath: clusterctlConfigPath,
265+
WorkloadFlavor: "aks",
266+
WorkloadKubernetesVersion: aksKubernetesVersion,
267+
ControlPlaneMachineCount: ptr.To[int64](0),
268+
BootstrapClusterProxy: bootstrapClusterProxy,
269+
ArtifactFolder: artifactFolder,
270+
SkipCleanup: skipCleanup,
271+
PreInit: getPreInitFunc(ctx),
272+
InitWithProvidersContract: "v1beta1",
273+
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
274+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
275+
},
276+
InitWithKubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersionAPIUpgradeFrom),
277+
InitWithBinary: fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.MustGetVariable(OldCAPIUpgradeVersion)),
278+
InitWithCoreProvider: "cluster-api:" + e2eConfig.MustGetVariable(OldCAPIUpgradeVersion),
279+
InitWithInfrastructureProviders: []string{"azure:" + e2eConfig.MustGetVariable(OldProviderUpgradeVersion)},
280+
Upgrades: []capi_e2e.ClusterctlUpgradeSpecInputUpgrade{
281+
{
282+
Contract: clusterv1.GroupVersion.Version,
283+
PostUpgrade: func(managementClusterProxy framework.ClusterProxy, clusterNamespace, clusterName string) {
284+
AKSMachinePoolPostUpgradeSpec(ctx, func() AKSMachinePoolPostUpgradeSpecInput {
285+
return AKSMachinePoolPostUpgradeSpecInput{
286+
MgmtCluster: managementClusterProxy,
287+
ClusterName: clusterName,
288+
ClusterNamespace: clusterNamespace,
289+
}
290+
})
291+
},
292+
},
293+
},
294+
}
295+
})
296+
})
297+
298+
Context("upgrade from the latest version of v1beta1 to current, and scale AKS workload clusters created in the old version", func() {
299+
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
300+
return capi_e2e.ClusterctlUpgradeSpecInput{
301+
E2EConfig: e2eConfig,
302+
ClusterctlConfigPath: clusterctlConfigPath,
303+
WorkloadFlavor: "aks",
304+
WorkloadKubernetesVersion: aksKubernetesVersion,
305+
ControlPlaneMachineCount: ptr.To[int64](0),
306+
BootstrapClusterProxy: bootstrapClusterProxy,
307+
ArtifactFolder: artifactFolder,
308+
SkipCleanup: skipCleanup,
309+
PreInit: getPreInitFunc(ctx),
310+
InitWithProvidersContract: "v1beta1",
311+
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
312+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
313+
},
314+
InitWithKubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersionAPIUpgradeFrom),
315+
InitWithBinary: fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.MustGetVariable(LatestCAPIUpgradeVersion)),
316+
InitWithCoreProvider: "cluster-api:" + e2eConfig.MustGetVariable(LatestCAPIUpgradeVersion),
317+
InitWithInfrastructureProviders: []string{"azure:" + e2eConfig.MustGetVariable(LatestProviderUpgradeVersion)},
318+
Upgrades: []capi_e2e.ClusterctlUpgradeSpecInputUpgrade{
319+
{
320+
Contract: clusterv1.GroupVersion.Version,
321+
PostUpgrade: func(managementClusterProxy framework.ClusterProxy, clusterNamespace, clusterName string) {
322+
AKSMachinePoolPostUpgradeSpec(ctx, func() AKSMachinePoolPostUpgradeSpecInput {
323+
return AKSMachinePoolPostUpgradeSpecInput{
324+
MgmtCluster: managementClusterProxy,
325+
ClusterName: clusterName,
326+
ClusterNamespace: clusterNamespace,
327+
}
328+
})
329+
},
330+
},
331+
},
332+
}
333+
})
334+
})
253335
})
254336
}
255337

‎test/e2e/config/azure-dev.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ providers:
9393
targetName: "cluster-template.yaml"
9494
- sourcePath: "../data/infrastructure-azure/v1.18.5/cluster-template-prow-machine-and-machine-pool.yaml"
9595
targetName: "cluster-template-machine-and-machine-pool.yaml"
96+
- sourcePath: "../data/infrastructure-azure/v1.18.5/cluster-template-aks.yaml"
97+
targetName: "cluster-template-aks.yaml"
9698
replacements:
9799
- old: "imagePullPolicy: Always"
98100
new: "imagePullPolicy: IfNotPresent"
@@ -106,6 +108,8 @@ providers:
106108
targetName: "cluster-template.yaml"
107109
- sourcePath: "../data/infrastructure-azure/v1.19.4/cluster-template-prow-machine-and-machine-pool.yaml"
108110
targetName: "cluster-template-machine-and-machine-pool.yaml"
111+
- sourcePath: "../data/infrastructure-azure/v1.19.4/cluster-template-aks.yaml"
112+
targetName: "cluster-template-aks.yaml"
109113
replacements:
110114
- old: "imagePullPolicy: Always"
111115
new: "imagePullPolicy: IfNotPresent"

‎test/e2e/data/infrastructure-azure/v1.18.5/cluster-template-aks.yaml‎

Lines changed: 202 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)