Skip to content

Commit 3b67cb7

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-sigs#9502 from willie-yao/cc-mp-upgrade
🌱 Add MachinePools to topology upgrade test
2 parents 8255149 + 02fbcb1 commit 3b67cb7

File tree

7 files changed

+49
-20
lines changed

7 files changed

+49
-20
lines changed

‎test/e2e/cluster_upgrade.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
161161
EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
162162
DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
163163
MachineDeployments: clusterResources.MachineDeployments,
164+
MachinePools: clusterResources.MachinePools,
164165
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
165166
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
166167
WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),

‎test/e2e/cluster_upgrade_runtimesdk.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
221221
Cluster: clusterResources.Cluster,
222222
ControlPlane: clusterResources.ControlPlane,
223223
MachineDeployments: clusterResources.MachineDeployments,
224+
MachinePools: clusterResources.MachinePools,
224225
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
225226
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
226227
WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
@@ -233,7 +234,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
233234
input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
234235
input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"))
235236
},
236-
PreWaitForMachineDeploymentToBeUpgraded: func() {
237+
PreWaitForWorkersToBeUpgraded: func() {
237238
machineSetPreflightChecksTestHandler(ctx,
238239
input.BootstrapClusterProxy.GetClient(),
239240
clusterRef)

‎test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/cluster-runtimesdk.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ spec:
2626
nodeDeletionTimeout: "30s"
2727
replicas: ${WORKER_MACHINE_COUNT}
2828
failureDomain: fd4
29+
machinePools:
30+
- class: "default-worker"
31+
name: "mp-0"
32+
nodeDeletionTimeout: "30s"
33+
nodeVolumeDetachTimeout: "5m"
34+
minReadySeconds: 5
35+
replicas: ${WORKER_MACHINE_COUNT}
36+
failureDomains:
37+
- fd4
2938
variables:
3039
- name: kubeadmControlPlaneMaxSurge
3140
value: "1"

‎test/e2e/data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk/kustomization.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
resources:
22
- ../bases/crs.yaml
3-
- ../bases/mp.yaml
43
- cluster-runtimesdk.yaml

‎test/e2e/data/infrastructure-docker/main/cluster-template-upgrades/kustomization.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
resources:
22
- ../bases/cluster-with-topology.yaml
33
- ../bases/crs.yaml
4-
- ../bases/mp.yaml

‎test/e2e/self_hosted.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
313313
EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
314314
DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
315315
MachineDeployments: clusterResources.MachineDeployments,
316+
MachinePools: clusterResources.MachinePools,
316317
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
317318
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
318319
WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),

‎test/framework/cluster_topology_helpers.go

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3030
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
31+
expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
3132
"sigs.k8s.io/cluster-api/test/framework/internal/log"
3233
"sigs.k8s.io/cluster-api/util/patch"
3334
)
@@ -59,19 +60,21 @@ func GetClusterClassByName(ctx context.Context, input GetClusterClassByNameInput
5960

6061
// UpgradeClusterTopologyAndWaitForUpgradeInput is the input type for UpgradeClusterTopologyAndWaitForUpgrade.
6162
type UpgradeClusterTopologyAndWaitForUpgradeInput struct {
62-
ClusterProxy ClusterProxy
63-
Cluster *clusterv1.Cluster
64-
ControlPlane *controlplanev1.KubeadmControlPlane
65-
EtcdImageTag string
66-
DNSImageTag string
67-
MachineDeployments []*clusterv1.MachineDeployment
68-
KubernetesUpgradeVersion string
69-
WaitForMachinesToBeUpgraded []interface{}
70-
WaitForKubeProxyUpgrade []interface{}
71-
WaitForDNSUpgrade []interface{}
72-
WaitForEtcdUpgrade []interface{}
73-
PreWaitForControlPlaneToBeUpgraded func()
74-
PreWaitForMachineDeploymentToBeUpgraded func()
63+
ClusterProxy ClusterProxy
64+
Cluster *clusterv1.Cluster
65+
ControlPlane *controlplanev1.KubeadmControlPlane
66+
EtcdImageTag string
67+
DNSImageTag string
68+
MachineDeployments []*clusterv1.MachineDeployment
69+
MachinePools []*expv1.MachinePool
70+
KubernetesUpgradeVersion string
71+
WaitForMachinesToBeUpgraded []interface{}
72+
WaitForMachinePoolToBeUpgraded []interface{}
73+
WaitForKubeProxyUpgrade []interface{}
74+
WaitForDNSUpgrade []interface{}
75+
WaitForEtcdUpgrade []interface{}
76+
PreWaitForControlPlaneToBeUpgraded func()
77+
PreWaitForWorkersToBeUpgraded func()
7578
}
7679

7780
// UpgradeClusterTopologyAndWaitForUpgrade upgrades a Cluster topology and waits for it to be upgraded.
@@ -82,6 +85,7 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
8285
Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade")
8386
Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade")
8487
Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade")
88+
Expect(input.MachinePools).ToNot(BeEmpty(), "Invalid argument. input.MachinePools can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade")
8589
Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade")
8690

8791
mgmtClient := input.ClusterProxy.GetClient()
@@ -150,12 +154,12 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
150154
}, input.WaitForEtcdUpgrade...)
151155
}
152156

153-
// Once the ControlPlane is upgraded we can run PreWaitForMachineDeploymentToBeUpgraded.
157+
// Once the ControlPlane is upgraded we can run PreWaitForWorkersToBeUpgraded.
154158
// Note: This can e.g. be used to verify the AfterControlPlaneUpgrade lifecycle hook is executed
155159
// and blocking correctly.
156-
if input.PreWaitForMachineDeploymentToBeUpgraded != nil {
157-
log.Logf("Calling PreWaitForMachineDeploymentToBeUpgraded")
158-
input.PreWaitForMachineDeploymentToBeUpgraded()
160+
if input.PreWaitForWorkersToBeUpgraded != nil {
161+
log.Logf("Calling PreWaitForWorkersToBeUpgraded")
162+
input.PreWaitForWorkersToBeUpgraded()
159163
}
160164

161165
for _, deployment := range input.MachineDeployments {
@@ -171,4 +175,19 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
171175
}, input.WaitForMachinesToBeUpgraded...)
172176
}
173177
}
178+
179+
for _, pool := range input.MachinePools {
180+
if *pool.Spec.Replicas > 0 {
181+
log.Logf("Waiting for Kubernetes versions of machines in MachinePool %s to be upgraded to %s",
182+
klog.KObj(pool), input.KubernetesUpgradeVersion)
183+
WaitForMachinePoolInstancesToBeUpgraded(ctx, WaitForMachinePoolInstancesToBeUpgradedInput{
184+
Getter: mgmtClient,
185+
WorkloadClusterGetter: input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient(),
186+
Cluster: input.Cluster,
187+
MachineCount: int(*pool.Spec.Replicas),
188+
KubernetesUpgradeVersion: input.KubernetesUpgradeVersion,
189+
MachinePool: pool,
190+
}, input.WaitForMachinePoolToBeUpgraded...)
191+
}
192+
}
174193
}

0 commit comments

Comments
 (0)