Skip to content

Commit 5a0c7a9

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #10715 from zhanggbj/e2e_support_extra_labels
🌱 Enhance E2E Tests to Filter Out Extra Provider-Specific Metadata
2 parents a7f5043 + ef5323b commit 5a0c7a9

File tree

1 file changed

+48
-25
lines changed

1 file changed

+48
-25
lines changed

‎test/e2e/clusterclass_rollout.go

Lines changed: 48 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,12 @@ type ClusterClassRolloutSpecInput struct {
7575
// Allows to inject a function to be run after test namespace is created.
7676
// If not specified, this is a no-op.
7777
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
78+
79+
// FilterMetadataBeforeValidation allows filtering out labels and annotations of Machines, InfraMachines,
80+
// BootstrapConfigs and Nodes before we validate them.
81+
// This can be e.g. used to filter out additional infrastructure provider specific labels that would
82+
// otherwise lead to a failed test.
83+
FilterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta
7884
}
7985

8086
// ClusterClassRolloutSpec implements a test that verifies the ClusterClass rollout behavior.
@@ -111,6 +117,14 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
111117
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
112118
Expect(input.E2EConfig.Variables).To(HaveValidVersion(input.E2EConfig.GetVariable(KubernetesVersion)))
113119

120+
// Set a default function to ensure that FilterMetadataBeforeValidation has a default behavior for
121+
// filtering metadata if it is not specified by infrastructure provider.
122+
if input.FilterMetadataBeforeValidation == nil {
123+
input.FilterMetadataBeforeValidation = func(obj client.Object) clusterv1.ObjectMeta {
124+
return clusterv1.ObjectMeta{Labels: obj.GetLabels(), Annotations: obj.GetAnnotations()}
125+
}
126+
}
127+
114128
// Set up a Namespace where to host objects for this spec and create a watcher for the namespace events.
115129
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
116130
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
@@ -142,7 +156,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
142156
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
143157
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
144158
}, clusterResources)
145-
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
159+
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)
146160

147161
By("Rolling out changes to control plane, MachineDeployments, and MachinePools (in-place)")
148162
machinesBeforeUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
@@ -218,7 +232,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
218232
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
219233
g.Expect(machinesAfterUpgrade.Equal(machinesBeforeUpgrade)).To(BeTrue(), "Machines must not be replaced through in-place rollout")
220234
}, 30*time.Second, 1*time.Second).Should(Succeed())
221-
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
235+
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)
222236

223237
By("Rolling out changes to control plane, MachineDeployments, and MachinePools (rollout)")
224238
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
@@ -259,7 +273,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
259273
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
260274
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout")
261275
}, input.E2EConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed())
262-
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
276+
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)
263277

264278
By("Rolling out control plane and MachineDeployment (rolloutAfter)")
265279
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
@@ -287,7 +301,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
287301
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
288302
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout with rolloutAfter")
289303
}, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...).Should(Succeed())
290-
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
304+
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)
291305

292306
By("PASSED!")
293307
})
@@ -299,7 +313,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
299313
}
300314

301315
// assertClusterObjects asserts cluster objects by checking that all objects have the right labels, annotations and selectors.
302-
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) {
316+
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
303317
By("Checking cluster objects have the right labels, annotations and selectors")
304318

305319
Eventually(func(g Gomega) {
@@ -312,12 +326,12 @@ func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterPro
312326

313327
// ControlPlane
314328
assertControlPlane(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
315-
assertControlPlaneMachines(g, clusterObjects, cluster)
329+
assertControlPlaneMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)
316330

317331
// MachineDeployments
318332
assertMachineDeployments(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
319333
assertMachineSets(g, clusterObjects, cluster)
320-
assertMachineSetsMachines(g, clusterObjects, cluster)
334+
assertMachineSetsMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)
321335

322336
// MachinePools
323337
assertMachinePools(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
@@ -430,13 +444,14 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust
430444
))
431445
}
432446

433-
func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
447+
func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
434448
controlPlaneMachineTemplateMetadata := mustMetadata(contract.ControlPlane().MachineTemplate().Metadata().Get(clusterObjects.ControlPlane))
435449
controlPlaneInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(clusterObjects.ControlPlaneInfrastructureMachineTemplate))
436450

437451
for _, machine := range clusterObjects.ControlPlaneMachines {
438452
// ControlPlane Machine.metadata
439-
g.Expect(machine.Labels).To(BeEquivalentTo(
453+
machineMetadata := filterMetadataBeforeValidation(machine)
454+
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
440455
union(
441456
map[string]string{
442457
clusterv1.ClusterNameLabel: cluster.Name,
@@ -449,17 +464,18 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
449464
))
450465
g.Expect(
451466
union(
452-
machine.Annotations,
467+
machineMetadata.Annotations,
453468
).without(g, controlplanev1.KubeadmClusterConfigurationAnnotation),
454469
).To(BeEquivalentTo(
455470
controlPlaneMachineTemplateMetadata.Annotations,
456471
))
457472

458473
// ControlPlane Machine InfrastructureMachine.metadata
459474
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
475+
infrastructureMachineMetadata := filterMetadataBeforeValidation(infrastructureMachine)
460476
controlPlaneMachineTemplateInfrastructureRef, err := contract.ControlPlane().MachineTemplate().InfrastructureRef().Get(clusterObjects.ControlPlane)
461477
g.Expect(err).ToNot(HaveOccurred())
462-
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
478+
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
463479
union(
464480
map[string]string{
465481
clusterv1.ClusterNameLabel: cluster.Name,
@@ -471,7 +487,7 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
471487
controlPlaneInfrastructureMachineTemplateTemplateMetadata.Labels,
472488
),
473489
))
474-
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
490+
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
475491
union(
476492
map[string]string{
477493
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(controlPlaneMachineTemplateInfrastructureRef),
@@ -484,7 +500,8 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
484500

485501
// ControlPlane Machine BootstrapConfig.metadata
486502
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
487-
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
503+
bootstrapConfigMetadata := filterMetadataBeforeValidation(bootstrapConfig)
504+
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
488505
union(
489506
map[string]string{
490507
clusterv1.ClusterNameLabel: cluster.Name,
@@ -497,16 +514,18 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
497514
))
498515
g.Expect(
499516
union(
500-
bootstrapConfig.GetAnnotations(),
517+
bootstrapConfigMetadata.Annotations,
501518
).without(g, clusterv1.MachineCertificatesExpiryDateAnnotation),
502519
).To(BeEquivalentTo(
503520
controlPlaneMachineTemplateMetadata.Annotations,
504521
))
505522

506523
// ControlPlane Machine Node.metadata
507524
node := clusterObjects.NodesByMachine[machine.Name]
508-
for k, v := range getManagedLabels(machine.Labels) {
509-
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
525+
nodeMetadata := filterMetadataBeforeValidation(node)
526+
527+
for k, v := range getManagedLabels(machineMetadata.Labels) {
528+
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
510529
}
511530
}
512531
}
@@ -790,7 +809,7 @@ func assertMachineSets(g Gomega, clusterObjects clusterObjects, cluster *cluster
790809
}
791810
}
792811

793-
func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
812+
func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
794813
for _, machineDeployment := range clusterObjects.MachineDeployments {
795814
mdTopology := getMDTopology(cluster, machineDeployment)
796815
infrastructureMachineTemplate := clusterObjects.InfrastructureMachineTemplateByMachineDeployment[machineDeployment.Name]
@@ -802,8 +821,9 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
802821
machineTemplateHash := machineSet.Labels[clusterv1.MachineDeploymentUniqueLabel]
803822

804823
for _, machine := range clusterObjects.MachinesByMachineSet[machineSet.Name] {
824+
machineMetadata := filterMetadataBeforeValidation(machine)
805825
// MachineDeployment MachineSet Machine.metadata
806-
g.Expect(machine.Labels).To(BeEquivalentTo(
826+
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
807827
union(
808828
map[string]string{
809829
clusterv1.ClusterNameLabel: cluster.Name,
@@ -816,13 +836,14 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
816836
machineSet.Spec.Template.Labels,
817837
),
818838
))
819-
g.Expect(machine.Annotations).To(BeEquivalentTo(
839+
g.Expect(machineMetadata.Annotations).To(BeEquivalentTo(
820840
machineSet.Spec.Template.Annotations,
821841
))
822842

823843
// MachineDeployment MachineSet Machine InfrastructureMachine.metadata
824844
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
825-
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
845+
infrastructureMachineMetadata := filterMetadataBeforeValidation(infrastructureMachine)
846+
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
826847
union(
827848
map[string]string{
828849
clusterv1.ClusterNameLabel: cluster.Name,
@@ -836,7 +857,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
836857
infrastructureMachineTemplateTemplateMetadata.Labels,
837858
),
838859
))
839-
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
860+
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
840861
union(
841862
map[string]string{
842863
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(&machineSet.Spec.Template.Spec.InfrastructureRef),
@@ -849,7 +870,8 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
849870

850871
// MachineDeployment MachineSet Machine BootstrapConfig.metadata
851872
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
852-
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
873+
bootstrapConfigMetadata := filterMetadataBeforeValidation(bootstrapConfig)
874+
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
853875
union(
854876
map[string]string{
855877
clusterv1.ClusterNameLabel: cluster.Name,
@@ -863,7 +885,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
863885
bootstrapConfigTemplateTemplateMetadata.Labels,
864886
),
865887
))
866-
g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo(
888+
g.Expect(bootstrapConfigMetadata.Annotations).To(BeEquivalentTo(
867889
union(
868890
map[string]string{
869891
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(machineSet.Spec.Template.Spec.Bootstrap.ConfigRef),
@@ -876,8 +898,9 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
876898

877899
// MachineDeployment MachineSet Machine Node.metadata
878900
node := clusterObjects.NodesByMachine[machine.Name]
879-
for k, v := range getManagedLabels(machine.Labels) {
880-
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
901+
nodeMetadata := filterMetadataBeforeValidation(node)
902+
for k, v := range getManagedLabels(machineMetadata.Labels) {
903+
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
881904
}
882905
}
883906
}

0 commit comments

Comments
 (0)