Skip to content

Commit 9644e46

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-sigs#9719 from willie-yao/fix-mp-cc-e2e
🌱 Follow-up fixes: Add MachinePools to Runtime SDK and Rollout tests
2 parents 9d5e127 + dbdf199 commit 9644e46

File tree

4 files changed

+165
-43
lines changed

4 files changed

+165
-43
lines changed

‎internal/contract/infrastructure_machinepool_template.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package contract
18+
19+
import (
20+
"sync"
21+
)
22+
23+
// InfrastructureMachinePoolTemplateContract encodes information about the Cluster API contract for InfrastructureMachinePoolTemplate objects
24+
// like DockerMachinePoolTemplates, AWSMachinePoolTemplates, etc.
25+
type InfrastructureMachinePoolTemplateContract struct{}
26+
27+
var infrastructureMachinePoolTemplate *InfrastructureMachinePoolTemplateContract
28+
var onceInfrastructureMachinePoolTemplate sync.Once
29+
30+
// InfrastructureMachinePoolTemplate provide access to the information about the Cluster API contract for InfrastructureMachinePoolTemplate objects.
31+
func InfrastructureMachinePoolTemplate() *InfrastructureMachinePoolTemplateContract {
32+
onceInfrastructureMachinePoolTemplate.Do(func() {
33+
infrastructureMachinePoolTemplate = &InfrastructureMachinePoolTemplateContract{}
34+
})
35+
return infrastructureMachinePoolTemplate
36+
}
37+
38+
// Template provides access to the template.
39+
func (c *InfrastructureMachinePoolTemplateContract) Template() *InfrastructureMachinePoolTemplateTemplate {
40+
return &InfrastructureMachinePoolTemplateTemplate{}
41+
}
42+
43+
// InfrastructureMachinePoolTemplateTemplate provides a helper struct for working with the template in an InfrastructureMachinePoolTemplate.
44+
type InfrastructureMachinePoolTemplateTemplate struct{}
45+
46+
// Metadata provides access to the metadata of a template.
47+
func (c *InfrastructureMachinePoolTemplateTemplate) Metadata() *Metadata {
48+
return &Metadata{
49+
path: Path{"spec", "template", "metadata"},
50+
}
51+
}

‎test/e2e/clusterclass_changes.go

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,24 @@ type ClusterClassChangesSpecInput struct {
8888
// "spec.template.spec.path.to.field": <value>,
8989
// }
9090
ModifyMachineDeploymentInfrastructureMachineTemplateFields map[string]interface{}
91+
92+
// ModifyMachinePoolBootstrapConfigTemplateFields are the fields which will be set on the
93+
// BootstrapConfigTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
94+
// The test verifies that these fields are rolled out to the MachinePools.
95+
// NOTE: The fields are configured in the following format:
96+
// map[string]interface{}{
97+
// "spec.template.spec.path.to.field": <value>,
98+
// }
99+
ModifyMachinePoolBootstrapConfigTemplateFields map[string]interface{}
100+
101+
// ModifyMachinePoolInfrastructureMachinePoolTemplateFields are the fields which will be set on the
102+
// InfrastructureMachinePoolTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
103+
// The test verifies that these fields are rolled out to the MachinePools.
104+
// NOTE: The fields are configured in the following format:
105+
// map[string]interface{}{
106+
// "spec.template.spec.path.to.field": <value>,
107+
// }
108+
ModifyMachinePoolInfrastructureMachinePoolTemplateFields map[string]interface{}
91109
}
92110

93111
// ClusterClassChangesSpec implements a test that verifies that ClusterClass changes are rolled out successfully.
@@ -181,6 +199,16 @@ func ClusterClassChangesSpec(ctx context.Context, inputGetter func() ClusterClas
181199
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
182200
})
183201

202+
By("Modifying the MachinePool configuration in ClusterClass and wait for changes to be applied to the MachinePool objects")
203+
modifyMachinePoolViaClusterClassAndWait(ctx, modifyMachinePoolViaClusterClassAndWaitInput{
204+
ClusterProxy: input.BootstrapClusterProxy,
205+
ClusterClass: clusterResources.ClusterClass,
206+
Cluster: clusterResources.Cluster,
207+
ModifyBootstrapConfigTemplateFields: input.ModifyMachinePoolBootstrapConfigTemplateFields,
208+
ModifyInfrastructureMachinePoolTemplateFields: input.ModifyMachinePoolInfrastructureMachinePoolTemplateFields,
209+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
210+
})
211+
184212
By("Rebasing the Cluster to a ClusterClass with a modified label for MachineDeployments and wait for changes to be applied to the MachineDeployment objects")
185213
rebaseClusterClassAndWait(ctx, rebaseClusterClassAndWaitInput{
186214
ClusterProxy: input.BootstrapClusterProxy,
@@ -535,13 +563,14 @@ func modifyMachinePoolViaClusterClassAndWait(ctx context.Context, input modifyMa
535563
}
536564

537565
// Get the corresponding InfrastructureMachinePoolTemplate.
538-
infrastructureMachinePoolTemplateRef := mp.Spec.Template.Spec.InfrastructureRef
539-
infrastructureMachinePoolTemplate, err := external.Get(ctx, mgmtClient, &infrastructureMachinePoolTemplateRef, input.Cluster.Namespace)
566+
infrastructureMachinePoolRef := mp.Spec.Template.Spec.InfrastructureRef
567+
infrastructureMachinePool, err := external.Get(ctx, mgmtClient, &infrastructureMachinePoolRef, input.Cluster.Namespace)
540568
g.Expect(err).ToNot(HaveOccurred())
541569

542570
// Verify that ModifyInfrastructureMachinePoolTemplateFields have been set.
543571
for fieldPath, expectedValue := range input.ModifyInfrastructureMachinePoolTemplateFields {
544-
currentValue, ok, err := unstructured.NestedFieldNoCopy(infrastructureMachinePoolTemplate.Object, strings.Split(fieldPath, ".")...)
572+
fieldPath = strings.TrimPrefix(fieldPath, "spec.template.")
573+
currentValue, ok, err := unstructured.NestedFieldNoCopy(infrastructureMachinePool.Object, strings.Split(fieldPath, ".")...)
545574
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get field %q", fieldPath))
546575
g.Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get field %q", fieldPath))
547576
g.Expect(currentValue).To(Equal(expectedValue), fmt.Sprintf("field %q should be equal", fieldPath))
@@ -619,8 +648,9 @@ func assertMachinePoolTopologyFields(g Gomega, mp expv1.MachinePool, mpTopology
619648
g.Expect(mp.Spec.MinReadySeconds).To(Equal(mpTopology.MinReadySeconds))
620649
}
621650

622-
if mpTopology.FailureDomains != nil && mp.Spec.Template.Spec.FailureDomain != nil {
623-
g.Expect(mpTopology.FailureDomains).To(ContainElement(mp.Spec.Template.Spec.FailureDomain))
651+
if mpTopology.FailureDomains != nil {
652+
g.Expect(mp.Spec.FailureDomains).NotTo(BeNil())
653+
g.Expect(mpTopology.FailureDomains).To(Equal(mp.Spec.FailureDomains))
624654
}
625655
}
626656

‎test/e2e/clusterclass_changes_test.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,29 @@ var _ = Describe("When testing ClusterClass changes [ClusterClass]", func() {
6363
},
6464
},
6565
},
66+
// ModifyMachinePoolBootstrapConfigTemplateFields are the fields which will be set on the
67+
// BootstrapConfigTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
68+
// The test verifies that these fields are rolled out to the MachinePools.
69+
ModifyMachinePoolBootstrapConfigTemplateFields: map[string]interface{}{
70+
"spec.template.spec.verbosity": int64(4),
71+
},
72+
// ModifyMachinePoolInfrastructureMachineTemplateFields are the fields which will be set on the
73+
// InfrastructureMachinePoolTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
74+
// The test verifies that these fields are rolled out to the MachinePools.
75+
ModifyMachinePoolInfrastructureMachinePoolTemplateFields: map[string]interface{}{
76+
"spec.template.spec.template.extraMounts": []interface{}{
77+
map[string]interface{}{
78+
"containerPath": "/var/run/docker.sock",
79+
"hostPath": "/var/run/docker.sock",
80+
},
81+
map[string]interface{}{
82+
// /tmp cannot be used as containerPath as
83+
// it already exists.
84+
"containerPath": "/test",
85+
"hostPath": "/tmp",
86+
},
87+
},
88+
},
6689
}
6790
})
6891
})

‎test/e2e/clusterclass_rollout.go

Lines changed: 56 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
187187
},
188188
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
189189
})
190-
By("Modifying the MachinePool configuration via Cluster topology and wait for changes to be applied to the MachinePool (in-place)")
190+
By("Modifying the MachinePool configuration via Cluster topology and wait for changes to be applied to the MachinePools (in-place)")
191191
modifyMachinePoolViaClusterAndWait(ctx, modifyMachinePoolViaClusterAndWaitInput{
192192
ClusterProxy: input.BootstrapClusterProxy,
193193
Cluster: clusterResources.Cluster,
@@ -204,7 +204,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
204204
topology.NodeVolumeDetachTimeout = &metav1.Duration{Duration: time.Duration(rand.Intn(20)) * time.Second} //nolint:gosec
205205
topology.MinReadySeconds = pointer.Int32(rand.Int31n(20)) //nolint:gosec
206206
},
207-
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
207+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
208208
})
209209
By("Verifying there are no unexpected rollouts through in-place rollout")
210210
Consistently(func(g Gomega) {
@@ -243,7 +243,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
243243
ModifyBootstrapConfigTemplateFields: map[string]interface{}{
244244
"spec.template.spec.verbosity": int64(4),
245245
},
246-
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
246+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
247247
})
248248
By("Verifying all Machines are replaced through rollout")
249249
Eventually(func(g Gomega) {
@@ -647,7 +647,7 @@ func assertMachinePools(g Gomega, clusterClassObjects clusterClassObjects, clust
647647
g.Expect(
648648
union(
649649
machinePool.Annotations,
650-
).without(g, clusterv1.RevisionAnnotation),
650+
),
651651
).To(BeEquivalentTo(
652652
union(
653653
mpTopology.Metadata.Annotations,
@@ -674,68 +674,52 @@ func assertMachinePools(g Gomega, clusterClassObjects clusterClassObjects, clust
674674
),
675675
))
676676

677-
// MachinePool InfrastructureMachineTemplate.metadata
678-
ccInfrastructureMachineTemplate := clusterClassObjects.InfrastructureMachinePoolTemplateByMachinePoolClass[mpClass.Class]
679-
ccInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(ccInfrastructureMachineTemplate))
680-
infrastructureMachineTemplate := clusterObjects.InfrastructureMachinePoolTemplateByMachinePool[machinePool.Name]
681-
infrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(infrastructureMachineTemplate))
682-
g.Expect(infrastructureMachineTemplate.GetLabels()).To(BeEquivalentTo(
677+
// MachinePool InfrastructureMachinePool.metadata
678+
ccInfrastructureMachinePoolTemplate := clusterClassObjects.InfrastructureMachinePoolTemplateByMachinePoolClass[mpClass.Class]
679+
ccInfrastructureMachinePoolTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachinePoolTemplate().Template().Metadata().Get(ccInfrastructureMachinePoolTemplate))
680+
infrastructureMachinePool := clusterObjects.InfrastructureMachinePoolByMachinePool[machinePool.Name]
681+
g.Expect(infrastructureMachinePool.GetLabels()).To(BeEquivalentTo(
683682
union(
684683
map[string]string{
685684
clusterv1.ClusterNameLabel: cluster.Name,
686685
clusterv1.ClusterTopologyOwnedLabel: "",
687686
clusterv1.ClusterTopologyMachinePoolNameLabel: mpTopology.Name,
688687
},
689-
ccInfrastructureMachineTemplate.GetLabels(),
688+
ccInfrastructureMachinePoolTemplateTemplateMetadata.Labels,
690689
),
691690
))
692-
g.Expect(infrastructureMachineTemplate.GetAnnotations()).To(BeEquivalentTo(
691+
g.Expect(infrastructureMachinePool.GetAnnotations()).To(BeEquivalentTo(
693692
union(
694693
map[string]string{
695694
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(mpClass.Template.Infrastructure.Ref),
696695
clusterv1.TemplateClonedFromNameAnnotation: mpClass.Template.Infrastructure.Ref.Name,
697696
},
698-
ccInfrastructureMachineTemplate.GetAnnotations(),
699-
).without(g, corev1.LastAppliedConfigAnnotation),
700-
))
701-
// MachinePool InfrastructureMachineTemplate.spec.template.metadata
702-
g.Expect(infrastructureMachineTemplateTemplateMetadata.Labels).To(BeEquivalentTo(
703-
ccInfrastructureMachineTemplateTemplateMetadata.Labels,
704-
))
705-
g.Expect(infrastructureMachineTemplateTemplateMetadata.Annotations).To(BeEquivalentTo(
706-
ccInfrastructureMachineTemplateTemplateMetadata.Annotations,
697+
ccInfrastructureMachinePoolTemplateTemplateMetadata.Annotations,
698+
),
707699
))
708700

709-
// MachinePool BootstrapConfigTemplate.metadata
701+
// MachinePool BootstrapConfig.metadata
710702
ccBootstrapConfigTemplate := clusterClassObjects.BootstrapConfigTemplateByMachinePoolClass[mpClass.Class]
711703
ccBootstrapConfigTemplateTemplateMetadata := mustMetadata(contract.BootstrapConfigTemplate().Template().Metadata().Get(ccBootstrapConfigTemplate))
712-
bootstrapConfigTemplate := clusterObjects.BootstrapConfigTemplateByMachinePool[machinePool.Name]
713-
bootstrapConfigTemplateTemplateMetadata := mustMetadata(contract.BootstrapConfigTemplate().Template().Metadata().Get(bootstrapConfigTemplate))
714-
g.Expect(bootstrapConfigTemplate.GetLabels()).To(BeEquivalentTo(
704+
bootstrapConfig := clusterObjects.BootstrapConfigByMachinePool[machinePool.Name]
705+
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
715706
union(
716707
map[string]string{
717708
clusterv1.ClusterNameLabel: cluster.Name,
718709
clusterv1.ClusterTopologyOwnedLabel: "",
719710
clusterv1.ClusterTopologyMachinePoolNameLabel: mpTopology.Name,
720711
},
721-
ccBootstrapConfigTemplate.GetLabels(),
712+
ccBootstrapConfigTemplateTemplateMetadata.Labels,
722713
),
723714
))
724-
g.Expect(bootstrapConfigTemplate.GetAnnotations()).To(BeEquivalentTo(
715+
g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo(
725716
union(
726717
map[string]string{
727718
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(mpClass.Template.Bootstrap.Ref),
728719
clusterv1.TemplateClonedFromNameAnnotation: mpClass.Template.Bootstrap.Ref.Name,
729720
},
730-
ccBootstrapConfigTemplate.GetAnnotations(),
731-
).without(g, corev1.LastAppliedConfigAnnotation),
732-
))
733-
// MachinePool BootstrapConfigTemplate.spec.template.metadata
734-
g.Expect(bootstrapConfigTemplateTemplateMetadata.Labels).To(BeEquivalentTo(
735-
ccBootstrapConfigTemplateTemplateMetadata.Labels,
736-
))
737-
g.Expect(bootstrapConfigTemplateTemplateMetadata.Annotations).To(BeEquivalentTo(
738-
ccBootstrapConfigTemplateTemplateMetadata.Annotations,
721+
ccBootstrapConfigTemplateTemplateMetadata.Annotations,
722+
),
739723
))
740724
}
741725
}
@@ -1047,6 +1031,8 @@ func getClusterClassObjects(ctx context.Context, g Gomega, clusterProxy framewor
10471031
res := clusterClassObjects{
10481032
InfrastructureMachineTemplateByMachineDeploymentClass: map[string]*unstructured.Unstructured{},
10491033
BootstrapConfigTemplateByMachineDeploymentClass: map[string]*unstructured.Unstructured{},
1034+
InfrastructureMachinePoolTemplateByMachinePoolClass: map[string]*unstructured.Unstructured{},
1035+
BootstrapConfigTemplateByMachinePoolClass: map[string]*unstructured.Unstructured{},
10501036
}
10511037
var err error
10521038

@@ -1069,6 +1055,16 @@ func getClusterClassObjects(ctx context.Context, g Gomega, clusterProxy framewor
10691055
res.BootstrapConfigTemplateByMachineDeploymentClass[mdClass.Class] = bootstrapConfigTemplate
10701056
}
10711057

1058+
for _, mpClass := range clusterClass.Spec.Workers.MachinePools {
1059+
infrastructureMachinePoolTemplate, err := external.Get(ctx, mgmtClient, mpClass.Template.Infrastructure.Ref, clusterClass.Namespace)
1060+
g.Expect(err).ToNot(HaveOccurred())
1061+
res.InfrastructureMachinePoolTemplateByMachinePoolClass[mpClass.Class] = infrastructureMachinePoolTemplate
1062+
1063+
bootstrapConfigTemplate, err := external.Get(ctx, mgmtClient, mpClass.Template.Bootstrap.Ref, clusterClass.Namespace)
1064+
g.Expect(err).ToNot(HaveOccurred())
1065+
res.BootstrapConfigTemplateByMachinePoolClass[mpClass.Class] = bootstrapConfigTemplate
1066+
}
1067+
10721068
return res
10731069
}
10741070

@@ -1089,8 +1085,8 @@ type clusterObjects struct {
10891085
InfrastructureMachineTemplateByMachineDeployment map[string]*unstructured.Unstructured
10901086
BootstrapConfigTemplateByMachineDeployment map[string]*unstructured.Unstructured
10911087

1092-
InfrastructureMachinePoolTemplateByMachinePool map[string]*unstructured.Unstructured
1093-
BootstrapConfigTemplateByMachinePool map[string]*unstructured.Unstructured
1088+
InfrastructureMachinePoolByMachinePool map[string]*unstructured.Unstructured
1089+
BootstrapConfigByMachinePool map[string]*unstructured.Unstructured
10941090

10951091
InfrastructureMachineByMachine map[string]*unstructured.Unstructured
10961092
BootstrapConfigByMachine map[string]*unstructured.Unstructured
@@ -1109,6 +1105,8 @@ func getClusterObjects(ctx context.Context, g Gomega, clusterProxy framework.Clu
11091105
InfrastructureMachineTemplateByMachineDeployment: map[string]*unstructured.Unstructured{},
11101106
BootstrapConfigByMachine: map[string]*unstructured.Unstructured{},
11111107
InfrastructureMachineByMachine: map[string]*unstructured.Unstructured{},
1108+
BootstrapConfigByMachinePool: map[string]*unstructured.Unstructured{},
1109+
InfrastructureMachinePoolByMachinePool: map[string]*unstructured.Unstructured{},
11121110
}
11131111
var err error
11141112

@@ -1177,6 +1175,26 @@ func getClusterObjects(ctx context.Context, g Gomega, clusterProxy framework.Clu
11771175
}
11781176
}
11791177

1178+
// MachinePools.
1179+
for _, mpTopology := range cluster.Spec.Topology.Workers.MachinePools {
1180+
// Get MachinePool for the current MachinePoolTopology.
1181+
mpList := &expv1.MachinePoolList{}
1182+
g.Expect(mgmtClient.List(ctx, mpList, client.InNamespace(cluster.Namespace), client.MatchingLabels{
1183+
clusterv1.ClusterTopologyMachinePoolNameLabel: mpTopology.Name,
1184+
})).To(Succeed())
1185+
g.Expect(mpList.Items).To(HaveLen(1), fmt.Sprintf("expected one MachinePool for topology %q, but got %d", mpTopology.Name, len(mpList.Items)))
1186+
mp := mpList.Items[0]
1187+
res.MachinePools = append(res.MachinePools, &mp)
1188+
1189+
bootstrapConfig, err := external.Get(ctx, mgmtClient, mp.Spec.Template.Spec.Bootstrap.ConfigRef, cluster.Namespace)
1190+
g.Expect(err).ToNot(HaveOccurred())
1191+
res.BootstrapConfigByMachinePool[mp.Name] = bootstrapConfig
1192+
1193+
infrastructureMachinePool, err := external.Get(ctx, mgmtClient, &mp.Spec.Template.Spec.InfrastructureRef, cluster.Namespace)
1194+
g.Expect(err).ToNot(HaveOccurred())
1195+
res.InfrastructureMachinePoolByMachinePool[mp.Name] = infrastructureMachinePool
1196+
}
1197+
11801198
return res
11811199
}
11821200

0 commit comments

Comments
 (0)