Skip to content

Commit dbdf199

Browse files
willie-yaoCecileRobertMichon
authored and
sbueringer
committed
Follow-up fixes: Add MachinePools to Runtime SDK and Rollout tests
Co-authored-by: Cecile Robert-Michon <[email protected]>
1 parent bb83615 commit dbdf199

File tree

4 files changed

+165
-43
lines changed

4 files changed

+165
-43
lines changed

‎internal/contract/infrastructure_machinepool_template.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package contract
18+
19+
import (
20+
"sync"
21+
)
22+
23+
// InfrastructureMachinePoolTemplateContract encodes information about the Cluster API contract for InfrastructureMachinePoolTemplate objects
24+
// like DockerMachinePoolTemplates, AWSMachinePoolTemplates, etc.
25+
type InfrastructureMachinePoolTemplateContract struct{}
26+
27+
var infrastructureMachinePoolTemplate *InfrastructureMachinePoolTemplateContract
28+
var onceInfrastructureMachinePoolTemplate sync.Once
29+
30+
// InfrastructureMachinePoolTemplate provide access to the information about the Cluster API contract for InfrastructureMachinePoolTemplate objects.
31+
func InfrastructureMachinePoolTemplate() *InfrastructureMachinePoolTemplateContract {
32+
onceInfrastructureMachinePoolTemplate.Do(func() {
33+
infrastructureMachinePoolTemplate = &InfrastructureMachinePoolTemplateContract{}
34+
})
35+
return infrastructureMachinePoolTemplate
36+
}
37+
38+
// Template provides access to the template.
39+
func (c *InfrastructureMachinePoolTemplateContract) Template() *InfrastructureMachinePoolTemplateTemplate {
40+
return &InfrastructureMachinePoolTemplateTemplate{}
41+
}
42+
43+
// InfrastructureMachinePoolTemplateTemplate provides a helper struct for working with the template in an InfrastructureMachinePoolTemplate.
44+
type InfrastructureMachinePoolTemplateTemplate struct{}
45+
46+
// Metadata provides access to the metadata of a template.
47+
func (c *InfrastructureMachinePoolTemplateTemplate) Metadata() *Metadata {
48+
return &Metadata{
49+
path: Path{"spec", "template", "metadata"},
50+
}
51+
}

‎test/e2e/clusterclass_changes.go

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,24 @@ type ClusterClassChangesSpecInput struct {
8888
// "spec.template.spec.path.to.field": <value>,
8989
// }
9090
ModifyMachineDeploymentInfrastructureMachineTemplateFields map[string]interface{}
91+
92+
// ModifyMachinePoolBootstrapConfigTemplateFields are the fields which will be set on the
93+
// BootstrapConfigTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
94+
// The test verifies that these fields are rolled out to the MachinePools.
95+
// NOTE: The fields are configured in the following format:
96+
// map[string]interface{}{
97+
// "spec.template.spec.path.to.field": <value>,
98+
// }
99+
ModifyMachinePoolBootstrapConfigTemplateFields map[string]interface{}
100+
101+
// ModifyMachinePoolInfrastructureMachinePoolTemplateFields are the fields which will be set on the
102+
// InfrastructureMachinePoolTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
103+
// The test verifies that these fields are rolled out to the MachinePools.
104+
// NOTE: The fields are configured in the following format:
105+
// map[string]interface{}{
106+
// "spec.template.spec.path.to.field": <value>,
107+
// }
108+
ModifyMachinePoolInfrastructureMachinePoolTemplateFields map[string]interface{}
91109
}
92110

93111
// ClusterClassChangesSpec implements a test that verifies that ClusterClass changes are rolled out successfully.
@@ -181,6 +199,16 @@ func ClusterClassChangesSpec(ctx context.Context, inputGetter func() ClusterClas
181199
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
182200
})
183201

202+
By("Modifying the MachinePool configuration in ClusterClass and wait for changes to be applied to the MachinePool objects")
203+
modifyMachinePoolViaClusterClassAndWait(ctx, modifyMachinePoolViaClusterClassAndWaitInput{
204+
ClusterProxy: input.BootstrapClusterProxy,
205+
ClusterClass: clusterResources.ClusterClass,
206+
Cluster: clusterResources.Cluster,
207+
ModifyBootstrapConfigTemplateFields: input.ModifyMachinePoolBootstrapConfigTemplateFields,
208+
ModifyInfrastructureMachinePoolTemplateFields: input.ModifyMachinePoolInfrastructureMachinePoolTemplateFields,
209+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
210+
})
211+
184212
By("Rebasing the Cluster to a ClusterClass with a modified label for MachineDeployments and wait for changes to be applied to the MachineDeployment objects")
185213
rebaseClusterClassAndWait(ctx, rebaseClusterClassAndWaitInput{
186214
ClusterProxy: input.BootstrapClusterProxy,
@@ -535,13 +563,14 @@ func modifyMachinePoolViaClusterClassAndWait(ctx context.Context, input modifyMa
535563
}
536564

537565
// Get the corresponding InfrastructureMachinePoolTemplate.
538-
infrastructureMachinePoolTemplateRef := mp.Spec.Template.Spec.InfrastructureRef
539-
infrastructureMachinePoolTemplate, err := external.Get(ctx, mgmtClient, &infrastructureMachinePoolTemplateRef, input.Cluster.Namespace)
566+
infrastructureMachinePoolRef := mp.Spec.Template.Spec.InfrastructureRef
567+
infrastructureMachinePool, err := external.Get(ctx, mgmtClient, &infrastructureMachinePoolRef, input.Cluster.Namespace)
540568
g.Expect(err).ToNot(HaveOccurred())
541569

542570
// Verify that ModifyInfrastructureMachinePoolTemplateFields have been set.
543571
for fieldPath, expectedValue := range input.ModifyInfrastructureMachinePoolTemplateFields {
544-
currentValue, ok, err := unstructured.NestedFieldNoCopy(infrastructureMachinePoolTemplate.Object, strings.Split(fieldPath, ".")...)
572+
fieldPath = strings.TrimPrefix(fieldPath, "spec.template.")
573+
currentValue, ok, err := unstructured.NestedFieldNoCopy(infrastructureMachinePool.Object, strings.Split(fieldPath, ".")...)
545574
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get field %q", fieldPath))
546575
g.Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get field %q", fieldPath))
547576
g.Expect(currentValue).To(Equal(expectedValue), fmt.Sprintf("field %q should be equal", fieldPath))
@@ -619,8 +648,9 @@ func assertMachinePoolTopologyFields(g Gomega, mp expv1.MachinePool, mpTopology
619648
g.Expect(mp.Spec.MinReadySeconds).To(Equal(mpTopology.MinReadySeconds))
620649
}
621650

622-
if mpTopology.FailureDomains != nil && mp.Spec.Template.Spec.FailureDomain != nil {
623-
g.Expect(mpTopology.FailureDomains).To(ContainElement(mp.Spec.Template.Spec.FailureDomain))
651+
if mpTopology.FailureDomains != nil {
652+
g.Expect(mp.Spec.FailureDomains).NotTo(BeNil())
653+
g.Expect(mpTopology.FailureDomains).To(Equal(mp.Spec.FailureDomains))
624654
}
625655
}
626656

‎test/e2e/clusterclass_changes_test.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,29 @@ var _ = Describe("When testing ClusterClass changes [ClusterClass]", func() {
6363
},
6464
},
6565
},
66+
// ModifyMachinePoolBootstrapConfigTemplateFields are the fields which will be set on the
67+
// BootstrapConfigTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
68+
// The test verifies that these fields are rolled out to the MachinePools.
69+
ModifyMachinePoolBootstrapConfigTemplateFields: map[string]interface{}{
70+
"spec.template.spec.verbosity": int64(4),
71+
},
72+
// ModifyMachinePoolInfrastructureMachineTemplateFields are the fields which will be set on the
73+
// InfrastructureMachinePoolTemplate of all MachinePoolClasses of the ClusterClass after the initial Cluster creation.
74+
// The test verifies that these fields are rolled out to the MachinePools.
75+
ModifyMachinePoolInfrastructureMachinePoolTemplateFields: map[string]interface{}{
76+
"spec.template.spec.template.extraMounts": []interface{}{
77+
map[string]interface{}{
78+
"containerPath": "/var/run/docker.sock",
79+
"hostPath": "/var/run/docker.sock",
80+
},
81+
map[string]interface{}{
82+
// /tmp cannot be used as containerPath as
83+
// it already exists.
84+
"containerPath": "/test",
85+
"hostPath": "/tmp",
86+
},
87+
},
88+
},
6689
}
6790
})
6891
})

‎test/e2e/clusterclass_rollout.go

Lines changed: 56 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
186186
},
187187
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
188188
})
189-
By("Modifying the MachinePool configuration via Cluster topology and wait for changes to be applied to the MachinePool (in-place)")
189+
By("Modifying the MachinePool configuration via Cluster topology and wait for changes to be applied to the MachinePools (in-place)")
190190
modifyMachinePoolViaClusterAndWait(ctx, modifyMachinePoolViaClusterAndWaitInput{
191191
ClusterProxy: input.BootstrapClusterProxy,
192192
Cluster: clusterResources.Cluster,
@@ -203,7 +203,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
203203
topology.NodeVolumeDetachTimeout = &metav1.Duration{Duration: time.Duration(rand.Intn(20)) * time.Second} //nolint:gosec
204204
topology.MinReadySeconds = pointer.Int32(rand.Int31n(20)) //nolint:gosec
205205
},
206-
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
206+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
207207
})
208208
By("Verifying there are no unexpected rollouts through in-place rollout")
209209
Consistently(func(g Gomega) {
@@ -242,7 +242,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
242242
ModifyBootstrapConfigTemplateFields: map[string]interface{}{
243243
"spec.template.spec.verbosity": int64(4),
244244
},
245-
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
245+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
246246
})
247247
By("Verifying all Machines are replaced through rollout")
248248
Eventually(func(g Gomega) {
@@ -644,7 +644,7 @@ func assertMachinePools(g Gomega, clusterClassObjects clusterClassObjects, clust
644644
g.Expect(
645645
union(
646646
machinePool.Annotations,
647-
).without(g, clusterv1.RevisionAnnotation),
647+
),
648648
).To(BeEquivalentTo(
649649
union(
650650
mpTopology.Metadata.Annotations,
@@ -671,68 +671,52 @@ func assertMachinePools(g Gomega, clusterClassObjects clusterClassObjects, clust
671671
),
672672
))
673673

674-
// MachinePool InfrastructureMachineTemplate.metadata
675-
ccInfrastructureMachineTemplate := clusterClassObjects.InfrastructureMachinePoolTemplateByMachinePoolClass[mpClass.Class]
676-
ccInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(ccInfrastructureMachineTemplate))
677-
infrastructureMachineTemplate := clusterObjects.InfrastructureMachinePoolTemplateByMachinePool[machinePool.Name]
678-
infrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(infrastructureMachineTemplate))
679-
g.Expect(infrastructureMachineTemplate.GetLabels()).To(BeEquivalentTo(
674+
// MachinePool InfrastructureMachinePool.metadata
675+
ccInfrastructureMachinePoolTemplate := clusterClassObjects.InfrastructureMachinePoolTemplateByMachinePoolClass[mpClass.Class]
676+
ccInfrastructureMachinePoolTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachinePoolTemplate().Template().Metadata().Get(ccInfrastructureMachinePoolTemplate))
677+
infrastructureMachinePool := clusterObjects.InfrastructureMachinePoolByMachinePool[machinePool.Name]
678+
g.Expect(infrastructureMachinePool.GetLabels()).To(BeEquivalentTo(
680679
union(
681680
map[string]string{
682681
clusterv1.ClusterNameLabel: cluster.Name,
683682
clusterv1.ClusterTopologyOwnedLabel: "",
684683
clusterv1.ClusterTopologyMachinePoolNameLabel: mpTopology.Name,
685684
},
686-
ccInfrastructureMachineTemplate.GetLabels(),
685+
ccInfrastructureMachinePoolTemplateTemplateMetadata.Labels,
687686
),
688687
))
689-
g.Expect(infrastructureMachineTemplate.GetAnnotations()).To(BeEquivalentTo(
688+
g.Expect(infrastructureMachinePool.GetAnnotations()).To(BeEquivalentTo(
690689
union(
691690
map[string]string{
692691
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(mpClass.Template.Infrastructure.Ref),
693692
clusterv1.TemplateClonedFromNameAnnotation: mpClass.Template.Infrastructure.Ref.Name,
694693
},
695-
ccInfrastructureMachineTemplate.GetAnnotations(),
696-
).without(g, corev1.LastAppliedConfigAnnotation),
697-
))
698-
// MachinePool InfrastructureMachineTemplate.spec.template.metadata
699-
g.Expect(infrastructureMachineTemplateTemplateMetadata.Labels).To(BeEquivalentTo(
700-
ccInfrastructureMachineTemplateTemplateMetadata.Labels,
701-
))
702-
g.Expect(infrastructureMachineTemplateTemplateMetadata.Annotations).To(BeEquivalentTo(
703-
ccInfrastructureMachineTemplateTemplateMetadata.Annotations,
694+
ccInfrastructureMachinePoolTemplateTemplateMetadata.Annotations,
695+
),
704696
))
705697

706-
// MachinePool BootstrapConfigTemplate.metadata
698+
// MachinePool BootstrapConfig.metadata
707699
ccBootstrapConfigTemplate := clusterClassObjects.BootstrapConfigTemplateByMachinePoolClass[mpClass.Class]
708700
ccBootstrapConfigTemplateTemplateMetadata := mustMetadata(contract.BootstrapConfigTemplate().Template().Metadata().Get(ccBootstrapConfigTemplate))
709-
bootstrapConfigTemplate := clusterObjects.BootstrapConfigTemplateByMachinePool[machinePool.Name]
710-
bootstrapConfigTemplateTemplateMetadata := mustMetadata(contract.BootstrapConfigTemplate().Template().Metadata().Get(bootstrapConfigTemplate))
711-
g.Expect(bootstrapConfigTemplate.GetLabels()).To(BeEquivalentTo(
701+
bootstrapConfig := clusterObjects.BootstrapConfigByMachinePool[machinePool.Name]
702+
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
712703
union(
713704
map[string]string{
714705
clusterv1.ClusterNameLabel: cluster.Name,
715706
clusterv1.ClusterTopologyOwnedLabel: "",
716707
clusterv1.ClusterTopologyMachinePoolNameLabel: mpTopology.Name,
717708
},
718-
ccBootstrapConfigTemplate.GetLabels(),
709+
ccBootstrapConfigTemplateTemplateMetadata.Labels,
719710
),
720711
))
721-
g.Expect(bootstrapConfigTemplate.GetAnnotations()).To(BeEquivalentTo(
712+
g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo(
722713
union(
723714
map[string]string{
724715
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(mpClass.Template.Bootstrap.Ref),
725716
clusterv1.TemplateClonedFromNameAnnotation: mpClass.Template.Bootstrap.Ref.Name,
726717
},
727-
ccBootstrapConfigTemplate.GetAnnotations(),
728-
).without(g, corev1.LastAppliedConfigAnnotation),
729-
))
730-
// MachinePool BootstrapConfigTemplate.spec.template.metadata
731-
g.Expect(bootstrapConfigTemplateTemplateMetadata.Labels).To(BeEquivalentTo(
732-
ccBootstrapConfigTemplateTemplateMetadata.Labels,
733-
))
734-
g.Expect(bootstrapConfigTemplateTemplateMetadata.Annotations).To(BeEquivalentTo(
735-
ccBootstrapConfigTemplateTemplateMetadata.Annotations,
718+
ccBootstrapConfigTemplateTemplateMetadata.Annotations,
719+
),
736720
))
737721
}
738722
}
@@ -1040,6 +1024,8 @@ func getClusterClassObjects(ctx context.Context, g Gomega, clusterProxy framewor
10401024
res := clusterClassObjects{
10411025
InfrastructureMachineTemplateByMachineDeploymentClass: map[string]*unstructured.Unstructured{},
10421026
BootstrapConfigTemplateByMachineDeploymentClass: map[string]*unstructured.Unstructured{},
1027+
InfrastructureMachinePoolTemplateByMachinePoolClass: map[string]*unstructured.Unstructured{},
1028+
BootstrapConfigTemplateByMachinePoolClass: map[string]*unstructured.Unstructured{},
10431029
}
10441030
var err error
10451031

@@ -1062,6 +1048,16 @@ func getClusterClassObjects(ctx context.Context, g Gomega, clusterProxy framewor
10621048
res.BootstrapConfigTemplateByMachineDeploymentClass[mdClass.Class] = bootstrapConfigTemplate
10631049
}
10641050

1051+
for _, mpClass := range clusterClass.Spec.Workers.MachinePools {
1052+
infrastructureMachinePoolTemplate, err := external.Get(ctx, mgmtClient, mpClass.Template.Infrastructure.Ref, clusterClass.Namespace)
1053+
g.Expect(err).ToNot(HaveOccurred())
1054+
res.InfrastructureMachinePoolTemplateByMachinePoolClass[mpClass.Class] = infrastructureMachinePoolTemplate
1055+
1056+
bootstrapConfigTemplate, err := external.Get(ctx, mgmtClient, mpClass.Template.Bootstrap.Ref, clusterClass.Namespace)
1057+
g.Expect(err).ToNot(HaveOccurred())
1058+
res.BootstrapConfigTemplateByMachinePoolClass[mpClass.Class] = bootstrapConfigTemplate
1059+
}
1060+
10651061
return res
10661062
}
10671063

@@ -1082,8 +1078,8 @@ type clusterObjects struct {
10821078
InfrastructureMachineTemplateByMachineDeployment map[string]*unstructured.Unstructured
10831079
BootstrapConfigTemplateByMachineDeployment map[string]*unstructured.Unstructured
10841080

1085-
InfrastructureMachinePoolTemplateByMachinePool map[string]*unstructured.Unstructured
1086-
BootstrapConfigTemplateByMachinePool map[string]*unstructured.Unstructured
1081+
InfrastructureMachinePoolByMachinePool map[string]*unstructured.Unstructured
1082+
BootstrapConfigByMachinePool map[string]*unstructured.Unstructured
10871083

10881084
InfrastructureMachineByMachine map[string]*unstructured.Unstructured
10891085
BootstrapConfigByMachine map[string]*unstructured.Unstructured
@@ -1102,6 +1098,8 @@ func getClusterObjects(ctx context.Context, g Gomega, clusterProxy framework.Clu
11021098
InfrastructureMachineTemplateByMachineDeployment: map[string]*unstructured.Unstructured{},
11031099
BootstrapConfigByMachine: map[string]*unstructured.Unstructured{},
11041100
InfrastructureMachineByMachine: map[string]*unstructured.Unstructured{},
1101+
BootstrapConfigByMachinePool: map[string]*unstructured.Unstructured{},
1102+
InfrastructureMachinePoolByMachinePool: map[string]*unstructured.Unstructured{},
11051103
}
11061104
var err error
11071105

@@ -1170,6 +1168,26 @@ func getClusterObjects(ctx context.Context, g Gomega, clusterProxy framework.Clu
11701168
}
11711169
}
11721170

1171+
// MachinePools.
1172+
for _, mpTopology := range cluster.Spec.Topology.Workers.MachinePools {
1173+
// Get MachinePool for the current MachinePoolTopology.
1174+
mpList := &expv1.MachinePoolList{}
1175+
g.Expect(mgmtClient.List(ctx, mpList, client.InNamespace(cluster.Namespace), client.MatchingLabels{
1176+
clusterv1.ClusterTopologyMachinePoolNameLabel: mpTopology.Name,
1177+
})).To(Succeed())
1178+
g.Expect(mpList.Items).To(HaveLen(1), fmt.Sprintf("expected one MachinePool for topology %q, but got %d", mpTopology.Name, len(mpList.Items)))
1179+
mp := mpList.Items[0]
1180+
res.MachinePools = append(res.MachinePools, &mp)
1181+
1182+
bootstrapConfig, err := external.Get(ctx, mgmtClient, mp.Spec.Template.Spec.Bootstrap.ConfigRef, cluster.Namespace)
1183+
g.Expect(err).ToNot(HaveOccurred())
1184+
res.BootstrapConfigByMachinePool[mp.Name] = bootstrapConfig
1185+
1186+
infrastructureMachinePool, err := external.Get(ctx, mgmtClient, &mp.Spec.Template.Spec.InfrastructureRef, cluster.Namespace)
1187+
g.Expect(err).ToNot(HaveOccurred())
1188+
res.InfrastructureMachinePoolByMachinePool[mp.Name] = infrastructureMachinePool
1189+
}
1190+
11731191
return res
11741192
}
11751193

0 commit comments

Comments
 (0)