Skip to content

Commit 777b6b0

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-sigs#9393 from willie-yao/cc-mp-e2e-quickstart
🌱 Add MachinePools to Topology Quickstart E2E Templates
2 parents ea26509 + d9147e4 commit 777b6b0

File tree

5 files changed

+165
-15
lines changed

5 files changed

+165
-15
lines changed

‎docs/book/src/reference/owner_references.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ These owner references are almost all tested in an [end-to-end test](https://git
5151
|----------------------------|--------------------|------------|--------------------------|
5252
| ClusterResourcesSet | None | | |
5353
| ClusterResourcesSetBinding | ClusterResourceSet | no | May have many CRS owners |
54-
| MachinePool | Cluster | unknown | Not tested in e2e |
54+
| MachinePool | Cluster | no | |
5555

5656

5757
## KubeadmControlPlane types
@@ -65,7 +65,7 @@ These owner references are almost all tested in an [end-to-end test](https://git
6565
| type | Owner | Controller | Note |
6666
|-----------------------|--------------|------------|-------------------------------------------------|
6767
| KubeadmConfig | Machine | yes | When created for Machine |
68-
| KubeadmConfig | MachinePool | unknown | When created for MachinePool. Not tested in e2e |
68+
| KubeadmConfig | MachinePool | yes | When created for MachinePool |
6969
| KubeadmConfigTemplate | Cluster | no | When referenced in MachineDeployment spec |
7070
| KubeadmConfigTemplate | ClusterClass | no | When referenced in ClusterClass |
7171

@@ -77,6 +77,6 @@ These owner references are almost all tested in an [end-to-end test](https://git
7777
| InfrastructureMachineTemplate | ClusterClass | no | When referenced in a ClusterClass |
7878
| InfrastructureCluster | Cluster | yes | |
7979
| InfrastructureClusterTemplate | ClusterClass | no | |
80-
| InfrastructureMachinePool | MachinePool | unknown | Not tested in e2e |
80+
| InfrastructureMachinePool | MachinePool | yes | |
8181

8282

‎test/e2e/data/infrastructure-docker/main/bases/cluster-with-topology.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,22 @@ spec:
4747
rollingUpdate:
4848
maxSurge: "20%"
4949
maxUnavailable: 0
50+
machinePools:
51+
- class: "default-worker"
52+
name: "mp-0"
53+
metadata:
54+
labels:
55+
Cluster.topology.machinePool.label: "Cluster.topology.machinePool.labelValue"
56+
# Note: this label is propagated to Nodes.
57+
Cluster.topology.machinePool.label.node.cluster.x-k8s.io: "Cluster.topology.machinePool.nodeLabelValue"
58+
annotations:
59+
Cluster.topology.machinePool.annotation: "Cluster.topology.machinePool.annotationValue"
60+
nodeDeletionTimeout: "30s"
61+
nodeVolumeDetachTimeout: "5m"
62+
minReadySeconds: 5
63+
replicas: ${WORKER_MACHINE_COUNT}
64+
failureDomains:
65+
- fd4
5066
variables:
5167
# We set an empty value to use the default tag kubeadm init is using.
5268
- name: etcdImageTag

‎test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,21 @@ spec:
3636
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
3737
kind: DockerMachineTemplate
3838
name: quick-start-default-worker-machinetemplate
39+
machinePools:
40+
- class: default-worker
41+
namingStrategy:
42+
template: "{{ .cluster.name }}-mp-{{ .machinePool.topologyName }}-{{ .random }}"
43+
template:
44+
bootstrap:
45+
ref:
46+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
47+
kind: KubeadmConfigTemplate
48+
name: quick-start-default-worker-bootstraptemplate
49+
infrastructure:
50+
ref:
51+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
52+
kind: DockerMachinePoolTemplate
53+
name: quick-start-default-worker-machinepooltemplate
3954
variables:
4055
# This variable is not used in any patch, but is here to ensure the variable discovered from the runtime hook is correctly
4156
# used.
@@ -122,6 +137,18 @@ spec:
122137
- containerPath: "/var/run/docker.sock"
123138
hostPath: "/var/run/docker.sock"
124139
---
140+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
141+
kind: DockerMachinePoolTemplate
142+
metadata:
143+
name: quick-start-default-worker-machinepooltemplate
144+
spec:
145+
template:
146+
spec:
147+
template:
148+
extraMounts:
149+
- containerPath: "/var/run/docker.sock"
150+
hostPath: "/var/run/docker.sock"
151+
---
125152
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
126153
kind: KubeadmConfigTemplate
127154
metadata:

‎test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ spec:
4242
ref:
4343
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
4444
kind: KubeadmConfigTemplate
45-
name: quick-start-default-worker-bootstraptemplate
45+
name: quick-start-md-default-worker-bootstraptemplate
4646
infrastructure:
4747
ref:
4848
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
@@ -54,6 +54,24 @@ spec:
5454
- type: e2e.remediation.condition
5555
status: "False"
5656
timeout: 20s
57+
machinePools:
58+
- class: default-worker
59+
template:
60+
metadata:
61+
labels:
62+
ClusterClass.machinePool.label: "ClusterClass.machinePool.labelValue"
63+
annotations:
64+
ClusterClass.machinePool.annotation: "ClusterClass.machinePool.annotationValue"
65+
bootstrap:
66+
ref:
67+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
68+
kind: KubeadmConfigTemplate
69+
name: quick-start-mp-default-worker-bootstraptemplate
70+
infrastructure:
71+
ref:
72+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
73+
kind: DockerMachinePoolTemplate
74+
name: quick-start-default-worker-machinepooltemplate
5775
variables:
5876
- name: lbImageRepository
5977
required: true
@@ -182,6 +200,19 @@ spec:
182200
valueFrom:
183201
template: |
184202
kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
203+
- selector:
204+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
205+
kind: DockerMachinePoolTemplate
206+
matchResources:
207+
machinePoolClass:
208+
names:
209+
- default-worker
210+
jsonPatches:
211+
- op: add
212+
path: "/spec/template/spec/template/customImage"
213+
valueFrom:
214+
template: |
215+
kindest/node:{{ .builtin.machinePool.version | replace "+" "_" }}
185216
- name: preloadImages
186217
description: |
187218
Sets the container images to preload to the node that is used for running dockerMachines.
@@ -201,6 +232,24 @@ spec:
201232
path: "/spec/template/spec/preLoadImages"
202233
valueFrom:
203234
variable: preLoadImages
235+
- name: preloadImagesMP
236+
description: |
237+
Sets the container images to preload to the node that is used for running dockerMachines.
238+
This is especially required for self-hosted e2e tests to ensure the required controller images to be available
239+
and reduce load to public registries.
240+
definitions:
241+
- selector:
242+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
243+
kind: DockerMachinePoolTemplate
244+
matchResources:
245+
machinePoolClass:
246+
names:
247+
- default-worker
248+
jsonPatches:
249+
- op: add
250+
path: "/spec/template/spec/template/preLoadImages"
251+
valueFrom:
252+
variable: preLoadImages
204253
- name: kubeadmControlPlaneMaxSurge
205254
description: "Sets the maxSurge value used for rolloutStrategy in the KubeadmControlPlane."
206255
enabledIf: '{{ ne .kubeadmControlPlaneMaxSurge "" }}'
@@ -264,6 +313,18 @@ spec:
264313
path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
265314
value:
266315
cloud-provider: "external"
316+
- selector:
317+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
318+
kind: KubeadmConfigTemplate
319+
matchResources:
320+
machinePoolClass:
321+
names:
322+
- '*-worker'
323+
jsonPatches:
324+
- op: add
325+
path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
326+
value:
327+
cloud-provider: "external"
267328
- name: localEndpointIPv6
268329
enabledIf: "{{ .ipv6Primary }}"
269330
description: "Configures KCP to use IPv6 for its localAPIEndpoint."
@@ -440,10 +501,31 @@ spec:
440501
- containerPath: "/var/run/docker.sock"
441502
hostPath: "/var/run/docker.sock"
442503
---
504+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
505+
kind: DockerMachinePoolTemplate
506+
metadata:
507+
name: quick-start-default-worker-machinepooltemplate
508+
labels:
509+
InfraMachinePoolTemplate.machinePool.label: "InfraMachinePoolTemplate.machinePool.labelValue"
510+
annotations:
511+
InfraMachinePoolTemplate.machinePool.annotation: "InfraMachinePoolTemplate.machinePool.annotationValue"
512+
spec:
513+
template:
514+
metadata:
515+
labels:
516+
InfraMachinePoolTemplate.machinePool.template.label: "InfraMachinePoolTemplate.machinePool.labelValue"
517+
annotations:
518+
InfraMachinePoolTemplate.machinePool.template.annotation: "InfraMachinePoolTemplate.machinePool.annotationValue"
519+
spec:
520+
template:
521+
extraMounts:
522+
- containerPath: "/var/run/docker.sock"
523+
hostPath: "/var/run/docker.sock"
524+
---
443525
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
444526
kind: KubeadmConfigTemplate
445527
metadata:
446-
name: quick-start-default-worker-bootstraptemplate
528+
name: quick-start-md-default-worker-bootstraptemplate
447529
labels:
448530
BootstrapConfigTemplate.machineDeployment.label: "BootstrapConfigTemplate.machineDeployment.labelValue"
449531
annotations:
@@ -460,3 +542,24 @@ spec:
460542
nodeRegistration: # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
461543
kubeletExtraArgs: # having a not empty kubeletExtraArgs is required for the externalCloudProvider to work
462544
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
545+
---
546+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
547+
kind: KubeadmConfigTemplate
548+
metadata:
549+
name: quick-start-mp-default-worker-bootstraptemplate
550+
labels:
551+
BootstrapConfigTemplate.machinePool.label: "BootstrapConfigTemplate.machinePool.template.labelValue"
552+
annotations:
553+
BootstrapConfigTemplate.machinePool.annotation: "BootstrapConfigTemplate.machinePool.template.annotationValue"
554+
spec:
555+
template:
556+
metadata:
557+
labels:
558+
BootstrapConfigTemplate.machinePool.template.label: "BootstrapConfigTemplate.machinePool.template.labelValue"
559+
annotations:
560+
BootstrapConfigTemplate.machinePool.template.annotation: "BootstrapConfigTemplate.machinePool.template.annotationValue"
561+
spec:
562+
joinConfiguration:
563+
nodeRegistration: # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
564+
kubeletExtraArgs: # having a not empty kubeletExtraArgs is required for the externalCloudProvider to work
565+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'

‎test/framework/ownerreference_helpers.go

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,6 @@ var (
183183
clusterResourceSetBindingKind = "ClusterResourceSetBinding"
184184
machinePoolKind = "MachinePool"
185185

186-
machinePoolOwner = metav1.OwnerReference{Kind: machinePoolKind, APIVersion: expv1.GroupVersion.String()}
187186
machinePoolController = metav1.OwnerReference{Kind: machinePoolKind, APIVersion: expv1.GroupVersion.String(), Controller: pointer.Bool(true)}
188187

189188
clusterResourceSetOwner = metav1.OwnerReference{Kind: clusterResourceSetKind, APIVersion: addonsv1.GroupVersion.String()}
@@ -270,7 +269,7 @@ var (
270269
var KubeadmBootstrapOwnerReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
271270
kubeadmConfigKind: func(owners []metav1.OwnerReference) error {
272271
// The KubeadmConfig must be owned and controlled by a Machine or MachinePool.
273-
return HasOneOfExactOwners(owners, []metav1.OwnerReference{machineController}, []metav1.OwnerReference{machinePoolController})
272+
return HasOneOfExactOwners(owners, []metav1.OwnerReference{machineController}, []metav1.OwnerReference{machinePoolController, clusterOwner})
274273
},
275274
kubeadmConfigTemplateKind: func(owners []metav1.OwnerReference) error {
276275
// The KubeadmConfigTemplate must be owned by a ClusterClass.
@@ -280,11 +279,12 @@ var KubeadmBootstrapOwnerReferenceAssertions = map[string]func([]metav1.OwnerRef
280279

281280
// Kinds for types in the Docker infrastructure package.
282281
var (
283-
dockerMachineKind = "DockerMachine"
284-
dockerMachineTemplateKind = "DockerMachineTemplate"
285-
dockerMachinePoolKind = "DockerMachinePool"
286-
dockerClusterKind = "DockerCluster"
287-
dockerClusterTemplateKind = "DockerClusterTemplate"
282+
dockerMachineKind = "DockerMachine"
283+
dockerMachineTemplateKind = "DockerMachineTemplate"
284+
dockerMachinePoolKind = "DockerMachinePool"
285+
dockerMachinePoolTemplateKind = "DockerMachinePoolTemplate"
286+
dockerClusterKind = "DockerCluster"
287+
dockerClusterTemplateKind = "DockerClusterTemplate"
288288
)
289289

290290
// DockerInfraOwnerReferenceAssertions maps Docker Infrastructure types to functions which return an error if the passed
@@ -310,8 +310,12 @@ var DockerInfraOwnerReferenceAssertions = map[string]func([]metav1.OwnerReferenc
310310
return HasExactOwners(owners, clusterClassOwner)
311311
},
312312
dockerMachinePoolKind: func(owners []metav1.OwnerReference) error {
313-
// DockerMachinePool must be owned by a MachinePool.
314-
return HasExactOwners(owners, machinePoolOwner)
313+
// DockerMachinePool must be owned and controlled by a MachinePool.
314+
return HasExactOwners(owners, machinePoolController, clusterOwner)
315+
},
316+
dockerMachinePoolTemplateKind: func(owners []metav1.OwnerReference) error {
317+
// DockerMachinePoolTemplate must be owned by a ClusterClass.
318+
return HasExactOwners(owners, clusterClassOwner)
315319
},
316320
}
317321

@@ -328,7 +332,7 @@ func HasExactOwners(gotOwners []metav1.OwnerReference, wantOwners ...metav1.Owne
328332
sort.Strings(wantComparable)
329333

330334
if !reflect.DeepEqual(gotComparable, wantComparable) {
331-
return fmt.Errorf("wanted %v, actual %v", wantOwners, gotOwners)
335+
return fmt.Errorf("wanted %v, actual %v", wantComparable, gotComparable)
332336
}
333337
return nil
334338
}

0 commit comments

Comments
 (0)