Skip to content

Commit 8334992

Browse files
sbueringersbueringer
authored
🌱 Add quickstart e2e test with v1beta1 with ClusterClass and RuntimeSDK (#12577)
* Add v1beta1 with ClusterClass and RuntimeSDK quickstart e2e test Signed-off-by: Stefan Büringer [email protected] * Fix review findings --------- Signed-off-by: Stefan Büringer [email protected]
1 parent 493d358 commit 8334992

File tree

10 files changed

+305
-11
lines changed

10 files changed

+305
-11
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ hack/tools/bin
1414
test/e2e/data/infrastructure-docker/**/cluster-template*.yaml
1515
!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start.yaml
1616
!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start-runtimesdk.yaml
17+
!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start-runtimesdk-v1beta1.yaml
1718
!test/e2e/data/infrastructure-docker/**/cluster-template-in-memory.yaml
1819
!test/e2e/data/infrastructure-docker/**/clusterclass-in-memory.yaml
1920
test/e2e/data/infrastructure-docker/**/clusterclass-*.yaml

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -627,6 +627,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
627627
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary.yaml
628628
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary.yaml
629629
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-no-workers --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-no-workers.yaml
630+
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-runtimesdk-v1beta1 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-runtimesdk-v1beta1.yaml
630631
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-kcp-only --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-kcp-only.yaml
631632
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler.yaml
632633
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology.yaml

‎test/e2e/config/docker.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,7 @@ providers:
343343
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary.yaml"
344344
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary.yaml"
345345
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-no-workers.yaml"
346+
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1.yaml"
346347
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-kcp-only.yaml"
347348
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-autoscaler.yaml"
348349
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml"
@@ -351,6 +352,7 @@ providers:
351352
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml"
352353
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml"
353354
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml"
355+
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml"
354356
- sourcePath: "../data/infrastructure-docker/main/clusterclass-in-memory.yaml"
355357
- sourcePath: "../data/shared/main/metadata.yaml"
356358

‎test/e2e/data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1/cluster-runtimesdk.yaml

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Note: This file is intentionally using v1beta1 for v1beta1 test coverage.
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: '${CLUSTER_NAME}'
6+
namespace: default
7+
labels:
8+
cni: "${CLUSTER_NAME}-crs-0"
9+
spec:
10+
clusterNetwork:
11+
services:
12+
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
13+
pods:
14+
cidrBlocks: ['${DOCKER_POD_CIDRS}']
15+
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
16+
topology:
17+
class: "quick-start-runtimesdk-v1beta1"
18+
classNamespace: '${CLUSTER_CLASS_NAMESPACE:-${NAMESPACE}}'
19+
version: "${KUBERNETES_VERSION}"
20+
controlPlane:
21+
nodeDeletionTimeout: 30s
22+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
23+
workers:
24+
machineDeployments:
25+
- class: "default-worker"
26+
name: "md-0"
27+
nodeDeletionTimeout: 30s
28+
nodeVolumeDetachTimeout: 300s
29+
minReadySeconds: 5
30+
replicas: ${WORKER_MACHINE_COUNT}
31+
failureDomain: fd4
32+
machinePools:
33+
- class: "default-worker"
34+
name: "mp-0"
35+
nodeDeletionTimeout: 30s
36+
nodeVolumeDetachTimeout: 300s
37+
minReadySeconds: 5
38+
replicas: ${WORKER_MACHINE_COUNT}
39+
failureDomains:
40+
- fd4
41+
variables:
42+
- name: kubeadmControlPlaneMaxSurge
43+
value: "1"
44+
- name: imageRepository
45+
value: "kindest"

‎test/e2e/data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1/kustomization.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
resources:
2+
- ../bases/crs.yaml
3+
- cluster-runtimesdk.yaml

‎test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# Note: This file is intentionally using v1beta1 for v1beta1 test coverage.
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: ClusterClass
4+
metadata:
5+
name: quick-start-runtimesdk-v1beta1
6+
spec:
7+
controlPlane:
8+
ref:
9+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
10+
kind: KubeadmControlPlaneTemplate
11+
name: quick-start-control-plane
12+
machineInfrastructure:
13+
ref:
14+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
15+
kind: DockerMachineTemplate
16+
name: quick-start-control-plane
17+
namingStrategy:
18+
template: "{{ .cluster.name }}-cp-{{ .random }}"
19+
infrastructure:
20+
ref:
21+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
22+
kind: DockerClusterTemplate
23+
name: quick-start-cluster
24+
infrastructureNamingStrategy:
25+
template: "{{ .cluster.name }}-infra-{{ .random }}"
26+
workers:
27+
machineDeployments:
28+
- class: default-worker
29+
namingStrategy:
30+
template: "{{ .cluster.name }}-md-{{ .machineDeployment.topologyName }}-{{ .random }}"
31+
template:
32+
bootstrap:
33+
ref:
34+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
35+
kind: KubeadmConfigTemplate
36+
name: quick-start-default-worker-bootstraptemplate
37+
infrastructure:
38+
ref:
39+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
40+
kind: DockerMachineTemplate
41+
name: quick-start-default-worker-machinetemplate
42+
machinePools:
43+
- class: default-worker
44+
namingStrategy:
45+
template: "{{ .cluster.name }}-mp-{{ .machinePool.topologyName }}-{{ .random }}"
46+
template:
47+
bootstrap:
48+
ref:
49+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
50+
kind: KubeadmConfigTemplate
51+
name: quick-start-default-worker-bootstraptemplate
52+
infrastructure:
53+
ref:
54+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
55+
kind: DockerMachinePoolTemplate
56+
name: quick-start-default-worker-machinepooltemplate
57+
patches:
58+
- name: test-patch
59+
external:
60+
generateExtension: generate-patches.${EXTENSION_CONFIG_NAME:-test-extension}
61+
validateExtension: validate-topology.${EXTENSION_CONFIG_NAME:-test-extension}
62+
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-test-extension}
63+
---
64+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
65+
kind: DockerClusterTemplate
66+
metadata:
67+
name: quick-start-cluster
68+
spec:
69+
template:
70+
spec:
71+
failureDomains:
72+
fd1:
73+
controlPlane: true
74+
fd2:
75+
controlPlane: true
76+
fd3:
77+
controlPlane: true
78+
fd4:
79+
controlPlane: false
80+
fd5:
81+
controlPlane: false
82+
fd6:
83+
controlPlane: false
84+
fd7:
85+
controlPlane: false
86+
fd8:
87+
controlPlane: false
88+
---
89+
kind: KubeadmControlPlaneTemplate
90+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
91+
metadata:
92+
name: quick-start-control-plane
93+
spec:
94+
template:
95+
spec:
96+
machineTemplate:
97+
nodeDrainTimeout: 1s
98+
kubeadmConfigSpec:
99+
clusterConfiguration:
100+
apiServer:
101+
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
102+
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
103+
initConfiguration:
104+
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
105+
joinConfiguration:
106+
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
107+
---
108+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
109+
kind: DockerMachineTemplate
110+
metadata:
111+
name: quick-start-control-plane
112+
spec:
113+
template:
114+
spec:
115+
extraMounts:
116+
- containerPath: "/var/run/docker.sock"
117+
hostPath: "/var/run/docker.sock"
118+
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
119+
---
120+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
121+
kind: DockerMachineTemplate
122+
metadata:
123+
name: quick-start-default-worker-machinetemplate
124+
spec:
125+
template:
126+
spec:
127+
extraMounts:
128+
- containerPath: "/var/run/docker.sock"
129+
hostPath: "/var/run/docker.sock"
130+
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
131+
---
132+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
133+
kind: DockerMachinePoolTemplate
134+
metadata:
135+
name: quick-start-default-worker-machinepooltemplate
136+
spec:
137+
template:
138+
spec:
139+
template:
140+
extraMounts:
141+
- containerPath: "/var/run/docker.sock"
142+
hostPath: "/var/run/docker.sock"
143+
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
144+
---
145+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
146+
kind: KubeadmConfigTemplate
147+
metadata:
148+
name: quick-start-default-worker-bootstraptemplate
149+
spec:
150+
template:
151+
spec:
152+
joinConfiguration:
153+
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.

‎test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ spec:
5454
patches:
5555
- name: test-patch
5656
external:
57-
generatePatchesExtension: generate-patches.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
58-
validateTopologyExtension: validate-topology.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
59-
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
57+
generatePatchesExtension: generate-patches.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
58+
validateTopologyExtension: validate-topology.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
59+
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
6060
---
6161
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
6262
kind: DockerClusterTemplate

‎test/e2e/quick_start.go

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"maps"
2323
"os"
2424
"path/filepath"
25+
"time"
2526

2627
. "github.com/onsi/ginkgo/v2"
2728
. "github.com/onsi/gomega"
@@ -70,6 +71,19 @@ type QuickStartSpecInput struct {
7071
// which unblocks CNI installation, and for the control plane machines to be ready (after CNI installation).
7172
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
7273

74+
// ExtensionConfigName is the name of the ExtensionConfig. Defaults to "quick-start".
75+
// This value is provided to clusterctl as "EXTENSION_CONFIG_NAME" variable and can be used to template the
76+
// name of the ExtensionConfig into the ClusterClass.
77+
ExtensionConfigName string
78+
79+
// ExtensionServiceNamespace is the namespace where the service for the Runtime Extension is located.
80+
// Note: This should only be set if a Runtime Extension is used.
81+
ExtensionServiceNamespace string
82+
83+
// ExtensionServiceNamespace is the name where the service for the Runtime Extension is located.
84+
// Note: This should only be set if a Runtime Extension is used.
85+
ExtensionServiceName string
86+
7387
// Allows to inject a function to be run after test namespace is created.
7488
// If not specified, this is a no-op.
7589
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
@@ -107,6 +121,12 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
107121

108122
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
109123

124+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
125+
if input.ExtensionConfigName == "" {
126+
input.ExtensionConfigName = specName
127+
}
128+
}
129+
110130
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
111131
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
112132

@@ -146,7 +166,36 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
146166
clusterName = *input.ClusterName
147167
}
148168

149-
variables := map[string]string{}
169+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
170+
// NOTE: test extension is already deployed in the management cluster. If for any reason in future we want
171+
// to make this test more self-contained this test should be modified in order to create an additional
172+
// management cluster; also the E2E test configuration should be modified introducing something like
173+
// optional:true allowing to define which providers should not be installed by default in
174+
// a management cluster.
175+
By("Deploy Test Extension ExtensionConfig")
176+
177+
// In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the
178+
// cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension
179+
// automatically creates the ConfigMap with non-blocking preloaded responses.
180+
defaultAllHandlersToBlocking := false
181+
// select on the current namespace
182+
// This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks
183+
// in other test namespaces.
184+
namespaces := []string{namespace.Name}
185+
if input.DeployClusterClassInSeparateNamespace {
186+
// Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace.
187+
namespaces = append(namespaces, clusterClassNamespace.Name)
188+
}
189+
extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...)
190+
Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
191+
extensionConfig)).
192+
To(Succeed(), "Failed to create the ExtensionConfig")
193+
}
194+
195+
variables := map[string]string{
196+
// This is used to template the name of the ExtensionConfig into the ClusterClass.
197+
"EXTENSION_CONFIG_NAME": input.ExtensionConfigName,
198+
}
150199
maps.Copy(variables, input.ClusterctlVariables)
151200

152201
if input.DeployClusterClassInSeparateNamespace {
@@ -200,11 +249,18 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
200249
AfterEach(func() {
201250
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
202251
framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
203-
if input.DeployClusterClassInSeparateNamespace && !input.SkipCleanup {
204-
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
205-
Deleter: input.BootstrapClusterProxy.GetClient(),
206-
Name: clusterClassNamespace.Name,
207-
})
252+
if !input.SkipCleanup {
253+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
254+
Eventually(func() error {
255+
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true))
256+
}, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed")
257+
}
258+
if input.DeployClusterClassInSeparateNamespace {
259+
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
260+
Deleter: input.BootstrapClusterProxy.GetClient(),
261+
Name: clusterClassNamespace.Name,
262+
})
263+
}
208264
}
209265
})
210266
}

‎test/e2e/quick_start_test.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,31 @@ var _ = Describe("When following the Cluster API quick-start with ClusterClass [
122122
})
123123
})
124124

125+
var _ = Describe("When following the Cluster API quick-start with v1beta1 ClusterClass [ClusterClass]", Label("ClusterClass"), func() {
126+
QuickStartSpec(ctx, func() QuickStartSpecInput {
127+
return QuickStartSpecInput{
128+
E2EConfig: e2eConfig,
129+
ClusterctlConfigPath: clusterctlConfigPath,
130+
BootstrapClusterProxy: bootstrapClusterProxy,
131+
ArtifactFolder: artifactFolder,
132+
SkipCleanup: skipCleanup,
133+
Flavor: ptr.To("topology-runtimesdk-v1beta1"),
134+
// The runtime extension gets deployed to the test-extension-system namespace and is exposed
135+
// by the test-extension-webhook-service.
136+
// The below values are used when creating the cluster-wide ExtensionConfig to refer
137+
// the actual service.
138+
ExtensionServiceNamespace: "test-extension-system",
139+
ExtensionServiceName: "test-extension-webhook-service",
140+
PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
141+
// This check ensures that the resourceVersions are stable, i.e. it verifies there are no
142+
// continuous reconciles when everything should be stable.
143+
By("Checking that resourceVersions are stable")
144+
framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
145+
},
146+
}
147+
})
148+
})
149+
125150
// NOTE: This test requires an IPv6 management cluster (can be configured via IP_FAMILY=IPv6).
126151
var _ = Describe("When following the Cluster API quick-start with IPv6 [IPv6]", Label("IPv6"), func() {
127152
QuickStartSpec(ctx, func() QuickStartSpecInput {

0 commit comments

Comments
 (0)