Skip to content

Commit e0cb9d5

Browse files
sbueringerk8s-infra-cherrypick-robot
sbueringer
authored and
k8s-infra-cherrypick-robot
committed
Add v1beta1 with ClusterClass and RuntimeSDK quickstart e2e test
Signed-off-by: Stefan Büringer [email protected]
1 parent 2fbf12b commit e0cb9d5

File tree

10 files changed

+302
-10
lines changed

10 files changed

+302
-10
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ hack/tools/bin
1414
test/e2e/data/infrastructure-docker/**/cluster-template*.yaml
1515
!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start.yaml
1616
!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start-runtimesdk.yaml
17+
!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start-runtimesdk-v1beta1.yaml
1718
!test/e2e/data/infrastructure-docker/**/cluster-template-in-memory.yaml
1819
!test/e2e/data/infrastructure-docker/**/clusterclass-in-memory.yaml
1920
test/e2e/data/infrastructure-docker/**/clusterclass-*.yaml

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -627,6 +627,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
627627
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary.yaml
628628
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary.yaml
629629
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-no-workers --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-no-workers.yaml
630+
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-runtimesdk-v1beta1 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-runtimesdk-v1beta1.yaml
630631
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-kcp-only --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-kcp-only.yaml
631632
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler.yaml
632633
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology.yaml

‎test/e2e/config/docker.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,7 @@ providers:
343343
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary.yaml"
344344
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary.yaml"
345345
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-no-workers.yaml"
346+
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1.yaml"
346347
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-kcp-only.yaml"
347348
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-autoscaler.yaml"
348349
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml"
@@ -351,6 +352,7 @@ providers:
351352
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml"
352353
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml"
353354
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml"
355+
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml"
354356
- sourcePath: "../data/infrastructure-docker/main/clusterclass-in-memory.yaml"
355357
- sourcePath: "../data/shared/main/metadata.yaml"
356358

‎test/e2e/data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1/cluster-runtimesdk.yaml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: Cluster
3+
metadata:
4+
name: '${CLUSTER_NAME}'
5+
namespace: default
6+
labels:
7+
cni: "${CLUSTER_NAME}-crs-0"
8+
spec:
9+
clusterNetwork:
10+
services:
11+
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
12+
pods:
13+
cidrBlocks: ['${DOCKER_POD_CIDRS}']
14+
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
15+
topology:
16+
class: "quick-start-runtimesdk-v1beta1"
17+
classNamespace: '${CLUSTER_CLASS_NAMESPACE:-${NAMESPACE}}'
18+
version: "${KUBERNETES_VERSION}"
19+
controlPlane:
20+
nodeDeletionTimeout: 30s
21+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
22+
workers:
23+
machineDeployments:
24+
- class: "default-worker"
25+
name: "md-0"
26+
nodeDeletionTimeout: 30s
27+
nodeVolumeDetachTimeout: 300s
28+
minReadySeconds: 5
29+
replicas: ${WORKER_MACHINE_COUNT}
30+
failureDomain: fd4
31+
machinePools:
32+
- class: "default-worker"
33+
name: "mp-0"
34+
nodeDeletionTimeout: 30s
35+
nodeVolumeDetachTimeout: 300s
36+
minReadySeconds: 5
37+
replicas: ${WORKER_MACHINE_COUNT}
38+
failureDomains:
39+
- fd4
40+
variables:
41+
- name: kubeadmControlPlaneMaxSurge
42+
value: "1"
43+
- name: imageRepository
44+
value: "kindest"

‎test/e2e/data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1/kustomization.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
resources:
2+
- ../bases/crs.yaml
3+
- cluster-runtimesdk.yaml

‎test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: ClusterClass
3+
metadata:
4+
name: quick-start-runtimesdk-v1beta1
5+
spec:
6+
controlPlane:
7+
ref:
8+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
9+
kind: KubeadmControlPlaneTemplate
10+
name: quick-start-control-plane
11+
machineInfrastructure:
12+
ref:
13+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
14+
kind: DockerMachineTemplate
15+
name: quick-start-control-plane
16+
namingStrategy:
17+
template: "{{ .cluster.name }}-cp-{{ .random }}"
18+
infrastructure:
19+
ref:
20+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
21+
kind: DockerClusterTemplate
22+
name: quick-start-cluster
23+
infrastructureNamingStrategy:
24+
template: "{{ .cluster.name }}-infra-{{ .random }}"
25+
workers:
26+
machineDeployments:
27+
- class: default-worker
28+
namingStrategy:
29+
template: "{{ .cluster.name }}-md-{{ .machineDeployment.topologyName }}-{{ .random }}"
30+
template:
31+
bootstrap:
32+
ref:
33+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
34+
kind: KubeadmConfigTemplate
35+
name: quick-start-default-worker-bootstraptemplate
36+
infrastructure:
37+
ref:
38+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
39+
kind: DockerMachineTemplate
40+
name: quick-start-default-worker-machinetemplate
41+
machinePools:
42+
- class: default-worker
43+
namingStrategy:
44+
template: "{{ .cluster.name }}-mp-{{ .machinePool.topologyName }}-{{ .random }}"
45+
template:
46+
bootstrap:
47+
ref:
48+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
49+
kind: KubeadmConfigTemplate
50+
name: quick-start-default-worker-bootstraptemplate
51+
infrastructure:
52+
ref:
53+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
54+
kind: DockerMachinePoolTemplate
55+
name: quick-start-default-worker-machinepooltemplate
56+
patches:
57+
- name: test-patch
58+
external:
59+
generateExtension: generate-patches.${EXTENSION_CONFIG_NAME:-test-extension}
60+
validateExtension: validate-topology.${EXTENSION_CONFIG_NAME:-test-extension}
61+
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-test-extension}
62+
---
63+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
64+
kind: DockerClusterTemplate
65+
metadata:
66+
name: quick-start-cluster
67+
spec:
68+
template:
69+
spec:
70+
failureDomains:
71+
fd1:
72+
controlPlane: true
73+
fd2:
74+
controlPlane: true
75+
fd3:
76+
controlPlane: true
77+
fd4:
78+
controlPlane: false
79+
fd5:
80+
controlPlane: false
81+
fd6:
82+
controlPlane: false
83+
fd7:
84+
controlPlane: false
85+
fd8:
86+
controlPlane: false
87+
---
88+
kind: KubeadmControlPlaneTemplate
89+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
90+
metadata:
91+
name: quick-start-control-plane
92+
spec:
93+
template:
94+
spec:
95+
machineTemplate:
96+
nodeDrainTimeout: 1s
97+
kubeadmConfigSpec:
98+
clusterConfiguration:
99+
apiServer:
100+
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
101+
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
102+
initConfiguration:
103+
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
104+
joinConfiguration:
105+
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
106+
---
107+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
108+
kind: DockerMachineTemplate
109+
metadata:
110+
name: quick-start-control-plane
111+
spec:
112+
template:
113+
spec:
114+
extraMounts:
115+
- containerPath: "/var/run/docker.sock"
116+
hostPath: "/var/run/docker.sock"
117+
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
118+
---
119+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
120+
kind: DockerMachineTemplate
121+
metadata:
122+
name: quick-start-default-worker-machinetemplate
123+
spec:
124+
template:
125+
spec:
126+
extraMounts:
127+
- containerPath: "/var/run/docker.sock"
128+
hostPath: "/var/run/docker.sock"
129+
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
130+
---
131+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
132+
kind: DockerMachinePoolTemplate
133+
metadata:
134+
name: quick-start-default-worker-machinepooltemplate
135+
spec:
136+
template:
137+
spec:
138+
template:
139+
extraMounts:
140+
- containerPath: "/var/run/docker.sock"
141+
hostPath: "/var/run/docker.sock"
142+
preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
143+
---
144+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
145+
kind: KubeadmConfigTemplate
146+
metadata:
147+
name: quick-start-default-worker-bootstraptemplate
148+
spec:
149+
template:
150+
spec:
151+
joinConfiguration:
152+
nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.

‎test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ spec:
5454
patches:
5555
- name: test-patch
5656
external:
57-
generatePatchesExtension: generate-patches.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
58-
validateTopologyExtension: validate-topology.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
59-
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
57+
generatePatchesExtension: generate-patches.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
58+
validateTopologyExtension: validate-topology.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
59+
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
6060
---
6161
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
6262
kind: DockerClusterTemplate

‎test/e2e/quick_start.go

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"maps"
2323
"os"
2424
"path/filepath"
25+
"time"
2526

2627
. "github.com/onsi/ginkgo/v2"
2728
. "github.com/onsi/gomega"
@@ -70,6 +71,19 @@ type QuickStartSpecInput struct {
7071
// which unblocks CNI installation, and for the control plane machines to be ready (after CNI installation).
7172
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
7273

74+
// ExtensionConfigName is the name of the ExtensionConfig. Defaults to "quick-start".
75+
// This value is provided to clusterctl as "EXTENSION_CONFIG_NAME" variable and can be used to template the
76+
// name of the ExtensionConfig into the ClusterClass.
77+
ExtensionConfigName string
78+
79+
// ExtensionServiceNamespace is the namespace where the service for the Runtime Extension is located.
80+
// Note: This should only be set if a Runtime Extension is used.
81+
ExtensionServiceNamespace string
82+
83+
// ExtensionServiceNamespace is the name where the service for the Runtime Extension is located.
84+
// Note: This should only be set if a Runtime Extension is used.
85+
ExtensionServiceName string
86+
7387
// Allows to inject a function to be run after test namespace is created.
7488
// If not specified, this is a no-op.
7589
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
@@ -107,6 +121,12 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
107121

108122
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
109123

124+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
125+
if input.ExtensionConfigName == "" {
126+
input.ExtensionConfigName = specName
127+
}
128+
}
129+
110130
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
111131
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
112132

@@ -146,7 +166,36 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
146166
clusterName = *input.ClusterName
147167
}
148168

149-
variables := map[string]string{}
169+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
170+
// NOTE: test extension is already deployed in the management cluster. If for any reason in future we want
171+
// to make this test more self-contained this test should be modified in order to create an additional
172+
// management cluster; also the E2E test configuration should be modified introducing something like
173+
// optional:true allowing to define which providers should not be installed by default in
174+
// a management cluster.
175+
By("Deploy Test Extension ExtensionConfig")
176+
177+
// In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the
178+
// cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension
179+
// automatically creates the ConfigMap with non-blocking preloaded responses.
180+
defaultAllHandlersToBlocking := false
181+
// select on the current namespace
182+
// This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks
183+
// in other test namespaces.
184+
namespaces := []string{namespace.Name}
185+
if input.DeployClusterClassInSeparateNamespace {
186+
// Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace.
187+
namespaces = append(namespaces, clusterClassNamespace.Name)
188+
}
189+
extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...)
190+
Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
191+
extensionConfig)).
192+
To(Succeed(), "Failed to create the ExtensionConfig")
193+
}
194+
195+
variables := map[string]string{
196+
// This is used to template the name of the ExtensionConfig into the ClusterClass.
197+
"EXTENSION_CONFIG_NAME": input.ExtensionConfigName,
198+
}
150199
maps.Copy(variables, input.ClusterctlVariables)
151200

152201
if input.DeployClusterClassInSeparateNamespace {
@@ -200,11 +249,18 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
200249
AfterEach(func() {
201250
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
202251
framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
203-
if input.DeployClusterClassInSeparateNamespace && !input.SkipCleanup {
204-
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
205-
Deleter: input.BootstrapClusterProxy.GetClient(),
206-
Name: clusterClassNamespace.Name,
207-
})
252+
if !input.SkipCleanup {
253+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
254+
Eventually(func() error {
255+
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true))
256+
}, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed")
257+
}
258+
if input.DeployClusterClassInSeparateNamespace {
259+
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
260+
Deleter: input.BootstrapClusterProxy.GetClient(),
261+
Name: clusterClassNamespace.Name,
262+
})
263+
}
208264
}
209265
})
210266
}

‎test/e2e/quick_start_test.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,31 @@ var _ = Describe("When following the Cluster API quick-start with ClusterClass [
122122
})
123123
})
124124

125+
var _ = Describe("When following the Cluster API quick-start with v1beta1 ClusterClass [ClusterClass]", Label("ClusterClass"), func() {
126+
QuickStartSpec(ctx, func() QuickStartSpecInput {
127+
return QuickStartSpecInput{
128+
E2EConfig: e2eConfig,
129+
ClusterctlConfigPath: clusterctlConfigPath,
130+
BootstrapClusterProxy: bootstrapClusterProxy,
131+
ArtifactFolder: artifactFolder,
132+
SkipCleanup: skipCleanup,
133+
Flavor: ptr.To("topology-runtimesdk-v1beta1"),
134+
// The runtime extension gets deployed to the test-extension-system namespace and is exposed
135+
// by the test-extension-webhook-service.
136+
// The below values are used when creating the cluster-wide ExtensionConfig to refer
137+
// the actual service.
138+
ExtensionServiceNamespace: "test-extension-system",
139+
ExtensionServiceName: "test-extension-webhook-service",
140+
PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
141+
// This check ensures that the resourceVersions are stable, i.e. it verifies there are no
142+
// continuous reconciles when everything should be stable.
143+
By("Checking that resourceVersions are stable")
144+
framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
145+
},
146+
}
147+
})
148+
})
149+
125150
// NOTE: This test requires an IPv6 management cluster (can be configured via IP_FAMILY=IPv6).
126151
var _ = Describe("When following the Cluster API quick-start with IPv6 [IPv6]", Label("IPv6"), func() {
127152
QuickStartSpec(ctx, func() QuickStartSpecInput {

0 commit comments

Comments
 (0)