Skip to content

Commit 58b0101

Browse files
tormath1tormath1
committed
test: add Flatcar e2e test
Signed-off-by: Mathieu Tortuyaux <[email protected]>
1 parent 84bd31a commit 58b0101

File tree

4 files changed

+318
-0
lines changed

4 files changed

+318
-0
lines changed

‎test/e2e/config/gcp-ci.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ providers:
7373
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml"
7474
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml"
7575
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml"
76+
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-flatcar.yaml"
7677

7778
variables:
7879
KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.31.0}"
@@ -86,6 +87,10 @@ variables:
8687
KUBERNETES_VERSION_UPGRADE_TO: "${KUBERNETES_VERSION_UPGRADE_TO:-v1.31.0}"
8788
KUBERNETES_VERSION_UPGRADE_FROM: "${KUBERNETES_VERSION_UPGRADE_FROM:-v1.30.5}"
8889
EXP_CLUSTER_RESOURCE_SET: "true"
90+
# Required for Ignition based images (e.g Flatcar)
91+
EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"
92+
# TODO: Set this based on latest available image
93+
FLATCAR_IMAGE_ID: "projects/kinvolk-public/global/images/flatcar-stable-4081-2-0"
8994
CLUSTER_TOPOLOGY: "true"
9095
# Cluster Addons
9196
CNI: "${PWD}/test/e2e/data/cni/calico/calico.yaml"

‎test/e2e/data/infrastructure-gcp/cluster-template-ci-flatcar.yaml

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: "${CLUSTER_NAME}"
6+
labels:
7+
cni: "${CLUSTER_NAME}-crs-cni"
8+
spec:
9+
clusterNetwork:
10+
pods:
11+
cidrBlocks: ["192.168.0.0/16"]
12+
infrastructureRef:
13+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
14+
kind: GCPCluster
15+
name: "${CLUSTER_NAME}"
16+
controlPlaneRef:
17+
kind: KubeadmControlPlane
18+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
19+
name: "${CLUSTER_NAME}-control-plane"
20+
---
21+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
22+
kind: GCPCluster
23+
metadata:
24+
name: "${CLUSTER_NAME}"
25+
spec:
26+
project: "${GCP_PROJECT}"
27+
region: "${GCP_REGION}"
28+
network:
29+
name: "${GCP_NETWORK_NAME}"
30+
---
31+
kind: KubeadmControlPlane
32+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
33+
metadata:
34+
name: "${CLUSTER_NAME}-control-plane"
35+
spec:
36+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
37+
machineTemplate:
38+
infrastructureRef:
39+
kind: GCPMachineTemplate
40+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
41+
name: "${CLUSTER_NAME}-control-plane"
42+
kubeadmConfigSpec:
43+
files: []
44+
format: ignition
45+
ignition:
46+
containerLinuxConfig:
47+
additionalConfig: |
48+
storage:
49+
links:
50+
- path: /etc/extensions/kubernetes.raw
51+
hard: false
52+
target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
53+
files:
54+
- path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf
55+
mode: 0644
56+
contents:
57+
remote:
58+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf
59+
- path: /etc/sysupdate.d/noop.conf
60+
mode: 0644
61+
contents:
62+
remote:
63+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
64+
- path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
65+
contents:
66+
remote:
67+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
68+
systemd:
69+
units:
70+
- name: systemd-sysupdate.service
71+
dropins:
72+
- name: kubernetes.conf
73+
contents: |
74+
[Service]
75+
ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
76+
ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
77+
ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
78+
ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi"
79+
- name: update-engine.service
80+
# Set this to 'false' if you want to enable Flatcar auto-update
81+
mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
82+
- name: locksmithd.service
83+
# NOTE: To coordinate the node reboot in this context, we recommend to use Kured.
84+
mask: true
85+
- name: systemd-sysupdate.timer
86+
# Set this to 'true' if you want to enable the Kubernetes auto-update.
87+
# NOTE: Only patches version will be pulled.
88+
enabled: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
89+
- name: [email protected]
90+
enabled: true
91+
- name: kubeadm.service
92+
enabled: true
93+
dropins:
94+
- name: 10-flatcar.conf
95+
contents: |
96+
[Unit]
97+
Requires=containerd.service coreos-metadata.service
98+
After=containerd.service coreos-metadata.service
99+
[Service]
100+
EnvironmentFile=/run/metadata/flatcar
101+
initConfiguration:
102+
nodeRegistration:
103+
name: $${COREOS_GCE_HOSTNAME}
104+
kubeletExtraArgs:
105+
cloud-provider: gce
106+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
107+
clusterConfiguration:
108+
apiServer:
109+
timeoutForControlPlane: 20m
110+
extraArgs:
111+
cloud-provider: gce
112+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
113+
controllerManager:
114+
extraArgs:
115+
cloud-provider: gce
116+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
117+
allocate-node-cidrs: "false"
118+
joinConfiguration:
119+
nodeRegistration:
120+
name: $${COREOS_GCE_HOSTNAME}
121+
kubeletExtraArgs:
122+
cloud-provider: gce
123+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
124+
preKubeadmCommands:
125+
- export COREOS_GCE_HOSTNAME=$${COREOS_GCE_HOSTNAME%%.*}
126+
- envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
127+
- mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
128+
version: "${KUBERNETES_VERSION}"
129+
---
130+
kind: GCPMachineTemplate
131+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
132+
metadata:
133+
name: "${CLUSTER_NAME}-control-plane"
134+
spec:
135+
template:
136+
spec:
137+
instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
138+
image: "${IMAGE_ID}"
139+
---
140+
apiVersion: cluster.x-k8s.io/v1beta1
141+
kind: MachineDeployment
142+
metadata:
143+
name: "${CLUSTER_NAME}-md-0"
144+
spec:
145+
clusterName: "${CLUSTER_NAME}"
146+
replicas: ${WORKER_MACHINE_COUNT}
147+
selector:
148+
matchLabels:
149+
template:
150+
spec:
151+
clusterName: "${CLUSTER_NAME}"
152+
version: "${KUBERNETES_VERSION}"
153+
bootstrap:
154+
configRef:
155+
name: "${CLUSTER_NAME}-md-0"
156+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
157+
kind: KubeadmConfigTemplate
158+
infrastructureRef:
159+
name: "${CLUSTER_NAME}-md-0"
160+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
161+
kind: GCPMachineTemplate
162+
---
163+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
164+
kind: GCPMachineTemplate
165+
metadata:
166+
name: "${CLUSTER_NAME}-md-0"
167+
spec:
168+
template:
169+
spec:
170+
instanceType: "${GCP_NODE_MACHINE_TYPE}"
171+
image: "${IMAGE_ID}"
172+
---
173+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
174+
kind: KubeadmConfigTemplate
175+
metadata:
176+
name: "${CLUSTER_NAME}-md-0"
177+
spec:
178+
template:
179+
spec:
180+
files: []
181+
format: ignition
182+
ignition:
183+
containerLinuxConfig:
184+
additionalConfig: |
185+
storage:
186+
links:
187+
- path: /etc/extensions/kubernetes.raw
188+
hard: false
189+
target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
190+
files:
191+
- path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf
192+
mode: 0644
193+
contents:
194+
remote:
195+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf
196+
- path: /etc/sysupdate.d/noop.conf
197+
mode: 0644
198+
contents:
199+
remote:
200+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
201+
- path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
202+
contents:
203+
remote:
204+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
205+
systemd:
206+
units:
207+
- name: systemd-sysupdate.service
208+
dropins:
209+
- name: kubernetes.conf
210+
contents: |
211+
[Service]
212+
ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
213+
ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
214+
ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
215+
ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi"
216+
- name: update-engine.service
217+
# Set this to 'false' if you want to enable Flatcar auto-update
218+
mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
219+
- name: locksmithd.service
220+
# NOTE: To coordinate the node reboot in this context, we recommend to use Kured.
221+
mask: true
222+
- name: systemd-sysupdate.timer
223+
# Set this to 'true' if you want to enable the Kubernetes auto-update.
224+
# NOTE: Only patches version will be pulled.
225+
enabled: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
226+
- name: [email protected]
227+
enabled: true
228+
- name: kubeadm.service
229+
enabled: true
230+
dropins:
231+
- name: 10-flatcar.conf
232+
contents: |
233+
[Unit]
234+
Requires=containerd.service coreos-metadata.service
235+
After=containerd.service coreos-metadata.service
236+
[Service]
237+
EnvironmentFile=/run/metadata/flatcar
238+
joinConfiguration:
239+
nodeRegistration:
240+
name: $${COREOS_GCE_HOSTNAME}
241+
kubeletExtraArgs:
242+
cloud-provider: gce
243+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
244+
preKubeadmCommands:
245+
- export COREOS_GCE_HOSTNAME=$${COREOS_GCE_HOSTNAME%%.*}
246+
- envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
247+
- mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
248+
---
249+
apiVersion: v1
250+
kind: ConfigMap
251+
metadata:
252+
name: "${CLUSTER_NAME}-crs-cni"
253+
data: ${CNI_RESOURCES}
254+
---
255+
apiVersion: addons.cluster.x-k8s.io/v1beta1
256+
kind: ClusterResourceSet
257+
metadata:
258+
name: "${CLUSTER_NAME}-crs-cni"
259+
spec:
260+
strategy: ApplyOnce
261+
clusterSelector:
262+
matchLabels:
263+
cni: "${CLUSTER_NAME}-crs-cni"
264+
resources:
265+
- name: "${CLUSTER_NAME}-crs-cni"
266+
kind: ConfigMap

‎test/e2e/e2e_test.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,4 +230,50 @@ var _ = Describe("Workload cluster creation", func() {
230230
}, result)
231231
})
232232
})
233+
234+
Context("Creating a single Flatcar control-plane cluster", func() {
235+
It("Should create a cluster with 1 Flatcar worker node and can be scaled", func() {
236+
By("Initializes with 1 Flatcar worker node")
237+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
238+
ClusterProxy: bootstrapClusterProxy,
239+
ConfigCluster: clusterctl.ConfigClusterInput{
240+
LogFolder: clusterctlLogFolder,
241+
ClusterctlConfigPath: clusterctlConfigPath,
242+
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
243+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
244+
Flavor: "ci-flatcar",
245+
Namespace: namespace.Name,
246+
ClusterName: clusterName,
247+
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
248+
ControlPlaneMachineCount: ptr.To[int64](1),
249+
WorkerMachineCount: ptr.To[int64](1),
250+
ClusterctlVariables: map[string]string{"IMAGE_ID": e2eConfig.GetVariable(FlatcarImageID)},
251+
},
252+
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
253+
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
254+
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
255+
}, result)
256+
257+
By("Scaling Flatcar worker node to 3")
258+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
259+
ClusterProxy: bootstrapClusterProxy,
260+
ConfigCluster: clusterctl.ConfigClusterInput{
261+
LogFolder: clusterctlLogFolder,
262+
ClusterctlConfigPath: clusterctlConfigPath,
263+
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
264+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
265+
Flavor: "ci-flatcar",
266+
Namespace: namespace.Name,
267+
ClusterName: clusterName,
268+
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
269+
ControlPlaneMachineCount: ptr.To[int64](1),
270+
WorkerMachineCount: ptr.To[int64](3),
271+
ClusterctlVariables: map[string]string{"IMAGE_ID": e2eConfig.GetVariable(FlatcarImageID)},
272+
},
273+
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
274+
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
275+
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
276+
}, result)
277+
})
278+
})
233279
})

‎test/e2e/suite_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
)
4444

4545
const (
46+
FlatcarImageID = "FLATCAR_IMAGE_ID"
4647
KubernetesVersion = "KUBERNETES_VERSION"
4748
KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT"
4849

0 commit comments

Comments
 (0)