Skip to content
This repository was archived by the owner on Aug 12, 2025. It is now read-only.

Commit 1b70171

Browse files
cpriviterecprivitere
committed
feat: add emlb templates for cluster generation and testing, update e2e tests to reference emlb template
Signed-off-by: Chris Privitere <[email protected]>
1 parent 6d5a0ba commit 1b70171

File tree

8 files changed

+509
-143
lines changed

8 files changed

+509
-143
lines changed

‎Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ e2e-test-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1bet
225225
mkdir -p $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/
226226
$(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template.yaml
227227
$(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-kube-vip.yaml
228+
$(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-emlb-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-emlb.yaml
228229
$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-kcp-scale-in.yaml
229230
$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-node-drain --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-node-drain.yaml
230231
$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-md-remediation.yaml
@@ -282,6 +283,8 @@ generate: ## Generate code
282283

283284
.PHONY: generate-templates
284285
generate-templates: $(KUSTOMIZE) ## Generate cluster templates
286+
$(KUSTOMIZE) build templates/experimental-emlb --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb.yaml
287+
$(KUSTOMIZE) build templates/experimental-emlb-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb-crs-cni.yaml
285288
$(KUSTOMIZE) build templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip-crs-cni.yaml
286289
$(KUSTOMIZE) build templates/experimental-kube-vip --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip.yaml
287290
$(KUSTOMIZE) build templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-crs-cni.yaml

‎templates/cluster-template-emlb-crs-cni.yaml

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
apiVersion: v1
2+
data: ${CNI_RESOURCES}
3+
kind: ConfigMap
4+
metadata:
5+
name: ${CLUSTER_NAME}-crs-cni
6+
---
7+
apiVersion: addons.cluster.x-k8s.io/v1beta1
8+
kind: ClusterResourceSet
9+
metadata:
10+
name: ${CLUSTER_NAME}-crs-cni
11+
spec:
12+
clusterSelector:
13+
matchLabels:
14+
cni: ${CLUSTER_NAME}-crs-cni
15+
resources:
16+
- kind: ConfigMap
17+
name: ${CLUSTER_NAME}-crs-cni
18+
strategy: ApplyOnce
19+
---
20+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
21+
kind: KubeadmConfigTemplate
22+
metadata:
23+
name: ${CLUSTER_NAME}-worker-a
24+
spec:
25+
template:
26+
spec:
27+
joinConfiguration:
28+
nodeRegistration:
29+
kubeletExtraArgs:
30+
cloud-provider: external
31+
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
32+
preKubeadmCommands:
33+
- |
34+
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
35+
swapoff -a
36+
mount -a
37+
cat <<EOF > /etc/modules-load.d/containerd.conf
38+
overlay
39+
br_netfilter
40+
EOF
41+
modprobe overlay
42+
modprobe br_netfilter
43+
cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
44+
net.bridge.bridge-nf-call-iptables = 1
45+
net.ipv4.ip_forward = 1
46+
net.bridge.bridge-nf-call-ip6tables = 1
47+
EOF
48+
sysctl --system
49+
export DEBIAN_FRONTEND=noninteractive
50+
apt-get update -y
51+
apt-get remove -y docker docker-engine containerd runc
52+
apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
53+
install -m 0755 -d /etc/apt/keyrings
54+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
55+
MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
56+
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
57+
chmod a+r /etc/apt/keyrings/docker.gpg
58+
chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
59+
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
60+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
61+
apt-get update -y
62+
TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
63+
RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
64+
apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
65+
cat <<EOF > /etc/crictl.yaml
66+
runtime-endpoint: unix:///run/containerd/containerd.sock
67+
image-endpoint: unix:///run/containerd/containerd.sock
68+
EOF
69+
containerd config default > /etc/containerd/config.toml
70+
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
71+
sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
72+
systemctl restart containerd
73+
---
74+
apiVersion: cluster.x-k8s.io/v1beta1
75+
kind: Cluster
76+
metadata:
77+
labels:
78+
cni: ${CLUSTER_NAME}-crs-cni
79+
name: ${CLUSTER_NAME}
80+
spec:
81+
clusterNetwork:
82+
pods:
83+
cidrBlocks:
84+
- ${POD_CIDR:=192.168.0.0/16}
85+
services:
86+
cidrBlocks:
87+
- ${SERVICE_CIDR:=172.26.0.0/16}
88+
controlPlaneRef:
89+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
90+
kind: KubeadmControlPlane
91+
name: ${CLUSTER_NAME}-control-plane
92+
infrastructureRef:
93+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
94+
kind: PacketCluster
95+
name: ${CLUSTER_NAME}
96+
---
97+
apiVersion: cluster.x-k8s.io/v1beta1
98+
kind: MachineDeployment
99+
metadata:
100+
labels:
101+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
102+
pool: worker-a
103+
name: ${CLUSTER_NAME}-worker-a
104+
spec:
105+
clusterName: ${CLUSTER_NAME}
106+
replicas: ${WORKER_MACHINE_COUNT}
107+
selector:
108+
matchLabels:
109+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
110+
pool: worker-a
111+
template:
112+
metadata:
113+
labels:
114+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
115+
pool: worker-a
116+
spec:
117+
bootstrap:
118+
configRef:
119+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
120+
kind: KubeadmConfigTemplate
121+
name: ${CLUSTER_NAME}-worker-a
122+
clusterName: ${CLUSTER_NAME}
123+
infrastructureRef:
124+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
125+
kind: PacketMachineTemplate
126+
name: ${CLUSTER_NAME}-worker-a
127+
version: ${KUBERNETES_VERSION}
128+
---
129+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
130+
kind: KubeadmControlPlane
131+
metadata:
132+
name: ${CLUSTER_NAME}-control-plane
133+
spec:
134+
kubeadmConfigSpec:
135+
clusterConfiguration:
136+
apiServer:
137+
extraArgs:
138+
cloud-provider: external
139+
controllerManager:
140+
extraArgs:
141+
cloud-provider: external
142+
initConfiguration:
143+
nodeRegistration:
144+
kubeletExtraArgs:
145+
cloud-provider: external
146+
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
147+
joinConfiguration:
148+
nodeRegistration:
149+
ignorePreflightErrors:
150+
- DirAvailable--etc-kubernetes-manifests
151+
kubeletExtraArgs:
152+
cloud-provider: external
153+
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
154+
postKubeadmCommands:
155+
- |-
156+
mkdir -p $HOME/.kube
157+
cp /etc/kubernetes/admin.conf $HOME/.kube/config
158+
echo "source <(kubectl completion bash)" >> $HOME/.bashrc
159+
echo "alias k=kubectl" >> $HOME/.bashrc
160+
echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc
161+
if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
162+
export KUBECONFIG=/etc/kubernetes/admin.conf
163+
export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.8.0}/deployment.yaml
164+
export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}"}'''
165+
kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
166+
kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
167+
fi
168+
preKubeadmCommands:
169+
- |
170+
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
171+
swapoff -a
172+
mount -a
173+
cat <<EOF > /etc/modules-load.d/containerd.conf
174+
overlay
175+
br_netfilter
176+
EOF
177+
modprobe overlay
178+
modprobe br_netfilter
179+
cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
180+
net.bridge.bridge-nf-call-iptables = 1
181+
net.ipv4.ip_forward = 1
182+
net.bridge.bridge-nf-call-ip6tables = 1
183+
EOF
184+
sysctl --system
185+
export DEBIAN_FRONTEND=noninteractive
186+
apt-get update -y
187+
apt-get remove -y docker docker-engine containerd runc
188+
apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
189+
major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
190+
if [ "$major_vers" -ge 20 ]; then
191+
apt-get install -y kubetail
192+
fi
193+
install -m 0755 -d /etc/apt/keyrings
194+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
195+
MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
196+
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
197+
chmod a+r /etc/apt/keyrings/docker.gpg
198+
chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
199+
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
200+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
201+
apt-get update -y
202+
TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
203+
RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
204+
apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
205+
containerd config default > /etc/containerd/config.toml
206+
cat <<EOF > /etc/crictl.yaml
207+
runtime-endpoint: unix:///run/containerd/containerd.sock
208+
image-endpoint: unix:///run/containerd/containerd.sock
209+
EOF
210+
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
211+
sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
212+
systemctl restart containerd
213+
if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
214+
ip addr add {{ .controlPlaneEndpoint }} dev lo
215+
fi
216+
machineTemplate:
217+
infrastructureRef:
218+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
219+
kind: PacketMachineTemplate
220+
name: ${CLUSTER_NAME}-control-plane
221+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
222+
version: ${KUBERNETES_VERSION}
223+
---
224+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
225+
kind: PacketCluster
226+
metadata:
227+
name: ${CLUSTER_NAME}
228+
spec:
229+
metro: ${METRO}
230+
projectID: ${PROJECT_ID}
231+
vipManager: EMLB
232+
---
233+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
234+
kind: PacketMachineTemplate
235+
metadata:
236+
name: ${CLUSTER_NAME}-control-plane
237+
spec:
238+
template:
239+
spec:
240+
billingCycle: hourly
241+
machineType: ${CONTROLPLANE_NODE_TYPE}
242+
os: ${NODE_OS:=ubuntu_20_04}
243+
sshKeys:
244+
- ${SSH_KEY}
245+
tags: []
246+
---
247+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
248+
kind: PacketMachineTemplate
249+
metadata:
250+
name: ${CLUSTER_NAME}-worker-a
251+
spec:
252+
template:
253+
spec:
254+
billingCycle: hourly
255+
machineType: ${WORKER_NODE_TYPE}
256+
os: ${NODE_OS:=ubuntu_20_04}
257+
sshKeys:
258+
- ${SSH_KEY}
259+
tags: []

0 commit comments

Comments
 (0)