Skip to content

Commit d584542

Browse files
Prajyot-ParabPrajyot-Parab
authored
Integrate MHC for worker nodes with e2e flow (#802)
Signed-off-by: Prajyot-Parab <[email protected]> Signed-off-by: Prajyot-Parab <[email protected]>
1 parent 101fbd9 commit d584542

File tree

2 files changed

+264
-1
lines changed

2 files changed

+264
-1
lines changed

‎test/e2e/config/ibmcloud-e2e-powervs.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,11 @@ providers:
4040
targetName: "metadata.yaml"
4141
- sourcePath: "${PWD}/templates/cluster-template-powervs.yaml"
4242
targetName: "cluster-template-powervs.yaml"
43+
- sourcePath: "${PWD}/test/e2e/data/templates/cluster-template-md-remediation.yaml"
44+
targetName: "cluster-template-md-remediation.yaml"
4345

4446
variables:
45-
KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.23.5}"
47+
KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.24.2}"
4648
# Below variable should be set based on the targeted environment
4749
SERVICE_ENDPOINT: "${SERVICE_ENDPOINT:-}"
4850
# Cluster Addons

‎test/e2e/data/templates/cluster-template-md-remediation.yaml

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: Cluster
3+
metadata:
4+
labels:
5+
cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}"
6+
name: "${CLUSTER_NAME}"
7+
spec:
8+
clusterNetwork:
9+
pods:
10+
cidrBlocks:
11+
- ${POD_CIDR:="192.168.0.0/16"}
12+
serviceDomain: ${SERVICE_DOMAIN:="cluster.local"}
13+
services:
14+
cidrBlocks:
15+
- ${SERVICE_CIDR:="10.128.0.0/12"}
16+
infrastructureRef:
17+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
18+
kind: IBMPowerVSCluster
19+
name: "${CLUSTER_NAME}"
20+
controlPlaneRef:
21+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
22+
kind: KubeadmControlPlane
23+
name: "${CLUSTER_NAME}-control-plane"
24+
---
25+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
26+
kind: IBMPowerVSCluster
27+
metadata:
28+
labels:
29+
cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}"
30+
name: "${CLUSTER_NAME}"
31+
spec:
32+
serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}"
33+
network:
34+
name: "${IBMPOWERVS_NETWORK_NAME}"
35+
controlPlaneEndpoint:
36+
host: "${IBMPOWERVS_VIP_EXTERNAL}"
37+
port: 6443
38+
---
39+
kind: KubeadmControlPlane
40+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
41+
metadata:
42+
name: "${CLUSTER_NAME}-control-plane"
43+
spec:
44+
version: "${KUBERNETES_VERSION}"
45+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
46+
machineTemplate:
47+
infrastructureRef:
48+
kind: IBMPowerVSMachineTemplate
49+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
50+
name: "${CLUSTER_NAME}-control-plane"
51+
kubeadmConfigSpec:
52+
clusterConfiguration:
53+
apiServer:
54+
certSANs:
55+
- "${IBMPOWERVS_VIP}"
56+
- "${IBMPOWERVS_VIP_EXTERNAL}"
57+
controlPlaneEndpoint: ${IBMPOWERVS_VIP}:6443
58+
controllerManager:
59+
extraArgs:
60+
enable-hostpath-provisioner: "true"
61+
initConfiguration:
62+
nodeRegistration:
63+
criSocket: /var/run/containerd/containerd.sock
64+
kubeletExtraArgs:
65+
cloud-provider: external
66+
provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}'
67+
eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
68+
name: '{{ v1.local_hostname }}'
69+
joinConfiguration:
70+
discovery:
71+
bootstrapToken:
72+
apiServerEndpoint: ${IBMPOWERVS_VIP}:6443
73+
token: ""
74+
caCertHashes: [ ]
75+
unsafeSkipCAVerification: false
76+
nodeRegistration:
77+
criSocket: /var/run/containerd/containerd.sock
78+
kubeletExtraArgs:
79+
cloud-provider: external
80+
provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}'
81+
eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
82+
name: '{{ v1.local_hostname }}'
83+
useExperimentalRetryJoin: true
84+
preKubeadmCommands:
85+
- hostname "{{ v1.local_hostname }}"
86+
- echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts
87+
- echo "127.0.0.1 localhost" >>/etc/hosts
88+
- echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts
89+
- echo "{{ v1.local_hostname }}" >/etc/hostname
90+
files:
91+
- path: /etc/kubernetes/manifests/kube-vip.yaml
92+
content: |
93+
apiVersion: v1
94+
kind: Pod
95+
metadata:
96+
creationTimestamp: null
97+
name: kube-vip
98+
namespace: kube-system
99+
spec:
100+
containers:
101+
- args:
102+
- manager
103+
env:
104+
- name: vip_arp
105+
value: "true"
106+
- name: port
107+
value: "6443"
108+
- name: vip_interface
109+
value: env2
110+
- name: vip_cidr
111+
value: "${IBMPOWERVS_VIP_CIDR}"
112+
- name: cp_enable
113+
value: "true"
114+
- name: cp_namespace
115+
value: kube-system
116+
- name: vip_ddns
117+
value: "false"
118+
- name: svc_enable
119+
value: "true"
120+
- name: vip_leaderelection
121+
value: "true"
122+
- name: vip_leaseduration
123+
value: "5"
124+
- name: vip_renewdeadline
125+
value: "3"
126+
- name: vip_retryperiod
127+
value: "1"
128+
- name: address
129+
value: "${IBMPOWERVS_VIP}"
130+
image: ghcr.io/kube-vip/kube-vip:v0.4.4
131+
imagePullPolicy: Always
132+
name: kube-vip
133+
resources: {}
134+
securityContext:
135+
capabilities:
136+
add:
137+
- NET_ADMIN
138+
- NET_RAW
139+
volumeMounts:
140+
- mountPath: /etc/kubernetes/admin.conf
141+
name: kubeconfig
142+
hostAliases:
143+
- hostnames:
144+
- kubernetes
145+
ip: 127.0.0.1
146+
hostNetwork: true
147+
volumes:
148+
- hostPath:
149+
path: /etc/kubernetes/admin.conf
150+
name: kubeconfig
151+
status: {}
152+
owner: "root:root"
153+
permissions: "0744"
154+
---
155+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
156+
kind: IBMPowerVSMachineTemplate
157+
metadata:
158+
name: "${CLUSTER_NAME}-control-plane"
159+
spec:
160+
template:
161+
spec:
162+
serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}"
163+
sshKey: "${IBMPOWERVS_SSHKEY_NAME}"
164+
image:
165+
name: "${IBMPOWERVS_IMAGE_NAME}"
166+
network:
167+
name: "${IBMPOWERVS_NETWORK_NAME}"
168+
memory: ${IBMPOWERVS_CONTROL_PLANE_MEMORY:="4"}
169+
processors: ${IBMPOWERVS_CONTROL_PLANE_PROCESSORS:="0.25"}
170+
sysType: ${IBMPOWERVS_CONTROL_PLANE_SYSTYPE:="s922"}
171+
procType: ${IBMPOWERVS_CONTROL_PLANE_PROCTYPE:="shared"}
172+
---
173+
apiVersion: cluster.x-k8s.io/v1beta1
174+
kind: MachineDeployment
175+
metadata:
176+
name: "${CLUSTER_NAME}-md-0"
177+
spec:
178+
clusterName: "${CLUSTER_NAME}"
179+
replicas: ${WORKER_MACHINE_COUNT}
180+
selector: {}
181+
template:
182+
metadata:
183+
labels:
184+
e2e.remediation.label: ""
185+
spec:
186+
clusterName: "${CLUSTER_NAME}"
187+
version: "${KUBERNETES_VERSION}"
188+
bootstrap:
189+
configRef:
190+
name: "${CLUSTER_NAME}-md-0"
191+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
192+
kind: KubeadmConfigTemplate
193+
infrastructureRef:
194+
name: "${CLUSTER_NAME}-md-0"
195+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
196+
kind: IBMPowerVSMachineTemplate
197+
---
198+
apiVersion: cluster.x-k8s.io/v1beta1
199+
kind: MachineHealthCheck
200+
metadata:
201+
name: "${CLUSTER_NAME}-mhc-0"
202+
spec:
203+
clusterName: "${CLUSTER_NAME}"
204+
maxUnhealthy: 100%
205+
nodeStartupTimeout: 20m
206+
selector:
207+
matchLabels:
208+
e2e.remediation.label: ""
209+
unhealthyConditions:
210+
- type: Ready
211+
status: "False"
212+
timeout: 30s
213+
---
214+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
215+
kind: IBMPowerVSMachineTemplate
216+
metadata:
217+
name: "${CLUSTER_NAME}-md-0"
218+
spec:
219+
template:
220+
spec:
221+
serviceInstanceID: "${IBMPOWERVS_SERVICE_INSTANCE_ID}"
222+
sshKey: "${IBMPOWERVS_SSHKEY_NAME}"
223+
image:
224+
name: "${IBMPOWERVS_IMAGE_NAME}"
225+
network:
226+
name: "${IBMPOWERVS_NETWORK_NAME}"
227+
memory: ${IBMPOWERVS_COMPUTE_MEMORY:="4"}
228+
processors: ${IBMPOWERVS_COMPUTE_PROCESSORS:="0.25"}
229+
sysType: ${IBMPOWERVS_COMPUTE_SYSTYPE:="s922"}
230+
procType: ${IBMPOWERVS_COMPUTE_PROCTYPE:="shared"}
231+
---
232+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
233+
kind: KubeadmConfigTemplate
234+
metadata:
235+
labels:
236+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
237+
cluster.x-k8s.io/control-plane: ""
238+
name: "${CLUSTER_NAME}-md-0"
239+
spec:
240+
template:
241+
spec:
242+
joinConfiguration:
243+
discovery:
244+
bootstrapToken:
245+
apiServerEndpoint: ${IBMPOWERVS_VIP}:6443
246+
token: ""
247+
caCertHashes: []
248+
unsafeSkipCAVerification: false
249+
nodeRegistration:
250+
criSocket: /var/run/containerd/containerd.sock
251+
kubeletExtraArgs:
252+
cloud-provider: external
253+
provider-id: ibmpowervs://${CLUSTER_NAME}/'{{ v1.local_hostname }}'
254+
eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
255+
name: '{{ v1.local_hostname }}'
256+
preKubeadmCommands:
257+
- hostname "{{ v1.local_hostname }}"
258+
- echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts
259+
- echo "127.0.0.1 localhost" >>/etc/hosts
260+
- echo "127.0.0.1 {{ v1.local_hostname }}" >>/etc/hosts
261+
- echo "{{ v1.local_hostname }}" >/etc/hostname

0 commit comments

Comments
 (0)