Skip to content

Commit d4f5e68

Browse files
alvaroalemanalvaroaleman
authored
Add healthchecking for kubelet and docker (#352)
1 parent 8b5b66e commit d4f5e68

17 files changed

+650
-43
lines changed

‎pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ write_files:
6262
{aws-config:true}
6363

6464
- path: "/usr/local/bin/setup"
65-
permissions: "0777"
65+
permissions: "0755"
6666
content: |
6767
#!/bin/bash
6868
set -xeuo pipefail
@@ -83,14 +83,31 @@ write_files:
8383
systemctl enable --now docker
8484
systemctl enable --now kubelet
8585

86-
kubeadm join \
87-
--token my-token \
88-
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
89-
--ignore-preflight-errors=CRI \
90-
server:443
86+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
87+
curl -Lfo /usr/local/bin/health-monitor.sh \
88+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
89+
chmod +x /usr/local/bin/health-monitor.sh
90+
fi
91+
92+
if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
93+
kubeadm join \
94+
--token my-token \
95+
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
96+
--ignore-preflight-errors=CRI \
97+
server:443
98+
fi
99+
100+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
101+
curl -Lfo /usr/local/bin/health-monitor.sh \
102+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
103+
chmod +x /usr/local/bin/health-monitor.sh
104+
fi
105+
106+
systemctl enable --now --no-block kubelet-healthcheck.service
107+
systemctl enable --now --no-block docker-healthcheck.service
91108

92109
- path: "/usr/local/bin/supervise.sh"
93-
permissions: "0777"
110+
permissions: "0755"
94111
content: |
95112
#!/bin/bash
96113
set -xeuo pipefail
@@ -112,5 +129,31 @@ write_files:
112129
RemainAfterExit=true
113130
ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup
114131

132+
- path: /etc/systemd/system/kubelet-healthcheck.service
133+
permissions: "0644"
134+
content: |
135+
[Unit]
136+
Requires=kubelet.service
137+
After=kubelet.service
138+
139+
[Service]
140+
ExecStart=/usr/local/bin/health-monitor.sh kubelet
141+
142+
[Install]
143+
WantedBy=multi-user.target
144+
145+
- path: /etc/systemd/system/docker-healthcheck.service
146+
permissions: "0644"
147+
content: |
148+
[Unit]
149+
Requires=docker.service
150+
After=docker.service
151+
152+
[Service]
153+
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
154+
155+
[Install]
156+
WantedBy=multi-user.target
157+
115158
runcmd:
116159
- systemctl enable --now setup.service

‎pkg/userdata/centos/testdata/kubelet-v1.11-aws.golden‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ write_files:
5858
{aws-config:true}
5959

6060
- path: "/usr/local/bin/setup"
61-
permissions: "0777"
61+
permissions: "0755"
6262
content: |
6363
#!/bin/bash
6464
set -xeuo pipefail
@@ -79,14 +79,31 @@ write_files:
7979
systemctl enable --now docker
8080
systemctl enable --now kubelet
8181

82-
kubeadm join \
83-
--token my-token \
84-
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
85-
--ignore-preflight-errors=CRI \
86-
server:443
82+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
83+
curl -Lfo /usr/local/bin/health-monitor.sh \
84+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
85+
chmod +x /usr/local/bin/health-monitor.sh
86+
fi
87+
88+
if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
89+
kubeadm join \
90+
--token my-token \
91+
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
92+
--ignore-preflight-errors=CRI \
93+
server:443
94+
fi
95+
96+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
97+
curl -Lfo /usr/local/bin/health-monitor.sh \
98+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
99+
chmod +x /usr/local/bin/health-monitor.sh
100+
fi
101+
102+
systemctl enable --now --no-block kubelet-healthcheck.service
103+
systemctl enable --now --no-block docker-healthcheck.service
87104

88105
- path: "/usr/local/bin/supervise.sh"
89-
permissions: "0777"
106+
permissions: "0755"
90107
content: |
91108
#!/bin/bash
92109
set -xeuo pipefail
@@ -108,5 +125,31 @@ write_files:
108125
RemainAfterExit=true
109126
ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup
110127

128+
- path: /etc/systemd/system/kubelet-healthcheck.service
129+
permissions: "0644"
130+
content: |
131+
[Unit]
132+
Requires=kubelet.service
133+
After=kubelet.service
134+
135+
[Service]
136+
ExecStart=/usr/local/bin/health-monitor.sh kubelet
137+
138+
[Install]
139+
WantedBy=multi-user.target
140+
141+
- path: /etc/systemd/system/docker-healthcheck.service
142+
permissions: "0644"
143+
content: |
144+
[Unit]
145+
Requires=docker.service
146+
After=docker.service
147+
148+
[Service]
149+
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
150+
151+
[Install]
152+
WantedBy=multi-user.target
153+
111154
runcmd:
112155
- systemctl enable --now setup.service

‎pkg/userdata/centos/testdata/kubelet-v1.12-aws.golden‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ write_files:
5858
{aws-config:true}
5959

6060
- path: "/usr/local/bin/setup"
61-
permissions: "0777"
61+
permissions: "0755"
6262
content: |
6363
#!/bin/bash
6464
set -xeuo pipefail
@@ -79,14 +79,31 @@ write_files:
7979
systemctl enable --now docker
8080
systemctl enable --now kubelet
8181

82-
kubeadm join \
83-
--token my-token \
84-
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
85-
--ignore-preflight-errors=CRI \
86-
server:443
82+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
83+
curl -Lfo /usr/local/bin/health-monitor.sh \
84+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
85+
chmod +x /usr/local/bin/health-monitor.sh
86+
fi
87+
88+
if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
89+
kubeadm join \
90+
--token my-token \
91+
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
92+
--ignore-preflight-errors=CRI \
93+
server:443
94+
fi
95+
96+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
97+
curl -Lfo /usr/local/bin/health-monitor.sh \
98+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
99+
chmod +x /usr/local/bin/health-monitor.sh
100+
fi
101+
102+
systemctl enable --now --no-block kubelet-healthcheck.service
103+
systemctl enable --now --no-block docker-healthcheck.service
87104

88105
- path: "/usr/local/bin/supervise.sh"
89-
permissions: "0777"
106+
permissions: "0755"
90107
content: |
91108
#!/bin/bash
92109
set -xeuo pipefail
@@ -108,5 +125,31 @@ write_files:
108125
RemainAfterExit=true
109126
ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup
110127

128+
- path: /etc/systemd/system/kubelet-healthcheck.service
129+
permissions: "0644"
130+
content: |
131+
[Unit]
132+
Requires=kubelet.service
133+
After=kubelet.service
134+
135+
[Service]
136+
ExecStart=/usr/local/bin/health-monitor.sh kubelet
137+
138+
[Install]
139+
WantedBy=multi-user.target
140+
141+
- path: /etc/systemd/system/docker-healthcheck.service
142+
permissions: "0644"
143+
content: |
144+
[Unit]
145+
Requires=docker.service
146+
After=docker.service
147+
148+
[Service]
149+
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
150+
151+
[Install]
152+
WantedBy=multi-user.target
153+
111154
runcmd:
112155
- systemctl enable --now setup.service

‎pkg/userdata/centos/testdata/kubelet-v1.9-aws.golden‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ write_files:
6262
{aws-config:true}
6363

6464
- path: "/usr/local/bin/setup"
65-
permissions: "0777"
65+
permissions: "0755"
6666
content: |
6767
#!/bin/bash
6868
set -xeuo pipefail
@@ -83,14 +83,31 @@ write_files:
8383
systemctl enable --now docker
8484
systemctl enable --now kubelet
8585

86-
kubeadm join \
87-
--token my-token \
88-
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
89-
--ignore-preflight-errors=CRI \
90-
server:443
86+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
87+
curl -Lfo /usr/local/bin/health-monitor.sh \
88+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
89+
chmod +x /usr/local/bin/health-monitor.sh
90+
fi
91+
92+
if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
93+
kubeadm join \
94+
--token my-token \
95+
--discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
96+
--ignore-preflight-errors=CRI \
97+
server:443
98+
fi
99+
100+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
101+
curl -Lfo /usr/local/bin/health-monitor.sh \
102+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
103+
chmod +x /usr/local/bin/health-monitor.sh
104+
fi
105+
106+
systemctl enable --now --no-block kubelet-healthcheck.service
107+
systemctl enable --now --no-block docker-healthcheck.service
91108

92109
- path: "/usr/local/bin/supervise.sh"
93-
permissions: "0777"
110+
permissions: "0755"
94111
content: |
95112
#!/bin/bash
96113
set -xeuo pipefail
@@ -112,5 +129,31 @@ write_files:
112129
RemainAfterExit=true
113130
ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup
114131

132+
- path: /etc/systemd/system/kubelet-healthcheck.service
133+
permissions: "0644"
134+
content: |
135+
[Unit]
136+
Requires=kubelet.service
137+
After=kubelet.service
138+
139+
[Service]
140+
ExecStart=/usr/local/bin/health-monitor.sh kubelet
141+
142+
[Install]
143+
WantedBy=multi-user.target
144+
145+
- path: /etc/systemd/system/docker-healthcheck.service
146+
permissions: "0644"
147+
content: |
148+
[Unit]
149+
Requires=docker.service
150+
After=docker.service
151+
152+
[Service]
153+
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
154+
155+
[Install]
156+
WantedBy=multi-user.target
157+
115158
runcmd:
116159
- systemctl enable --now setup.service

‎pkg/userdata/centos/userdata.go‎

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ write_files:
209209
{{ if ne .CloudConfig "" }}{{ .CloudConfig | indent 4 }}{{ end }}
210210
211211
- path: "/usr/local/bin/setup"
212-
permissions: "0777"
212+
permissions: "0755"
213213
content: |
214214
#!/bin/bash
215215
set -xeuo pipefail
@@ -230,16 +230,33 @@ write_files:
230230
systemctl enable --now docker
231231
systemctl enable --now kubelet
232232
233-
kubeadm join \
234-
--token {{ .BoostrapToken }} \
235-
--discovery-token-ca-cert-hash sha256:{{ .KubeadmCACertHash }} \
236-
{{- if semverCompare ">=1.9.X" .KubeletVersion }}
237-
--ignore-preflight-errors=CRI \
238-
{{- end }}
239-
{{ .ServerAddr }}
233+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
234+
curl -Lfo /usr/local/bin/health-monitor.sh \
235+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
236+
chmod +x /usr/local/bin/health-monitor.sh
237+
fi
238+
239+
if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
240+
kubeadm join \
241+
--token {{ .BoostrapToken }} \
242+
--discovery-token-ca-cert-hash sha256:{{ .KubeadmCACertHash }} \
243+
{{- if semverCompare ">=1.9.X" .KubeletVersion }}
244+
--ignore-preflight-errors=CRI \
245+
{{- end }}
246+
{{ .ServerAddr }}
247+
fi
248+
249+
if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
250+
curl -Lfo /usr/local/bin/health-monitor.sh \
251+
https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
252+
chmod +x /usr/local/bin/health-monitor.sh
253+
fi
254+
255+
systemctl enable --now --no-block kubelet-healthcheck.service
256+
systemctl enable --now --no-block docker-healthcheck.service
240257
241258
- path: "/usr/local/bin/supervise.sh"
242-
permissions: "0777"
259+
permissions: "0755"
243260
content: |
244261
#!/bin/bash
245262
set -xeuo pipefail
@@ -261,6 +278,32 @@ write_files:
261278
RemainAfterExit=true
262279
ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup
263280
281+
- path: /etc/systemd/system/kubelet-healthcheck.service
282+
permissions: "0644"
283+
content: |
284+
[Unit]
285+
Requires=kubelet.service
286+
After=kubelet.service
287+
288+
[Service]
289+
ExecStart=/usr/local/bin/health-monitor.sh kubelet
290+
291+
[Install]
292+
WantedBy=multi-user.target
293+
294+
- path: /etc/systemd/system/docker-healthcheck.service
295+
permissions: "0644"
296+
content: |
297+
[Unit]
298+
Requires=docker.service
299+
After=docker.service
300+
301+
[Service]
302+
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
303+
304+
[Install]
305+
WantedBy=multi-user.target
306+
264307
runcmd:
265308
- systemctl enable --now setup.service
266309
`

0 commit comments

Comments
 (0)