Skip to content

Commit 5bf2d8b

Browse files
shishir-a412edshishir-a412ed
authored
Merge pull request #17 from Roblox/readme
Add README.md and setup.sh
2 parents 42b99a1 + 7ffced5 commit 5bf2d8b

File tree

4 files changed

+309
-2
lines changed

4 files changed

+309
-2
lines changed

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
BINARY ?= containerd-driver
2-
GOLANG ?= /usr/local/go/bin/go
2+
ifndef $(GOLANG)
3+
GOLANG=$(shell which go)
4+
export GOLANG
5+
endif
36

47
export GO111MODULE=on
58

‎README.md‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,71 @@
11
# nomad-driver-containerd
2-
Nomad driver for launching containers using containerd.
2+
Nomad task driver for launching containers using containerd.
3+
4+
**Containerd** [`(containerd.io)`](https://containerd.io) is a lightweight container daemon for
5+
running and managing container lifecycle.<br/>
6+
Docker daemon also uses containerd.
7+
8+
```
9+
dockerd (docker daemon) --> containerd --> containerd-shim --> runc
10+
```
11+
12+
**nomad-driver-containerd** enables nomad client to launch containers directly using containerd, without docker!<br/>
13+
Docker daemon is not required on the host system.
14+
15+
## nomad-driver-containerd architecture
16+
<img src="images/nomad_driver_containerd.png" width="850" height="475" />
17+
18+
## Requirements
19+
20+
- [Nomad](https://www.nomadproject.io/downloads.html) >=v0.11
21+
- [Go](https://golang.org/doc/install) >=v1.11
22+
- [Containerd](https://containerd.io/downloads/) >=1.3
23+
24+
## Building nomad-driver-containerd
25+
26+
Make sure your **$GOPATH** is setup correctly.
27+
```
28+
$ mkdir -p $GOPATH/src/github.com/Roblox
29+
$ cd $GOPATH/src/github.com/Roblox
30+
$ git clone [email protected]:Roblox/nomad-driver-containerd.git
31+
$ cd nomad-driver-containerd
32+
$ make build (This will build your containerd-driver binary)
33+
```
34+
## Wanna try it out!?
35+
36+
```
37+
./setup.sh
38+
```
39+
The setup script will setup `containerd 1.3.4` and `nomad server+nomad-driver-containerd` (nomad server/client should already be installed on your system, and `setup.sh` only builds the driver) on your system, so you can try out [`example`](https://github.com/Roblox/nomad-driver-containerd/tree/readme/example) jobs.
40+
41+
Once `setup.sh` is complete and the nomad server is up and running, you can check the registered task drivers (which will also show `containerd-driver`) using:
42+
```
43+
$ nomad node status (Note down the <node_id>)
44+
$ nomad node status <node_id>
45+
```
46+
47+
## Run Example jobs.
48+
49+
There are few example jobs in the [`example`](https://github.com/Roblox/nomad-driver-containerd/tree/readme/example) directory.
50+
51+
```
52+
$ nomad job run <job_name.nomad>
53+
```
54+
will launch the job.<br/>
55+
56+
**NOTE:** You need to run `setup.sh` before trying out the example jobs.<br/>
57+
More detailed instructions are in the [`example README.md`](https://github.com/Roblox/nomad-driver-containerd/tree/readme/example)
58+
59+
## Tests
60+
```
61+
$ make test
62+
```
63+
**NOTE**: These are destructive tests and can leave the system in a changed state.<br/>
64+
It is highly recommended to run these tests either as part of a CI/CD system or on
65+
a immutable infrastructure e.g VMs.
66+
67+
## Cleanup
68+
```
69+
make clean
70+
```
71+
This will delete your binary: `containerd-driver`

‎images/nomad_driver_containerd.png‎

2.7 MB
Loading

‎setup.sh‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
export CONTAINERD_VERSION=1.3.4
6+
7+
main() {
8+
echo "INFO: Welcome! nomad-driver-containerd setup."
9+
check_root
10+
check_os
11+
check_nomad
12+
check_golang
13+
14+
echo "WARN: Some installation steps are time consuming. Please be patient!"
15+
16+
# Save present working directory (pwd).
17+
curr_dir=$(echo $PWD)
18+
19+
if systemctl -q is-active "containerd.service"; then
20+
echo "WARN: Containerd detected on the system."
21+
read -p "INFO: Backup existing containerd and deploy containerd-${CONTAINERD_VERSION} (Y/N)? Press Y to continue. " -n 1 -r
22+
echo
23+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
24+
echo "INFO: Aborting setup..."
25+
exit 0
26+
fi
27+
systemctl stop containerd
28+
if [ -f "/lib/systemd/system/containerd.service" ]; then
29+
echo "INFO: Backup containerd systemd unit /lib/systemd/system/containerd.service."
30+
mv /lib/systemd/system/containerd.service /lib/systemd/system/containerd.service.bkp
31+
echo "WARN: Backup file saved at: /lib/systemd/system/containerd.service.bkp"
32+
fi
33+
fi
34+
setup_containerd
35+
36+
read -p "INFO: Setup nomad server + nomad-driver-containerd (Y/N)? Press Y to continue. " -n 1 -r
37+
echo
38+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
39+
echo "INFO: Aborting setup..."
40+
cleanup
41+
exit 0
42+
fi
43+
echo "INFO: Cleanup any old binaries."
44+
make clean >/dev/null 2>&1
45+
echo "INFO: Build nomad-driver-containerd binary: containerd-driver."
46+
make build >/dev/null 2>&1 || (cleanup && exit 1)
47+
echo "INFO: Create plugin-dir for containerd-driver: /tmp/nomad-driver-containerd."
48+
mkdir -p /tmp/nomad-driver-containerd || (cleanup && exit 1)
49+
echo "INFO: Move containerd-driver to /tmp/nomad-driver-containerd."
50+
mv containerd-driver /tmp/nomad-driver-containerd || (cleanup && exit 1)
51+
drop_nomad_unit_file $curr_dir
52+
echo "INFO: Reload nomad.service systemd unit."
53+
systemctl daemon-reload
54+
echo "INFO: Starting nomad server + nomad-driver-containerd."
55+
systemctl start nomad || (cleanup && exit 1)
56+
if ! systemctl -q is-active "nomad.service"; then
57+
echo "ERROR: nomad.service didn't come up. journalctl -u nomad.service for more info."
58+
exit 1
59+
fi
60+
echo "INFO: Setup finished successfully."
61+
}
62+
63+
cleanup() {
64+
echo "INFO: Starting cleanup."
65+
pushd $curr_dir >/dev/null 2>&1
66+
if [ -f "/lib/systemd/system/containerd.service.bkp" ]; then
67+
if systemctl -q is-active "containerd.service"; then
68+
echo "INFO: Stopping containerd."
69+
systemctl stop containerd.service
70+
fi
71+
72+
if [ -f "/tmp/containerd.service" ]; then
73+
echo "INFO: Cleanup /tmp/containerd.service."
74+
rm -f /tmp/containerd.service
75+
fi
76+
77+
if [ -f "/lib/systemd/system/containerd.service" ]; then
78+
echo "INFO: Cleanup: /lib/systemd/system/containerd.service."
79+
rm -f /lib/systemd/system/containerd.service
80+
fi
81+
82+
if [ -f "/tmp/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz" ]; then
83+
echo "INFO: Cleanup: /tmp/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz."
84+
rm -f /tmp/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
85+
fi
86+
fi
87+
88+
if systemctl -q is-active "nomad.service"; then
89+
echo "INFO: Stopping nomad server+nomad-driver-containerd."
90+
systemctl stop nomad
91+
fi
92+
93+
if [ -f "$curr_dir/nomad.service" ]; then
94+
echo "INFO: Cleanup $curr_dir/nomad.service."
95+
rm -f $curr_dir/nomad.service
96+
fi
97+
98+
if [ -f "/lib/systemd/system/nomad.service" ]; then
99+
echo "INFO: Cleanup: /lib/systemd/system/nomad.service."
100+
rm -f /lib/systemd/system/nomad.service
101+
fi
102+
103+
echo "INFO: Cleanup /tmp/nomad-driver-containerd."
104+
rm -rf /tmp/nomad-driver-containerd
105+
106+
echo "INFO: Cleanup containerd-driver binary."
107+
make clean >/dev/null 2>&1
108+
popd >/dev/null 2>&1
109+
echo "INFO: Cleanup complete."
110+
}
111+
112+
setup_containerd() {
113+
read -p "INFO: Download containerd (Y/N)? Press Y to continue. " -n 1 -r
114+
echo
115+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
116+
echo "INFO: Aborting setup..."
117+
cleanup
118+
exit 0
119+
fi
120+
pushd /tmp >/dev/null 2>&1
121+
curl -L --silent -o containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz || (cleanup && exit 1)
122+
tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz || (cleanup && exit 1)
123+
rm -f containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
124+
read -p "INFO: Drop systemd unit containerd.service into /lib/systemd/system/containerd.service (Y/N)? Press Y to continue. " -n 1 -r
125+
echo
126+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
127+
echo "INFO: Aborting setup..."
128+
cleanup
129+
exit 0
130+
fi
131+
drop_containerd_unit_file
132+
echo "INFO: Reload containerd.service systemd unit."
133+
systemctl daemon-reload
134+
echo "INFO: Starting containerd daemon."
135+
systemctl start containerd || (cleanup && exit 1)
136+
popd >/dev/null 2>&1
137+
if ! systemctl -q is-active "containerd.service"; then
138+
echo "ERROR: containerd.service didn't come up. journalctl -u containerd.service for more info."
139+
exit 1
140+
fi
141+
}
142+
143+
drop_nomad_unit_file() {
144+
local nomad=$(which nomad)
145+
# Drop nomad server (dev) + nomad-driver-containerd systemd unit file into /lib/systemd/system.
146+
cat << EOF > nomad.service
147+
# /lib/systemd/system/nomad.service
148+
[Unit]
149+
Description=nomad server (dev) + nomad-driver-containerd
150+
Documentation=https://nomadproject.io
151+
After=network.target containerd.service
152+
153+
[Service]
154+
ExecStart=$nomad agent -dev -config=$1/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd
155+
KillMode=process
156+
Delegate=yes
157+
LimitNOFILE=1048576
158+
LimitNPROC=infinity
159+
LimitCORE=infinity
160+
TasksMax=infinity
161+
162+
[Install]
163+
WantedBy=multi-user.target
164+
EOF
165+
mv nomad.service /lib/systemd/system/nomad.service
166+
}
167+
168+
drop_containerd_unit_file() {
169+
# Drop containerd systemd unit file into /lib/systemd/system.
170+
cat << EOF > containerd.service
171+
# /lib/systemd/system/containerd.service
172+
[Unit]
173+
Description=containerd container runtime
174+
Documentation=https://containerd.io
175+
After=network.target
176+
177+
[Service]
178+
ExecStartPre=-/sbin/modprobe overlay
179+
ExecStart=/usr/local/bin/containerd
180+
KillMode=process
181+
Delegate=yes
182+
LimitNOFILE=1048576
183+
# Having non-zero Limit*s causes performance problems due to accounting overhead
184+
# in the kernel. We recommend using cgroups to do container-local accounting.
185+
LimitNPROC=infinity
186+
LimitCORE=infinity
187+
TasksMax=infinity
188+
189+
[Install]
190+
WantedBy=multi-user.target
191+
EOF
192+
mv containerd.service /lib/systemd/system/containerd.service
193+
}
194+
195+
check_golang() {
196+
set +e
197+
go version >/dev/null 2>&1
198+
rc=$?
199+
set -e
200+
if [ $rc -ne 0 ];then
201+
echo "ERROR: Golang is missing. Please install golang >=1.11 to continue with the setup."
202+
exit 1
203+
fi
204+
}
205+
206+
check_nomad() {
207+
set +e
208+
which nomad >/dev/null 2>&1
209+
rc=$?
210+
set -e
211+
if [ $rc -ne 0 ];then
212+
echo "ERROR: Nomad is missing. Please install nomad >=0.11 to continue with the setup."
213+
exit 1
214+
fi
215+
}
216+
217+
check_root() {
218+
if [ $(id -u) != 0 ]; then
219+
echo "ERROR: Run as root user."
220+
exit 1
221+
fi
222+
}
223+
224+
check_os() {
225+
set +e
226+
cat /etc/os-release|grep -q -i "Ubuntu"
227+
rc=$?
228+
set -e
229+
if [ $rc -ne 0 ];then
230+
echo "ERROR: Unsupported host OS. Run tests on Ubuntu."
231+
exit 1
232+
fi
233+
}
234+
235+
main "$@"

0 commit comments

Comments
 (0)