Merge pull request #66 from Roblox/local_tests

Support running integration tests in local vagrant VM.

Author: shishir-a412ed

README.md

@@ -24,7 +24,7 @@ Docker daemon is not required on the host system.
 
 ## Requirements
 
-- [Nomad](https://www.nomadproject.io/downloads.html) >=v0.11
+- [Nomad](https://www.nomadproject.io/downloads.html) >=v1.0
 - [Go](https://golang.org/doc/install) >=v1.11
 - [Containerd](https://containerd.io/downloads/) >=1.3
 - [Vagrant](https://www.vagrantup.com/downloads.html) >=v2.2
@@ -90,14 +90,14 @@ More detailed instructions are in the [`example README.md`](https://github.com/R
 | **cwd** | string | no | Specify the current working directory for your container process. If the directory does not exist, one will be created for you. |
 | **privileged** | bool | no | Run container in privileged mode. Your container will have all linux capabilities when running in privileged mode. |
 | **host_dns** | bool | no | Default (`true`). By default, a container launched using `containerd-driver` will use host `/etc/resolv.conf`. This is similar to [`docker behavior`](https://docs.docker.com/config/containers/container-networking/#dns-services). However, if you don't want to use host DNS, you can turn off this flag by setting `host_dns=false`. |
-| **seccomp** | bool | no | Enable default seccomp profile. List of [`allowed syscalls`](https://github.com/containerd/containerd/blob/master/contrib/seccomp/seccomp_default.go#L51-L390). |
+| **seccomp** | bool | no | Enable default seccomp profile. List of [`allowed syscalls`](https://github.com/containerd/containerd/blob/master/contrib/seccomp/seccomp_default.go#L51-L395). |
 | **seccomp_profile** | string | no | Path to custom seccomp profile. `seccomp` must be set to `true` in order to use `seccomp_profile`. The default `docker` seccomp profile found [`here`](https://github.com/moby/moby/blob/master/profiles/seccomp/default.json) can be used as a reference, and modified to create a custom seccomp profile. |
 | **readonly_rootfs** | bool | no | Container root filesystem will be read-only. |
 | **host_network** | bool | no | Enable host network. This is equivalent to `--net=host` in docker. |
 | **cap_add** | []string | no | Add individual capabilities. |
 | **cap_drop** | []string | no | Drop invidual capabilities. |
 | **devices** | []string | no | A list of devices to be exposed to the container. |
-| **mounts** | []block | no | A list of mounts to be mounted in the container. Volume, bind and tmpfs type mounts are supported. fstab style [`mount options`](https://github.com/containerd/containerd/blob/master/mount/mount_linux.go#L187-L211) are supported. |
+| **mounts** | []block | no | A list of mounts to be mounted in the container. Volume, bind and tmpfs type mounts are supported. fstab style [`mount options`](https://github.com/containerd/containerd/blob/master/mount/mount_linux.go#L211-L235) are supported. |
 
 **Mount block**<br/>
        &emsp;&emsp;\{<br/>
@@ -219,12 +219,17 @@ A [`service`](https://www.nomadproject.io/docs/job-specification/service) stanza
 The service stanza instructs Nomad to register a service with Consul.
 
 ## Tests
+
+If you are running the tests locally, use the [`vagrant VM`](Vagrantfile) provided in the repository.
+
 ```
-$ make test
+$ vagrant up
+$ vagrant ssh containerd-linux
+$ sudo make test
 ```
 **NOTE**: These are destructive tests and can leave the system in a changed state.<br/>
-It is highly recommended to run these tests either as part of a CI/CD system or on
-a immutable infrastructure e.g VMs.
+It is highly recommended to run these tests either as part of a CI/CD system e.g. circleci or on
+a immutable infrastructure e.g vagrant VMs.
 
 ## Cleanup
 ```

Vagrantfile

@@ -6,8 +6,8 @@ VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure("2") do |config|
   config.vm.define "containerd-linux"
   config.vm.box = "hashicorp/bionic64"
-  config.vm.synced_folder ".", "/home/vagrant/go/src/nomad-driver-containerd"
-  config.ssh.extra_args = ["-t", "cd /home/vagrant/go/src/nomad-driver-containerd; bash --login"]
+  config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/Roblox/nomad-driver-containerd"
+  config.ssh.extra_args = ["-t", "cd /home/vagrant/go/src/github.com/Roblox/nomad-driver-containerd; bash --login"]
   config.vm.network "forwarded_port", guest: 4646, host: 4646, host_ip: "127.0.0.1"
   config.vm.provider "virtualbox" do |vb|
       vb.name = "containerd-linux"
@@ -16,7 +16,7 @@ Vagrant.configure("2") do |config|
   end
   config.vm.provision "shell", inline: <<-SHELL
     apt-get update
-    apt-get install -y unzip gcc runc
+    apt-get install -y unzip gcc runc jq
     echo "export GOPATH=/home/vagrant/go" >> /home/vagrant/.bashrc
     echo "export PATH=$PATH:/usr/local/go/bin" >> /home/vagrant/.bashrc
     source /home/vagrant/.bashrc
@@ -29,12 +29,12 @@ Vagrant.configure("2") do |config|
       rm -f go1.14.3.linux-amd64.tar.gz
     fi
 
-    # Install nomad-0.11.3
+    # Install nomad-1.0.2
     if [ ! -f "/usr/bin/nomad" ]; then
-      wget --quiet https://releases.hashicorp.com/nomad/0.11.3/nomad_0.11.3_linux_amd64.zip
-      unzip nomad_0.11.3_linux_amd64.zip -d /usr/bin
+      wget --quiet https://releases.hashicorp.com/nomad/1.0.2/nomad_1.0.2_linux_amd64.zip
+      unzip nomad_1.0.2_linux_amd64.zip -d /usr/bin
       chmod +x /usr/bin/nomad
-      rm -f nomad_0.11.3_linux_amd64.zip
+      rm -f nomad_1.0.2_linux_amd64.zip
     fi
 
     # Install containerd-1.3.4
@@ -47,8 +47,11 @@ Vagrant.configure("2") do |config|
     # Create source directory for privileged.nomad example job.
     mkdir -p /tmp/s1
 
+    # Prepare nomad host volume
+    mkdir -p /tmp/host_volume/s1
+
     # Run setup
-    cd /home/vagrant/go/src/nomad-driver-containerd/vagrant
+    cd /home/vagrant/go/src/github.com/Roblox/nomad-driver-containerd/vagrant
     ./setup.sh
   SHELL
 end

example/agent.hcl

@@ -7,3 +7,10 @@ plugin "containerd-driver" {
     stats_interval = "5s"
   }
 }
+
+client {
+  host_volume "s1" {
+    path = "/tmp/host_volume/s1"
+    read_only = false
+  }
+}

example/agent_tests.hcl

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source $SRCDIR/utils.sh
+
 test_redis_nomad_job() {
     pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
 
@@ -16,7 +18,7 @@ test_redis_nomad_job() {
     # The actual container process might not be running yet.
     # We need to wait for actual container to start running before trying exec.
     echo "INFO: Wait for redis container to get into RUNNING state, before trying exec."
-    is_redis_container_active
+    is_container_active redis false
 
     echo "INFO: Inspecting redis job."
     redis_status=$(nomad job inspect redis|jq -r '.Job .Status')
@@ -53,24 +55,4 @@ test_redis_nomad_job() {
     popd
 }
 
-is_redis_container_active() {
-        i="0"
-        while test $i -lt 5
-        do
-                sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING
-                if [ $? -eq 0 ]; then
-                        echo "INFO: redis container is up and running"
-                        break
-                fi
-                echo "INFO: redis container is down, sleep for 4 seconds."
-                sleep 4s
-                i=$[$i+1]
-        done
-
-        if [ $i -ge 5 ]; then
-                echo "ERROR: redis container didn't come up. exit 1."
-                exit 1
-        fi
-}
-
 test_redis_nomad_job

tests/001-test-redis.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source $SRCDIR/utils.sh
+
 test_signal_handler_nomad_job() {
     pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
 
@@ -24,7 +26,7 @@ test_signal_handler_nomad_job() {
     # The actual container process might not be running yet.
     # We need to wait for actual container to start running before trying to send invalid signal.
     echo "INFO: Wait for signal container to get into RUNNING state, before trying to send invalid signal."
-    is_signal_container_active
+    is_container_active signal false
 
     echo "INFO: Test invalid signal."
     alloc_id=$(nomad job status signal|awk 'END{print}'|cut -d ' ' -f 1)
@@ -55,24 +57,4 @@ cleanup() {
   rm $tmpfile > /dev/null 2>&1
 }
 
-is_signal_container_active() {
-        i="0"
-        while test $i -lt 5
-        do
-                sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING
-                if [ $? -eq 0 ]; then
-                        echo "INFO: signal container is up and running"
-                        break
-                fi
-                echo "INFO: signal container is down, sleep for 4 seconds."
-                sleep 4s
-                i=$[$i+1]
-        done
-
-        if [ $i -ge 5 ]; then
-                echo "ERROR: signal container didn't come up. exit 1."
-                exit 1
-        fi
-}
-
 test_signal_handler_nomad_job

tests/002-test-signal-handler.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source $SRCDIR/utils.sh
+
 # readonly_rootfs, cap_add and cap_drop flags are tested as part of this test.
 test_capabilities_nomad_job() {
     pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
@@ -18,7 +20,7 @@ test_capabilities_nomad_job() {
     # The actual container process might not be running yet.
     # We need to wait for actual container to start running before trying exec.
     echo "INFO: Wait for capabilities container to get into RUNNING state, before trying exec."
-    is_capabilities_container_active
+    is_container_active capabilities true
 
     echo "INFO: Inspecting capabilities job."
     cap_status=$(nomad job inspect capabilities|jq -r '.Job .Status')
@@ -27,17 +29,6 @@ test_capabilities_nomad_job() {
         exit 1
     fi
 
-    # Check if readonly_rootfs is set to true.
-    echo "INFO: Checking if readonly_rootfs is set to true."
-    local outfile=$(mktemp /tmp/capabilities.XXXXXX)
-    nomad alloc exec -job capabilities touch /tmp/file.txt >> $outfile 2>&1
-    if ! grep -q "Read-only file system" $outfile; then
-        echo "ERROR: readonly_rootfs is not set to true."
-        cleanup "$outfile"
-        exit 1
-    fi
-    cleanup "$outfile"
-
     # Check if CAP_SYS_ADMIN was added.
     echo "INFO: Checking if CAP_SYS_ADMIN is added."
     nomad alloc exec -job capabilities capsh --print|grep cap_sys_admin >/dev/null 2>&1
@@ -56,6 +47,17 @@ test_capabilities_nomad_job() {
         exit 1
     fi
 
+    # Check if readonly_rootfs is set to true.
+    echo "INFO: Checking if readonly_rootfs is set to true."
+    local outfile=$(mktemp /tmp/capabilities.XXXXXX)
+    nomad alloc exec -job capabilities touch /tmp/file.txt >> $outfile 2>&1
+    if ! grep -q "Read-only file system" $outfile; then
+        echo "ERROR: readonly_rootfs is not set to true."
+        cleanup "$outfile"
+        exit 1
+    fi
+    cleanup "$outfile"
+
     echo "INFO: Stopping nomad capabilities job."
     nomad job stop capabilities
     cap_status=$(nomad job status -short capabilities|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@@ -74,25 +76,4 @@ cleanup() {
   rm $tmpfile > /dev/null 2>&1
 }
 
-is_capabilities_container_active() {
-        i="0"
-        while test $i -lt 5
-        do
-                sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING
-                if [ $? -eq 0 ]; then
-                        echo "INFO: capabilities container is up and running"
-                        sleep 5s
-                        break
-                fi
-                echo "INFO: capabilities container is down, sleep for 4 seconds."
-                sleep 4s
-                i=$[$i+1]
-        done
-
-        if [ $i -ge 5 ]; then
-                echo "ERROR: capabilities container didn't come up. exit 1."
-                exit 1
-        fi
-}
-
 test_capabilities_nomad_job

tests/003-test-capabilities.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source $SRCDIR/utils.sh
+
 # privileged mode, devices and mounts are tested as part of this test.
 test_privileged_nomad_job() {
     pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
@@ -20,7 +22,7 @@ test_privileged_nomad_job() {
     # The actual container process might not be running yet.
     # We need to wait for actual container to start running before trying exec.
     echo "INFO: Wait for privileged container to get into RUNNING state, before trying exec."
-    is_privileged_container_active
+    is_container_active privileged true
 
     echo "INFO: Inspecting privileged job."
     job_status=$(nomad job inspect privileged|jq -r '.Job .Status')
@@ -73,25 +75,4 @@ setup_bind_source() {
     echo hello > /tmp/s1/bind.txt
 }
 
-is_privileged_container_active() {
-        i="0"
-        while test $i -lt 5
-        do
-                sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING
-                if [ $? -eq 0 ]; then
-                        echo "INFO: privileged container is up and running"
-                        sleep 5s
-                        break
-                fi
-                echo "INFO: privileged container is down, sleep for 4 seconds."
-                sleep 4s
-                i=$[$i+1]
-        done
-
-        if [ $i -ge 5 ]; then
-                echo "ERROR: privileged container didn't come up. exit 1."
-                exit 1
-        fi
-}
-
 test_privileged_nomad_job

tests/004-test-privileged.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source $SRCDIR/utils.sh
+
 job_name=volume_mount
 host_volume_path=/tmp/host_volume/s1
 
@@ -16,7 +18,7 @@ test_volume_mount_nomad_job() {
     # The actual container process might not be running yet.
     # We need to wait for actual container to start running before trying exec.
     echo "INFO: Wait for ${job_name} container to get into RUNNING state, before trying exec."
-    is_${job_name}_container_active
+    is_container_active ${job_name} true
 
     echo "INFO: Checking status of $job_name job."
     job_status=$(nomad job status -short $job_name|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@@ -71,25 +73,4 @@ setup_bind_source() {
     echo hello > ${host_volume_path}/bind.txt
 }
 
-is_volume_mount_container_active() {
-        i="0"
-        while test $i -lt 5
-        do
-                sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING
-                if [ $? -eq 0 ]; then
-                        echo "INFO: ${job_name} container is up and running"
-                        sleep 5s
-                        break
-                fi
-                echo "INFO: ${job_name} container is down, sleep for 4 seconds."
-                sleep 4s
-                i=$[$i+1]
-        done
-
-        if [ $i -ge 5 ]; then
-                echo "ERROR: ${job_name} container didn't come up. exit 1."
-                exit 1
-        fi
-}
-
 test_volume_mount_nomad_job