Merge pull request #40 from Roblox/consul_templates

Add support for consul templates, port forwarding and consul service discovery.

Author: chuckyz

README.md

@@ -151,6 +151,62 @@ You need to install CNI plugins on Nomad client nodes under `/opt/cni/bin` befor
  $ sudo mkdir -p /opt/cni/bin
  $ sudo tar -C /opt/cni/bin -xzf cni-plugins.tgz
 ```
+Also, ensure your Linux operating system distribution has been configured to allow container traffic through the bridge network to be routed via iptables. These tunables can be set as follows:
+
+```
+$ echo 1 > /proc/sys/net/bridge/bridge-nf-call-arptables
+$ echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables
+$ echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
+```
+To preserve these settings on startup of a nomad client node, add a file including the following to `/etc/sysctl.d/` or remove the file your Linux distribution puts in that directory.
+
+```
+net.bridge.bridge-nf-call-arptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables = 1
+```
+
+## Port forwarding
+
+nomad supports both **static** and **dynamic** port mapping.
+
+1. **Static ports**
+
+Static port mapping can be added in the `network` stanza.
+```
+network {
+  mode = "bridge"
+  port "lb" {
+    static = 8889
+    to     = 8889
+  }
+}
+```
+Here, `host` port `8889` is mapped to `container` port `8889`.<br/>
+**NOTE**: static ports are usually not recommended, except for `system` or specialized jobs like load balancers.
+
+2. **Dynamic ports**
+
+Dynamic port mapping is also enabled in the `network` stanza.
+```
+network {
+  mode = "bridge"
+  port "http" {
+    to = 8080
+  }
+}
+```
+Here, nomad will allocate a dynamic port on the `host` and that port will be mapped to `8080` in the container.
+
+You can also read more about `network stanza` in the [`nomad official documentation`](https://www.nomadproject.io/docs/job-specification/network)
+
+## Service discovery
+
+Nomad schedules workloads of various types across a cluster of generic hosts. Because of this, placement is not known in advance and you will need to use service discovery to connect tasks to other services deployed across your cluster. Nomad integrates with Consul to provide service discovery and monitoring.
+
+A [`service`](https://www.nomadproject.io/docs/job-specification/service) stanza can be added to your job spec, to enable service discovery. 
+
+The service stanza instructs Nomad to register a service with Consul.
 
 ## Tests
 ```
@@ -174,15 +230,6 @@ This will destroy your vagrant VM.
 ## Currently supported environments
 Ubuntu (>= 16.04)
 
-## Limitations
-
-`nomad-driver-containerd` [`v0.1`](https://github.com/Roblox/nomad-driver-containerd/releases/tag/v0.1) is **not** production ready.
-There are some open items which are currently being worked on.
-
-1) **Port forwarding**: The ability to map a host port to a container port. This is currently not supported, but could be supported in future.
-
-2) **Consul connect**: When a user launches a job in `nomad`, s/he can add a [`service stanza`](https://www.nomadproject.io/docs/job-specification/service) which will instruct `nomad` to register the service with `consul` for service discovery. This is currently not supported.
-
 ## License
 
 Copyright 2020 Roblox Corporation

containerd/containerd.go

@@ -41,7 +41,7 @@ func (d *Driver) pullImage(imageName string) (containerd.Image, error) {
 	return d.client.Pull(d.ctxContainerd, imageName, containerd.WithPullUnpack)
 }
 
-func (d *Driver) createContainer(image containerd.Image, containerName, containerSnapshotName, containerdRuntime, netnsPath string, env []string, config *TaskConfig) (containerd.Container, error) {
+func (d *Driver) createContainer(image containerd.Image, containerName, containerSnapshotName, containerdRuntime, netnsPath, secretsDir, taskDir string, env []string, config *TaskConfig) (containerd.Container, error) {
 	if config.Command == "" && len(config.Args) > 0 {
 		return nil, fmt.Errorf("Command is empty. Cannot set --args without --command.")
 	}
@@ -119,14 +119,22 @@ func (d *Driver) createContainer(image containerd.Image, containerName, containe
 			return nil, fmt.Errorf("Options cannot be empty for mount type: %s. You need to atleast pass rbind and ro.", mount.Type)
 		}
 
-		m := specs.Mount{}
-		m.Type = mount.Type
-		m.Destination = mount.Target
-		m.Source = mount.Source
-		m.Options = mount.Options
+		m := buildMountpoint(mount.Type, mount.Target, mount.Source, mount.Options)
 		mounts = append(mounts, m)
 	}
 
+	// Setup "/secrets" (NOMAD_SECRETS_DIR) in the container.
+	if secretsDir != "" {
+		secretsMount := buildMountpoint("bind", "/secrets", secretsDir, []string{"rbind", "ro"})
+		mounts = append(mounts, secretsMount)
+	}
+
+	// Setup "/local" (NOMAD_TASK_DIR) in the container.
+	if taskDir != "" {
+		taskMount := buildMountpoint("bind", "/local", taskDir, []string{"rbind", "ro"})
+		mounts = append(mounts, taskMount)
+	}
+
 	if len(mounts) > 0 {
 		opts = append(opts, oci.WithMounts(mounts))
 	}
@@ -150,6 +158,16 @@ func (d *Driver) createContainer(image containerd.Image, containerName, containe
 	)
 }
 
+// buildMountpoint builds the mount point for the container.
+func buildMountpoint(mountType, mountTarget, mountSource string, mountOptions []string) specs.Mount {
+	m := specs.Mount{}
+	m.Type = mountType
+	m.Destination = mountTarget
+	m.Source = mountSource
+	m.Options = mountOptions
+	return m
+}
+
 func (d *Driver) loadContainer(id string) (containerd.Container, error) {
 	return d.client.LoadContainer(d.ctxContainerd, id)
 }

containerd/driver.go

@@ -357,10 +357,17 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 
 	// Setup environment variables.
 	var env []string
+	var secretsDir, taskDir string
 	for key, val := range cfg.Env {
 		if skipOverride(key) {
 			continue
 		}
+		if key == "NOMAD_SECRETS_DIR" {
+			secretsDir = val
+		}
+		if key == "NOMAD_TASK_DIR" {
+			taskDir = val
+		}
 		env = append(env, fmt.Sprintf("%s=%s", key, val))
 	}
 
@@ -370,7 +377,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		netnsPath = cfg.NetworkIsolation.Path
 	}
 
-	container, err := d.createContainer(image, containerName, containerSnapshotName, d.config.ContainerdRuntime, netnsPath, env, &driverConfig)
+	container, err := d.createContainer(image, containerName, containerSnapshotName, d.config.ContainerdRuntime, netnsPath, secretsDir, taskDir, env, &driverConfig)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Error in creating container: %v", err)
 	}