Adding support for jailer

Adds support for firecracker's jailer. This allows for more secure
virtualization through creation of namespaces and cgroups. The jailer
can be turned on by the setting the EnableJailer field in the config.
This also implements a very naive mapper and will require update to
support a smarter one.

Signed-off-by: xibz <[email protected]>

Author: xibz

.buildkite/pipeline.yml

@@ -31,6 +31,17 @@ steps:
       - 'ln -s /var/lib/fc-ci/vmlinux.bin testdata/vmlinux'
       - 'ln -s /usr/local/bin/firecracker testdata/firecracker'
       - 'ln -s /usr/local/bin/jailer testdata/jailer'
-      - "make test EXTRAGOARGS='-v -count=1'"
+      - 'sudo ip tuntap add fc-test-tap${BUILDKITE_BUILD_NUMBER} mode tap user $(sudo id -u buildkite-agent)'
+      - "DISABLE_ROOT_TESTS=true FC_TEST_TAP=fc-test-tap${BUILDKITE_BUILD_NUMBER} make test EXTRAGOARGS='-v -count=1'"
+      - 'sudo ip tuntap del fc-test-tap${BUILDKITE_BUILD_NUMBER} mode tap'
+
+  - label: ':hammer: root tests'
+    commands:
+      - 'ln -s /var/lib/fc-ci/vmlinux.bin testdata/vmlinux'
+      - 'ln -s /usr/local/bin/firecracker testdata/firecracker'
+      - 'ln -s /usr/local/bin/jailer testdata/jailer'
+      - 'sudo ip tuntap add fc-root-tap${BUILDKITE_BUILD_NUMBER} mode tap user $(sudo id -u buildkite-agent)'
+      - "sudo FC_TEST_TAP=fc-root-tap${BUILDKITE_BUILD_NUMBER} make test EXTRAGOARGS='-v -count=1'"
+      - 'sudo ip tuntap del fc-root-tap${BUILDKITE_BUILD_NUMBER} mode tap'
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"

HACKING.md

@@ -6,13 +6,17 @@ Testing
 
 Tests are written using Go's testing framework and can be run with the standard
 `go test` tool.  If you prefer to use the Makefile, `make test EXTRAGOARGS=-v`
-will run tests in verbose mode.
+will run tests in verbose mode. By default, the unit tests require root
+privileged access. This can be disabled by setting the `DISABLE_ROOT_TESTS`
+environment variable.
 
 You need some external resources in order to run the tests, as described below:
 
-1. A firecracker binary (tested with 0.10.1), but any recent version should
-   work.  Must either be installed as `./testdata/firecracker` or the path must
-   be specified through the `FC_TEST_BIN` environment variable.
+1. A firecracker and jailer binary (tested with 0.12.0) must either be
+   installed as `./testdata/firecracker` or the path must be specified through
+   the `FC_TEST_BIN` environment variable. The jailer requires go test to be
+   run with sudo and can also be turned off by setting the `DISABLE_ROOT_TESTS`
+   env flag.
 2. Access to hardware virtualization via `/dev/kvm` and `/dev/vhost-vsock`
    (ensure you have mode `+rw`!)
 3. An uncompressed Linux kernel binary that can boot in Firecracker VM (Must be

command_builder.go

@@ -20,8 +20,10 @@ import (
 	"os/exec"
 )
 
+const defaultFcBin = "firecracker"
+
 var defaultFirecrackerVMMCommandBuilder = VMCommandBuilder{}.
-	WithBin("firecracker").
+	WithBin(defaultFcBin).
 	WithStdin(os.Stdin).
 	WithStdout(os.Stdout).
 	WithStderr(os.Stderr)
@@ -55,8 +57,13 @@ func (b VMCommandBuilder) AddArgs(args ...string) VMCommandBuilder {
 	return b
 }
 
-// Bin return the bin that was set
+// Bin returns the bin that was set. If bin had not been set, then the default
+// will be returned.
 func (b VMCommandBuilder) Bin() string {
+	if len(b.bin) == 0 {
+		return defaultFcBin
+	}
+
 	return b.bin
 }
 

command_builder_test.go

@@ -29,18 +29,13 @@ func TestVMCommandBuilder(t *testing.T) {
 func testVMCommandBuilderImmutable(t *testing.T) {
 	b := VMCommandBuilder{}
 	b.WithSocketPath("foo").
-		WithBin("bar").
 		WithArgs([]string{"baz", "qux"}).
 		AddArgs("moo", "cow")
 
 	if e, a := []string(nil), b.SocketPath(); !reflect.DeepEqual(e, a) {
 		t.Errorf("expected immutable builder, but socket path was set: %q", a)
 	}
 
-	if e, a := "", b.Bin(); e != a {
-		t.Errorf("bin had been set with %q, but should have been empty", a)
-	}
-
 	if e, a := ([]string)(nil), b.Args(); !reflect.DeepEqual(e, a) {
 		t.Errorf("args should have been empty, but received %v", a)
 	}

example_test.go

@@ -19,6 +19,13 @@ func ExampleWithProcessRunner_logging() {
 		MachineCfg: models.MachineConfiguration{
 			VcpuCount: 1,
 		},
+		JailerCfg: firecracker.JailerConfig{
+			GID:      firecracker.Int(100),
+			UID:      firecracker.Int(100),
+			ID:       "my-micro-vm",
+			NumaNode: firecracker.Int(0),
+			ExecFile: "/path/to/firecracker",
+		},
 	}
 
 	// stdout will be directed to this file
@@ -212,3 +219,52 @@ func ExampleNetworkInterface_rateLimiting() {
 		panic(err)
 	}
 }
+
+func ExampleJailerCommandBuilder() {
+	ctx := context.Background()
+	// Creates a jailer command using the JailerCommandBuilder.
+	b := firecracker.NewJailerCommandBuilder().
+		WithID("my-test-id").
+		WithUID(123).
+		WithGID(100).
+		WithNumaNode(0).
+		WithExecFile("/usr/local/bin/firecracker").
+		WithChrootBaseDir("/tmp").
+		WithStdout(os.Stdout).
+		WithStderr(os.Stderr)
+
+	const socketPath = "/tmp/firecracker/my-test-id/api.socket"
+	cfg := firecracker.Config{
+		SocketPath:      socketPath,
+		KernelImagePath: "./vmlinux",
+		Drives: []models.Drive{
+			models.Drive{
+				DriveID:      firecracker.String("1"),
+				IsRootDevice: firecracker.Bool(true),
+				IsReadOnly:   firecracker.Bool(false),
+				PathOnHost:   firecracker.String("/path/to/root/drive"),
+			},
+		},
+		MachineCfg: models.MachineConfiguration{
+			VcpuCount: 1,
+		},
+		DisableValidation: true,
+	}
+
+	// Passes the custom jailer command into the constructor
+	m, err := firecracker.NewMachine(ctx, cfg, firecracker.WithProcessRunner(b.Build(ctx)))
+	if err != nil {
+		panic(fmt.Errorf("failed to create new machine: %v", err))
+	}
+
+	// This does not copy any of the files over to the rootfs since a process
+	// runner was specified. This examples assumes that the files have been
+	// properly mounted.
+	if err := m.Start(ctx); err != nil {
+		panic(err)
+	}
+
+	tCtx, cancelFn := context.WithTimeout(ctx, time.Minute)
+	defer cancelFn()
+	m.Wait(tCtx)
+}

fctesting/utils.go

@@ -0,0 +1,45 @@
+// Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package fctesting
+
+import (
+	"os"
+	"testing"
+)
+
+const rootDisableEnvName = "DISABLE_ROOT_TESTS"
+
+var rootDisabled bool
+
+func init() {
+	if v := os.Getenv(rootDisableEnvName); len(v) != 0 {
+		rootDisabled = true
+	}
+}
+
+// RequiresRoot will ensure that tests that require root access are actually
+// root. In addition, this will skip root tests if the DISABLE_ROOT_TESTS is
+// set to true
+func RequiresRoot(t testing.TB) {
+	if rootDisabled {
+		t.Skip("skipping test that requires root")
+	}
+
+	if e, a := 0, os.Getuid(); e != a {
+		t.Fatalf("This test must be run as root. "+
+			"To disable tests that require root, "+
+			"run the tests with the %s environment variable set.",
+			rootDisableEnvName)
+	}
+}

handlers.go

@@ -1,7 +1,21 @@
+// Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
 package firecracker
 
 import (
 	"context"
+	"fmt"
 )
 
 // Handler name constants
@@ -14,10 +28,76 @@ const (
 	CreateNetworkInterfacesHandlerName = "fcinit.CreateNetworkInterfaces"
 	AddVsocksHandlerName               = "fcinit.AddVsocks"
 	SetMetadataHandlerName             = "fcinit.SetMetadata"
+	LinkFilesToRootFSHandlerName       = "fcinit.LinkFilesToRootFS"
 
-	ValidateCfgHandlerName = "validate.Cfg"
+	ValidateCfgHandlerName       = "validate.Cfg"
+	ValidateJailerCfgHandlerName = "validate.JailerCfg"
 )
 
+// HandlersAdapter is an interface used to modify a given set of handlers.
+type HandlersAdapter interface {
+	AdaptHandlers(*Handlers) error
+}
+
+// ConfigValidationHandler is used to validate that required fields are
+// present. This validator is to be used when the jailer is turned off.
+var ConfigValidationHandler = Handler{
+	Name: ValidateCfgHandlerName,
+	Fn: func(ctx context.Context, m *Machine) error {
+		// ensure that the configuration is valid for the FcInit handlers.
+		return m.cfg.Validate()
+	},
+}
+
+// JailerConfigValidationHandler is used to validate that required fields are
+// present.
+var JailerConfigValidationHandler = Handler{
+	Name: ValidateJailerCfgHandlerName,
+	Fn: func(ctx context.Context, m *Machine) error {
+		if !m.cfg.EnableJailer {
+			return nil
+		}
+
+		hasRoot := false
+		for _, drive := range m.cfg.Drives {
+			if BoolValue(drive.IsRootDevice) {
+				hasRoot = true
+				break
+			}
+		}
+
+		if !hasRoot {
+			return fmt.Errorf("A root drive must be present in the drive list")
+		}
+
+		if m.cfg.JailerCfg.ChrootStrategy == nil {
+			return fmt.Errorf("ChrootStrategy cannot be nil")
+		}
+
+		if len(m.cfg.JailerCfg.ExecFile) == 0 {
+			return fmt.Errorf("exec file must be specified when using jailer mode")
+		}
+
+		if len(m.cfg.JailerCfg.ID) == 0 {
+			return fmt.Errorf("id must be specified when using jailer mode")
+		}
+
+		if m.cfg.JailerCfg.GID == nil {
+			return fmt.Errorf("GID must be specified when using jailer mode")
+		}
+
+		if m.cfg.JailerCfg.UID == nil {
+			return fmt.Errorf("UID must be specified when using jailer mode")
+		}
+
+		if m.cfg.JailerCfg.NumaNode == nil {
+			return fmt.Errorf("ID must be specified when using jailer mode")
+		}
+
+		return nil
+	},
+}
+
 // StartVMMHandler is a named handler that will handle starting of the VMM.
 // This handler will also set the exit channel on completion.
 var StartVMMHandler = Handler{
@@ -98,17 +178,6 @@ func NewSetMetadataHandler(metadata interface{}) Handler {
 	}
 }
 
-var defaultValidationHandlerList = HandlerList{}.Append(
-	Handler{
-		Name: ValidateCfgHandlerName,
-		Fn: func(ctx context.Context, m *Machine) error {
-			// ensure that the configuration is valid for the
-			// FcInit handlers.
-			return m.cfg.Validate()
-		},
-	},
-)
-
 var defaultFcInitHandlerList = HandlerList{}.Append(
 	StartVMMHandler,
 	BootstrapLoggingHandler,
@@ -120,8 +189,7 @@ var defaultFcInitHandlerList = HandlerList{}.Append(
 )
 
 var defaultHandlers = Handlers{
-	Validation: defaultValidationHandlerList,
-	FcInit:     defaultFcInitHandlerList,
+	FcInit: defaultFcInitHandlerList,
 }
 
 // Handler represents a named handler that contains a name and a function which
@@ -140,9 +208,12 @@ type Handlers struct {
 // Run will execute all handlers in the Handlers object by flattening the lists
 // into a single list and running.
 func (h Handlers) Run(ctx context.Context, m *Machine) error {
-	l := HandlerList{}.Append(
-		h.Validation.list...,
-	).Append(
+	l := HandlerList{}
+	if !m.cfg.DisableValidation {
+		l = l.Append(h.Validation.list...)
+	}
+
+	l = l.Append(
 		h.FcInit.list...,
 	)
 
@@ -162,6 +233,21 @@ func (l HandlerList) Append(handlers ...Handler) HandlerList {
 	return l
 }
 
+// AppendAfter will append a given handler after the specified handler.
+func (l HandlerList) AppendAfter(name string, handler Handler) HandlerList {
+	newList := HandlerList{}
+	for _, h := range l.list {
+		if h.Name == name {
+			newList = newList.Append(h, handler)
+			continue
+		}
+
+		newList = newList.Append(h)
+	}
+
+	return newList
+}
+
 // Len return the length of the given handler list
 func (l HandlerList) Len() int {
 	return len(l.list)
@@ -229,6 +315,7 @@ func (l HandlerList) Run(ctx context.Context, m *Machine) error {
 	for _, handler := range l.list {
 		m.logger.Debugf("Running handler %s", handler.Name)
 		if err := handler.Fn(ctx, m); err != nil {
+			m.logger.Warnf("Failed handler %q: %v", handler.Name, err)
 			return err
 		}
 	}