Merge pull request #573 from Kern--/vm-mounts

Add ability to launch containers with vm local rootfs.

Author: Kern--

agent/service.go

@@ -189,6 +189,16 @@ func (ts *TaskService) Create(requestCtx context.Context, req *taskAPI.CreateTas
 		return nil
 	})
 
+	isVmLocalRootFs := len(req.Rootfs) > 0 && vm.IsLocalMount(req.Rootfs[0])
+
+	// If the rootfs is inside the VM, then the DriveMount call didn't happen and therefore
+	// the bundledir was not created. Create it here.
+	if isVmLocalRootFs {
+		if err := os.MkdirAll(bundleDir.RootfsPath(), 0700); err != nil {
+			return nil, errors.Wrapf(err, "Failed to create bundle's rootfs path from inside the vm %q", bundleDir.RootfsPath())
+		}
+	}
+
 	// check the rootfs dir has been created (presumed to be by a previous MountDrive call)
 	rootfsStat, err := os.Stat(bundleDir.RootfsPath())
 	if err != nil {
@@ -204,8 +214,24 @@ func (ts *TaskService) Create(requestCtx context.Context, req *taskAPI.CreateTas
 		}
 		return nil
 	})
-
-	err = bundleDir.OCIConfig().Write(extraData.JsonSpec)
+	specData := extraData.JsonSpec
+
+	// If the rootfs is inside the VM then:
+	// a) the rootfs mount type has a prefix that we used to identify this which needs to be stripped before passing to runc
+	// b) we were not able to inspect the container's rootfs from the client when setting up the spec. Do that here.
+	if isVmLocalRootFs {
+		req.Rootfs[0] = vm.StripLocalMountIdentifier(req.Rootfs[0])
+		rootfsMount := mount.Mount{
+			Type:    req.Rootfs[0].Type,
+			Source:  req.Rootfs[0].Source,
+			Options: req.Rootfs[0].Options,
+		}
+		specData, err = vm.UpdateUserInSpec(requestCtx, specData, rootfsMount)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to update spec")
+		}
+	}
+	err = bundleDir.OCIConfig().Write(specData)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to write oci config file")
 	}

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/awslabs/tc-redirect-tap v0.0.0-20211025175357-e30dfca224c2
 	github.com/bits-and-blooms/bitset v1.2.1 // indirect
 	github.com/containerd/containerd v1.5.9
-	github.com/containerd/continuity v0.2.0 // indirect
+	github.com/containerd/continuity v0.2.0
 	github.com/containerd/fifo v1.0.0
 	github.com/containerd/go-cni v1.1.1 // indirect
 	github.com/containerd/go-runc v1.0.0
@@ -25,6 +25,7 @@ require (
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/mdlayher/vsock v1.1.0
 	github.com/miekg/dns v1.1.25
+	github.com/opencontainers/image-spec v1.0.2
 	github.com/opencontainers/runc v1.0.3
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210910115017-0d6cc581aeea
 	github.com/opencontainers/selinux v1.8.5 // indirect

internal/vm/mount.go

@@ -0,0 +1,53 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package vm
+
+import (
+	"strings"
+
+	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/mount"
+)
+
+const vmLocalMountTypePrefix = "vm:"
+
+// IsLocalMount returns true if the mount source is inside the VM as opposed to the
+// default assumption that the mount source is a block device on the host.
+func IsLocalMount(mount *types.Mount) bool {
+	return mount != nil && strings.HasPrefix(mount.Type, vmLocalMountTypePrefix)
+}
+
+// AddLocalMountIdentifier adds an identifier to a mount so that it can be detected by IsLocalMount.
+// This is intended to be used by cooperating snapshotters to mark mounts as inside the VM so they
+// can be plumbed at the proper points inside/outside the VM.
+func AddLocalMountIdentifier(mnt mount.Mount) mount.Mount {
+	return mount.Mount{
+		Type:    vmLocalMountTypePrefix + mnt.Type,
+		Source:  mnt.Source,
+		Options: mnt.Options,
+	}
+}
+
+// StripLocalMountIdentifier removes the identifier that signals that a mount
+// is inside the VM. This is used before passing the mount information to runc
+func StripLocalMountIdentifier(mnt *types.Mount) *types.Mount {
+	options := make([]string, len(mnt.Options))
+	copy(options, mnt.Options)
+	return &types.Mount{
+		Type:    strings.Replace(mnt.Type, vmLocalMountTypePrefix, "", 1),
+		Source:  mnt.Source,
+		Target:  mnt.Target,
+		Options: options,
+	}
+}

internal/vm/mount_test.go

@@ -0,0 +1,71 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package vm
+
+import (
+	"testing"
+
+	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/mount"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsLocalMount(t *testing.T) {
+	mnt := types.Mount{
+		Type:    "bind",
+		Source:  "/tmp/snapshots/1/fs",
+		Options: []string{"rbind"},
+	}
+	assert.False(t, IsLocalMount(&mnt), "Standard mount was considered vm local")
+}
+
+func TestAddLocalMountIdentifier(t *testing.T) {
+	mnt := mount.Mount{
+		Type:    "bind",
+		Source:  "/tmp/snapshots/1/fs",
+		Options: []string{"rbind"},
+	}
+	localMnt := AddLocalMountIdentifier(mnt)
+
+	mntProto := types.Mount{
+		Type:    mnt.Type,
+		Source:  mnt.Source,
+		Options: mnt.Options,
+	}
+	localMntProto := types.Mount{
+		Type:    localMnt.Type,
+		Source:  localMnt.Source,
+		Options: localMnt.Options,
+	}
+
+	assert.False(t, IsLocalMount(&mntProto), "Standard mount was considered vm local")
+	assert.True(t, IsLocalMount(&localMntProto), "Mount was not vm local after adding the local mount identifier")
+}
+
+func TestStripLocalMountIdentifier(t *testing.T) {
+	mnt := mount.Mount{
+		Type:    "bind",
+		Source:  "/tmp/snapshots/1/fs",
+		Options: []string{"rbind"},
+	}
+	localMnt := AddLocalMountIdentifier(mnt)
+	localMntProto := &types.Mount{
+		Type:    localMnt.Type,
+		Source:  localMnt.Source,
+		Options: localMnt.Options,
+	}
+	assert.True(t, IsLocalMount(localMntProto), "Mount was not vm local after adding the local mount identifier")
+	localMntProto = StripLocalMountIdentifier(localMntProto)
+	assert.False(t, IsLocalMount(localMntProto), "Mount was vm local after stripping the local mount identifier")
+}