volume: support in-VM snapshotters

With demux snapshotter, firecracker-containerd can run snapshotters
inside VMs. This will allow firecracker-containerd run FUSE-backed
lazy-loading snapshotters such as stargz snapshotter.

This change supports in-VM snapshotters in volume package.

Signed-off-by: Kazuyoshi Kato <[email protected]>

Author: kzys

.buildkite/pipeline.yml

@@ -101,10 +101,11 @@ steps:
       DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
       NUMBER_OF_VMS: 10
       EXTRAGOARGS: "-v -count=1 -race"
+      FICD_DM_VOLUME_GROUP: fcci-vg
     artifact_paths:
       - "snapshotter/logs/*"
     command:
-      - make -C snapshotter integ-test
+      - make -C snapshotter integ-test FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_snapshotter
 
   - wait
 

Makefile

@@ -11,7 +11,7 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-SUBDIRS:=agent runtime examples firecracker-control/cmd/containerd snapshotter docker-credential-mmds
+SUBDIRS:=agent runtime examples firecracker-control/cmd/containerd snapshotter docker-credential-mmds volume
 TEST_SUBDIRS:=$(addprefix test-,$(SUBDIRS))
 INTEG_TEST_SUBDIRS:=$(addprefix integ-test-,$(SUBDIRS))
 
@@ -147,9 +147,10 @@ files_ephemeral: $(RUNC_BIN) agent-in-docker
 	cp agent/agent tools/image-builder/files_ephemeral/usr/local/bin
 	touch tools/image-builder/files_ephemeral
 
-files_ephemeral_stargz: $(STARGZ_BIN) docker-credential-mmds-in-docker
+files_ephemeral_stargz: $(STARGZ_BIN) docker-credential-mmds-in-docker volume-in-docker
 	mkdir -p tools/image-builder/files_ephemeral_stargz/usr/local/bin
 	cp docker-credential-mmds/docker-credential-mmds tools/image-builder/files_ephemeral_stargz/usr/local/bin
+	cp volume/volume-init tools/image-builder/files_ephemeral_stargz/usr/local/bin
 	cp $(STARGZ_BIN) tools/image-builder/files_ephemeral_stargz/usr/local/bin
 
 image: files_ephemeral

snapshotter/Makefile

@@ -20,6 +20,7 @@ GOSUM := $(GOMOD:.mod=.sum)
 DOCKER_IMAGE_TAG?=latest
 GO_CACHE_VOLUME_NAME?=gocache
 FIRECRACKER_CONTAINERD_TEST_IMAGE?=localhost/firecracker-containerd-test
+FICD_DM_POOL?=fc-test-thinpool
 
 REVISION := $(shell git rev-parse HEAD)
 
@@ -51,6 +52,7 @@ integ-test:
 	$(MAKE) $(addprefix integ-test-,$(INTEG_TESTNAMES))
 
 integ-test-%: logs
+	$(CURDIR)/../tools/thinpool.sh reset "$(FICD_DM_POOL)"
 	docker run --rm -it \
 		--privileged \
 		--ipc=host \
@@ -65,6 +67,8 @@ integ-test-%: logs
 		--env GOSUMDB=off \
 		--env GO111MODULE=on \
 		--env NUMBER_OF_VMS=$(NUMBER_OF_VMS) \
+		--env FICD_DM_VOLUME_GROUP=$(FICD_DM_VOLUME_GROUP) \
+		--env FICD_DM_POOL=$(FICD_DM_POOL) \
 		--workdir="/src/snapshotter" \
 		--init \
 		$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \

snapshotter/service_integ_test.go

@@ -34,7 +34,7 @@ import (
 const (
 	snapshotterName = "demux"
 
-	imageRef = "ghcr.io/firecracker-microvm/firecracker-containerd/amazonlinux:latest-esgz"
+	al2stargz = "ghcr.io/firecracker-microvm/firecracker-containerd/amazonlinux:latest-esgz"
 
 	noAuth = ""
 
@@ -138,10 +138,10 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string)
 		return fmt.Errorf("Failed to configure VM metadata for registry resolution [%v]", err)
 	}
 
-	image, err := client.Pull(ctx, imageRef,
+	image, err := client.Pull(ctx, al2stargz,
 		containerd.WithPullUnpack,
 		containerd.WithPullSnapshotter(snapshotterName),
-		containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(imageRef, 10*1024*1024)),
+		containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*1024*1024)),
 	)
 	if err != nil {
 		return fmt.Errorf("Failed to pull image for VM: %s [%v]", vmID, err)

snapshotter/volume_integ_test.go

@@ -0,0 +1,145 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/oci"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
+	"github.com/firecracker-microvm/firecracker-containerd/proto"
+	"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
+	"github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source"
+	"github.com/firecracker-microvm/firecracker-containerd/volume"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const mib = 1024 * 1024
+
+func TestGuestVolumeFrom_Isolated(t *testing.T) {
+	integtest.Prepare(t, integtest.WithDefaultNetwork())
+	const (
+		vmID     = "default"
+		postgres = "docker.io/library/postgres:14.3"
+		alpine   = "docker.io/library/alpine:3.10.1"
+		runtime  = "aws.firecracker"
+	)
+
+	ctx := namespaces.WithNamespace(context.Background(), vmID)
+
+	client, err := containerd.New(integtest.ContainerdSockPath, containerd.WithDefaultRuntime(runtime))
+	require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", integtest.ContainerdSockPath)
+	defer client.Close()
+	fcClient, err := integtest.NewFCControlClient(integtest.ContainerdSockPath)
+	require.NoError(t, err, "Failed to create fccontrol client")
+
+	vs := volume.NewSet(runtime)
+
+	// Add a non-stargz image with volumes.
+	localImage := volume.FromImage(client, postgres, "postgres-snapshot", volume.WithSnapshotter("devmapper"))
+	err = vs.AddFrom(ctx, localImage)
+	require.NoError(t, err)
+
+	// Add a stargz image.
+	// The volume directories must be specified since the host's containerd doesn't know about the image.
+	remoteImage := volume.FromGuestImage(
+		client, vmID, al2stargz, "al2-snapshot", []string{"/etc/yum"},
+		volume.WithSnapshotter("demux"),
+	)
+	mount, err := vs.PrepareDriveMount(ctx, 10*mib)
+	require.NoError(t, err)
+
+	_, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{
+		VMID: vmID,
+		RootDrive: &proto.FirecrackerRootDrive{
+			HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img",
+		},
+		NetworkInterfaces: []*proto.FirecrackerNetworkInterface{
+			{
+				AllowMMDS: true,
+				CNIConfig: &proto.CNIConfiguration{
+					NetworkName:   "fcnet",
+					InterfaceName: "veth0",
+				},
+			},
+		},
+		MachineCfg: &proto.FirecrackerMachineConfiguration{
+			VcpuCount:  2,
+			MemSizeMib: 2048,
+		},
+		ContainerCount: 1,
+		DriveMounts:    []*proto.FirecrackerDriveMount{mount},
+	})
+	require.NoErrorf(t, err, "Failed to create microVM[%s]", vmID)
+	defer fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: vmID})
+
+	_, err = fcClient.SetVMMetadata(ctx, &proto.SetVMMetadataRequest{
+		VMID:     vmID,
+		Metadata: fmt.Sprintf(dockerMetadataTemplate, "ghcr.io", noAuth, noAuth),
+	})
+	require.NoError(t, err, "Failed to configure VM metadata for registry resolution")
+
+	err = remoteImage.Pull(ctx,
+		containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib)),
+	)
+	require.NoError(t, err)
+
+	err = remoteImage.Copy(ctx, "image")
+	require.NoError(t, err)
+
+	err = vs.AddFrom(ctx, remoteImage)
+	require.NoError(t, err)
+
+	image, err := client.Pull(ctx,
+		alpine,
+		containerd.WithPullUnpack,
+		containerd.WithPullSnapshotter("devmapper"),
+	)
+	require.NoError(t, err)
+
+	mountsFromAL2, err := vs.WithMountsFromProvider(al2stargz)
+	require.NoError(t, err)
+
+	mountsFromPostgres, err := vs.WithMountsFromProvider(postgres)
+	require.NoError(t, err)
+
+	name := "cat"
+	snapshotName := fmt.Sprintf("%s-snapshot", name)
+	container, err := client.NewContainer(ctx,
+		name,
+		containerd.WithSnapshotter("devmapper"),
+		containerd.WithNewSnapshot(snapshotName, image),
+		containerd.WithNewSpec(
+			firecrackeroci.WithVMID(vmID),
+			oci.WithProcessArgs("sh", "-c", "ls -d /var/lib/postgresql/data; ls /etc/yum"),
+			oci.WithDefaultPathEnv,
+			mountsFromAL2,
+			mountsFromPostgres,
+		),
+	)
+	require.NoError(t, err, "failed to create container %s", name)
+	defer container.Delete(ctx, containerd.WithSnapshotCleanup)
+
+	result, err := integtest.RunTask(ctx, container)
+	require.NoError(t, err)
+
+	assert.Equal(t, uint32(0), result.ExitCode)
+	assert.Equal(t, "/var/lib/postgresql/data\nfssnap.d\npluginconf.d\nprotected.d\nvars\nversion-groups.conf\n", result.Stdout)
+	assert.Equal(t, "", result.Stderr)
+}

tools/image-builder/files_debootstrap/sbin/overlay-init

@@ -57,6 +57,10 @@ fi
 
 do_overlay
 
+# firecracker-containerd itself doesn't need /volumes but volume package
+# uses that to share files between in-VM snapshotters.
+mkdir /volumes
+
 # invoke the actual system init program and procede with the boot
 # process.
 exec /usr/sbin/init $@

volume/Makefile

@@ -0,0 +1,25 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+volume-init: $(shell find ./cmd -name '*.go')
+	go build -o $@ ./cmd/volume-init
+
+test:
+	go test ./...
+
+clean:
+	rm -f volume-init
+
+install:
+
+.PHONY: test clean install

volume/cmd/volume-init/main.go

@@ -0,0 +1,78 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/containerd/continuity/fs"
+	"github.com/firecracker-microvm/firecracker-containerd/volume"
+)
+
+func copy(in volume.GuestVolumeImageInput) error {
+	for _, v := range in.Volumes {
+		to, err := fs.RootPath(in.To, v)
+		if err != nil {
+			return fmt.Errorf("failed to join %q and %q: %w", in.To, v, err)
+		}
+
+		from, err := fs.RootPath(in.From, v)
+		if err != nil {
+			return fmt.Errorf("failed to join %q and %q: %w", in.From, v, err)
+		}
+
+		err = os.MkdirAll(to, 0700)
+		if err != nil {
+			return err
+		}
+
+		err = fs.CopyDir(to, from)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func realMain() error {
+	b, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		return err
+	}
+
+	var in volume.GuestVolumeImageInput
+	err = json.Unmarshal(b, &in)
+	if err != nil {
+		return err
+	}
+
+	return copy(in)
+}
+
+func main() {
+	err := realMain()
+	if err != nil {
+		out := volume.GuestVolumeImageOutput{Error: err.Error()}
+		b, err := json.Marshal(out)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to unmarshal %+v: %s", out, err)
+			os.Exit(2)
+		}
+		fmt.Printf("%s", string(b))
+		os.Exit(1)
+	}
+}

volume/cmd/volume-init/main_test.go

@@ -0,0 +1,51 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/firecracker-microvm/firecracker-containerd/volume"
+	"github.com/stretchr/testify/require"
+)
+
+func mkdirAndTouch(path, content string) error {
+	err := os.MkdirAll(filepath.Dir(path), 0700)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(path, []byte(content), 0600)
+}
+
+func TestCopy(t *testing.T) {
+	from := t.TempDir()
+	to := t.TempDir()
+
+	err := mkdirAndTouch(filepath.Join(from, "/etc/foobar/hello"), "helloworld")
+	require.NoError(t, err)
+
+	err = copy(volume.GuestVolumeImageInput{
+		From:    from,
+		To:      to,
+		Volumes: []string{"/etc/foobar"},
+	})
+	require.NoError(t, err)
+
+	b, err := os.ReadFile(filepath.Join(to, "/etc/foobar/hello"))
+	require.NoError(t, err)
+	require.Equal(t, "helloworld", string(b))
+}