feat: add demultiplexing snapshotter

This change adds a demultiplexing snapshotter application which allows
for proxying remote snapshotter requests across the microVM threshold.

The snapshotter includes a vsock lookup mechanism via namespace
specified via configuration file. Also includes a cache mechanism
for vsock lookup results.

Signed-off-by: Austin Vazquez <[email protected]>

Author: austinvazquez

Makefile

@@ -11,7 +11,7 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-SUBDIRS:=agent runtime examples firecracker-control/cmd/containerd
+SUBDIRS:=agent runtime examples firecracker-control/cmd/containerd snapshotter
 TEST_SUBDIRS:=$(addprefix test-,$(SUBDIRS))
 INTEG_TEST_SUBDIRS:=$(addprefix integ-test-,$(SUBDIRS))
 

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5
 	github.com/opencontainers/runc v1.1.0
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210910115017-0d6cc581aeea
+	github.com/pelletier/go-toml v1.9.3
 	github.com/pkg/errors v0.9.1
 	github.com/shirou/gopsutil v2.18.12+incompatible
 	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect

snapshotter/.gitignore

@@ -0,0 +1,2 @@
+demux-snapshotter
+http-resolver

snapshotter/Makefile

@@ -0,0 +1,42 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+EXTRAGOARGS?=
+
+SOURCES := $(shell find . -name '*.go')
+GOMOD := $(shell go env GOMOD)
+GOSUM := $(GOMOD:.mod=.sum)
+
+REVISION := $(shell git rev-parse HEAD)
+
+all: demux-snapshotter
+
+demux-snapshotter: $(SOURCES) $(GOMOD) $(GOSUM)
+	go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@
+
+http-resolver: $(SOURCES) $(GOMOD) $(GOSUM)
+	go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ internal/http_address_resolver.go
+
+test:
+	go test ./... $(EXTRAGOARGS)
+
+clean:
+	- rm -f demux-snapshotter
+	- rm -f http-resolver
+
+distclean: clean
+
+# Install is a noop here, for now.
+install:
+
+.PHONY: all install test clean distclean

snapshotter/app/service.go

@@ -0,0 +1,152 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package app
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+
+	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
+	"github.com/containerd/containerd/contrib/snapshotservice"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/snapshots"
+	"github.com/firecracker-microvm/firecracker-go-sdk/vsock"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc"
+
+	"github.com/firecracker-microvm/firecracker-containerd/snapshotter/config"
+	"github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux"
+	"github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/cache"
+	"github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy"
+	proxyaddress "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy/address"
+)
+
+// Run the demultiplexing snapshotter service.
+//
+// The snapshotter server will be running on the
+// network address and port specified in listener config.
+func Run(config config.Config) error {
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE, syscall.SIGHUP, syscall.SIGQUIT)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	group, ctx := errgroup.WithContext(ctx)
+
+	snapshotter, err := initSnapshotter(ctx, config)
+	if err != nil {
+		log.G(ctx).WithFields(
+			logrus.Fields{"resolver": config.Snapshotter.Proxy.Address.Resolver.Type},
+		).WithError(err).Fatal("failed creating socket resolver")
+		return err
+	}
+
+	grpcServer := grpc.NewServer()
+	service := snapshotservice.FromSnapshotter(snapshotter)
+	snapshotsapi.RegisterSnapshotsServer(grpcServer, service)
+
+	listenerConfig := config.Snapshotter.Listener
+	listener, err := net.Listen(listenerConfig.Network, listenerConfig.Address)
+	if err != nil {
+		log.G(ctx).WithFields(
+			logrus.Fields{
+				"network": listenerConfig.Network,
+				"address": listenerConfig.Address,
+			},
+		).WithError(err).Fatal("failed creating listener")
+		return err
+	}
+
+	group.Go(func() error {
+		return grpcServer.Serve(listener)
+	})
+
+	group.Go(func() error {
+		defer func() {
+			log.G(ctx).Info("stopping server")
+			grpcServer.Stop()
+
+			if err := snapshotter.Close(); err != nil {
+				log.G(ctx).WithError(err).Error("failed to close snapshotter")
+			}
+		}()
+
+		for {
+			select {
+			case <-stop:
+				cancel()
+				return nil
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+	})
+
+	if err := group.Wait(); err != nil {
+		log.G(ctx).WithError(err).Error("demux snapshotter error")
+		return err
+	}
+
+	log.G(ctx).Info("done")
+	return nil
+}
+
+func initResolver(config config.Config) (proxyaddress.Resolver, error) {
+	resolverConfig := config.Snapshotter.Proxy.Address.Resolver
+	switch resolverConfig.Type {
+	case "http":
+		return proxyaddress.NewHTTPResolver(resolverConfig.Address), nil
+	default:
+		return nil, fmt.Errorf("invalid resolver type: %s", resolverConfig.Type)
+	}
+}
+
+const base10 = 10
+const bits32 = 32
+
+func initSnapshotter(ctx context.Context, config config.Config) (snapshots.Snapshotter, error) {
+	resolver, err := initResolver(config)
+	if err != nil {
+		return nil, err
+	}
+
+	newProxySnapshotterFunc := func(ctx context.Context, namespace string) (snapshots.Snapshotter, error) {
+		r := resolver
+		response, err := r.Get(namespace)
+		if err != nil {
+			return nil, err
+		}
+		host, portstr, err := net.SplitHostPort(response.Address)
+		if err != nil {
+			return nil, err
+		}
+		port, err := strconv.ParseUint(portstr, base10, bits32)
+		if err != nil {
+			return nil, err
+		}
+		snapshotterDialer := func(ctx context.Context, namespace string) (net.Conn, error) {
+			return vsock.DialContext(ctx, host, uint32(port), vsock.WithLogger(log.G(ctx)))
+		}
+		return proxy.NewProxySnapshotter(ctx, host, snapshotterDialer)
+	}
+
+	return demux.NewSnapshotter(cache.NewSnapshotterCache(), newProxySnapshotterFunc), nil
+}

snapshotter/config/config.go

@@ -0,0 +1,88 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package config
+
+import (
+	"io"
+	"os"
+
+	"github.com/pelletier/go-toml"
+)
+
+// Config contains metadata necessary to forward snapshotter requests
+// to the destined remote snapshotter.
+//
+// The default location for the configuration file is '/etc/demux-snapshotter/config.toml'
+type Config struct {
+	Snapshotter snapshotter `toml:"snapshotter"`
+
+	Debug debug `toml:"debug"`
+}
+
+type snapshotter struct {
+	Listener listener `toml:"listener"`
+	Proxy    proxy    `toml:"proxy"`
+}
+
+type listener struct {
+	Network string `toml:"network" default:"unix"`
+	Address string `toml:"address" default:"/var/lib/demux-snapshotter/snapshotter.sock"`
+}
+
+type proxy struct {
+	Address address `toml:"address"`
+}
+
+type address struct {
+	Resolver resolver `toml:"resolver"`
+}
+
+type resolver struct {
+	Type    string `toml:"type"`
+	Address string `toml:"address"`
+}
+
+type debug struct {
+	LogLevel string `toml:"logLevel" default:"info"`
+}
+
+// Load parses application configuration from a specified file path.
+func Load(filePath string) (Config, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return Config{}, err
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return Config{}, err
+	}
+
+	return load(file, fileInfo.Size())
+}
+
+func load(reader io.Reader, readSize int64) (Config, error) {
+	buffer := make([]byte, readSize)
+	readBytes, err := reader.Read(buffer)
+	if int64(readBytes) != readSize || err != nil {
+		return Config{}, err
+	}
+
+	config := Config{}
+	if err = toml.Unmarshal(buffer, &config); err != nil {
+		return Config{}, err
+	}
+	return config, nil
+}

snapshotter/config/config.toml.example

@@ -0,0 +1,10 @@
+[snapshotter.listener]
+  type = "unix"
+  address = "/var/lib/demux-snapshotter/snapshotter.sock"
+
+[snapshotter.proxy.address.resolver]
+  type = "http"
+  address = "127.0.0.1:10001"
+
+[debug]
+  logLevel = "info"