Add support for remote Firecracker snapshots

- When remote snapshots are enabled, after committing the snapshot, it is uploaded to a MinIO instance. When loading from a snapshot, if it is not available locally, it checks if it is available in MinIO and fetches it.
- Remote Firecracker snapshots are currently only supported using the Stargz snapshotter (there are some container corruption issues when using devmapper).

Signed-off-by: André Jesus <[email protected]>

Author: andre-j3sus

Makefile

@@ -29,7 +29,7 @@ EXTRATESTFILES:=vhive_test.go stats.go vhive.go functions.go
 # WITHLAZY:=-lazyTest
 WITHUPF:=
 WITHLAZY:=
-WITHSNAPSHOTS:=-snapshotsTest
+WITHSNAPSHOTS:=-snapshotsTest 'local'
 CTRDLOGDIR:=/tmp/ctrd-logs
 
 vhive: proto

bench_test.go

@@ -63,7 +63,7 @@ func TestBenchParallelServe(t *testing.T) {
 	imageName, isPresent := images[*funcName]
 	require.True(t, isPresent, "Function is not supported")
 
-	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, "disabled")
 
 	createResultsDir()
 
@@ -136,7 +136,7 @@ func TestBenchWarmServe(t *testing.T) {
 	imageName, isPresent := images[*funcName]
 	require.True(t, isPresent, "Function is not supported")
 
-	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, "disabled")
 
 	createResultsDir()
 
@@ -201,7 +201,7 @@ func TestBenchServe(t *testing.T) {
 	imageName, isPresent := images[*funcName]
 	require.True(t, isPresent, "Function is not supported")
 
-	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, "disabled")
 
 	createResultsDir()
 

configs/.wordlist.txt

@@ -465,6 +465,7 @@ SMI
 sms
 SMT
 snapshotted
+snapshotters
 snapshotting
 SoC
 SOCACHE

cri/firecracker/coordinator.go

@@ -35,6 +35,9 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/vhive-serverless/vhive/ctriface"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
 type coordinator struct {
@@ -66,11 +69,32 @@ func newFirecrackerCoordinator(orch *ctriface.Orchestrator, opts ...coordinatorO
 		opt(c)
 	}
 
+	snapshotsMode := "disabled"
 	snapshotsDir := "/fccd/test/snapshots"
+
+	var minioClient *minio.Client
+	snapshotsBucket := "snapshots"
+	minioAddr := "localhost:50052"
+	minioAccessKey := "minio"
+	minioSecretKey := "minio123"
+
 	if !c.withoutOrchestrator {
+		snapshotsMode = orch.GetSnapshotMode()
 		snapshotsDir = orch.GetSnapshotsDir()
+		snapshotsBucket = orch.GetSnapshotsBucket()
+		minioAddr = orch.GetMinioAddr()
+		minioAccessKey = orch.GetMinioAccessKey()
+		minioSecretKey = orch.GetMinioSecretKey()
 	}
-	c.snapshotManager = snapshotting.NewSnapshotManager(snapshotsDir)
+
+	if snapshotsMode == "remote" {
+		minioClient, _ = minio.New(minioAddr, &minio.Options{
+			Creds:  credentials.NewStaticV4(minioAccessKey, minioSecretKey, ""),
+			Secure: false,
+		})
+	}
+
+	c.snapshotManager = snapshotting.NewSnapshotManager(snapshotsDir, snapshotsBucket, minioClient)
 
 	return c
 }
@@ -80,13 +104,27 @@ func (c *coordinator) startVM(ctx context.Context, image, revision string) (*fun
 }
 
 func (c *coordinator) startVMWithEnvironment(ctx context.Context, image, revision string, environment []string) (*funcInstance, error) {
-	if c.orch != nil && c.orch.GetSnapshotsEnabled() {
-		// Check if snapshot is available
-		if snap, err := c.snapshotManager.AcquireSnapshot(revision); err == nil {
+	if c.orch != nil && c.orch.GetSnapshotMode() != "disabled" {
+		if snap, err := c.snapshotManager.AcquireSnapshot(revision); snap == nil {
+			log.Printf("failed to acquire snapshot: %v", err)
+			if c.orch.GetSnapshotMode() == "remote" {
+				log.Printf("downloading snapshot from remote storage")
+				if _, err := c.snapshotManager.DownloadSnapshot(revision); err != nil {
+					log.Printf("failed to download snapshot from remote storage: %v", err)
+					_ = c.snapshotManager.DeleteSnapshot(revision) // TODO only for testing. Remove later
+				} else {
+					log.Printf("downloaded snapshot from remote storage")
+				}
+			}
+		}
+
+		if snap, _ := c.snapshotManager.AcquireSnapshot(revision); snap != nil {
+			log.Printf("loading snapshot %s", snap.GetId())
 			return c.orchLoadInstance(ctx, snap)
 		}
 	}
 
+	log.Printf("creating fresh instance")
 	return c.orchStartVM(ctx, image, revision, environment)
 }
 
@@ -102,7 +140,7 @@ func (c *coordinator) stopVM(ctx context.Context, containerID string) error {
 		return nil
 	}
 
-	if c.orch != nil && c.orch.GetSnapshotsEnabled() && !fi.SnapBooted {
+	if c.orch != nil && c.orch.GetSnapshotMode() != "disabled" && !fi.SnapBooted {
 		err := c.orchCreateSnapshot(ctx, fi)
 		if err != nil {
 			log.Printf("Err creating snapshot %s\n", err)
@@ -199,6 +237,7 @@ func (c *coordinator) orchLoadInstance(ctx context.Context, snap *snapshotting.S
 
 func (c *coordinator) orchCreateSnapshot(ctx context.Context, fi *funcInstance) error {
 	var err error
+	log.Printf("creating snapshot for %s\n", fi.Revision)
 
 	snap, err := c.snapshotManager.InitSnapshot(fi.Revision, fi.Image)
 	if err != nil {
@@ -230,11 +269,23 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, fi *funcInstance)
 		}
 	}
 
+	if err := snap.SerializeSnapInfo(); err != nil {
+		fi.Logger.WithError(err).Error("failed to serialize snapshot info")
+		return err
+	}
+
 	if err := c.snapshotManager.CommitSnapshot(fi.Revision); err != nil {
 		fi.Logger.WithError(err).Error("failed to commit snapshot")
 		return err
 	}
 
+	if c.orch.GetSnapshotMode() == "remote" {
+		fi.Logger.Debug("uploading snapshot to remote storage")
+		if err := c.snapshotManager.UploadSnapshot(fi.Revision); err != nil {
+			fi.Logger.WithError(err).Error("failed to upload snapshot")
+		}
+	}
+
 	return nil
 }
 

ctriface/orch.go

@@ -98,16 +98,21 @@ type Orchestrator struct {
 	imageManager      *image.ImageManager
 	dockerCredentials DockerCredentials
 	// store *skv.KVStore
-	snapshotsEnabled bool
-	isUPFEnabled     bool
-	isLazyMode       bool
-	snapshotsDir     string
-	isMetricsMode    bool
-	netPoolSize      int
+	snapshotMode    string
+	isUPFEnabled    bool
+	isLazyMode      bool
+	snapshotsDir    string
+	snapshotsBucket string
+	isMetricsMode   bool
+	netPoolSize     int
 
 	vethPrefix  string
 	clonePrefix string
 
+	minioAddr      string
+	minioAccessKey string
+	minioSecretKey string
+
 	memoryManager *manager.MemoryManager
 }
 
@@ -119,9 +124,13 @@ func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption)
 	o.cachedImages = make(map[string]containerd.Image)
 	o.snapshotter = snapshotter
 	o.snapshotsDir = "/fccd/snapshots"
+	o.snapshotsBucket = "snapshots"
 	o.netPoolSize = 10
 	o.vethPrefix = "172.17"
 	o.clonePrefix = "172.18"
+	o.minioAddr = "10.96.0.46:9000"
+	o.minioAccessKey = "minio"
+	o.minioSecretKey = "minio123"
 
 	for _, opt := range opts {
 		opt(o)
@@ -187,9 +196,9 @@ func (o *Orchestrator) Cleanup() {
 	}
 }
 
-// GetSnapshotsEnabled Returns the snapshots mode of the orchestrator
-func (o *Orchestrator) GetSnapshotsEnabled() bool {
-	return o.snapshotsEnabled
+// GetSnapshotMode Returns the snapshots mode of the orchestrator
+func (o *Orchestrator) GetSnapshotMode() string {
+	return o.snapshotMode
 }
 
 // GetUPFEnabled Returns the UPF mode of the orchestrator
@@ -252,6 +261,27 @@ func (o *Orchestrator) GetDockerCredentials() string {
 	return string(data)
 }
 
+// GetSnapshotsBucket returns the S3 bucket name used by the orchestrator for storing remote snapshots.
+func (o *Orchestrator) GetSnapshotsBucket() string {
+	return o.snapshotsBucket
+}
+
+// GetMinioAddr returns the address (endpoint) of the MinIO server used by the orchestrator.
+func (o *Orchestrator) GetMinioAddr() string {
+	return o.minioAddr
+}
+
+// GetMinioAccessKey returns the access key used to authenticate with the MinIO server.
+func (o *Orchestrator) GetMinioAccessKey() string {
+	return o.minioAccessKey
+}
+
+// GetMinioSecretKey returns the secret key used to authenticate with the MinIO server.
+// This should be handled securely and never exposed in logs or error messages.
+func (o *Orchestrator) GetMinioSecretKey() string {
+	return o.minioSecretKey
+}
+
 func (o *Orchestrator) setupHeartbeat() {
 	heartbeat := time.NewTicker(60 * time.Second)
 

ctriface/orch_options.go

@@ -40,10 +40,10 @@ func WithTestModeOn(testModeOn bool) OrchestratorOption {
 	}
 }
 
-// WithSnapshots Sets the snapshot mode on or off
-func WithSnapshots(snapshotsEnabled bool) OrchestratorOption {
+// WithSnapshotMode Sets the snapshot mode
+func WithSnapshotMode(snapshotMode string) OrchestratorOption {
 	return func(o *Orchestrator) {
-		o.snapshotsEnabled = snapshotsEnabled
+		o.snapshotMode = snapshotMode
 	}
 }
 
@@ -111,3 +111,26 @@ func WithDockerCredentials(dockerCredentials string) OrchestratorOption {
 		o.dockerCredentials = creds
 	}
 }
+
+// WithMinioAddr Sets the MinIO server address (endpoint)
+func WithMinioAddr(minioAddr string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioAddr = minioAddr
+	}
+}
+
+// WithMinioAccessKey Sets the MinIO access key
+// Used in conjunction with the secret key for authentication with the MinIO server
+func WithMinioAccessKey(minioAccessKey string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioAccessKey = minioAccessKey
+	}
+}
+
+// WithMinioSecretKey Sets the MinIO secret key
+// Used in conjunction with the access key for authentication with the MinIO server
+func WithMinioSecretKey(minioSecretKey string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioSecretKey = minioSecretKey
+	}
+}

docs/quickstart_guide.md

@@ -172,11 +172,11 @@ Another option is to install using official instructions: [https://golang.org/do
     # EITHER
     sudo screen -dmS vhive bash -c "./vhive > >(tee -a /tmp/vhive-logs/vhive.stdout) 2> >(tee -a /tmp/vhive-logs/vhive.stderr >&2)"
     # OR
-    sudo screen -dmS vhive bash -c "./vhive -snapshots > >(tee -a /tmp/vhive-logs/vhive.stdout) 2> >(tee -a /tmp/vhive-logs/vhive.stderr >&2)"
+    sudo screen -dmS vhive bash -c "./vhive -snapshots 'local' > >(tee -a /tmp/vhive-logs/vhive.stdout) 2> >(tee -a /tmp/vhive-logs/vhive.stderr >&2)"
     ```
     > **Note:**
     >
-    > By default, the microVMs are booted, `-snapshots` enables snapshots after the 2nd invocation of each function.
+    > By default, the microVMs are booted, `-snapshots <local|remote>` enables snapshots after the 2nd invocation of each function.
     >
     > If `-snapshots` and `-upf` are specified, the snapshots are accelerated with the Record-and-Prefetch (REAP)
     technique that we described in our ASPLOS'21
@@ -186,6 +186,8 @@ Another option is to install using official instructions: [https://golang.org/do
     >
     > If you are using `stargz` with `firecracker`, you also need to set the `-dockerCredentials` flag to be able to [pull the images
     from inside the microVMs](https://github.com/firecracker-microvm/firecracker-containerd/blob/main/docker-credential-mmds/README.md#docker-credential-helper-mmds).
+    >
+    > Remote snapshots are only supported in the `firecracker` mode using `stargz`. Check the [snapshot guide](../docs/snapshots.md) for more details on how to set up remote snapshots.
 
 ### 3. Configure Master Node
 **On the master node**, execute the following instructions below **as a non-root user with sudo rights** using **bash**:
@@ -297,7 +299,7 @@ Execute the following below **as a non-root user with sudo rights** using **bash
 
     > **Note:**
     >
-    > By default, the microVMs are booted, `-snapshots` enables snapshots after the 2nd invocation of each function.
+    > By default, the microVMs are booted, `-snapshots <local|remote>` enables snapshots after the 2nd invocation of each function.
     >
     > If `-snapshots` and `-upf` are specified, the snapshots are accelerated with the Record-and-Prefetch (REAP)
     technique that we described in our ASPLOS'21
@@ -307,6 +309,8 @@ Execute the following below **as a non-root user with sudo rights** using **bash
     >
     > If you are using `stargz` with `firecracker`, you also need to set the `-dockerCredentials` flag to be able to [pull the images
     from inside the microVMs](https://github.com/firecracker-microvm/firecracker-containerd/blob/main/docker-credential-mmds/README.md#docker-credential-helper-mmds).
+    >
+    > Remote snapshots are only supported in the `firecracker` mode using `stargz`. Check the [snapshot guide](../docs/snapshots.md) for more details on how to set up remote snapshots.
 
 6. Run the single node cluster setup script:
     ```bash