Add support for remote Firecracker snapshots

- When remote snapshots are enabled, after committing the snapshot, it is uploaded to a MinIO instance. When loading from a snapshot, if it is not available locally, it checks if it is available in MinIO and fetches it.
- Remote Firecracker snapshots are currently only supported using the Stargz snapshotter (there are some container corruption issues when using devmapper).

Signed-off-by: André Jesus <[email protected]>

Author: andre-j3sus

.github/workflows/unit_tests.yml

@@ -27,14 +27,32 @@ jobs:
       fail-fast: false
       matrix:
         module: [misc, networking, snapshotting]
+    services:
+      minio: # MinIO service for testing remote snapshots
+        image: lazybit/minio # Can't use minio/minio because there's still no support for the jobs.<job_id>.services.<service_id>.command option in GH Actions
+        ports:
+          - 9000:9000
+        options: >-
+          --health-cmd "curl -f http://localhost:9000/minio/health/live || exit 1"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        volumes:
+          - ${{ github.workspace }}/data:/data
+        env:
+          MINIO_ROOT_USER: minio
+          MINIO_ROOT_PASSWORD: minio123
+        # command: server /data
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v4
+      with:
+        path: ${{ github.sha }}
 
     - name: Set up Go version in go.mod file
       uses: actions/setup-go@v5
       with:
-        go-version-file: ${{ github.workspace }}/go.mod
+        go-version-file: ${{ github.workspace }}/${{ github.sha }}/go.mod
         cache-dependency-path: |
           **/go.sum
           **/go.mod
@@ -45,27 +63,27 @@ jobs:
         python-version: '3.x'
 
     - name: Build setup scripts
-      run: pushd scripts && go build -o setup_tool && popd
+      run: pushd ${{ github.sha }}/scripts && go build -o setup_tool && popd
 
     - name: Add rsync
       run: |
         sudo apt update
         sudo apt install rsync -y
 
     - name: Setup System
-      run: ./scripts/setup_tool setup_system
+      run: ./${{ github.sha }}/scripts/setup_tool setup_system
 
     - name: Build
-      run: go build -race -v -a ./...
+      run: cd ${{ github.sha }} && go build -race -v -a ./...
 
     - name: Run tests in submodules
       env:
           MODULE: ${{ matrix.module }}
           AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
           AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
       run: |
-        make -C $MODULE test
-        make -C $MODULE test-man
+        make -C ${{ github.sha }}/${{ matrix.module }} test
+        make -C ${{ github.sha }}/${{ matrix.module }} test-man
   
   profile-unit-test:
     name: "Unit test: profile unit test"

Makefile

@@ -29,7 +29,7 @@ EXTRATESTFILES:=vhive_test.go stats.go vhive.go functions.go
 # WITHLAZY:=-lazyTest
 WITHUPF:=
 WITHLAZY:=
-WITHSNAPSHOTS:=-snapshotsTest
+WITHSNAPSHOTS:=-snapshotsTest 'local'
 CTRDLOGDIR:=/tmp/ctrd-logs
 
 vhive: proto

bench_test.go

@@ -63,7 +63,7 @@ func TestBenchParallelServe(t *testing.T) {
 	imageName, isPresent := images[*funcName]
 	require.True(t, isPresent, "Function is not supported")
 
-	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, *snapshotTestMode)
 
 	createResultsDir()
 
@@ -136,7 +136,7 @@ func TestBenchWarmServe(t *testing.T) {
 	imageName, isPresent := images[*funcName]
 	require.True(t, isPresent, "Function is not supported")
 
-	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, *snapshotTestMode)
 
 	createResultsDir()
 
@@ -201,7 +201,7 @@ func TestBenchServe(t *testing.T) {
 	imageName, isPresent := images[*funcName]
 	require.True(t, isPresent, "Function is not supported")
 
-	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, *snapshotTestMode)
 
 	createResultsDir()
 

configs/.wordlist.txt

@@ -465,6 +465,7 @@ SMI
 sms
 SMT
 snapshotted
+snapshotters
 snapshotting
 SoC
 SOCACHE

cri/firecracker/coordinator.go

@@ -35,6 +35,9 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/vhive-serverless/vhive/ctriface"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
 type coordinator struct {
@@ -67,10 +70,22 @@ func newFirecrackerCoordinator(orch *ctriface.Orchestrator, opts ...coordinatorO
 	}
 
 	snapshotsDir := "/fccd/test/snapshots"
+	snapshotsBucket := "snapshots"
+	var minioClient *minio.Client
+
 	if !c.withoutOrchestrator {
 		snapshotsDir = orch.GetSnapshotsDir()
+		snapshotsBucket = orch.GetSnapshotsBucket()
+
+		if orch.GetSnapshotMode() == "remote" {
+			minioClient, _ = minio.New(orch.GetMinioAddr(), &minio.Options{
+				Creds:  credentials.NewStaticV4(orch.GetMinioAccessKey(), orch.GetMinioSecretKey(), ""),
+				Secure: false,
+			})
+		}
 	}
-	c.snapshotManager = snapshotting.NewSnapshotManager(snapshotsDir)
+
+	c.snapshotManager = snapshotting.NewSnapshotManager(snapshotsDir, snapshotsBucket, minioClient)
 
 	return c
 }
@@ -80,9 +95,18 @@ func (c *coordinator) startVM(ctx context.Context, image, revision string) (*fun
 }
 
 func (c *coordinator) startVMWithEnvironment(ctx context.Context, image, revision string, environment []string) (*funcInstance, error) {
-	if c.orch != nil && c.orch.GetSnapshotsEnabled() {
-		// Check if snapshot is available
-		if snap, err := c.snapshotManager.AcquireSnapshot(revision); err == nil {
+	if c.orch != nil && c.orch.GetSnapshotMode() != "disabled" {
+		if snap, _ := c.snapshotManager.AcquireSnapshot(revision); snap == nil {
+			if c.orch.GetSnapshotMode() == "remote" {
+				if _, err := c.snapshotManager.DownloadSnapshot(revision); err != nil {
+					log.WithError(err).Errorf("failed to download snapshot %s from remote storage", revision)
+				} else {
+					log.Printf("downloaded snapshot %s from remote storage", revision)
+				}
+			}
+		}
+
+		if snap, _ := c.snapshotManager.AcquireSnapshot(revision); snap != nil {
 			return c.orchLoadInstance(ctx, snap)
 		}
 	}
@@ -102,7 +126,7 @@ func (c *coordinator) stopVM(ctx context.Context, containerID string) error {
 		return nil
 	}
 
-	if c.orch != nil && c.orch.GetSnapshotsEnabled() && !fi.SnapBooted {
+	if c.orch != nil && c.orch.GetSnapshotMode() != "disabled" && !fi.SnapBooted {
 		err := c.orchCreateSnapshot(ctx, fi)
 		if err != nil {
 			log.Printf("Err creating snapshot %s\n", err)
@@ -230,11 +254,23 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, fi *funcInstance)
 		}
 	}
 
+	if err := snap.SerializeSnapInfo(); err != nil {
+		fi.Logger.WithError(err).Error("failed to serialize snapshot info")
+		return err
+	}
+
 	if err := c.snapshotManager.CommitSnapshot(fi.Revision); err != nil {
 		fi.Logger.WithError(err).Error("failed to commit snapshot")
 		return err
 	}
 
+	if !c.withoutOrchestrator && c.orch.GetSnapshotMode() == "remote" {
+		fi.Logger.Debug("uploading snapshot to remote storage")
+		if err := c.snapshotManager.UploadSnapshot(fi.Revision); err != nil {
+			fi.Logger.WithError(err).Error("failed to upload snapshot")
+		}
+	}
+
 	return nil
 }
 

ctriface/orch.go

@@ -98,16 +98,21 @@ type Orchestrator struct {
 	imageManager      *image.ImageManager
 	dockerCredentials DockerCredentials
 	// store *skv.KVStore
-	snapshotsEnabled bool
-	isUPFEnabled     bool
-	isLazyMode       bool
-	snapshotsDir     string
-	isMetricsMode    bool
-	netPoolSize      int
+	snapshotMode    string
+	isUPFEnabled    bool
+	isLazyMode      bool
+	snapshotsDir    string
+	snapshotsBucket string
+	isMetricsMode   bool
+	netPoolSize     int
 
 	vethPrefix  string
 	clonePrefix string
 
+	minioAddr      string
+	minioAccessKey string
+	minioSecretKey string
+
 	memoryManager *manager.MemoryManager
 }
 
@@ -119,9 +124,13 @@ func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption)
 	o.cachedImages = make(map[string]containerd.Image)
 	o.snapshotter = snapshotter
 	o.snapshotsDir = "/fccd/snapshots"
+	o.snapshotsBucket = "snapshots"
 	o.netPoolSize = 10
 	o.vethPrefix = "172.17"
 	o.clonePrefix = "172.18"
+	o.minioAddr = "10.96.0.46:9000"
+	o.minioAccessKey = "minio"
+	o.minioSecretKey = "minio123"
 
 	for _, opt := range opts {
 		opt(o)
@@ -187,9 +196,9 @@ func (o *Orchestrator) Cleanup() {
 	}
 }
 
-// GetSnapshotsEnabled Returns the snapshots mode of the orchestrator
-func (o *Orchestrator) GetSnapshotsEnabled() bool {
-	return o.snapshotsEnabled
+// GetSnapshotMode Returns the snapshots mode of the orchestrator
+func (o *Orchestrator) GetSnapshotMode() string {
+	return o.snapshotMode
 }
 
 // GetUPFEnabled Returns the UPF mode of the orchestrator
@@ -252,6 +261,27 @@ func (o *Orchestrator) GetDockerCredentials() string {
 	return string(data)
 }
 
+// GetSnapshotsBucket returns the S3 bucket name used by the orchestrator for storing remote snapshots.
+func (o *Orchestrator) GetSnapshotsBucket() string {
+	return o.snapshotsBucket
+}
+
+// GetMinioAddr returns the address (endpoint) of the MinIO server used by the orchestrator.
+func (o *Orchestrator) GetMinioAddr() string {
+	return o.minioAddr
+}
+
+// GetMinioAccessKey returns the access key used to authenticate with the MinIO server.
+func (o *Orchestrator) GetMinioAccessKey() string {
+	return o.minioAccessKey
+}
+
+// GetMinioSecretKey returns the secret key used to authenticate with the MinIO server.
+// This should be handled securely and never exposed in logs or error messages.
+func (o *Orchestrator) GetMinioSecretKey() string {
+	return o.minioSecretKey
+}
+
 func (o *Orchestrator) setupHeartbeat() {
 	heartbeat := time.NewTicker(60 * time.Second)
 

ctriface/orch_options.go

@@ -40,10 +40,10 @@ func WithTestModeOn(testModeOn bool) OrchestratorOption {
 	}
 }
 
-// WithSnapshots Sets the snapshot mode on or off
-func WithSnapshots(snapshotsEnabled bool) OrchestratorOption {
+// WithSnapshotMode Sets the snapshot mode
+func WithSnapshotMode(snapshotMode string) OrchestratorOption {
 	return func(o *Orchestrator) {
-		o.snapshotsEnabled = snapshotsEnabled
+		o.snapshotMode = snapshotMode
 	}
 }
 
@@ -111,3 +111,26 @@ func WithDockerCredentials(dockerCredentials string) OrchestratorOption {
 		o.dockerCredentials = creds
 	}
 }
+
+// WithMinioAddr Sets the MinIO server address (endpoint)
+func WithMinioAddr(minioAddr string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioAddr = minioAddr
+	}
+}
+
+// WithMinioAccessKey Sets the MinIO access key
+// Used in conjunction with the secret key for authentication with the MinIO server
+func WithMinioAccessKey(minioAccessKey string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioAccessKey = minioAccessKey
+	}
+}
+
+// WithMinioSecretKey Sets the MinIO secret key
+// Used in conjunction with the access key for authentication with the MinIO server
+func WithMinioSecretKey(minioSecretKey string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.minioSecretKey = minioSecretKey
+	}
+}

docs/quickstart_guide.md

@@ -172,11 +172,11 @@ Another option is to install using official instructions: [https://golang.org/do
     # EITHER
     sudo screen -dmS vhive bash -c "./vhive > >(tee -a /tmp/vhive-logs/vhive.stdout) 2> >(tee -a /tmp/vhive-logs/vhive.stderr >&2)"
     # OR
-    sudo screen -dmS vhive bash -c "./vhive -snapshots > >(tee -a /tmp/vhive-logs/vhive.stdout) 2> >(tee -a /tmp/vhive-logs/vhive.stderr >&2)"
+    sudo screen -dmS vhive bash -c "./vhive -snapshots 'local' > >(tee -a /tmp/vhive-logs/vhive.stdout) 2> >(tee -a /tmp/vhive-logs/vhive.stderr >&2)"
     ```
     > **Note:**
     >
-    > By default, the microVMs are booted, `-snapshots` enables snapshots after the 2nd invocation of each function.
+    > By default, the microVMs are booted, `-snapshots <local|remote>` enables snapshots after the 2nd invocation of each function.
     >
     > If `-snapshots` and `-upf` are specified, the snapshots are accelerated with the Record-and-Prefetch (REAP)
     technique that we described in our ASPLOS'21
@@ -186,6 +186,8 @@ Another option is to install using official instructions: [https://golang.org/do
     >
     > If you are using `stargz` with `firecracker`, you also need to set the `-dockerCredentials` flag to be able to [pull the images
     from inside the microVMs](https://github.com/firecracker-microvm/firecracker-containerd/blob/main/docker-credential-mmds/README.md#docker-credential-helper-mmds).
+    >
+    > Remote snapshots are only supported in the `firecracker` mode using `stargz`. Check the [snapshot guide](../docs/snapshots.md) for more details on how to set up remote snapshots.
 
 ### 3. Configure Master Node
 **On the master node**, execute the following instructions below **as a non-root user with sudo rights** using **bash**:
@@ -297,7 +299,7 @@ Execute the following below **as a non-root user with sudo rights** using **bash
 
     > **Note:**
     >
-    > By default, the microVMs are booted, `-snapshots` enables snapshots after the 2nd invocation of each function.
+    > By default, the microVMs are booted, `-snapshots <local|remote>` enables snapshots after the 2nd invocation of each function.
     >
     > If `-snapshots` and `-upf` are specified, the snapshots are accelerated with the Record-and-Prefetch (REAP)
     technique that we described in our ASPLOS'21
@@ -307,6 +309,8 @@ Execute the following below **as a non-root user with sudo rights** using **bash
     >
     > If you are using `stargz` with `firecracker`, you also need to set the `-dockerCredentials` flag to be able to [pull the images
     from inside the microVMs](https://github.com/firecracker-microvm/firecracker-containerd/blob/main/docker-credential-mmds/README.md#docker-credential-helper-mmds).
+    >
+    > Remote snapshots are only supported in the `firecracker` mode using `stargz`. Check the [snapshot guide](../docs/snapshots.md) for more details on how to set up remote snapshots.
 
 6. Run the single node cluster setup script:
     ```bash