add separate option for new snapshots

Signed-off-by: Amory Hoste <[email protected]>

Author: amohoste

bin/containerd-shim-aws-firecracker

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e290046f2e24c117ef450a3bef6c8f8e3b1ec387decc76ccc936e1f54c827327
-size 26355405
+oid sha256:5b48fcdff74c342e8b4f65659139056dea1c27fdb99a0c2f267070b6b3b97b0b
+size 26530283

bin/firecracker

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:561cff75b2e1d768d2a4e7dad01cffb3eaff194e1b1696ad3ede5284c404fb0c
-size 4010736
+oid sha256:d42ddb2c3d970d6a234e0d3f92980e085fc04a9ae17e29e05bb4ca73debfe0b8
+size 4016240

bin/firecracker-containerd

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89c20c096978dafa7f3ba3b1d66a9e574f2fd89f3781ee0537da30120aea6455
-size 46999272
+oid sha256:cc908873170a25ca713ca2e80323cf1496d5d9b7a3449778d0018a84825dd0f7
+size 47224352

bin/firecracker-ctr

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b0bab69371a224e9eaed86edb26dd57e2a0b04eaa7e9b4da7e3e8c7c38e0016
-size 34476496
+oid sha256:51a994f7cb2cd48087a4b5a27476577c60d9fd6ce34a470435de5f33c2fb3508
+size 34510472

bin/jailer

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:375abd369c55ad8057ec6cd39ee77e8f68933fd7a97e1d1901881805f22815f8
-size 3060760
+oid sha256:80284269eb8b44483b84a527bf7f4012932d94410e32bd20e4d25d09303336ea
+size 3072784

cri/firecracker/coordinator.go

@@ -25,25 +25,30 @@ package firecracker
 import (
 	"context"
 	"fmt"
+	"github.com/ease-lab/vhive/ctriface"
 	"github.com/ease-lab/vhive/metrics"
 	"github.com/ease-lab/vhive/snapshotting"
+	"github.com/ease-lab/vhive/snapshotting/deduplicated"
+	"github.com/ease-lab/vhive/snapshotting/regular"
 	"github.com/pkg/errors"
 	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/ease-lab/vhive/ctriface"
 	log "github.com/sirupsen/logrus"
 )
 
 const snapshotsDir = "/fccd/snapshots"
 
+// TODO: interface for orchestrator
+
 type coordinator struct {
 	sync.Mutex
 	orch   *ctriface.Orchestrator
 	nextID uint64
 	isSparseSnaps bool
+	isDeduplicatedSnaps bool
 
 	activeInstances     map[string]*FuncInstance
 	snapshotManager     *snapshotting.SnapshotManager
@@ -59,12 +64,18 @@ func withoutOrchestrator() coordinatorOption {
 	}
 }
 
-func newFirecrackerCoordinator(orch *ctriface.Orchestrator, snapsCapacityMiB int64, isSparseSnaps bool, opts ...coordinatorOption) *coordinator {
+func newFirecrackerCoordinator(orch *ctriface.Orchestrator, snapsCapacityMiB int64, isSparseSnaps bool, isDeduplicatedSnaps bool, opts ...coordinatorOption) *coordinator {
 	c := &coordinator{
 		activeInstances: make(map[string]*FuncInstance),
 		orch:            orch,
-		snapshotManager: snapshotting.NewSnapshotManager(snapshotsDir, snapsCapacityMiB),
-		isSparseSnaps: isSparseSnaps,
+		isSparseSnaps:   isSparseSnaps,
+		isDeduplicatedSnaps: isDeduplicatedSnaps,
+	}
+
+	if isDeduplicatedSnaps {
+		c.snapshotManager = snapshotting.NewSnapshotManager(deduplicated.NewSnapshotManager(snapshotsDir, snapsCapacityMiB))
+	} else {
+		c.snapshotManager = snapshotting.NewSnapshotManager(regular.NewRegularSnapshotManager(snapshotsDir))
 	}
 
 	for _, opt := range opts {
@@ -76,12 +87,24 @@ func newFirecrackerCoordinator(orch *ctriface.Orchestrator, snapsCapacityMiB int
 
 func (c *coordinator) startVM(ctx context.Context, image string, revision string, memSizeMib, vCPUCount uint32) (*FuncInstance, error) {
 	if c.orch != nil && c.orch.GetSnapshotsEnabled()  {
+		id := image
+		if c.isDeduplicatedSnaps {
+			id = revision
+		}
+
 		// Check if snapshot is available
-		if snap, err := c.snapshotManager.AcquireSnapshot(revision); err == nil {
+		if snap, err := c.snapshotManager.AcquireSnapshot(id); err == nil {
 			if snap.MemSizeMib != memSizeMib || snap.VCPUCount != vCPUCount {
 				return nil, errors.New("Please create a new revision when updating uVM memory size or vCPU count")
 			} else {
-				return c.orchStartVMSnapshot(ctx, snap, memSizeMib, vCPUCount)
+				vmID := ""
+				if c.isDeduplicatedSnaps {
+					vmID = strconv.Itoa(int(atomic.AddUint64(&c.nextID, 1)))
+				} else {
+					vmID = snap.GetId()
+				}
+
+				return c.orchStartVMSnapshot(ctx, snap, memSizeMib, vCPUCount, vmID)
 			}
 		} else {
 			return c.orchStartVM(ctx, image, revision, memSizeMib, vCPUCount)
@@ -106,17 +129,30 @@ func (c *coordinator) stopVM(ctx context.Context, containerID string) error {
 		return nil
 	}
 
+	if c.orch == nil || ! c.orch.GetSnapshotsEnabled() {
+		return c.orchStopVM(ctx, fi)
+	}
+
+	id := fi.vmID
+	if c.isDeduplicatedSnaps {
+		id = fi.revisionId
+	}
+
 	if fi.snapBooted {
-		defer c.snapshotManager.ReleaseSnapshot(fi.revisionId)
-	} else if c.orch != nil && c.orch.GetSnapshotsEnabled() {
+		defer c.snapshotManager.ReleaseSnapshot(id)
+	} else {
 		// Create snapshot
 		err := c.orchCreateSnapshot(ctx, fi)
 		if err != nil {
 			log.Printf("Err creating snapshot %s\n", err)
 		}
 	}
 
-	return c.orchStopVM(ctx, fi)
+	if c.isDeduplicatedSnaps {
+		return c.orchStopVM(ctx, fi)
+	} else {
+		return c.orchOffloadVM(ctx, fi)
+	}
 }
 
 // for testing
@@ -178,9 +214,8 @@ func (c *coordinator) orchStartVM(ctx context.Context, image, revision string, m
 	return fi, err
 }
 
-func (c *coordinator) orchStartVMSnapshot(ctx context.Context, snap *snapshotting.Snapshot, memSizeMib, vCPUCount uint32) (*FuncInstance, error) {
+func (c *coordinator) orchStartVMSnapshot(ctx context.Context, snap *snapshotting.Snapshot, memSizeMib, vCPUCount uint32, vmID string) (*FuncInstance, error) {
 	tStartCold := time.Now()
-	vmID := strconv.Itoa(int(atomic.AddUint64(&c.nextID, 1)))
 	logger := log.WithFields(
 		log.Fields{
 			"vmID":  vmID,
@@ -210,7 +245,7 @@ func (c *coordinator) orchStartVMSnapshot(ctx context.Context, snap *snapshottin
 	}
 
 	coldStartTimeMs := metrics.ToMs(time.Since(tStartCold))
-	fi := NewFuncInstance(vmID, snap.GetImage(), snap.GetRevisionId(), resp, true, memSizeMib, vCPUCount, coldStartTimeMs)
+	fi := NewFuncInstance(vmID, snap.GetImage(), snap.GetId(), resp, true, memSizeMib, vCPUCount, coldStartTimeMs)
 	logger.Debug("successfully loaded instance from snapshot")
 
 	return fi, err
@@ -224,17 +259,23 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, fi *FuncInstance)
 		},
 	)
 
-	removeContainerSnaps, snap, err := c.snapshotManager.InitSnapshot(fi.revisionId, fi.image, fi.coldStartTimeMs, fi.memSizeMib, fi.vCPUCount, c.isSparseSnaps)
+	id := fi.vmID
+	if c.isDeduplicatedSnaps {
+		id = fi.revisionId
+	}
+
+	removeContainerSnaps, snap, err := c.snapshotManager.InitSnapshot(id, fi.image, fi.coldStartTimeMs, fi.memSizeMib, fi.vCPUCount, c.isSparseSnaps)
+
 	if err != nil {
 		if fmt.Sprint(err) == "There is not enough free space available" {
 			fi.logger.Info(fmt.Sprintf("There is not enough space available for snapshots of %s", fi.revisionId))
 		}
 		return nil
 	}
 
-	if removeContainerSnaps != nil {
+	if c.isDeduplicatedSnaps && removeContainerSnaps != nil {
 		for _, cleanupSnapId := range *removeContainerSnaps {
-			if err := c.orch.CleanupRevisionSnapshot(ctx, cleanupSnapId); err != nil {
+			if err := c.orch.CleanupSnapshot(ctx, cleanupSnapId); err != nil {
 				return errors.Wrap(err, "removing devmapper revision snapshot")
 			}
 		}
@@ -257,7 +298,7 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, fi *FuncInstance)
 		return nil
 	}
 
-	if err := c.snapshotManager.CommitSnapshot(fi.revisionId); err != nil {
+	if err := c.snapshotManager.CommitSnapshot(id); err != nil {
 		fi.logger.WithError(err).Error("failed to commit snapshot")
 		return err
 	}
@@ -277,3 +318,16 @@ func (c *coordinator) orchStopVM(ctx context.Context, fi *FuncInstance) error {
 
 	return nil
 }
+
+func (c *coordinator) orchOffloadVM(ctx context.Context, fi *FuncInstance) error {
+	if c.withoutOrchestrator {
+		return nil
+	}
+
+	if err := c.orch.OffloadVM(ctx, fi.vmID); err != nil {
+		fi.logger.WithError(err).Error("failed to offload VM")
+		return err
+	}
+
+	return nil
+}

cri/firecracker/coordinator_test.go

@@ -42,7 +42,7 @@ var (
 )
 
 func TestMain(m *testing.M) {
-	coord = newFirecrackerCoordinator(nil, 10240, false, withoutOrchestrator())
+	coord = newFirecrackerCoordinator(nil, 10240, false, false, withoutOrchestrator())
 
 	ret := m.Run()
 	os.Exit(ret)

cri/firecracker/service.go

@@ -25,11 +25,11 @@ package firecracker
 import (
 	"context"
 	"errors"
+	"github.com/ease-lab/vhive/ctriface"
 	"strconv"
 	"sync"
 
 	"github.com/ease-lab/vhive/cri"
-	"github.com/ease-lab/vhive/ctriface"
 	log "github.com/sirupsen/logrus"
 	criapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
@@ -61,15 +61,15 @@ type VMConfig struct {
 	guestPort string
 }
 
-func NewFirecrackerService(orch *ctriface.Orchestrator, snapsCapacityMiB int64, isSparseSnaps bool) (*FirecrackerService, error) {
+func NewFirecrackerService(orch *ctriface.Orchestrator, snapsCapacityMiB int64, isSparseSnaps, isDeduplicatedSnaps bool) (*FirecrackerService, error) {
 	fs := new(FirecrackerService)
 	stockRuntimeClient, err := cri.NewStockRuntimeServiceClient()
 	if err != nil {
 		log.WithError(err).Error("failed to create new stock runtime service client")
 		return nil, err
 	}
 	fs.stockRuntimeClient = stockRuntimeClient
-	fs.coordinator = newFirecrackerCoordinator(orch, snapsCapacityMiB, isSparseSnaps)
+	fs.coordinator = newFirecrackerCoordinator(orch, snapsCapacityMiB, isSparseSnaps, isDeduplicatedSnaps)
 	fs.vmConfigs = make(map[string]*VMConfig)
 	return fs, nil
 }

ctriface/bench_test.go

@@ -24,6 +24,7 @@ package ctriface
 
 import (
 	"context"
+	"github.com/ease-lab/vhive/ctriface/regular"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -43,6 +44,22 @@ const (
 )
 
 func TestBenchmarkStart(t *testing.T) {
+	orch := NewOrchestrator(regular.NewRegOrchestrator(
+		"devmapper",
+		"",
+		"fc-dev-thinpool",
+		"",
+		10,
+		regular.WithTestModeOn(true),
+		regular.WithUPF(*isUPFEnabled),
+	))
+
+	benchCount := 10
+	vmID := 0
+	benchmarkStart(t, orch, benchCount, vmID)
+}
+
+func benchmarkStart(t *testing.T, orch *Orchestrator, benchCount, vmID int) {
 	log.SetFormatter(&log.TextFormatter{
 		TimestampFormat: ctrdlog.RFC3339NanoFixed,
 		FullTimestamp:   true,
@@ -53,14 +70,10 @@ func TestBenchmarkStart(t *testing.T) {
 	log.SetLevel(log.InfoLevel)
 
 	testTimeout := 2000 * time.Second
-	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout)
+	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), regular.NamespaceName), testTimeout)
 	defer cancel()
 
-	orch := NewOrchestrator("devmapper", "", "fc-dev-thinpool","",10, WithTestModeOn(true), WithUPF(*isUPFEnabled))
-
 	images := getAllImages()
-	benchCount := 10
-	vmID := 0
 
 	createResultsDir()
 
@@ -69,7 +82,7 @@ func TestBenchmarkStart(t *testing.T) {
 		startMetrics := make([]*metrics.Metric, benchCount)
 
 		// Pull image
-		_, err := orch.imageManager.GetImage(ctx, imageName)
+		_, err := orch.GetImage(ctx, imageName)
 		require.NoError(t, err, "Failed to pull image "+imageName)
 
 		for i := 0; i < benchCount; i++ {