Add thinpool detection if not specified

amohoste · amohoste · commit 862a3a4665ee · 2022-05-16T21:16:24.000+01:00
Signed-off-by: Amory Hoste &lt;amory.hoste@gmail.com&gt;
diff --git a/.github/workflows/cri_test.yml b/.github/workflows/cri_test.yml
@@ -54,6 +54,11 @@ jobs:
     - name: Setup firecracker-containerd
       run: ./scripts/setup_firecracker_containerd.sh
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build
 
diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml
@@ -31,6 +31,11 @@ jobs:
         sudo apt update
         sudo apt install git -y
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
       with:
@@ -55,7 +60,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vhive_args: ["-dbg", "-dbg -snapshots", "-dbg -snapshots -upf"]
+        vhive_args: ["-dbg", "-dbg -snapshots", "-dbg -snapshots -upf", "-dbg -snapshots -fulllocal"]
     env:
         GITHUB_RUN_ID: ${{ github.run_id }}
         GITHUB_JOB: ${{ github.job }}
@@ -83,6 +88,11 @@ jobs:
     - name: Setup firecracker-containerd
       run: ./scripts/setup_firecracker_containerd.sh
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build
 
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -26,7 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        module: [taps, misc, profile]
+        module: [taps, misc, profile, devmapper, networking, snapshotting, ctriface/image]
     steps:
 
     - name: Set up Go 1.18
@@ -48,6 +48,11 @@ jobs:
     - name: Install PMU tools
       run: ./scripts/install_pmutools.sh
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build -race -v -a ./...
 
@@ -84,6 +89,11 @@ jobs:
     - name: Pull binaries
       run: ./scripts/setup_firecracker_containerd.sh
 
+    - name: Add rsync # TODO: add dependencies to vHive
+      run: |
+        sudo apt update
+        sudo apt install rsync -y
+
     - name: Build
       run: go build -race -v -a ./...
 
diff --git a/bin/containerd-shim-aws-firecracker b/bin/containerd-shim-aws-firecracker
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3d8525300d6ce747c63847f0f688d56d61be927648f19a86abee2e8f1e9e0e4
-size 26534379
+oid sha256:2621670b622850e209a25b189c7f7f6f9b4cc3b2e161f28a1f98072e70452cc8
+size 23910923
diff --git a/bin/firecracker b/bin/firecracker
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d42ddb2c3d970d6a234e0d3f92980e085fc04a9ae17e29e05bb4ca73debfe0b8
+oid sha256:59c71f6e860d67c87f3d76cedebd0b72cb0dda3918dbae8a1f9a6cc7d57672dd
 size 4016240
diff --git a/bin/firecracker-containerd b/bin/firecracker-containerd
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17da34088e3c544328545e39037110bb1ab2c09543e9f25614d5425ea90793ad
-size 47224352
+oid sha256:b46f59950e1c760c243a1aa1ce52c017a06fc847cb6fa4b9ac81ebe535118856
+size 43720504
diff --git a/bin/firecracker-ctr b/bin/firecracker-ctr
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d61c35b77178fbabc4c996f4e2411f565b6974ef40773b68d465db17e725bb99
-size 34510472
+oid sha256:78efef1c7049ca2735e29f9cba3782ea4fdfd6433fa9805076818f03150f01eb
+size 32104096
diff --git a/bin/jailer b/bin/jailer
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80284269eb8b44483b84a527bf7f4012932d94410e32bd20e4d25d09303336ea
+oid sha256:28614cb54e9b0a959822da0018c81d7de000c4cd2253de026e1c4b4689f3d236
 size 3072784
diff --git a/cri/firecracker/coordinator.go b/cri/firecracker/coordinator.go
@@ -316,7 +316,10 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, funcInst *FuncInst
 	// TODO: StopVM does not work for fullLocal snapshots without resuming. Might be the same for offloaded since
 	// those are never stopped
 	if c.isFullLocal {
-		_, err = c.orch.ResumeVM(ctx, funcInst.vmID)
+		if _, err := c.orch.ResumeVM(ctx, funcInst.vmID); err != nil {
+			funcInst.logger.WithError(err).Error("failed to resume VM")
+			return err
+		}
 	}
 
 	if err := c.snapshotManager.CommitSnapshot(id); err != nil {
diff --git a/ctriface/bench_test.go b/ctriface/bench_test.go
@@ -59,7 +59,7 @@ func TestBenchmarkStart(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
diff --git a/ctriface/failing_test.go b/ctriface/failing_test.go
@@ -55,7 +55,7 @@ func TestStartSnapStop(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
diff --git a/ctriface/iface.go b/ctriface/iface.go
@@ -26,8 +26,10 @@ import (
 	"context"
 	"github.com/ease-lab/vhive/devmapper"
 	"github.com/ease-lab/vhive/snapshotting"
+	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
@@ -377,9 +379,9 @@ func (o *Orchestrator) StopActiveVMs() error {
 	log.Info("waiting done")
 
 	log.Info("Closing fcClient")
-	o.fcClient.Close()
+	defer func() { _ = o.fcClient.Close() }()
 	log.Info("Closing containerd client")
-	o.client.Close()
+	defer func() { _ = o.client.Close() }()
 
 	return nil
 }
@@ -579,6 +581,29 @@ func (o *Orchestrator) LoadSnapshot(
 
 		if _, loadErr = o.fcClient.LoadSnapshot(ctx, req); loadErr != nil {
 			logger.Error("Failed to load snapshot of the VM: ", loadErr)
+			logger.Errorf("snapFilePath: %s, memFilePath: %s, newSnapshotPath: %s", snapFilePath, memFilePath, containerSnap.GetDevicePath())
+			files, err := ioutil.ReadDir(filepath.Dir(snapFilePath))
+			if err != nil {
+				logger.Error(err)
+			}
+
+			snapFiles := ""
+			for _, f := range files {
+				snapFiles += f.Name() + ", "
+			}
+
+			logger.Error(snapFiles)
+
+			files, _ = ioutil.ReadDir(filepath.Dir(containerSnap.GetDevicePath()))
+			if err != nil {
+				logger.Error(err)
+			}
+
+			snapFiles = ""
+			for _, f := range files {
+				snapFiles += f.Name() + ", "
+			}
+			logger.Error(snapFiles)
 		}
 	}()
 
diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go
@@ -67,7 +67,7 @@ func TestPauseSnapResume(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
@@ -121,7 +121,7 @@ func TestStartStopSerial(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
-		"fc-dev-thinpool",
+		"",
 		"",
 		"",
 		10,
@@ -158,7 +158,7 @@ func TestPauseResumeSerial(t *testing.T) {
 
 	orch := NewOrchestrator(
 		"devmapper",
-		"fc-dev-thinpool",
+		"",
 		"",
 		"",
 		10,
@@ -203,7 +203,7 @@ func TestStartStopParallel(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
@@ -265,7 +265,7 @@ func TestPauseResumeParallel(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
diff --git a/ctriface/manual_cleanup_test.go b/ctriface/manual_cleanup_test.go
@@ -57,7 +57,7 @@ func TestSnapLoad(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		1,
 		WithTestModeOn(true),
@@ -81,7 +81,7 @@ func TestSnapLoad(t *testing.T) {
 
 	snap := snapshotting.NewSnapshot(snapId, "/fccd/snapshots", TestImageName, 256, 1, *isSparseSnaps)
 	if *isFullLocal {
-		err = snap.CreateSnapDir()
+		_ = snap.CreateSnapDir()
 	}
 
 	err = orch.CreateSnapshot(ctx, vmID, snap)
@@ -129,7 +129,7 @@ func TestSnapLoadMultiple(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
@@ -220,7 +220,7 @@ func TestParallelSnapLoad(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
@@ -309,7 +309,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
 	orch := NewOrchestrator(
 		"devmapper",
 		"",
-		"fc-dev-thinpool",
+		"",
 		"",
 		10,
 		WithTestModeOn(true),
diff --git a/devmapper/devicemapper.go b/devmapper/devicemapper.go
@@ -23,6 +23,7 @@
 package devmapper
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"github.com/containerd/containerd"
@@ -31,18 +32,34 @@ import (
 	"github.com/ease-lab/vhive/devmapper/thindelta"
 	"github.com/opencontainers/image-spec/identity"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
 )
 
-// Own managed snapshots in snapmanager and in containerd identified by snapkey. Has name deviceName.
-// snapshotId used internally by containerd, needed for thin_delta. Once committed, snapshots identified
-// by snapName within containerd?
+const defaultPoolName = "fc-dev-thinpool"
 
-// Only remove snapshots created through createSnapshot. Use key used there to delete them.
+// getHostIfaceName returns the default host network interface name.
+func getThinPoolName() string {
+	b, err := ioutil.ReadFile("/proc/1/cpuset")
+	if err != nil {
+		log.Warnf("Failed to fetch thin-pool name. Falling back to default %v\n", err)
+		return defaultPoolName
+	}
+
+	containerId := filepath.Base(strings.TrimSuffix(string(b), "\n"))
+
+	//  Check if running inside container (containerID should be 64 characters)
+	if len(containerId) != 64 {
+		return defaultPoolName
+	}
+
+	return containerId + "_thinpool"
+}
 
 // DeviceMapper creates and manages device snapshots used to store container images.
 type DeviceMapper struct {
@@ -61,6 +78,9 @@ type DeviceMapper struct {
 func NewDeviceMapper(client *containerd.Client, poolName, metadataDev string) *DeviceMapper {
 	devMapper := new(DeviceMapper)
 	devMapper.poolName = poolName
+	if devMapper.poolName == "" {
+		devMapper.poolName = getThinPoolName()
+	}
 	devMapper.thinDelta = thindelta.NewThinDelta(poolName, metadataDev)
 	devMapper.snapDevices = make(map[string]*DeviceSnapshot)
 	devMapper.snapshotService = client.SnapshotService("devmapper")
@@ -262,10 +282,14 @@ func (dmpr *DeviceMapper) CreatePatch(ctx context.Context, patchPath, containerS
 // writes the differences to the supplied patchPath.
 func extractPatch(imageMountPath, containerMountPath, patchPath string) error {
 	patchArg := fmt.Sprintf("--only-write-batch=%s", patchPath)
+
+	var errb bytes.Buffer
 	cmd := exec.Command("sudo", "rsync", "-ar", patchArg, addTrailingSlash(imageMountPath), addTrailingSlash(containerMountPath))
+	cmd.Stderr = &errb
 	err := cmd.Run()
+
 	if err != nil {
-		return errors.Wrapf(err, "creating patch between %s and %s at %s", imageMountPath, containerMountPath, patchPath)
+		return errors.Wrapf(err, "creating patch between %s and %s at %s: %s", imageMountPath, containerMountPath, patchPath, errb.String())
 	}
 
 	err = os.Remove(patchPath + ".sh") // Remove unnecessary script output
diff --git a/devmapper/devicemapper_test.go b/devmapper/devicemapper_test.go
@@ -40,7 +40,6 @@ import (
 
 const (
 	containerdAddress = "/run/firecracker-containerd/containerd.sock"
-	poolName          = "fc-dev-thinpool"
 	NamespaceName     = "firecracker-containerd"
 	TestImageName     = "ghcr.io/ease-lab/helloworld:var_workload"
 )
@@ -98,29 +97,29 @@ func TestDevmapper(t *testing.T) {
 
 	// Create containerd client
 	client, err := containerd.New(containerdAddress)
-	defer client.Close()
+	defer func() { _ = client.Close() }()
 	require.NoError(t, err, "Containerd client creation returned error")
 
 	// Create image manager
 	mgr := image.NewImageManager(client, "devmapper")
 
 	// Create devmapper
-	dmpr := devmapper.NewDeviceMapper(client, poolName, "")
+	dmpr := devmapper.NewDeviceMapper(client, "", "")
 
 	testDevmapper(t, mgr, dmpr, snapKey, TestImageName)
 }
 
 func TestDevmapperConcurrent(t *testing.T) {
 	// Create containerd client
 	client, err := containerd.New(containerdAddress)
-	defer client.Close()
+	defer func() { _ = client.Close() }()
 	require.NoError(t, err, "Containerd client creation returned error")
 
 	// Create image manager
 	mgr := image.NewImageManager(client, "devmapper")
 
 	// Create devmapper
-	dmpr := devmapper.NewDeviceMapper(client, poolName, "")
+	dmpr := devmapper.NewDeviceMapper(client, "", "")
 
 	// Test concurrent devmapper
 	var wg sync.WaitGroup
diff --git a/networking/networkconfig.go b/networking/networkconfig.go
diff --git a/networking/networking.go b/networking/networking.go
diff --git a/vhive_test.go b/vhive_test.go
