Skip to content

Commit b62820a

Browse files
amohosteamohoste
committed
Add tests and compatibility with offloaded snapshots
Signed-off-by: Amory Hoste <[email protected]>
1 parent d6827fb commit b62820a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1912
-287
lines changed

‎bin/containerd-shim-aws-firecracker‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:5b48fcdff74c342e8b4f65659139056dea1c27fdb99a0c2f267070b6b3b97b0b
3-
size 26530283
2+
oid sha256:b3d8525300d6ce747c63847f0f688d56d61be927648f19a86abee2e8f1e9e0e4
3+
size 26534379

‎bin/firecracker-containerd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:cc908873170a25ca713ca2e80323cf1496d5d9b7a3449778d0018a84825dd0f7
2+
oid sha256:17da34088e3c544328545e39037110bb1ab2c09543e9f25614d5425ea90793ad
33
size 47224352

‎bin/firecracker-ctr‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:51a994f7cb2cd48087a4b5a27476577c60d9fd6ce34a470435de5f33c2fb3508
2+
oid sha256:d61c35b77178fbabc4c996f4e2411f565b6974ef40773b68d465db17e725bb99
33
size 34510472

‎cri/firecracker/coordinator.go‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
"github.com/ease-lab/vhive/ctriface"
2929
"github.com/ease-lab/vhive/metrics"
3030
"github.com/ease-lab/vhive/snapshotting"
31-
"github.com/ease-lab/vhive/snapshotting/deduplicated"
31+
"github.com/ease-lab/vhive/snapshotting/fulllocal"
3232
"github.com/ease-lab/vhive/snapshotting/regular"
3333
"github.com/pkg/errors"
3434
"strconv"
@@ -77,9 +77,9 @@ func newFirecrackerCoordinator(
7777
}
7878

7979
if isFullLocal {
80-
c.snapshotManager = snapshotting.NewSnapshotManager(deduplicated.NewSnapshotManager(snapshotsDir, snapsCapacityMiB))
80+
c.snapshotManager = snapshotting.NewSnapshotManager(fulllocal.NewSnapshotManager(snapshotsDir, snapsCapacityMiB))
8181
} else {
82-
c.snapshotManager = snapshotting.NewSnapshotManager(regular.NewRegularSnapshotManager(snapshotsDir))
82+
c.snapshotManager = snapshotting.NewSnapshotManager(regular.NewSnapshotManager(snapshotsDir))
8383
}
8484

8585
for _, opt := range opts {
@@ -142,7 +142,8 @@ func (c *coordinator) stopVM(ctx context.Context, containerID string) error {
142142
}
143143

144144
if funcInst.snapBooted {
145-
defer c.snapshotManager.ReleaseSnapshot(id)
145+
// Release snapshot after the VM has been stopped / offloaded
146+
defer func() { _ = c.snapshotManager.ReleaseSnapshot(id) }()
146147
} else {
147148
// Create snapshot
148149
err := c.orchCreateSnapshot(ctx, funcInst)
@@ -204,7 +205,7 @@ func (c *coordinator) orchStartVM(ctx context.Context, image, revision string, m
204205

205206
if !c.withoutOrchestrator {
206207
trackDirtyPages := c.isSparseSnaps
207-
resp, _, err = c.orch.StartVM(ctxTimeout, vmID, image, memSizeMib, vCPUCount, trackDirtyPages, c.isFullLocal)
208+
resp, _, err = c.orch.StartVM(ctxTimeout, vmID, image, memSizeMib, vCPUCount, trackDirtyPages)
208209
if err != nil {
209210
logger.WithError(err).Error("coordinator failed to start VM")
210211
}
@@ -242,7 +243,7 @@ func (c *coordinator) orchStartVMSnapshot(
242243
ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
243244
defer cancel()
244245

245-
resp, _, err = c.orch.LoadSnapshot(ctxTimeout, vmID, snap, c.isFullLocal)
246+
resp, _, err = c.orch.LoadSnapshot(ctxTimeout, vmID, snap)
246247
if err != nil {
247248
logger.WithError(err).Error("failed to load VM")
248249
return nil, err
@@ -273,7 +274,7 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, funcInst *FuncInst
273274
id = funcInst.revisionId
274275
}
275276

276-
removeContainerSnaps, snap, err := c.snapshotManager.InitSnapshot(
277+
_, snap, err := c.snapshotManager.InitSnapshot(
277278
id,
278279
funcInst.image,
279280
funcInst.coldStartTimeMs,
@@ -286,13 +287,14 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, funcInst *FuncInst
286287
return nil
287288
}
288289

289-
if c.isFullLocal && removeContainerSnaps != nil {
290+
// This call is only necessary if the alternative approach in devicemapper with thin-delta is used.
291+
/*if c.isFullLocal && removeContainerSnaps != nil {
290292
for _, cleanupSnapId := range *removeContainerSnaps {
291293
if err := c.orch.CleanupSnapshot(ctx, cleanupSnapId); err != nil {
292294
return errors.Wrap(err, "removing devmapper revision snapshot")
293295
}
294296
}
295-
}
297+
}*/
296298

297299
ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*60)
298300
defer cancel()
@@ -305,12 +307,18 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, funcInst *FuncInst
305307
return nil
306308
}
307309

308-
err = c.orch.CreateSnapshot(ctxTimeout, funcInst.vmID, snap, c.isFullLocal)
310+
err = c.orch.CreateSnapshot(ctxTimeout, funcInst.vmID, snap)
309311
if err != nil {
310312
funcInst.logger.WithError(err).Error("failed to create snapshot")
311313
return nil
312314
}
313315

316+
// TODO: StopVM does not work for fullLocal snapshots without resuming. Might be the same for offloaded since
317+
// those are never stopped
318+
if c.isFullLocal {
319+
_, err = c.orch.ResumeVM(ctx, funcInst.vmID)
320+
}
321+
314322
if err := c.snapshotManager.CommitSnapshot(id); err != nil {
315323
funcInst.logger.WithError(err).Error("failed to commit snapshot")
316324
return err
@@ -324,7 +332,7 @@ func (c *coordinator) orchOffloadVM(ctx context.Context, funcInst *FuncInstance)
324332
return nil
325333
}
326334

327-
if err := c.orch.OffloadVM(ctx, funcInst.vmID, c.isFullLocal); err != nil {
335+
if err := c.orch.OffloadVM(ctx, funcInst.vmID); err != nil {
328336
funcInst.logger.WithError(err).Error("failed to offload VM")
329337
return err
330338
}
@@ -337,7 +345,7 @@ func (c *coordinator) orchStopVM(ctx context.Context, funcInst *FuncInstance) er
337345
return nil
338346
}
339347

340-
if err := c.orch.StopSingleVM(ctx, funcInst.vmID, c.isFullLocal); err != nil {
348+
if err := c.orch.StopSingleVM(ctx, funcInst.vmID); err != nil {
341349
funcInst.logger.WithError(err).Error("failed to stop VM for instance")
342350
return err
343351
}

‎cri/firecracker/coordinator_test.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ package firecracker
2424

2525
import (
2626
"context"
27+
"flag"
2728
"fmt"
2829
"os"
2930
"strconv"
@@ -37,13 +38,18 @@ const (
3738
testImageName = "ghcr.io/ease-lab/helloworld:var_workload"
3839
)
3940

41+
var (
42+
isFullLocal = flag.Bool("fulllocal", false, "Set full local snapshots")
43+
isSparseSnaps = flag.Bool("sparsesnaps", false, "Use sparse snapshots")
44+
)
45+
4046
var (
4147
coord *coordinator
4248
)
4349

4450
func TestMain(m *testing.M) {
45-
coord = newFirecrackerCoordinator(nil, 10240, false, false, withoutOrchestrator())
46-
51+
coord = newFirecrackerCoordinator(nil, 10240, *isSparseSnaps, *isFullLocal, withoutOrchestrator())
52+
flag.Parse()
4753
ret := m.Run()
4854
os.Exit(ret)
4955
}

‎cri/firecracker/service.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ func NewFirecrackerService(orch *ctriface.Orchestrator, snapsCapacityMiB int64,
7171
return nil, err
7272
}
7373
fs.stockRuntimeClient = stockRuntimeClient
74-
7574
fs.coordinator = newFirecrackerCoordinator(orch, snapsCapacityMiB, isSparseSnaps, isFullLocal)
7675
fs.vmConfigs = make(map[string]*VMConfig)
7776
return fs, nil

‎ctriface/Makefile‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@
2121
# SOFTWARE.
2222

2323
EXTRAGOARGS:=-v -race -cover
24-
EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go
24+
EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go types.go
2525
BENCHFILES:=bench_test.go iface.go orch_options.go orch.go
2626
WITHUPF:=-upf
27+
WITHFULLLOCAL:=-fulllocal
2728
WITHLAZY:=-lazy
29+
WITHSPARSESNAPS:=-sparsesnaps
2830
GOBENCH:=-v -timeout 1500s
2931
CTRDLOGDIR:=/tmp/ctrd-logs
3032

@@ -34,6 +36,12 @@ test:
3436
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS)
3537
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHUPF)
3638
./../scripts/clean_fcctr.sh
39+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
40+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHFULLLOCAL)
41+
./../scripts/clean_fcctr.sh
42+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
43+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
44+
./../scripts/clean_fcctr.sh
3745

3846
test-man:
3947
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_noupf_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_noupf_man_travis.err &
@@ -45,7 +53,12 @@ test-man:
4553
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err &
4654
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHUPF) $(WITHLAZY)
4755
./../scripts/clean_fcctr.sh
48-
56+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err &
57+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHFULLLOCAL)
58+
./../scripts/clean_fcctr.sh
59+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err &
60+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
61+
./../scripts/clean_fcctr.sh
4962
test-skip:
5063
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_noupf_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_noupf_man_skip.err &
5164
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad
@@ -59,8 +72,14 @@ test-skip:
5972
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHUPF) $(WITHLAZY)
6073
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHUPF) $(WITHLAZY)
6174
./../scripts/clean_fcctr.sh
62-
63-
75+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.err &
76+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHFULLLOCAL)
77+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHFULLLOCAL)
78+
./../scripts/clean_fcctr.sh
79+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.err &
80+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
81+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
82+
./../scripts/clean_fcctr.sh
6483
bench:
6584
sudo env "PATH=$(PATH)" go test $(BENCHFILES) $(GOBENCH)
6685
./../scripts/clean_fcctr.sh

‎ctriface/bench_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,9 @@ func TestBenchmarkStart(t *testing.T) {
6464
10,
6565
WithTestModeOn(true),
6666
WithUPF(*isUPFEnabled),
67+
WithFullLocal(*isFullLocal),
6768
)
69+
defer orch.Cleanup()
6870

6971
images := getAllImages()
7072
benchCount := 10
@@ -83,11 +85,11 @@ func TestBenchmarkStart(t *testing.T) {
8385
for i := 0; i < benchCount; i++ {
8486
dropPageCache()
8587

86-
_, metric, err := orch.StartVM(ctx, vmIDString, imageName, 256, 1, false, false)
88+
_, metric, err := orch.StartVM(ctx, vmIDString, imageName, 256, 1, false)
8789
require.NoError(t, err, "Failed to start VM")
8890
startMetrics[i] = metric
8991

90-
err = orch.StopSingleVM(ctx, vmIDString, false)
92+
err = orch.StopSingleVM(ctx, vmIDString)
9193
require.NoError(t, err, "Failed to stop VM")
9294
}
9395

@@ -99,7 +101,6 @@ func TestBenchmarkStart(t *testing.T) {
99101

100102
}
101103

102-
orch.Cleanup()
103104
}
104105

105106
func dropPageCache() {

‎ctriface/failing_test.go‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,29 +59,31 @@ func TestStartSnapStop(t *testing.T) {
5959
"",
6060
10,
6161
WithTestModeOn(true),
62+
WithFullLocal(*isFullLocal),
6263
)
64+
defer orch.Cleanup()
6365

6466
vmID := "2"
65-
revisionID := "myrev-2"
6667

67-
_, _, err := orch.StartVM(ctx, vmID, TestImageName, 256, 1, false, false)
68+
_, _, err := orch.StartVM(ctx, vmID, TestImageName, 256, 1, false)
6869
require.NoError(t, err, "Failed to start VM")
6970

7071
err = orch.PauseVM(ctx, vmID)
7172
require.NoError(t, err, "Failed to pause VM")
7273

73-
snap := snapshotting.NewSnapshot(revisionID, "/fccd/snapshots", TestImageName, 0, 0, false)
74-
err = orch.CreateSnapshot(ctx, vmID, snap, false)
74+
snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", TestImageName, 256, 1, false)
75+
err = orch.CreateSnapshot(ctx, vmID, snap)
7576
require.NoError(t, err, "Failed to create snapshot of VM")
7677

77-
_, _, err = orch.LoadSnapshot(ctx, vmID, snap, false)
78+
err = orch.OffloadVM(ctx, vmID)
79+
require.NoError(t, err, "Failed to offload VM")
80+
81+
_, _, err = orch.LoadSnapshot(ctx, vmID, snap)
7882
require.NoError(t, err, "Failed to load snapshot of VM")
7983

8084
_, err = orch.ResumeVM(ctx, vmID)
8185
require.NoError(t, err, "Failed to resume VM")
8286

83-
err = orch.StopSingleVM(ctx, vmID, false)
87+
err = orch.StopSingleVM(ctx, vmID)
8488
require.NoError(t, err, "Failed to stop VM")
85-
86-
orch.Cleanup()
8789
}

0 commit comments

Comments
 (0)