Skip to content

Commit f6a953d

Browse files
amohosteamohoste
committed
add compatibility for offload snapshotting
Signed-off-by: Amory Hoste <[email protected]>
1 parent 24ba3ba commit f6a953d

28 files changed

+973
-185
lines changed

‎bin/containerd-shim-aws-firecracker‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:5b48fcdff74c342e8b4f65659139056dea1c27fdb99a0c2f267070b6b3b97b0b
3-
size 26530283
2+
oid sha256:b3d8525300d6ce747c63847f0f688d56d61be927648f19a86abee2e8f1e9e0e4
3+
size 26534379

‎bin/firecracker-containerd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:cc908873170a25ca713ca2e80323cf1496d5d9b7a3449778d0018a84825dd0f7
2+
oid sha256:17da34088e3c544328545e39037110bb1ab2c09543e9f25614d5425ea90793ad
33
size 47224352

‎bin/firecracker-ctr‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:51a994f7cb2cd48087a4b5a27476577c60d9fd6ce34a470435de5f33c2fb3508
2+
oid sha256:d61c35b77178fbabc4c996f4e2411f565b6974ef40773b68d465db17e725bb99
33
size 34510472

‎cri/firecracker/coordinator.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ func (c *coordinator) orchStartVM(ctx context.Context, image, revision string, m
205205

206206
if !c.withoutOrchestrator {
207207
trackDirtyPages := c.isSparseSnaps
208-
resp, _, err = c.orch.StartVM(ctxTimeout, vmID, image, memSizeMib, vCPUCount, trackDirtyPages, c.isFullLocal)
208+
resp, _, err = c.orch.StartVM(ctxTimeout, vmID, image, memSizeMib, vCPUCount, trackDirtyPages)
209209
if err != nil {
210210
logger.WithError(err).Error("coordinator failed to start VM")
211211
}
@@ -243,7 +243,7 @@ func (c *coordinator) orchStartVMSnapshot(
243243
ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
244244
defer cancel()
245245

246-
resp, _, err = c.orch.LoadSnapshot(ctxTimeout, vmID, snap, c.isFullLocal)
246+
resp, _, err = c.orch.LoadSnapshot(ctxTimeout, vmID, snap)
247247
if err != nil {
248248
logger.WithError(err).Error("failed to load VM")
249249
return nil, err
@@ -307,13 +307,17 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, funcInst *FuncInst
307307
return nil
308308
}
309309

310-
err = c.orch.CreateSnapshot(ctxTimeout, funcInst.vmID, snap, c.isFullLocal)
310+
err = c.orch.CreateSnapshot(ctxTimeout, funcInst.vmID, snap)
311311
if err != nil {
312312
funcInst.logger.WithError(err).Error("failed to create snapshot")
313313
return nil
314314
}
315315

316-
_, err = c.orch.ResumeVM(ctx, funcInst.vmID)
316+
// TODO: StopVM does not work for fullLocal snapshots without resuming. Might be the same for offloaded since
317+
// those are never stopped
318+
if c.isFullLocal {
319+
_, err = c.orch.ResumeVM(ctx, funcInst.vmID)
320+
}
317321

318322
if err := c.snapshotManager.CommitSnapshot(id); err != nil {
319323
funcInst.logger.WithError(err).Error("failed to commit snapshot")
@@ -328,7 +332,7 @@ func (c *coordinator) orchOffloadVM(ctx context.Context, funcInst *FuncInstance)
328332
return nil
329333
}
330334

331-
if err := c.orch.OffloadVM(ctx, funcInst.vmID, c.isFullLocal); err != nil {
335+
if err := c.orch.OffloadVM(ctx, funcInst.vmID); err != nil {
332336
funcInst.logger.WithError(err).Error("failed to offload VM")
333337
return err
334338
}
@@ -341,7 +345,7 @@ func (c *coordinator) orchStopVM(ctx context.Context, funcInst *FuncInstance) er
341345
return nil
342346
}
343347

344-
if err := c.orch.StopSingleVM(ctx, funcInst.vmID, c.isFullLocal); err != nil {
348+
if err := c.orch.StopSingleVM(ctx, funcInst.vmID); err != nil {
345349
funcInst.logger.WithError(err).Error("failed to stop VM for instance")
346350
return err
347351
}

‎cri/firecracker/coordinator_test.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ package firecracker
2424

2525
import (
2626
"context"
27+
"flag"
2728
"fmt"
2829
"os"
2930
"strconv"
@@ -37,13 +38,18 @@ const (
3738
testImageName = "ghcr.io/ease-lab/helloworld:var_workload"
3839
)
3940

41+
var (
42+
isFullLocal = flag.Bool("fulllocal", false, "Set full local snapshots")
43+
isSparseSnaps = flag.Bool("sparsesnaps", false, "Use sparse snapshots")
44+
)
45+
4046
var (
4147
coord *coordinator
4248
)
4349

4450
func TestMain(m *testing.M) {
45-
coord = newFirecrackerCoordinator(nil, 10240, false, false, withoutOrchestrator())
46-
51+
coord = newFirecrackerCoordinator(nil, 10240, *isSparseSnaps, *isFullLocal, withoutOrchestrator())
52+
flag.Parse()
4753
ret := m.Run()
4854
os.Exit(ret)
4955
}

‎cri/firecracker/service.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ func NewFirecrackerService(orch *ctriface.Orchestrator, snapsCapacityMiB int64,
7171
return nil, err
7272
}
7373
fs.stockRuntimeClient = stockRuntimeClient
74-
7574
fs.coordinator = newFirecrackerCoordinator(orch, snapsCapacityMiB, isSparseSnaps, isFullLocal)
7675
fs.vmConfigs = make(map[string]*VMConfig)
7776
return fs, nil

‎ctriface/Makefile‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@
2121
# SOFTWARE.
2222

2323
EXTRAGOARGS:=-v -race -cover
24-
EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go
24+
EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go types.go
2525
BENCHFILES:=bench_test.go iface.go orch_options.go orch.go
2626
WITHUPF:=-upf
2727
WITHFULLLOCAL:=-fulllocal
2828
WITHLAZY:=-lazy
29+
WITHSPARSESNAPS:=-sparsesnaps
2930
GOBENCH:=-v -timeout 1500s
3031
CTRDLOGDIR:=/tmp/ctrd-logs
3132

@@ -34,8 +35,13 @@ test:
3435
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
3536
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS)
3637
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHUPF)
38+
./../scripts/clean_fcctr.sh
39+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
3740
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHFULLLOCAL)
3841
./../scripts/clean_fcctr.sh
42+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
43+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
44+
./../scripts/clean_fcctr.sh
3945

4046
test-man:
4147
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_noupf_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_noupf_man_travis.err &
@@ -47,7 +53,12 @@ test-man:
4753
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err &
4854
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHUPF) $(WITHLAZY)
4955
./../scripts/clean_fcctr.sh
50-
56+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err &
57+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHFULLLOCAL)
58+
./../scripts/clean_fcctr.sh
59+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err &
60+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
61+
./../scripts/clean_fcctr.sh
5162
test-skip:
5263
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_noupf_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_noupf_man_skip.err &
5364
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad
@@ -61,8 +72,14 @@ test-skip:
6172
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHUPF) $(WITHLAZY)
6273
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHUPF) $(WITHLAZY)
6374
./../scripts/clean_fcctr.sh
64-
65-
75+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.err &
76+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHFULLLOCAL)
77+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHFULLLOCAL)
78+
./../scripts/clean_fcctr.sh
79+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.err &
80+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
81+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHFULLLOCAL) $(WITHSPARSESNAPS)
82+
./../scripts/clean_fcctr.sh
6683
bench:
6784
sudo env "PATH=$(PATH)" go test $(BENCHFILES) $(GOBENCH)
6885
./../scripts/clean_fcctr.sh

‎ctriface/bench_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,9 @@ func TestBenchmarkStart(t *testing.T) {
6464
10,
6565
WithTestModeOn(true),
6666
WithUPF(*isUPFEnabled),
67+
WithFullLocal(*isFullLocal),
6768
)
69+
defer orch.Cleanup()
6870

6971
images := getAllImages()
7072
benchCount := 10
@@ -83,11 +85,11 @@ func TestBenchmarkStart(t *testing.T) {
8385
for i := 0; i < benchCount; i++ {
8486
dropPageCache()
8587

86-
_, metric, err := orch.StartVM(ctx, vmIDString, imageName, 256, 1, false, false)
88+
_, metric, err := orch.StartVM(ctx, vmIDString, imageName, 256, 1, false)
8789
require.NoError(t, err, "Failed to start VM")
8890
startMetrics[i] = metric
8991

90-
err = orch.StopSingleVM(ctx, vmIDString, false)
92+
err = orch.StopSingleVM(ctx, vmIDString)
9193
require.NoError(t, err, "Failed to stop VM")
9294
}
9395

@@ -99,7 +101,6 @@ func TestBenchmarkStart(t *testing.T) {
99101

100102
}
101103

102-
orch.Cleanup()
103104
}
104105

105106
func dropPageCache() {

‎ctriface/failing_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,31 +59,31 @@ func TestStartSnapStop(t *testing.T) {
5959
"",
6060
10,
6161
WithTestModeOn(true),
62+
WithFullLocal(*isFullLocal),
6263
)
64+
defer orch.Cleanup()
6365

6466
vmID := "2"
6567

66-
_, _, err := orch.StartVM(ctx, vmID, TestImageName, 256, 1, false, false)
68+
_, _, err := orch.StartVM(ctx, vmID, TestImageName, 256, 1, false)
6769
require.NoError(t, err, "Failed to start VM")
6870

6971
err = orch.PauseVM(ctx, vmID)
7072
require.NoError(t, err, "Failed to pause VM")
7173

7274
snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", TestImageName, 256, 1, false)
73-
err = orch.CreateSnapshot(ctx, vmID, snap, false)
75+
err = orch.CreateSnapshot(ctx, vmID, snap)
7476
require.NoError(t, err, "Failed to create snapshot of VM")
7577

76-
err = orch.OffloadVM(ctx, vmID, false)
78+
err = orch.OffloadVM(ctx, vmID)
7779
require.NoError(t, err, "Failed to offload VM")
7880

79-
_, _, err = orch.LoadSnapshot(ctx, vmID, snap, false)
81+
_, _, err = orch.LoadSnapshot(ctx, vmID, snap)
8082
require.NoError(t, err, "Failed to load snapshot of VM")
8183

8284
_, err = orch.ResumeVM(ctx, vmID)
8385
require.NoError(t, err, "Failed to resume VM")
8486

85-
err = orch.StopSingleVM(ctx, vmID, false)
87+
err = orch.StopSingleVM(ctx, vmID)
8688
require.NoError(t, err, "Failed to stop VM")
87-
88-
orch.Cleanup()
8989
}

0 commit comments

Comments
 (0)