Skip to content

Commit 74ea326

Browse files
moharakamoharaka
authored
Merge branch 'vhive-serverless:main' into migrating-remaining-unit-test
2 parents 3cacaf8 + 3de16e7 commit 74ea326

File tree

9 files changed

+95
-46
lines changed

9 files changed

+95
-46
lines changed

‎.github/actions/lfs-cache/action.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: 'LFS Cache Handler'
2+
description: 'Caches and restores LFS objects to reduce bandwidth usage'
3+
inputs:
4+
lfs_enabled:
5+
description: 'Whether LFS is used in this repository'
6+
required: false
7+
default: 'true'
8+
ref: # Add this new input
9+
description: 'Git ref to checkout'
10+
required: false
11+
runs:
12+
using: "composite"
13+
steps:
14+
- name: Checkout (LFS disabled)
15+
if: ${{ inputs.lfs_enabled == 'true' }}
16+
uses: actions/checkout@v4
17+
with:
18+
lfs: false
19+
ref: ${{ inputs.ref }}
20+
21+
- name: Generate LFS hash list
22+
if: ${{ inputs.lfs_enabled == 'true' }}
23+
shell: bash
24+
run: git lfs ls-files -l | cut -d' ' -f1 | sort > .git/lfs-hashes.txt
25+
26+
- name: Restore LFS cache
27+
if: ${{ inputs.lfs_enabled == 'true' }}
28+
id: lfs-cache
29+
uses: actions/cache@v4
30+
with:
31+
path: .git/lfs
32+
key: ${{ runner.os }}-lfs-${{ hashFiles('.git/lfs-hashes.txt') }}
33+
restore-keys: |
34+
${{ runner.os }}-lfs-
35+
36+
- name: Fetch LFS objects
37+
if: ${{ inputs.lfs_enabled == 'true' }}
38+
shell: bash
39+
run: |
40+
git lfs fetch --prune
41+
git lfs checkout

‎.github/workflows/build_docker_images.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ jobs:
3636
image: [github_runner, github_runner_cri, cri_dev_env]
3737
steps:
3838
- uses: actions/checkout@v4
39+
- name: Handle LFS Caching
40+
uses: ./.github/actions/lfs-cache
3941
with:
42+
lfs_enabled: 'true'
4043
ref: main
41-
- name: pull binaries
42-
run: |
43-
git lfs pull
4444
- name: docker build images
4545
run: |
46-
DOCKER_BUILDKIT=1 docker build --file ./scripts/github_runner/Dockerfile.${{ matrix.image }} .
46+
DOCKER_BUILDKIT=1 docker build --file ./scripts/github_runner/Dockerfile.${{ matrix.image }} .

‎.github/workflows/integration_tests.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,11 @@ jobs:
3030
matrix:
3131
test-name: [test, test-man-bench]
3232
steps:
33-
34-
- name: Check out code into the Go module directory
35-
uses: actions/checkout@v4
33+
- uses: actions/checkout@v4
34+
- name: Handle LFS Caching
35+
uses: ./.github/actions/lfs-cache
3636
with:
37-
lfs: true
37+
lfs_enabled: 'true'
3838

3939
- name: Set up Go version in go.mod file
4040
uses: actions/setup-go@v5
@@ -78,4 +78,4 @@ jobs:
7878
7979
- name: Cleaning
8080
if: ${{ always() }}
81-
run: ./scripts/setup_tool clean_fcctr
81+
run: ./scripts/setup_tool clean_fcctr

‎.github/workflows/nightly_tests.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,11 @@ jobs:
2020
- name: Setup TMPDIR
2121
run: mkdir -p $TMPDIR
2222

23-
- name: Check out code into the Go module directory
24-
uses: actions/checkout@v4
23+
- uses: actions/checkout@v4
24+
- name: Handle LFS Caching
25+
uses: ./.github/actions/lfs-cache
2526
with:
26-
lfs: true
27+
lfs_enabled: 'true'
2728

2829
- name: Set up Go version in go.mod file
2930
uses: actions/setup-go@v5
@@ -109,4 +110,4 @@ jobs:
109110
110111
- name: Cleaning
111112
if: ${{ always() }}
112-
run: ./scripts/github_runner/clean_cri_runner.sh
113+
run: ./scripts/github_runner/clean_cri_runner.sh

‎.github/workflows/stargz_tests.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ jobs:
3737

3838
steps:
3939
- uses: actions/checkout@v4
40+
- name: Handle LFS Caching
41+
uses: ./.github/actions/lfs-cache
4042
with:
41-
lfs: "true"
42-
- name: Checkout LFS objects
43-
run: git lfs checkout
43+
lfs_enabled: 'true'
4444

4545
- name: Set up Go version in go.mod file
4646
uses: actions/setup-go@v5
@@ -82,4 +82,4 @@ jobs:
8282
run: kn service apply stargz-test -f ./configs/knative_workloads/stargz-node.yaml --concurrency-target 1
8383

8484
- name: Curl container
85-
run: curl http://stargz-test.default.127.0.0.1.sslip.io | grep "Hello World"
85+
run: curl http://stargz-test.default.127.0.0.1.sslip.io | grep "Hello World"

‎.github/workflows/unit_tests.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,10 +125,11 @@ jobs:
125125
matrix:
126126
module: [ ctriface, ctriface/image, devmapper ]
127127
steps:
128-
- name: Check out code into the Go module directory
129-
uses: actions/checkout@v4
128+
- uses: actions/checkout@v4
129+
- name: Handle LFS Caching (Check out code into the Go module directory)
130+
uses: ./.github/actions/lfs-cache
130131
with:
131-
lfs: true
132+
lfs_enabled: 'true'
132133

133134
- name: Set up Go version in go.mod file
134135
uses: actions/setup-go@v5

‎ctriface/Makefile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,6 @@ test:
4141
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
4242
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHUPF)
4343
./../scripts/clean_fcctr.sh
44-
sudo env "PATH=$(PATH)" /usr/local/bin/http-address-resolver &
45-
sudo env "PATH=$(PATH)" /bin/bash -c 'while true; do /usr/local/bin/demux-snapshotter; done' &
46-
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
47-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(STARGZ) $(DOCKER_CREDENTIALS)
48-
./../scripts/clean_fcctr.sh
4944

5045
test-man:
5146
./../scripts/clean_fcctr.sh
@@ -67,6 +62,11 @@ test-man:
6762
# Loads the remote snapshot.
6863
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestRemoteSnapLoad
6964
./../scripts/clean_fcctr.sh
65+
sudo env "PATH=$(PATH)" /usr/local/bin/http-address-resolver &
66+
sudo env "PATH=$(PATH)" /bin/bash -c 'while true; do /usr/local/bin/demux-snapshotter; done' &
67+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
68+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(STARGZ) $(DOCKER_CREDENTIALS)
69+
./../scripts/clean_fcctr.sh
7070

7171
test-skip:
7272
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_noupf_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_noupf_man_skip.err &

‎ctriface/manual_cleanup_test.go‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -60,23 +60,24 @@ func TestSnapLoad(t *testing.T) {
6060
defer cancel()
6161

6262
orch := NewOrchestrator(
63-
"devmapper",
63+
*snapshotter,
6464
"",
6565
WithTestModeOn(true),
6666
WithUPF(*isUPFEnabled),
6767
WithLazyMode(*isLazyMode),
68+
WithDockerCredentials(*dockerCredentials),
6869
)
6970

7071
vmID := "1"
7172
revision := "myrev-1"
7273

73-
_, _, err := orch.StartVM(ctx, vmID, testImageName)
74+
_, _, err := orch.StartVM(ctx, vmID, *testImage)
7475
require.NoError(t, err, "Failed to start VM")
7576

7677
err = orch.PauseVM(ctx, vmID)
7778
require.NoError(t, err, "Failed to pause VM")
7879

79-
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", testImageName)
80+
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", *testImage)
8081
err = snap.CreateSnapDir()
8182
require.NoError(t, err, "Failed to create snapshots directory")
8283

@@ -121,23 +122,24 @@ func TestSnapLoadMultiple(t *testing.T) {
121122
defer cancel()
122123

123124
orch := NewOrchestrator(
124-
"devmapper",
125+
*snapshotter,
125126
"",
126127
WithTestModeOn(true),
127128
WithUPF(*isUPFEnabled),
128129
WithLazyMode(*isLazyMode),
130+
WithDockerCredentials(*dockerCredentials),
129131
)
130132

131133
vmID := "3"
132134
revision := "myrev-3"
133135

134-
_, _, err := orch.StartVM(ctx, vmID, testImageName)
136+
_, _, err := orch.StartVM(ctx, vmID, *testImage)
135137
require.NoError(t, err, "Failed to start VM")
136138

137139
err = orch.PauseVM(ctx, vmID)
138140
require.NoError(t, err, "Failed to pause VM")
139141

140-
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", testImageName)
142+
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", *testImage)
141143
err = snap.CreateSnapDir()
142144
require.NoError(t, err, "Failed to create snapshots directory")
143145

@@ -196,15 +198,16 @@ func TestParallelSnapLoad(t *testing.T) {
196198
vmIDBase := 6
197199

198200
orch := NewOrchestrator(
199-
"devmapper",
201+
*snapshotter,
200202
"",
201203
WithTestModeOn(true),
202204
WithUPF(*isUPFEnabled),
203205
WithLazyMode(*isLazyMode),
206+
WithDockerCredentials(*dockerCredentials),
204207
)
205208

206209
// Pull image
207-
_, err := orch.getImage(ctx, testImageName)
210+
_, err := orch.getImage(ctx, *testImage)
208211
require.NoError(t, err, "Failed to pull image "+testImageName)
209212

210213
var vmGroup sync.WaitGroup
@@ -215,13 +218,13 @@ func TestParallelSnapLoad(t *testing.T) {
215218
vmID := fmt.Sprintf("%d", i+vmIDBase)
216219
revision := fmt.Sprintf("myrev-%d", i+vmIDBase)
217220

218-
_, _, err := orch.StartVM(ctx, vmID, testImageName)
221+
_, _, err := orch.StartVM(ctx, vmID, *testImage)
219222
require.NoError(t, err, "Failed to start VM, "+vmID)
220223

221224
err = orch.PauseVM(ctx, vmID)
222225
require.NoError(t, err, "Failed to pause VM, "+vmID)
223226

224-
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", testImageName)
227+
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", *testImage)
225228
err = snap.CreateSnapDir()
226229
require.NoError(t, err, "Failed to create snapshots directory")
227230

@@ -274,15 +277,16 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
274277
vmIDBase := 16
275278

276279
orch := NewOrchestrator(
277-
"devmapper",
280+
*snapshotter,
278281
"",
279282
WithTestModeOn(true),
280283
WithUPF(*isUPFEnabled),
281284
WithLazyMode(*isLazyMode),
285+
WithDockerCredentials(*dockerCredentials),
282286
)
283287

284288
// Pull image
285-
_, err := orch.getImage(ctx, testImageName)
289+
_, err := orch.getImage(ctx, *testImage)
286290
require.NoError(t, err, "Failed to pull image "+testImageName)
287291

288292
{
@@ -292,7 +296,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
292296
go func(i int) {
293297
defer vmGroup.Done()
294298
vmID := fmt.Sprintf("%d", i+vmIDBase)
295-
_, _, err := orch.StartVM(ctx, vmID, testImageName)
299+
_, _, err := orch.StartVM(ctx, vmID, *testImage)
296300
require.NoError(t, err, "Failed to start VM, "+vmID)
297301
}(i)
298302
}
@@ -321,7 +325,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
321325
defer vmGroup.Done()
322326
vmID := fmt.Sprintf("%d", i+vmIDBase)
323327
revision := fmt.Sprintf("myrev-%d", i+vmIDBase)
324-
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", testImageName)
328+
snap := snapshotting.NewSnapshot(revision, "/fccd/snapshots", *testImage)
325329
err = snap.CreateSnapDir()
326330
require.NoError(t, err, "Failed to create snapshots directory")
327331

@@ -356,7 +360,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
356360
go func(i int) {
357361
defer vmGroup.Done()
358362
vmID := fmt.Sprintf("%d", i+vmIDBase)
359-
snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", testImageName)
363+
snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", *testImage)
360364
vmIDInt, _ := strconv.Atoi(vmID)
361365
vmID = strconv.Itoa(vmIDInt + 1)
362366
_, _, err := orch.LoadSnapshot(ctx, vmID, snap)
@@ -420,20 +424,21 @@ func TestRemoteSnapCreate(t *testing.T) {
420424
require.NoError(t, err, "Failed to create remote snapshots directory")
421425

422426
orch := NewOrchestrator(
423-
"devmapper",
427+
*snapshotter,
424428
"",
425429
WithTestModeOn(true),
426430
WithUPF(*isUPFEnabled),
427431
WithLazyMode(*isLazyMode),
432+
WithDockerCredentials(*dockerCredentials),
428433
)
429434

430-
_, _, err = orch.StartVM(ctx, vmID, testImageName)
435+
_, _, err = orch.StartVM(ctx, vmID, *testImage)
431436
require.NoError(t, err, "Failed to start VM")
432437

433438
err = orch.PauseVM(ctx, vmID)
434439
require.NoError(t, err, "Failed to pause VM")
435440

436-
snap := snapshotting.NewSnapshot(revision, remoteSnapshotsDir, testImageName)
441+
snap := snapshotting.NewSnapshot(revision, remoteSnapshotsDir, *testImage)
437442
_ = snap.Cleanup()
438443
err = snap.CreateSnapDir()
439444
require.NoError(t, err, "Failed to create remote snapshots directory")
@@ -473,14 +478,15 @@ func TestRemoteSnapLoad(t *testing.T) {
473478
require.NoError(t, err, "Failed to stat remote snapshots directory")
474479

475480
orch := NewOrchestrator(
476-
"devmapper",
481+
*snapshotter,
477482
"",
478483
WithTestModeOn(true),
479484
WithUPF(*isUPFEnabled),
480485
WithLazyMode(*isLazyMode),
486+
WithDockerCredentials(*dockerCredentials),
481487
)
482488

483-
snap := snapshotting.NewSnapshot(revision, remoteSnapshotsDir, testImageName)
489+
snap := snapshotting.NewSnapshot(revision, remoteSnapshotsDir, *testImage)
484490

485491
_, _, err = orch.LoadSnapshot(ctx, vmID, snap)
486492
require.NoError(t, err, "Failed to load remote snapshot of VM")

‎scripts/stargz/setup_stargz.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func SetupStargz(sandbox string) error {
3333
if sandbox == "firecracker" {
3434
_, err = utils.ExecVHiveBashScript("scripts/stargz/setup_demux_snapshotter.sh")
3535

36-
bashCmd := `sudo sysctl --quiet -w net.ipv4.conf.all.forwarding=1 && ` +
36+
bashCmd := `sudo sysctl --quiet -w net.ipv4.conf.all.forwarding=1 && ` +
3737
`sudo sysctl --quiet -w net.ipv4.ip_forward=1 && ` +
3838
`sudo sysctl --quiet --system`
3939

0 commit comments

Comments
 (0)