Skip to content

Commit c9c6721

Browse files
kzyskzys
authored
Merge pull request #610 from kzys/volume2
Support volumes
2 parents 62ae251 + 6bf556d commit c9c6721

File tree

5 files changed

+446
-1
lines changed

5 files changed

+446
-1
lines changed

‎.github/workflows/build.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@ jobs:
4343
- run: make deps
4444
- run: make
4545
- run: make lint
46-
- run: make test
46+
- name: make test
47+
run: DISABLE_ROOT_TESTS=1 make test
48+
- name: make test as root
49+
run: EXTRAGOARGS='-exec sudo' make test
4750
- run: |
4851
make tidy
4952
git diff --exit-code

‎runtime/volume_integ_test.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package main
15+
16+
import (
17+
"bytes"
18+
"context"
19+
"fmt"
20+
"os"
21+
"path/filepath"
22+
"strconv"
23+
"testing"
24+
25+
"github.com/containerd/containerd"
26+
"github.com/containerd/containerd/cio"
27+
"github.com/containerd/containerd/namespaces"
28+
"github.com/containerd/containerd/oci"
29+
"github.com/firecracker-microvm/firecracker-containerd/proto"
30+
"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
31+
"github.com/firecracker-microvm/firecracker-containerd/volume"
32+
"github.com/stretchr/testify/assert"
33+
"github.com/stretchr/testify/require"
34+
)
35+
36+
const mib = 1024 * 1024
37+
38+
func TestVolumes_Isolated(t *testing.T) {
39+
prepareIntegTest(t)
40+
41+
const vmID = 0
42+
43+
ctx := namespaces.WithNamespace(context.Background(), "default")
44+
45+
client, err := containerd.New(containerdSockPath, containerd.WithDefaultRuntime(firecrackerRuntime))
46+
require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", containerdSockPath)
47+
defer client.Close()
48+
49+
image, err := alpineImage(ctx, client, defaultSnapshotterName)
50+
require.NoError(t, err, "failed to get alpine image")
51+
52+
fcClient, err := newFCControlClient(containerdSockPath)
53+
require.NoError(t, err, "failed to create fccontrol client")
54+
55+
// Make volumes.
56+
path, err := os.MkdirTemp("", t.Name())
57+
require.NoError(t, err)
58+
59+
f, err := os.Create(filepath.Join(path, "hello.txt"))
60+
require.NoError(t, err)
61+
62+
_, err = f.Write([]byte("hello from host\n"))
63+
require.NoError(t, err)
64+
65+
const volName = "volume1"
66+
vs := volume.NewSet()
67+
vs.Add(volume.FromHost(volName, path))
68+
69+
// Since CreateVM doesn't take functional options, we need to explicitly create
70+
// a FirecrackerDriveMount
71+
mount, err := vs.PrepareDriveMount(ctx, 10*mib)
72+
require.NoError(t, err)
73+
74+
containers := []string{"c1", "c2"}
75+
76+
_, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{
77+
VMID: strconv.Itoa(vmID),
78+
ContainerCount: int32(len(containers)),
79+
DriveMounts: []*proto.FirecrackerDriveMount{mount},
80+
})
81+
require.NoError(t, err, "failed to create VM")
82+
83+
// Make containers with the volume.
84+
dir := "/path/in/container"
85+
mpOpt, err := vs.WithMounts([]volume.Mount{{Source: volName, Destination: dir, ReadOnly: false}})
86+
require.NoError(t, err)
87+
88+
for _, name := range containers {
89+
snapshotName := fmt.Sprintf("%s-snapshot", name)
90+
91+
sh := fmt.Sprintf("echo hello from %s >> %s/hello.txt", name, dir)
92+
container, err := client.NewContainer(ctx,
93+
name,
94+
containerd.WithSnapshotter(defaultSnapshotterName),
95+
containerd.WithNewSnapshot(snapshotName, image),
96+
containerd.WithNewSpec(
97+
firecrackeroci.WithVMID(strconv.Itoa(vmID)),
98+
oci.WithProcessArgs("sh", "-c", sh),
99+
oci.WithDefaultPathEnv,
100+
mpOpt,
101+
),
102+
)
103+
require.NoError(t, err, "failed to create container %s", name)
104+
105+
var stdout, stderr bytes.Buffer
106+
107+
task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, &stdout, &stderr)))
108+
require.NoError(t, err, "failed to create task for container %s", name)
109+
110+
exitCh, err := task.Wait(ctx)
111+
require.NoError(t, err, "failed to wait on task for container %s", name)
112+
113+
err = task.Start(ctx)
114+
require.NoError(t, err, "failed to start task for container %s", name)
115+
116+
exit := <-exitCh
117+
_, err = task.Delete(ctx)
118+
require.NoError(t, err)
119+
120+
assert.Equalf(t, uint32(0), exit.ExitCode(), "stdout=%q stderr=%q", stdout.String(), stderr.String())
121+
}
122+
123+
name := "cat"
124+
snapshotName := fmt.Sprintf("%s-snapshot", name)
125+
container, err := client.NewContainer(ctx,
126+
name,
127+
containerd.WithSnapshotter(defaultSnapshotterName),
128+
containerd.WithNewSnapshot(snapshotName, image),
129+
containerd.WithNewSpec(
130+
firecrackeroci.WithVMID(strconv.Itoa(vmID)),
131+
oci.WithProcessArgs("cat", fmt.Sprintf("%s/hello.txt", dir)),
132+
oci.WithDefaultPathEnv,
133+
mpOpt,
134+
),
135+
)
136+
require.NoError(t, err, "failed to create container %s", name)
137+
138+
var stdout, stderr bytes.Buffer
139+
task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, &stdout, &stderr)))
140+
require.NoError(t, err, "failed to create task for container %s", name)
141+
142+
exitCh, err := task.Wait(ctx)
143+
require.NoError(t, err, "failed to wait on task for container %s", name)
144+
145+
err = task.Start(ctx)
146+
require.NoError(t, err, "failed to start task for container %s", name)
147+
148+
exit := <-exitCh
149+
_, err = task.Delete(ctx)
150+
require.NoError(t, err)
151+
152+
assert.Equal(t, uint32(0), exit.ExitCode())
153+
assert.Equal(t, "hello from host\nhello from c1\nhello from c2\n", stdout.String())
154+
assert.Equal(t, "", stderr.String())
155+
}

‎volume/set.go

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package volume
15+
16+
import (
17+
"context"
18+
"fmt"
19+
"os"
20+
"os/exec"
21+
"path/filepath"
22+
23+
"github.com/containerd/containerd/containers"
24+
"github.com/containerd/containerd/mount"
25+
"github.com/containerd/containerd/oci"
26+
"github.com/containerd/continuity/fs"
27+
"github.com/firecracker-microvm/firecracker-containerd/proto"
28+
"github.com/hashicorp/go-multierror"
29+
"github.com/opencontainers/runtime-spec/specs-go"
30+
)
31+
32+
const (
33+
vmVolumePath = "/vmv"
34+
fsType = "ext4"
35+
)
36+
37+
// Set is a set of volumes.
38+
type Set struct {
39+
volumes map[string]*Volume
40+
tempDir string
41+
}
42+
43+
// NewSet returns a new volume set.
44+
func NewSet() *Set {
45+
return NewSetWithTempDir(os.TempDir())
46+
}
47+
48+
// NewSetWithTempDir returns a new volume set and creates all temporary files under the tempDir.
49+
func NewSetWithTempDir(tempDir string) *Set {
50+
return &Set{volumes: make(map[string]*Volume), tempDir: tempDir}
51+
}
52+
53+
// Add a volume to the set.
54+
func (vs *Set) Add(v *Volume) {
55+
vs.volumes[v.name] = v
56+
}
57+
58+
func (vs *Set) createDiskImage(ctx context.Context, size int64) (path string, retErr error) {
59+
f, err := os.CreateTemp(vs.tempDir, "createDiskImage")
60+
if err != nil {
61+
retErr = err
62+
return
63+
}
64+
defer func() {
65+
// The file must be closed even in the success case.
66+
err := f.Close()
67+
if err != nil {
68+
retErr = multierror.Append(retErr, err)
69+
}
70+
// But the file must not be deleted in the success case.
71+
if retErr != nil {
72+
err = os.Remove(path)
73+
if err != nil {
74+
retErr = multierror.Append(retErr, err)
75+
}
76+
}
77+
}()
78+
79+
err = f.Truncate(size)
80+
if err != nil {
81+
retErr = err
82+
return
83+
}
84+
85+
out, err := exec.CommandContext(ctx, "mkfs."+fsType, "-F", f.Name()).CombinedOutput()
86+
if err != nil {
87+
retErr = fmt.Errorf("failed to execute mkfs.%s: %s: %w", fsType, out, err)
88+
return
89+
}
90+
91+
path = f.Name()
92+
return
93+
}
94+
95+
func mountDiskImage(source, target string) error {
96+
return mount.All([]mount.Mount{{Type: fsType, Source: source, Options: []string{"loop"}}}, target)
97+
}
98+
99+
// PrepareDriveMount returns a FirecrackerDriveMount that could be used with CreateVM.
100+
func (vs *Set) PrepareDriveMount(ctx context.Context, size int64) (dm *proto.FirecrackerDriveMount, retErr error) {
101+
path, err := vs.createDiskImage(ctx, size)
102+
if err != nil {
103+
retErr = err
104+
return
105+
}
106+
defer func() {
107+
// The file must not be deleted in the success case.
108+
if retErr != nil {
109+
err := os.Remove(path)
110+
if err != nil {
111+
retErr = multierror.Append(retErr, err)
112+
}
113+
}
114+
}()
115+
116+
dir, err := os.MkdirTemp(vs.tempDir, "PrepareDriveMount")
117+
if err != nil {
118+
retErr = err
119+
return
120+
}
121+
defer func() {
122+
err := os.Remove(dir)
123+
if err != nil {
124+
retErr = multierror.Append(retErr, err)
125+
}
126+
}()
127+
128+
err = mountDiskImage(path, dir)
129+
if err != nil {
130+
retErr = err
131+
return
132+
}
133+
defer func() {
134+
err := mount.Unmount(dir, 0)
135+
if err != nil {
136+
retErr = multierror.Append(retErr, err)
137+
}
138+
}()
139+
140+
for _, v := range vs.volumes {
141+
path := filepath.Join(dir, v.name)
142+
if v.hostPath == "" {
143+
continue
144+
}
145+
err := fs.CopyDir(path, v.hostPath)
146+
if err != nil {
147+
retErr = fmt.Errorf("failed to copy volume %q: %w", v.name, err)
148+
return
149+
}
150+
}
151+
152+
dm = &proto.FirecrackerDriveMount{
153+
HostPath: path,
154+
VMPath: vmVolumePath,
155+
FilesystemType: fsType,
156+
IsWritable: true,
157+
}
158+
return
159+
}
160+
161+
// Mount is used to expose volumes to containers.
162+
type Mount struct {
163+
// Source is the name of a volume.
164+
Source string
165+
// Destination is the path inside the container where the volume is mounted.
166+
Destination string
167+
// ReadOnly is true if the volume is mounted as read-only.
168+
ReadOnly bool
169+
}
170+
171+
// WithMounts expose given volumes to the container.
172+
func (vs *Set) WithMounts(mountpoints []Mount) (oci.SpecOpts, error) {
173+
mounts := []specs.Mount{}
174+
175+
for _, mp := range mountpoints {
176+
v, ok := vs.volumes[mp.Source]
177+
if !ok {
178+
return nil, fmt.Errorf("failed to find volume %q", mp.Source)
179+
}
180+
181+
options := []string{"bind"}
182+
if mp.ReadOnly {
183+
options = append(options, "ro")
184+
}
185+
186+
mounts = append(mounts, specs.Mount{
187+
// TODO: for volumes that are provided by the guest (e.g. in-VM snapshotters)
188+
// We may be able to have bind-mounts from in-VM snapshotters' mount points.
189+
Source: filepath.Join(vmVolumePath, v.name),
190+
Destination: mp.Destination,
191+
Type: "bind",
192+
Options: options,
193+
})
194+
}
195+
196+
return func(ctx context.Context, client oci.Client, container *containers.Container, s *oci.Spec) error {
197+
s.Mounts = append(s.Mounts, mounts...)
198+
return nil
199+
}, nil
200+
}

0 commit comments

Comments
 (0)