Skip to content

Commit 6bf556d

Browse files
kzyskzys
committed
Support volumes
Since firecracker-containerd puts containers in VMs, mounting the host's directories is not straightforward. This commit adds "volume" package that makes an empty ext4 image to pack and create helper functions around to make the image usable from containers. Signed-off-by: Kazuyoshi Kato <[email protected]>
1 parent ecee4df commit 6bf556d

File tree

4 files changed

+442
-0
lines changed

4 files changed

+442
-0
lines changed

‎runtime/volume_integ_test.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package main
15+
16+
import (
17+
"bytes"
18+
"context"
19+
"fmt"
20+
"os"
21+
"path/filepath"
22+
"strconv"
23+
"testing"
24+
25+
"github.com/containerd/containerd"
26+
"github.com/containerd/containerd/cio"
27+
"github.com/containerd/containerd/namespaces"
28+
"github.com/containerd/containerd/oci"
29+
"github.com/firecracker-microvm/firecracker-containerd/proto"
30+
"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
31+
"github.com/firecracker-microvm/firecracker-containerd/volume"
32+
"github.com/stretchr/testify/assert"
33+
"github.com/stretchr/testify/require"
34+
)
35+
36+
const mib = 1024 * 1024
37+
38+
func TestVolumes_Isolated(t *testing.T) {
39+
prepareIntegTest(t)
40+
41+
const vmID = 0
42+
43+
ctx := namespaces.WithNamespace(context.Background(), "default")
44+
45+
client, err := containerd.New(containerdSockPath, containerd.WithDefaultRuntime(firecrackerRuntime))
46+
require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", containerdSockPath)
47+
defer client.Close()
48+
49+
image, err := alpineImage(ctx, client, defaultSnapshotterName)
50+
require.NoError(t, err, "failed to get alpine image")
51+
52+
fcClient, err := newFCControlClient(containerdSockPath)
53+
require.NoError(t, err, "failed to create fccontrol client")
54+
55+
// Make volumes.
56+
path, err := os.MkdirTemp("", t.Name())
57+
require.NoError(t, err)
58+
59+
f, err := os.Create(filepath.Join(path, "hello.txt"))
60+
require.NoError(t, err)
61+
62+
_, err = f.Write([]byte("hello from host\n"))
63+
require.NoError(t, err)
64+
65+
const volName = "volume1"
66+
vs := volume.NewSet()
67+
vs.Add(volume.FromHost(volName, path))
68+
69+
// Since CreateVM doesn't take functional options, we need to explicitly create
70+
// a FirecrackerDriveMount
71+
mount, err := vs.PrepareDriveMount(ctx, 10*mib)
72+
require.NoError(t, err)
73+
74+
containers := []string{"c1", "c2"}
75+
76+
_, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{
77+
VMID: strconv.Itoa(vmID),
78+
ContainerCount: int32(len(containers)),
79+
DriveMounts: []*proto.FirecrackerDriveMount{mount},
80+
})
81+
require.NoError(t, err, "failed to create VM")
82+
83+
// Make containers with the volume.
84+
dir := "/path/in/container"
85+
mpOpt, err := vs.WithMounts([]volume.Mount{{Source: volName, Destination: dir, ReadOnly: false}})
86+
require.NoError(t, err)
87+
88+
for _, name := range containers {
89+
snapshotName := fmt.Sprintf("%s-snapshot", name)
90+
91+
sh := fmt.Sprintf("echo hello from %s >> %s/hello.txt", name, dir)
92+
container, err := client.NewContainer(ctx,
93+
name,
94+
containerd.WithSnapshotter(defaultSnapshotterName),
95+
containerd.WithNewSnapshot(snapshotName, image),
96+
containerd.WithNewSpec(
97+
firecrackeroci.WithVMID(strconv.Itoa(vmID)),
98+
oci.WithProcessArgs("sh", "-c", sh),
99+
oci.WithDefaultPathEnv,
100+
mpOpt,
101+
),
102+
)
103+
require.NoError(t, err, "failed to create container %s", name)
104+
105+
var stdout, stderr bytes.Buffer
106+
107+
task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, &stdout, &stderr)))
108+
require.NoError(t, err, "failed to create task for container %s", name)
109+
110+
exitCh, err := task.Wait(ctx)
111+
require.NoError(t, err, "failed to wait on task for container %s", name)
112+
113+
err = task.Start(ctx)
114+
require.NoError(t, err, "failed to start task for container %s", name)
115+
116+
exit := <-exitCh
117+
_, err = task.Delete(ctx)
118+
require.NoError(t, err)
119+
120+
assert.Equalf(t, uint32(0), exit.ExitCode(), "stdout=%q stderr=%q", stdout.String(), stderr.String())
121+
}
122+
123+
name := "cat"
124+
snapshotName := fmt.Sprintf("%s-snapshot", name)
125+
container, err := client.NewContainer(ctx,
126+
name,
127+
containerd.WithSnapshotter(defaultSnapshotterName),
128+
containerd.WithNewSnapshot(snapshotName, image),
129+
containerd.WithNewSpec(
130+
firecrackeroci.WithVMID(strconv.Itoa(vmID)),
131+
oci.WithProcessArgs("cat", fmt.Sprintf("%s/hello.txt", dir)),
132+
oci.WithDefaultPathEnv,
133+
mpOpt,
134+
),
135+
)
136+
require.NoError(t, err, "failed to create container %s", name)
137+
138+
var stdout, stderr bytes.Buffer
139+
task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, &stdout, &stderr)))
140+
require.NoError(t, err, "failed to create task for container %s", name)
141+
142+
exitCh, err := task.Wait(ctx)
143+
require.NoError(t, err, "failed to wait on task for container %s", name)
144+
145+
err = task.Start(ctx)
146+
require.NoError(t, err, "failed to start task for container %s", name)
147+
148+
exit := <-exitCh
149+
_, err = task.Delete(ctx)
150+
require.NoError(t, err)
151+
152+
assert.Equal(t, uint32(0), exit.ExitCode())
153+
assert.Equal(t, "hello from host\nhello from c1\nhello from c2\n", stdout.String())
154+
assert.Equal(t, "", stderr.String())
155+
}

‎volume/set.go

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package volume
15+
16+
import (
17+
"context"
18+
"fmt"
19+
"os"
20+
"os/exec"
21+
"path/filepath"
22+
23+
"github.com/containerd/containerd/containers"
24+
"github.com/containerd/containerd/mount"
25+
"github.com/containerd/containerd/oci"
26+
"github.com/containerd/continuity/fs"
27+
"github.com/firecracker-microvm/firecracker-containerd/proto"
28+
"github.com/hashicorp/go-multierror"
29+
"github.com/opencontainers/runtime-spec/specs-go"
30+
)
31+
32+
const (
33+
vmVolumePath = "/vmv"
34+
fsType = "ext4"
35+
)
36+
37+
// Set is a set of volumes.
38+
type Set struct {
39+
volumes map[string]*Volume
40+
tempDir string
41+
}
42+
43+
// NewSet returns a new volume set.
44+
func NewSet() *Set {
45+
return NewSetWithTempDir(os.TempDir())
46+
}
47+
48+
// NewSetWithTempDir returns a new volume set and creates all temporary files under the tempDir.
49+
func NewSetWithTempDir(tempDir string) *Set {
50+
return &Set{volumes: make(map[string]*Volume), tempDir: tempDir}
51+
}
52+
53+
// Add a volume to the set.
54+
func (vs *Set) Add(v *Volume) {
55+
vs.volumes[v.name] = v
56+
}
57+
58+
func (vs *Set) createDiskImage(ctx context.Context, size int64) (path string, retErr error) {
59+
f, err := os.CreateTemp(vs.tempDir, "createDiskImage")
60+
if err != nil {
61+
retErr = err
62+
return
63+
}
64+
defer func() {
65+
// The file must be closed even in the success case.
66+
err := f.Close()
67+
if err != nil {
68+
retErr = multierror.Append(retErr, err)
69+
}
70+
// But the file must not be deleted in the success case.
71+
if retErr != nil {
72+
err = os.Remove(path)
73+
if err != nil {
74+
retErr = multierror.Append(retErr, err)
75+
}
76+
}
77+
}()
78+
79+
err = f.Truncate(size)
80+
if err != nil {
81+
retErr = err
82+
return
83+
}
84+
85+
out, err := exec.CommandContext(ctx, "mkfs."+fsType, "-F", f.Name()).CombinedOutput()
86+
if err != nil {
87+
retErr = fmt.Errorf("failed to execute mkfs.%s: %s: %w", fsType, out, err)
88+
return
89+
}
90+
91+
path = f.Name()
92+
return
93+
}
94+
95+
func mountDiskImage(source, target string) error {
96+
return mount.All([]mount.Mount{{Type: fsType, Source: source, Options: []string{"loop"}}}, target)
97+
}
98+
99+
// PrepareDriveMount returns a FirecrackerDriveMount that could be used with CreateVM.
100+
func (vs *Set) PrepareDriveMount(ctx context.Context, size int64) (dm *proto.FirecrackerDriveMount, retErr error) {
101+
path, err := vs.createDiskImage(ctx, size)
102+
if err != nil {
103+
retErr = err
104+
return
105+
}
106+
defer func() {
107+
// The file must not be deleted in the success case.
108+
if retErr != nil {
109+
err := os.Remove(path)
110+
if err != nil {
111+
retErr = multierror.Append(retErr, err)
112+
}
113+
}
114+
}()
115+
116+
dir, err := os.MkdirTemp(vs.tempDir, "PrepareDriveMount")
117+
if err != nil {
118+
retErr = err
119+
return
120+
}
121+
defer func() {
122+
err := os.Remove(dir)
123+
if err != nil {
124+
retErr = multierror.Append(retErr, err)
125+
}
126+
}()
127+
128+
err = mountDiskImage(path, dir)
129+
if err != nil {
130+
retErr = err
131+
return
132+
}
133+
defer func() {
134+
err := mount.Unmount(dir, 0)
135+
if err != nil {
136+
retErr = multierror.Append(retErr, err)
137+
}
138+
}()
139+
140+
for _, v := range vs.volumes {
141+
path := filepath.Join(dir, v.name)
142+
if v.hostPath == "" {
143+
continue
144+
}
145+
err := fs.CopyDir(path, v.hostPath)
146+
if err != nil {
147+
retErr = fmt.Errorf("failed to copy volume %q: %w", v.name, err)
148+
return
149+
}
150+
}
151+
152+
dm = &proto.FirecrackerDriveMount{
153+
HostPath: path,
154+
VMPath: vmVolumePath,
155+
FilesystemType: fsType,
156+
IsWritable: true,
157+
}
158+
return
159+
}
160+
161+
// Mount is used to expose volumes to containers.
162+
type Mount struct {
163+
// Source is the name of a volume.
164+
Source string
165+
// Destination is the path inside the container where the volume is mounted.
166+
Destination string
167+
// ReadOnly is true if the volume is mounted as read-only.
168+
ReadOnly bool
169+
}
170+
171+
// WithMounts expose given volumes to the container.
172+
func (vs *Set) WithMounts(mountpoints []Mount) (oci.SpecOpts, error) {
173+
mounts := []specs.Mount{}
174+
175+
for _, mp := range mountpoints {
176+
v, ok := vs.volumes[mp.Source]
177+
if !ok {
178+
return nil, fmt.Errorf("failed to find volume %q", mp.Source)
179+
}
180+
181+
options := []string{"bind"}
182+
if mp.ReadOnly {
183+
options = append(options, "ro")
184+
}
185+
186+
mounts = append(mounts, specs.Mount{
187+
// TODO: for volumes that are provided by the guest (e.g. in-VM snapshotters)
188+
// We may be able to have bind-mounts from in-VM snapshotters' mount points.
189+
Source: filepath.Join(vmVolumePath, v.name),
190+
Destination: mp.Destination,
191+
Type: "bind",
192+
Options: options,
193+
})
194+
}
195+
196+
return func(ctx context.Context, client oci.Client, container *containers.Container, s *oci.Spec) error {
197+
s.Mounts = append(s.Mounts, mounts...)
198+
return nil
199+
}, nil
200+
}

‎volume/volume.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
// Package volume provides volumes like Docker and Amazon ECS.
15+
// Volumes are specicial directories that could be shared by multiple containers.
16+
//
17+
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#volumes
18+
package volume
19+
20+
// Volume is a special directory that could be shared between containers.
21+
type Volume struct {
22+
name string
23+
hostPath string
24+
}
25+
26+
// New returns an empty volume.
27+
func New(name string) *Volume {
28+
return &Volume{name: name}
29+
}
30+
31+
// FromHost returns a volume which has the files from the given path.
32+
func FromHost(name, path string) *Volume {
33+
return &Volume{name: name, hostPath: path}
34+
}

0 commit comments

Comments
 (0)