Skip to content

Commit fe1dc17

Browse files
authored
Add FUSE-based Virtual Block Device package (#4866)
1 parent 72e6ea1 commit fe1dc17

File tree

3 files changed

+317
-0
lines changed

3 files changed

+317
-0
lines changed
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
package(default_visibility = ["//enterprise:__subpackages__"])
4+
5+
go_library(
6+
name = "vbd",
7+
srcs = ["vbd.go"],
8+
importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/vbd",
9+
deps = [
10+
"//server/util/log",
11+
"//server/util/status",
12+
"@com_github_hanwen_go_fuse_v2//fs",
13+
"@com_github_hanwen_go_fuse_v2//fuse",
14+
],
15+
)
16+
17+
go_test(
18+
name = "vbd_test",
19+
srcs = ["vbd_test.go"],
20+
exec_properties = {
21+
# Test requires mount() privileges.
22+
"test.workload-isolation-type": "firecracker",
23+
},
24+
target_compatible_with = [
25+
"@platforms//os:linux",
26+
],
27+
deps = [
28+
":vbd",
29+
"@com_github_stretchr_testify//require",
30+
],
31+
)
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package vbd
2+
3+
import (
4+
"context"
5+
"io"
6+
"os"
7+
"syscall"
8+
"time"
9+
10+
"github.com/buildbuddy-io/buildbuddy/server/util/log"
11+
"github.com/buildbuddy-io/buildbuddy/server/util/status"
12+
"github.com/hanwen/go-fuse/v2/fuse"
13+
14+
fusefs "github.com/hanwen/go-fuse/v2/fs"
15+
)
16+
17+
const (
18+
// FileName is the name of the single file exposed under the mount dir.
19+
FileName = "file"
20+
)
21+
22+
// BlockDevice is the interface backing VBD IO operations.
23+
type BlockDevice interface {
24+
io.ReaderAt
25+
io.WriterAt
26+
SizeBytes() (int64, error)
27+
}
28+
29+
// FS represents a handle on a VBD FS. Once mounted, the mounted directory
30+
// exposes a single file. The file name is the const FileName. IO operations on
31+
// the file are backed by the wrapped BlockDevice.
32+
type FS struct {
33+
store BlockDevice
34+
root *Node
35+
server *fuse.Server
36+
mountPath string
37+
}
38+
39+
// New returns a new FS serving the given file.
40+
func New(store BlockDevice) (*FS, error) {
41+
f := &FS{store: store}
42+
f.root = &Node{fs: f}
43+
return f, nil
44+
}
45+
46+
func (f *FS) SetFile(file BlockDevice) {
47+
f.store = file
48+
}
49+
50+
// Mount mounts the FS to the given directory path.
51+
// It exposes a single file "store" which points to the backing store.
52+
func (f *FS) Mount(path string) error {
53+
if f.mountPath != "" {
54+
return status.InternalErrorf("vbd is already mounted")
55+
}
56+
f.mountPath = path
57+
58+
if err := os.MkdirAll(path, 0755); err != nil {
59+
return err
60+
}
61+
62+
nodeAttrTimeout := 6 * time.Hour
63+
opts := &fusefs.Options{
64+
EntryTimeout: &nodeAttrTimeout,
65+
AttrTimeout: &nodeAttrTimeout,
66+
MountOptions: fuse.MountOptions{
67+
AllowOther: true,
68+
// Debug: true,
69+
DisableXAttrs: true,
70+
// Don't depend on `fusermount`.
71+
DirectMount: true,
72+
FsName: "vbd",
73+
MaxWrite: fuse.MAX_KERNEL_WRITE,
74+
},
75+
}
76+
nodeFS := fusefs.NewNodeFS(f.root, opts)
77+
server, err := fuse.NewServer(nodeFS, path, &opts.MountOptions)
78+
if err != nil {
79+
return status.UnavailableErrorf("could not mount VBD to %q: %s", path, err)
80+
}
81+
82+
go server.Serve()
83+
if err := server.WaitMount(); err != nil {
84+
return status.UnavailableErrorf("waiting for VBD mount failed: %s", err)
85+
}
86+
87+
f.server = server
88+
89+
attr := fusefs.StableAttr{Mode: fuse.S_IFREG}
90+
child := &Node{fs: f, file: f.store}
91+
inode := f.root.NewPersistentInode(context.TODO(), child, attr)
92+
f.root.AddChild(FileName, inode, false /*=overwrite*/)
93+
94+
return nil
95+
}
96+
97+
func (f *FS) Unmount() error {
98+
err := f.server.Unmount()
99+
f.server.Wait()
100+
f.server = nil
101+
if err := os.Remove(f.mountPath); err != nil {
102+
log.Errorf("Failed to unmount vbd: %s", err)
103+
}
104+
log.Debugf("Unmounted %s", f.mountPath)
105+
return err
106+
}
107+
108+
type Node struct {
109+
fusefs.Inode
110+
fs *FS
111+
file BlockDevice
112+
}
113+
114+
var _ fusefs.NodeOpener = (*Node)(nil)
115+
var _ fusefs.NodeGetattrer = (*Node)(nil)
116+
117+
func (n *Node) Open(ctx context.Context, flags uint32) (fusefs.FileHandle, uint32, syscall.Errno) {
118+
if n.file == nil {
119+
log.CtxErrorf(ctx, "open root dir: not supported")
120+
return nil, 0, syscall.EOPNOTSUPP
121+
}
122+
return &fileHandle{file: n.file}, 0, 0
123+
}
124+
125+
func (n *Node) Getattr(ctx context.Context, _ fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
126+
if n.file != nil {
127+
size, err := n.file.SizeBytes()
128+
if err != nil {
129+
log.CtxErrorf(ctx, "VBD size failed: %s", err)
130+
return syscall.EIO
131+
}
132+
out.Size = uint64(size)
133+
}
134+
return fusefs.OK
135+
}
136+
137+
type fileHandle struct {
138+
file BlockDevice
139+
}
140+
141+
var _ fusefs.FileReader = (*fileHandle)(nil)
142+
var _ fusefs.FileWriter = (*fileHandle)(nil)
143+
144+
func (h *fileHandle) Read(ctx context.Context, p []byte, off int64) (res fuse.ReadResult, errno syscall.Errno) {
145+
return &reader{h.file, off, len(p)}, 0
146+
}
147+
148+
func (h *fileHandle) Write(ctx context.Context, p []byte, off int64) (uint32, syscall.Errno) {
149+
n, err := h.file.WriteAt(p, off)
150+
if err != nil {
151+
log.CtxErrorf(ctx, "VBD write failed: %s", err)
152+
return uint32(n), syscall.EIO
153+
}
154+
return uint32(n), 0
155+
}
156+
157+
type reader struct {
158+
file BlockDevice
159+
off int64
160+
size int
161+
}
162+
163+
var _ fuse.ReadResult = (*reader)(nil)
164+
165+
func (r *reader) Bytes(p []byte) ([]byte, fuse.Status) {
166+
length := r.size
167+
if len(p) < length {
168+
length = len(p)
169+
}
170+
_, err := r.file.ReadAt(p[:length], r.off)
171+
if err != nil {
172+
log.Errorf("VBD read failed: %s", err)
173+
return nil, fuse.EIO
174+
}
175+
return p[:length], fuse.OK
176+
}
177+
178+
func (r *reader) Size() int {
179+
return r.size
180+
}
181+
182+
func (r *reader) Done() {}
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package vbd_test
2+
3+
import (
4+
"bytes"
5+
"math/rand"
6+
"os"
7+
"path/filepath"
8+
"testing"
9+
10+
"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/vbd"
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func TestVBD(t *testing.T) {
15+
if os.Getuid() != 0 {
16+
t.Skipf("test must be run as root")
17+
}
18+
19+
const root = "/tmp/vbd_test"
20+
err := os.MkdirAll(root, 0755)
21+
require.NoError(t, err)
22+
23+
// Set up backing file
24+
f, err := os.CreateTemp(root, "vbd-*")
25+
require.NoError(t, err)
26+
t.Cleanup(func() { f.Close() })
27+
const fSize = 1024 * 512
28+
b := make([]byte, fSize)
29+
copy(b, []byte("Hello world!"))
30+
_, err = f.Write(b)
31+
require.NoError(t, err)
32+
33+
// Mount as VBD
34+
v, err := vbd.New(&FileBlockDevice{f})
35+
require.NoError(t, err)
36+
dir, err := os.MkdirTemp(root, "mount-*")
37+
require.NoError(t, err)
38+
err = v.Mount(dir)
39+
require.NoError(t, err)
40+
t.Cleanup(func() {
41+
err := v.Unmount()
42+
require.NoError(t, err)
43+
})
44+
45+
// Try stat() on the virtual file
46+
s, err := os.Stat(filepath.Join(dir, vbd.FileName))
47+
require.NoError(t, err)
48+
require.Equal(t, int64(fSize), s.Size())
49+
50+
// Try random reads and writes to the virtual file
51+
{
52+
f, err := os.OpenFile(filepath.Join(dir, vbd.FileName), os.O_RDWR, 0)
53+
require.NoError(t, err)
54+
t.Cleanup(func() {
55+
err := f.Close()
56+
require.NoError(t, err)
57+
})
58+
59+
for i := 1; i <= 100; i++ {
60+
offset, length := randSubslice(len(b))
61+
p := make([]byte, length)
62+
if shouldRead := rand.Intn(2) == 0; shouldRead {
63+
// Read, randomly choosing between using Seek+Read / ReadAt.
64+
if shouldSeek := rand.Intn(2) == 0; shouldSeek {
65+
_, err := f.Seek(int64(offset), 0)
66+
require.NoError(t, err)
67+
_, err = f.Read(p)
68+
require.NoError(t, err)
69+
require.True(t, bytes.Equal(p, b[offset:offset+length]))
70+
} else {
71+
_, err := f.ReadAt(p, int64(offset))
72+
require.NoError(t, err)
73+
require.True(t, bytes.Equal(p, b[offset:offset+length]))
74+
}
75+
} else {
76+
// Write
77+
_, err := rand.Read(p)
78+
require.NoError(t, err)
79+
_, err = f.WriteAt(p, int64(offset))
80+
require.NoError(t, err)
81+
// Update our expected buffer contents
82+
copy(b[offset:offset+length], p)
83+
}
84+
}
85+
}
86+
}
87+
88+
type FileBlockDevice struct{ *os.File }
89+
90+
func (f *FileBlockDevice) SizeBytes() (int64, error) {
91+
s, err := f.File.Stat()
92+
if err != nil {
93+
return 0, err
94+
}
95+
return s.Size(), nil
96+
}
97+
98+
// Picks a uniform random subslice of a slice with a given length.
99+
// Returns the offset and length of the subslice.
100+
func randSubslice(sliceLength int) (offset, length int) {
101+
length = rand.Intn(sliceLength + 1)
102+
offset = rand.Intn(sliceLength - length + 1)
103+
return
104+
}

0 commit comments

Comments
 (0)