Skip to content

feat: add support for mount options #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: leafty/exp-doi-zenodo
Choose a base branch
from
Draft

feat: add support for mount options #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ebbdcb0
feat: add support for mount options
leafty Apr 9, 2025
14961c4
improve log
leafty Apr 9, 2025
438d434
nicer code for defaults
leafty Apr 9, 2025
cb3a4e7
improve logs
leafty Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ARG RCLONE_IMAGE_REPOSITORY="ghcr.io/swissdatasciencecenter/rclone"
ARG RCLONE_IMAGE_TAG="sha-316bdfc"
ARG RCLONE_IMAGE_TAG="sha-1a32af4"
FROM ${RCLONE_IMAGE_REPOSITORY}:${RCLONE_IMAGE_TAG} AS rclone

FROM golang:1.23.0-bookworm AS build
Expand Down
99 changes: 83 additions & 16 deletions pkg/rclone/rclone.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,62 @@ type MountRequest struct {
MountOpt MountOpt `json:"mountOpt"`
}

// VfsOpt is options for creating the vfs
//
// Note that the `Daemon` option has been removed as it is not accepted for rc calls.
type VfsOpt struct {
CacheMode string `json:"cacheMode"`
DirCacheTime time.Duration `json:"dirCacheTime"`
ReadOnly bool `json:"readOnly"`
NoSeek bool `json:",omitempty"` // don't allow seeking if set
NoChecksum bool `json:",omitempty"` // don't check checksums if set
ReadOnly bool `json:",omitempty"` // if set VFS is read only
NoModTime bool `json:",omitempty"` // don't read mod times for files
DirCacheTime time.Duration `json:",omitempty"` // how long to consider directory listing cache valid
Refresh bool `json:",omitempty"` // refreshes the directory listing recursively on start
PollInterval time.Duration `json:",omitempty"`
Umask int `json:",omitempty"`
UID uint32 `json:",omitempty"`
GID uint32 `json:",omitempty"`
DirPerms os.FileMode `json:",omitempty"`
FilePerms os.FileMode `json:",omitempty"`
ChunkSize int64 `json:",omitempty"` // if > 0 read files in chunks
ChunkSizeLimit int64 `json:",omitempty"` // if > ChunkSize double the chunk size after each chunk until reached
CacheMode string `json:",omitempty"`
CacheMaxAge time.Duration `json:",omitempty"`
CacheMaxSize int64 `json:",omitempty"`
CacheMinFreeSpace int64 `json:",omitempty"`
CachePollInterval time.Duration `json:",omitempty"`
CaseInsensitive bool `json:",omitempty"`
WriteWait time.Duration `json:",omitempty"` // time to wait for in-sequence write
ReadWait time.Duration `json:",omitempty"` // time to wait for in-sequence read
WriteBack time.Duration `json:",omitempty"` // time to wait before writing back dirty files
ReadAhead int64 `json:",omitempty"` // bytes to read ahead in cache mode "full"
UsedIsSize bool `json:",omitempty"` // if true, use the `rclone size` algorithm for Used size
FastFingerprint bool `json:",omitempty"` // if set use fast fingerprints
DiskSpaceTotalSize int64 `json:",omitempty"`
}

// Options for creating the mount
//
// Note that options not supported on Linux have been removed.
type MountOpt struct {
AllowNonEmpty bool `json:"allowNonEmpty"`
AllowOther bool `json:"allowOther"`
DebugFUSE bool `json:",omitempty"`
AllowNonEmpty bool `json:",omitempty"`
AllowRoot bool `json:",omitempty"`
AllowOther bool `json:",omitempty"`
DefaultPermissions bool `json:",omitempty"`
WritebackCache bool `json:",omitempty"`
DaemonWait time.Duration `json:",omitempty"` // time to wait for ready mount from daemon, maximum on Linux or constant on macOS/BSD
MaxReadAhead int64 `json:",omitempty"`
ExtraOptions []string `json:",omitempty"`
ExtraFlags []string `json:",omitempty"`
AttrTimeout time.Duration `json:",omitempty"` // how long the kernel caches attribute for
DeviceName string `json:",omitempty"`
VolumeName string `json:",omitempty"`
NoAppleDouble bool `json:",omitempty"`
NoAppleXattr bool `json:",omitempty"`
AsyncRead bool `json:",omitempty"`
CaseInsensitive string `json:",omitempty"`
}

type ConfigCreateRequest struct {
Name string `json:"name"`
Parameters map[string]string `json:"parameters"`
Expand Down Expand Up @@ -105,7 +152,7 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa
Parameters: params,
Opt: map[string]interface{}{"obscure": true},
}
klog.Infof("executing create config command args=%v, targetpath=%s", configName, targetPath)
klog.Infof("executing create config command name=%s, storageType=%s", configName, configOpts.StorageType)
postBody, err := json.Marshal(configOpts)
if err != nil {
return fmt.Errorf("mounting failed: couldn't create request body: %s", err)
Expand All @@ -121,31 +168,51 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa
}
klog.Infof("created config: %s", configName)

// VFS Mount parameters
vfsOpt := VfsOpt{
CacheMode: "writes",
DirCacheTime: 60 * time.Second,
}
vfsOptStr := parameters["vfsOpt"]
if vfsOptStr != "" {
err = json.Unmarshal([]byte(vfsOptStr), &vfsOpt)
if err != nil {
return fmt.Errorf("could not parse vfsOpt: %w", err)
}
}
// The `ReadOnly` option is specified in the PVC
vfsOpt.ReadOnly = readOnly
// Mount parameters
mountOpt := MountOpt{
AllowNonEmpty: true,
AllowOther: true,
}
mountOptStr := parameters["mountOpt"]
if mountOptStr != "" {
err = json.Unmarshal([]byte(mountOptStr), &mountOpt)
if err != nil {
return fmt.Errorf("could not parse mountOpt: %w", err)
}
}

remoteWithPath := fmt.Sprintf("%s:%s", configName, rcloneVolume.RemotePath)
mountArgs := MountRequest{
Fs: remoteWithPath,
MountPoint: targetPath,
VfsOpt: VfsOpt{
CacheMode: "writes",
DirCacheTime: 60 * time.Second,
ReadOnly: readOnly,
},
MountOpt: MountOpt{
AllowNonEmpty: true,
AllowOther: true,
},
VfsOpt: vfsOpt,
MountOpt: mountOpt,
}

// create target, os.Mkdirall is noop if it exists
err = os.MkdirAll(targetPath, 0750)
if err != nil {
return err
}
klog.Infof("executing mount command args=%v, targetpath=%s", mountArgs, targetPath)
postBody, err = json.Marshal(mountArgs)
if err != nil {
return fmt.Errorf("mounting failed: couldn't create request body: %s", err)
}
klog.Infof("executing mount command args=%s", string(postBody))
requestBody = bytes.NewBuffer(postBody)
resp, err = http.Post(fmt.Sprintf("http://localhost:%d/mount/mount", r.port), "application/json", requestBody)
if err != nil {
Expand Down