feat(local): read filesystem I/O block size for optimal operations

Query the filesystem to determine optimal I/O block size rather than
using a hardcoded 4096 value. This improves alignment with underlying
storage characteristics.

Block size detection (Unix):
- statfs.Bsize: Filesystem's fundamental block size
- stat.Blksize: Preferred I/O block size for the file/directory
- Use minimum of both with bounds checking (0 < size <= MaxInt32)
- Default to 4096 if detection fails

The IOBlockSize is exposed through fs.Usage and propagated to:
- FUSE statfs responses (Bsize, Frsize fields)
- File attribute block calculations
- VFS via GetBlockSizes() returning (dataBlockSize=512, ioBlockSize)

Union backend aggregates by taking minimum across all upstreams,
ensuring conservative alignment for heterogeneous storage.

Windows defaults to 4096 as the statfs equivalent doesn't provide
preferred I/O size information.

Signed-off-by: Anagh Kumar Baranwal <[email protected]>

Author: darthShadow

backend/local/about_unix.go

@@ -5,27 +5,67 @@ package local
 import (
 	"context"
 	"fmt"
+	"math"
 	"os"
-	"syscall"
 
 	"github.com/rclone/rclone/fs"
+	"golang.org/x/sys/unix"
 )
 
+// calculateOptimalBlockSize determines the optimal block size for I/O operations
+// by examining both statfs and stat block sizes and applying bounds checking
+func calculateOptimalBlockSize(statfsBlockSize, statBlockSize int64) int32 {
+	const defaultBlockSize = 4096      // reasonable default for modern filesystems
+	const maxBlockSize = math.MaxInt32 // max int32 value (~2GB)
+
+	// Apply bounds checking for statfs block size
+	if statfsBlockSize <= 0 {
+		statfsBlockSize = defaultBlockSize
+	} else if statfsBlockSize > maxBlockSize {
+		statfsBlockSize = maxBlockSize
+	}
+
+	// Apply bounds checking for stat block size
+	if statBlockSize <= 0 {
+		statBlockSize = defaultBlockSize
+	} else if statBlockSize > maxBlockSize {
+		statBlockSize = maxBlockSize
+	}
+
+	// Use minimum of both block sizes for optimal IO
+	return int32(min(statfsBlockSize, statBlockSize))
+}
+
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
-	var s syscall.Statfs_t
-	err := syscall.Statfs(f.root, &s)
+	var unixStatfs unix.Statfs_t
+	err := unix.Statfs(f.root, &unixStatfs)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, fs.ErrorDirNotFound
 		}
 		return nil, fmt.Errorf("failed to read disk usage: %w", err)
 	}
-	bs := int64(s.Bsize) // nolint: unconvert
+
+	// Get statfs block size (filesystem block size)
+	statfsBlockSize := int64(unixStatfs.Bsize) // nolint: unconvert
+
+	// Get stat block size from a file in the directory (preferred I/O block size)
+	var statBlockSize int64
+	var unixStat unix.Stat_t
+	err = unix.Stat(f.root, &unixStat)
+	if err == nil {
+		statBlockSize = int64(unixStat.Blksize) // nolint: unconvert
+	}
+
+	// Calculate optimal block size with bounds checking
+	bs := calculateOptimalBlockSize(statfsBlockSize, statBlockSize)
+
 	usage := &fs.Usage{
-		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         //nolint: unconvert // quota of bytes that can be used
-		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), //nolint: unconvert // bytes in use
-		Free:  fs.NewUsageValue(bs * int64(s.Bavail)),         //nolint: unconvert // bytes which can be uploaded before reaching the quota
+		Total:       fs.NewUsageValue(statfsBlockSize * int64(unixStatfs.Blocks)),                  //nolint: unconvert // quota of bytes that can be used
+		Used:        fs.NewUsageValue(statfsBlockSize * int64(unixStatfs.Blocks-unixStatfs.Bfree)), //nolint: unconvert // bytes in use
+		Free:        fs.NewUsageValue(statfsBlockSize * int64(unixStatfs.Bavail)),                  //nolint: unconvert // bytes which can be uploaded before reaching the quota
+		IOBlockSize: fs.NewUsageValue32(bs),                                                        // preferred IO block size for this filesystem
 	}
 	return usage, nil
 }

backend/local/about_windows.go

@@ -30,9 +30,10 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e1)
 	}
 	usage := &fs.Usage{
-		Total: fs.NewUsageValue(total),        // quota of bytes that can be used
-		Used:  fs.NewUsageValue(total - free), // bytes in use
-		Free:  fs.NewUsageValue(available),    // bytes which can be uploaded before reaching the quota
+		Total:       fs.NewUsageValue(total),        // quota of bytes that can be used
+		Used:        fs.NewUsageValue(total - free), // bytes in use
+		Free:        fs.NewUsageValue(available),    // bytes which can be uploaded before reaching the quota
+		IOBlockSize: fs.NewUsageValue32(4096),       // use 4KB default for Windows
 	}
 	return usage, nil
 }

backend/union/union.go

@@ -644,12 +644,13 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // About gets quota information from the Fs
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	usage := &fs.Usage{
-		Total:   new(int64),
-		Used:    new(int64),
-		Trashed: new(int64),
-		Other:   new(int64),
-		Free:    new(int64),
-		Objects: new(int64),
+		Total:       new(int64),
+		Used:        new(int64),
+		Trashed:     new(int64),
+		Other:       new(int64),
+		Free:        new(int64),
+		Objects:     new(int64),
+		IOBlockSize: new(int32),
 	}
 	for _, u := range f.upstreams {
 		usg, err := u.About(ctx)
@@ -689,6 +690,15 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 		} else {
 			usage.Objects = nil
 		}
+		if usg.IOBlockSize != nil && usage.IOBlockSize != nil {
+			*usage.IOBlockSize = min(*usage.IOBlockSize, *usg.IOBlockSize)
+		} else {
+			usage.IOBlockSize = nil
+		}
+	}
+	// If no upstream provided IOBlockSize, set to nil
+	if usage.IOBlockSize != nil && *usage.IOBlockSize == 0 {
+		usage.IOBlockSize = nil
 	}
 	return usage, nil
 }

cmd/cmount/fs.go

@@ -138,8 +138,9 @@ func (fsys *FS) getNode(path string, fh uint64) (node vfs.Node, handle vfs.Handl
 
 // stat fills up the stat block for Node
 func (fsys *FS) stat(node vfs.Node, stat *fuse.Stat_t) (errc int) {
-	Size := uint64(node.Size())
-	Blocks := (Size + 511) / 512
+	size := uint64(node.Size())
+	dataBlockSize, ioBlockSize := fsys.VFS.GetBlockSizes()
+	blocks := (size + uint64(dataBlockSize) - 1) / uint64(dataBlockSize)
 	modTime := node.ModTime()
 	//stat.Dev = 1
 	stat.Ino = node.Inode() // FIXME do we need to set the inode number?
@@ -148,13 +149,13 @@ func (fsys *FS) stat(node vfs.Node, stat *fuse.Stat_t) (errc int) {
 	stat.Uid = fsys.VFS.Opt.UID
 	stat.Gid = fsys.VFS.Opt.GID
 	//stat.Rdev
-	stat.Size = int64(Size)
+	stat.Size = int64(size)
 	t := fuse.NewTimespec(modTime)
 	stat.Atim = t
 	stat.Mtim = t
 	stat.Ctim = t
-	stat.Blksize = 512
-	stat.Blocks = int64(Blocks)
+	stat.Blocks = int64(blocks)
+	stat.Blksize = int64(ioBlockSize)
 	stat.Birthtim = t
 	// fs.Debugf(nil, "stat = %+v", *stat)
 	return 0
@@ -258,16 +259,17 @@ func (fsys *FS) Releasedir(path string, fh uint64) (errc int) {
 // Statfs reads overall stats on the filesystem
 func (fsys *FS) Statfs(path string, stat *fuse.Statfs_t) (errc int) {
 	defer log.Trace(path, "")("stat=%+v, errc=%d", stat, &errc)
-	const blockSize = 4096
+	_, ioBlockSize := fsys.VFS.GetBlockSizes()
+	blockSize := uint64(ioBlockSize)
 	total, _, free := fsys.VFS.Statfs()
 	stat.Blocks = uint64(total) / blockSize // Total data blocks in file system.
 	stat.Bfree = uint64(free) / blockSize   // Free blocks in file system.
 	stat.Bavail = stat.Bfree                // Free blocks in file system if you're not root.
 	stat.Files = 1e9                        // Total files in file system.
 	stat.Ffree = 1e9                        // Free files in file system.
-	stat.Bsize = blockSize                  // Block size
+	stat.Bsize = uint64(blockSize)          // Block size
 	stat.Namemax = 255                      // Maximum file name length?
-	stat.Frsize = blockSize                 // Fragment size, smallest addressable data size in the file system.
+	stat.Frsize = uint64(blockSize)         // Fragment size, smallest addressable data size in the file system.
 	mountlib.ClipBlocks(&stat.Blocks)
 	mountlib.ClipBlocks(&stat.Bfree)
 	mountlib.ClipBlocks(&stat.Bavail)

cmd/mount/file.go

@@ -28,16 +28,18 @@ func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) {
 	defer log.Trace(f, "")("a=%+v, err=%v", a, &err)
 	a.Valid = time.Duration(f.fsys.opt.AttrTimeout)
 	modTime := f.File.ModTime()
-	Size := uint64(f.File.Size())
-	Blocks := (Size + 511) / 512
+	size := uint64(f.File.Size())
+	dataBlockSize, ioBlockSize := f.VFS().GetBlockSizes()
+	blocks := (size + uint64(dataBlockSize) - 1) / uint64(dataBlockSize)
 	a.Gid = f.VFS().Opt.GID
 	a.Uid = f.VFS().Opt.UID
 	a.Mode = f.File.Mode() &^ os.ModeAppend
-	a.Size = Size
+	a.Size = size
 	a.Atime = modTime
 	a.Mtime = modTime
 	a.Ctime = modTime
-	a.Blocks = Blocks
+	a.Blocks = blocks
+	a.BlockSize = uint32(ioBlockSize)
 	return nil
 }
 

cmd/mount/fs.go

@@ -55,16 +55,17 @@ var _ fusefs.FSStatfser = (*FS)(nil)
 // It should write that data to resp.
 func (f *FS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) (err error) {
 	defer log.Trace("", "")("stat=%+v, err=%v", resp, &err)
-	const blockSize = 4096
+	_, ioBlockSize := f.VFS.GetBlockSizes()
+	blockSize := uint64(ioBlockSize)
 	total, _, free := f.VFS.Statfs()
 	resp.Blocks = uint64(total) / blockSize // Total data blocks in file system.
 	resp.Bfree = uint64(free) / blockSize   // Free blocks in file system.
 	resp.Bavail = resp.Bfree                // Free blocks in file system if you're not root.
 	resp.Files = 1e9                        // Total files in file system.
 	resp.Ffree = 1e9                        // Free files in file system.
-	resp.Bsize = blockSize                  // Block size
+	resp.Bsize = uint32(blockSize)          // Block size
 	resp.Namelen = 255                      // Maximum file name length?
-	resp.Frsize = blockSize                 // Fragment size, smallest addressable data size in the file system.
+	resp.Frsize = uint32(blockSize)         // Fragment size, smallest addressable data size in the file system.
 	mountlib.ClipBlocks(&resp.Blocks)
 	mountlib.ClipBlocks(&resp.Bfree)
 	mountlib.ClipBlocks(&resp.Bavail)

cmd/mount2/fs.go

@@ -87,20 +87,20 @@ func getMode(node os.FileInfo) uint32 {
 }
 
 // fill in attr from node
-func setAttr(node vfs.Node, attr *fuse.Attr) {
-	Size := uint64(node.Size())
-	const BlockSize = 512
-	Blocks := (Size + BlockSize - 1) / BlockSize
+func (f *FS) setAttr(node vfs.Node, attr *fuse.Attr) {
+	size := uint64(node.Size())
+	vfs := node.VFS()
+	dataBlockSize, ioBlockSize := vfs.GetBlockSizes()
+	blocks := (size + uint64(dataBlockSize) - 1) / uint64(dataBlockSize)
 	modTime := node.ModTime()
 	// set attributes
-	vfs := node.VFS()
 	attr.Owner.Gid = vfs.Opt.GID
 	attr.Owner.Uid = vfs.Opt.UID
 	attr.Mode = getMode(node)
-	attr.Size = Size
+	attr.Size = size
 	attr.Nlink = 1
-	attr.Blocks = Blocks
-	// attr.Blksize = BlockSize // not supported in freebsd/darwin, defaults to 4k if not set
+	attr.Blocks = blocks
+	attr.Blksize = uint32(ioBlockSize)
 	s := uint64(modTime.Unix())
 	ns := uint32(modTime.Nanosecond())
 	attr.Atime = s
@@ -114,13 +114,13 @@ func setAttr(node vfs.Node, attr *fuse.Attr) {
 
 // fill in AttrOut from node
 func (f *FS) setAttrOut(node vfs.Node, out *fuse.AttrOut) {
-	setAttr(node, &out.Attr)
+	f.setAttr(node, &out.Attr)
 	out.SetTimeout(time.Duration(f.opt.AttrTimeout))
 }
 
 // fill in EntryOut from node
 func (f *FS) setEntryOut(node vfs.Node, out *fuse.EntryOut) {
-	setAttr(node, &out.Attr)
+	f.setAttr(node, &out.Attr)
 	out.SetEntryTimeout(time.Duration(f.opt.AttrTimeout))
 	out.SetAttrTimeout(time.Duration(f.opt.AttrTimeout))
 }

cmd/mount2/node.go

@@ -101,16 +101,17 @@ func (n *Node) lookupDir(leaf string) (*vfs.Dir, syscall.Errno) {
 // will not work.
 func (n *Node) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
 	defer log.Trace(n, "")("out=%+v", &out)
-	const blockSize = 4096
 	total, _, free := n.VFS().Statfs()
+	_, ioBlockSize := n.VFS().GetBlockSizes()
+	blockSize := uint64(ioBlockSize)
 	out.Blocks = uint64(total) / blockSize // Total data blocks in file system.
 	out.Bfree = uint64(free) / blockSize   // Free blocks in file system.
 	out.Bavail = out.Bfree                 // Free blocks in file system if you're not root.
 	out.Files = 1e9                        // Total files in file system.
 	out.Ffree = 1e9                        // Free files in file system.
-	out.Bsize = blockSize                  // Block size
+	out.Bsize = uint32(blockSize)          // Block size
 	out.NameLen = 255                      // Maximum file name length?
-	out.Frsize = blockSize                 // Fragment size, smallest addressable data size in the file system.
+	out.Frsize = uint32(blockSize)         // Fragment size, smallest addressable data size in the file system.
 	mountlib.ClipBlocks(&out.Blocks)
 	mountlib.ClipBlocks(&out.Bfree)
 	mountlib.ClipBlocks(&out.Bavail)

fs/types.go

@@ -365,16 +365,32 @@ func NewUsageValue[T interface {
 	return p
 }
 
+// NewUsageValue32 makes a valid int32 value
+func NewUsageValue32[T interface {
+	int64 | int32 | int
+}](value T) *int32 {
+	p := new(int32)
+	if value > T(int32(math.MaxInt32)) {
+		*p = math.MaxInt32
+	} else if value < 0 {
+		*p = 0
+	} else {
+		*p = int32(value)
+	}
+	return p
+}
+
 // Usage is returned by the About call
 //
 // If a value is nil then it isn't supported by that backend
 type Usage struct {
-	Total   *int64 `json:"total,omitempty"`   // quota of bytes that can be used
-	Used    *int64 `json:"used,omitempty"`    // bytes in use
-	Trashed *int64 `json:"trashed,omitempty"` // bytes in trash
-	Other   *int64 `json:"other,omitempty"`   // other usage e.g. gmail in drive
-	Free    *int64 `json:"free,omitempty"`    // bytes which can be uploaded before reaching the quota
-	Objects *int64 `json:"objects,omitempty"` // objects in the storage system
+	Total       *int64 `json:"total,omitempty"`       // quota of bytes that can be used
+	Used        *int64 `json:"used,omitempty"`        // bytes in use
+	Trashed     *int64 `json:"trashed,omitempty"`     // bytes in trash
+	Other       *int64 `json:"other,omitempty"`       // other usage e.g. gmail in drive
+	Free        *int64 `json:"free,omitempty"`        // bytes which can be uploaded before reaching the quota
+	Objects     *int64 `json:"objects,omitempty"`     // objects in the storage system
+	IOBlockSize *int32 `json:"ioblocksize,omitempty"` // preferred IO block size for this filesystem
 }
 
 // WriterAtCloser wraps io.WriterAt and io.Closer

vfs/vfs.go

@@ -702,6 +702,28 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
 	return
 }
 
+// GetBlockSizes returns both the data block size and IO block size.
+// DataBlockSize is always 512 for POSIX compatibility.
+// IOBlockSize is obtained from the filesystem via About() call, or 4096 as default.
+func (vfs *VFS) GetBlockSizes() (int32, int32) {
+	const dataBlockSize = int32(512)
+
+	// Call Statfs to ensure we have up-to-date usage information
+	_, _, _ = vfs.Statfs()
+
+	vfs.usageMu.Lock()
+	defer vfs.usageMu.Unlock()
+
+	var ioBlockSize int32 = 4096 // Default
+	if vfs.usage != nil && vfs.usage.IOBlockSize != nil {
+		if *vfs.usage.IOBlockSize > 0 {
+			ioBlockSize = *vfs.usage.IOBlockSize
+		}
+	}
+
+	return dataBlockSize, ioBlockSize
+}
+
 // Remove removes the named file or (empty) directory.
 func (vfs *VFS) Remove(name string) error {
 	node, err := vfs.Stat(name)