vfs: do lazy revalidation of the directory cache per the poll interval

local: added custom bufferpool-backed path.Join & filepath.Join implementations to reduce allocations

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

Author: darthShadow

backend/drive/drive.go

@@ -42,6 +42,7 @@ import (
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
+	"github.com/rclone/rclone/lib/join"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -2024,7 +2025,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 	var iErr error
 	_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
-		entry, err := f.itemToDirEntry(ctx, path.Join(dir, item.Name), item)
+		entry, err := f.itemToDirEntry(ctx, join.PathJoin(dir, item.Name), item)
 		if err != nil {
 			iErr = err
 			return true
@@ -2136,7 +2137,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 						continue
 					}
 				}
-				remote := path.Join(paths[i], item.Name)
+				remote := join.PathJoin(paths[i], item.Name)
 				entry, err := f.itemToDirEntry(ctx, remote, item)
 				if err != nil {
 					iErr = err
@@ -2716,7 +2717,7 @@ func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 // purgeCheck removes the dir directory, if check is set then it
 // refuses to do so if it has anything in
 func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
-	root := path.Join(f.root, dir)
+	root := join.PathJoin(f.root, dir)
 	dc := f.dirCache
 	directoryID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
@@ -2919,7 +2920,7 @@ func (r cleanupResult) Error() string {
 
 func (f *Fs) cleanupTeamDrive(ctx context.Context, dir string, directoryID string) (r cleanupResult, err error) {
 	_, err = f.list(ctx, []string{directoryID}, "", false, false, true, false, func(item *drive.File) bool {
-		remote := path.Join(dir, item.Name)
+		remote := join.PathJoin(dir, item.Name)
 		if item.ExplicitlyTrashed { // description is wrong - can also be set for folders - no need to recurse them
 			err := f.delete(ctx, item.Id, false)
 			if err != nil {
@@ -3437,7 +3438,7 @@ func (f *Fs) unTrash(ctx context.Context, dir string, directoryID string, recurs
 	directoryID = actualID(directoryID)
 	fs.Debugf(dir, "finding trash to restore in directory %q", directoryID)
 	_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
-		remote := path.Join(dir, item.Name)
+		remote := join.PathJoin(dir, item.Name)
 		if item.ExplicitlyTrashed {
 			fs.Infof(remote, "restoring %q", item.Id)
 			if operations.SkipDestructive(ctx, remote, "restore") {

backend/drive/drive_notify.go

@@ -2,9 +2,9 @@ package drive
 
 import (
 	"context"
-	"path"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/join"
 	"google.golang.org/api/drive/v3"
 )
 
@@ -169,7 +169,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 						// translate the parent dir of this object
 						if parentPath, ok := f.dirCache.GetInv(parent); ok {
 							// and append the drive file name to compute the full file name
-							newPath := path.Join(parentPath, change.File.Name)
+							newPath := join.PathJoin(parentPath, change.File.Name)
 							// this will now clear the actual file too
 							pathsToClear = append(pathsToClear, entryType{
 								path: newPath, entryType: changeType})

backend/local/local.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -27,6 +26,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
+	"github.com/rclone/rclone/lib/join"
 	"github.com/rclone/rclone/lib/readers"
 	"golang.org/x/text/unicode/norm"
 )
@@ -261,9 +261,9 @@ cause disk fragmentation and can be slow to work with.`,
 				Help: `Disable setting modtime.
 
 Normally rclone updates modification time of files after they are done
-uploading. This can cause permissions issues on Linux platforms when 
+uploading. This can cause permissions issues on Linux platforms when
 the user rclone is running as does not own the file uploaded, such as
-when copying to a CIFS mount owned by another user. If this option is 
+when copying to a CIFS mount owned by another user. If this option is
 enabled, rclone will no longer update the modtime after copying a file.`,
 				Default:  false,
 				Advanced: true,
@@ -451,6 +451,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
+
 	return f, nil
 }
 
@@ -632,8 +633,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				break
 			}
 			if err == nil {
+				// TODO: Parallelise this loop
 				for _, name := range names {
-					namepath := filepath.Join(fsDirPath, name)
+					namepath := join.FilePathJoin(fsDirPath, name)
 					fi, fierr := os.Lstat(namepath)
 					if os.IsNotExist(fierr) {
 						// skip entry removed by a concurrent goroutine
@@ -660,13 +662,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			return nil, fmt.Errorf("failed to read directory entry: %w", err)
 		}
 
+		// TODO: Parallelise this loop
 		for _, fi := range fis {
 			name := fi.Name()
 			mode := fi.Mode()
 			newRemote := f.cleanRemote(dir, name)
 			// Follow symlinks if required
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
-				localPath := filepath.Join(fsDirPath, name)
+				localPath := join.FilePathJoin(fsDirPath, name)
 				fi, err = os.Stat(localPath)
 				// Quietly skip errors on excluded files and directories
 				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
@@ -718,7 +721,8 @@ func (f *Fs) cleanRemote(dir, filename string) (remote string) {
 	if f.opt.UTFNorm {
 		filename = norm.NFC.String(filename)
 	}
-	remote = path.Join(dir, f.opt.Enc.ToStandardName(filename))
+
+	remote = join.PathJoin(dir, f.opt.Enc.ToStandardName(filename))
 
 	if !utf8.ValidString(filename) {
 		f.warnedMu.Lock()
@@ -728,11 +732,12 @@ func (f *Fs) cleanRemote(dir, filename string) (remote string) {
 		}
 		f.warnedMu.Unlock()
 	}
+
 	return
 }
 
 func (f *Fs) localPath(name string) string {
-	return filepath.Join(f.root, filepath.FromSlash(f.opt.Enc.FromStandardPath(name)))
+	return join.FilePathJoin(f.root, filepath.FromSlash(f.opt.Enc.FromStandardPath(name)))
 }
 
 // Put the Object to the local filesystem

backend/local/metadata.go

@@ -68,7 +68,7 @@ func (o *Object) parseMetadataTime(m fs.Metadata, key string) (t time.Time, ok b
 	return t, ok
 }
 
-// parse am int from metadata with key and base
+// parse an int from metadata with key and base
 func (o *Object) parseMetadataInt(m fs.Metadata, key string, base int) (result int, ok bool) {
 	value, ok := m[key]
 	if ok {

backend/local/metadata_linux.go

@@ -107,7 +107,7 @@ func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) {
 	}
 	setTime := func(key string, t unix.Timespec) {
 		// The types of t.Sec and t.Nsec vary from int32 to int64 on
-		// different Linux architectures so we need to cast them to
+		// different Linux architectures, so we need to cast them to
 		// int64 here and hence need to quiet the linter about
 		// unnecessary casts.
 		//

fs/asyncreader/asyncreader.go

@@ -125,7 +125,6 @@ func (a *AsyncReader) getBuffer() *buffer {
 	}
 }
 
-// Read will return the next available data.
 func (a *AsyncReader) fill() (err error) {
 	if a.cur.isEmpty() {
 		if a.cur != nil {

fs/rc/internal.go

@@ -162,7 +162,10 @@ memory problems.
 
 // Do a garbage collection run
 func rcGc(ctx context.Context, in Params) (out Params, err error) {
+	oldGCPercent := debug.SetGCPercent(25)
 	runtime.GC()
+	debug.FreeOSMemory()
+	_ = debug.SetGCPercent(oldGCPercent)
 	return nil, nil
 }
 

fs/types.go

@@ -95,7 +95,7 @@ type Object interface {
 	// return an error or update the object properly (rather than e.g. calling panic).
 	Update(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) error
 
-	// Removes this object
+	// Remove this object
 	Remove(ctx context.Context) error
 }
 

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/klauspost/compress v1.17.11
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
+	github.com/libp2p/go-buffer-pool v0.1.0
 	github.com/mattn/go-colorable v0.1.14
 	github.com/mattn/go-runewidth v0.0.16
 	github.com/minio/minio-go/v7 v7.0.83

lib/debug/common.go

@@ -16,3 +16,9 @@ func SetGCPercent(percent int) int {
 func SetMemoryLimit(limit int64) int64 {
 	return debug.SetMemoryLimit(limit)
 }
+
+// FreeOSMemory calls the runtime/debug.FreeOSMemory function to free memory
+// that is no longer in use.
+func FreeOSMemory() {
+	debug.FreeOSMemory()
+}