wip: vfs: use specialized stat interface when available rather than

listing

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

Author: darthShadow

backend/drive/drive.go

@@ -3798,16 +3798,16 @@ Use the --interactive/-i or --dry-run flag to see what would be moved beforehand
 Usage:
 
     rclone backend query drive: query
-    
-The query syntax is documented at [Google Drive Search query terms and 
+
+The query syntax is documented at [Google Drive Search query terms and
 operators](https://developers.google.com/drive/api/guides/ref-search-terms).
 
 For example:
 
 	rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
 
 If the query contains literal ' or \ characters, these need to be escaped with
-\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a 
+\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a
 file named "foo ' \.txt":
 
 	rclone backend query drive: "name = 'foo \' \\\.txt'"

backend/local/local.go

@@ -1123,6 +1123,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	}
 }
 
+// Stat returns the DirEntry for the given name
+//
+// It should return fs.ErrorNotImplemented if it can't return a DirEntry
+func (f *Fs) Stat(ctx context.Context, dir string, leaf string) (fs.DirEntry, error) {
+	return nil, fs.ErrorNotImplemented
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1768,6 +1775,7 @@ var (
 	_ fs.OpenWriterAter  = &Fs{}
 	_ fs.DirSetModTimer  = &Fs{}
 	_ fs.MkdirMetadataer = &Fs{}
+	_ fs.Stater          = &Fs{}
 	_ fs.Object          = &Object{}
 	_ fs.Metadataer      = &Object{}
 	_ fs.SetMetadataer   = &Object{}

backend/onedrive/onedrive.go

@@ -356,7 +356,7 @@ block download of the file.
 
 In this case you will see a message like this
 
-    server reports this file is infected with a virus - use --onedrive-av-override to download anyway: Infected (name of virus): 403 Forbidden: 
+    server reports this file is infected with a virus - use --onedrive-av-override to download anyway: Infected (name of virus): 403 Forbidden:
 
 If you are 100% sure you want to download this file anyway then use
 the --onedrive-av-override flag, or av_override = true in the config
@@ -2988,13 +2988,13 @@ var (
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
-	_ fs.MimeTyper       = &Object{}
-	_ fs.IDer            = &Object{}
+	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 	_ fs.Metadataer      = (*Object)(nil)
 	_ fs.Metadataer      = (*Directory)(nil)
 	_ fs.SetModTimer     = (*Directory)(nil)
 	_ fs.SetMetadataer   = (*Directory)(nil)
-	_ fs.MimeTyper       = &Directory{}
+	_ fs.MimeTyper       = (*Directory)(nil)
 	_ fs.DirSetModTimer  = (*Fs)(nil)
 	_ fs.MkdirMetadataer = (*Fs)(nil)
 )

backend/union/union.go

@@ -911,6 +911,13 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return errs.Err()
 }
 
+// Stat returns the DirEntry for the given name
+//
+// It should return fs.ErrorNotImplemented if it can't return a DirEntry
+func (f *Fs) Stat(ctx context.Context, dir string, leaf string) (fs.DirEntry, error) {
+	return nil, fs.ErrorNotImplemented
+}
+
 // NewFs constructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
@@ -1093,4 +1100,5 @@ var (
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
+	_ fs.Stater          = (*Fs)(nil)
 )

fs/types.go

@@ -218,6 +218,14 @@ type Fder interface {
 	Fd(ctx context.Context, flags int) (uintptr, error)
 }
 
+// Stater is an optional interface for Directory
+type Stater interface {
+	// Stat returns the DirEntry for the given leaf
+	//
+	// It should return fs.ErrorNotImplemented if it can't return a DirEntry
+	Stat(ctx context.Context, dir string, leaf string) (DirEntry, error)
+}
+
 // SetModTimer is an optional interface for Directory.
 //
 // Object implements this as part of its requires set of interfaces.

vfs/dir.go

@@ -33,15 +33,16 @@ type Dir struct {
 	f            fs.Fs       // read only
 	cleanupTimer *time.Timer // read only: timer to call cacheCleanup
 
-	mu      sync.RWMutex // protects the following
-	parent  *Dir         // parent, nil for root
-	path    string
-	entry   fs.Directory
-	read    time.Time         // time directory entry last read
-	polled  time.Time         // time directory entry last polled
-	items   map[string]Node   // directory entries - can be empty but not nil
-	virtual map[string]vState // virtual directory entries - may be nil
-	sys     atomic.Value      // user defined info to be attached here
+	mu       sync.RWMutex // protects the following
+	parent   *Dir         // parent, nil for root
+	path     string
+	entry    fs.Directory
+	read     time.Time            // time directory entry last read
+	polled   time.Time            // time directory entry last polled
+	items    map[string]Node      // directory entries - can be empty but not nil
+	virtual  map[string]vState    // virtual directory entries - may be nil
+	statRead map[string]time.Time // time directory entry last read
+	sys      atomic.Value         // user defined info to be attached here
 
 	modTimeMu        sync.Mutex // protects the following
 	modTime          time.Time
@@ -65,14 +66,15 @@ const (
 
 func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
 	d := &Dir{
-		vfs:     vfs,
-		f:       f,
-		parent:  parent,
-		entry:   fsDir,
-		path:    fsDir.Remote(),
-		modTime: fsDir.ModTime(context.TODO()),
-		inode:   newInode(),
-		items:   make(map[string]Node),
+		vfs:      vfs,
+		f:        f,
+		parent:   parent,
+		entry:    fsDir,
+		path:     fsDir.Remote(),
+		modTime:  fsDir.ModTime(context.TODO()),
+		inode:    newInode(),
+		items:    make(map[string]Node),
+		statRead: make(map[string]time.Time),
 	}
 	// Set timer up like this to avoid race of d.cacheCleanup being called
 	// before d.cleanupTimer is assigned to
@@ -759,59 +761,69 @@ func (mv manageVirtuals) end(d *Dir) {
 	}
 }
 
-// update d.items and if dirTree is not nil update each dir in the DirTree below this one and
-// set the last read time - must be called with the lock held
-func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree, when time.Time) error {
+func (d *Dir) _processEntry(entry fs.DirEntry, mv manageVirtuals, dirTree dirtree.DirTree, when time.Time) error {
 	var err error
-	mv := d._newManageVirtuals()
-	for _, entry := range entries {
-		name := path.Base(entry.Remote())
-		if name == "." || name == ".." {
-			continue
-		}
-		if d.vfs.Opt.Links {
-			name, _ = strings.CutSuffix(name, fs.LinkSuffix)
+	name := path.Base(entry.Remote())
+	if name == "." || name == ".." {
+		return nil
+	}
+	if d.vfs.Opt.Links {
+		name, _ = strings.CutSuffix(name, fs.LinkSuffix)
+	}
+	if mv.add(d, name) {
+		return nil
+	}
+
+	node := d.items[name]
+
+	switch item := entry.(type) {
+	case fs.Object:
+		obj := item
+		// Reuse old file value if it exists
+		if file, ok := node.(*File); node != nil && ok {
+			file.setObjectNoUpdate(obj)
+		} else {
+			node = newFile(d, d.path, obj, name)
 		}
-		node := d.items[name]
-		if mv.add(d, name) {
-			continue
+	case fs.Directory:
+		// Reuse old dir value if it exists
+		if node == nil || !node.IsDir() {
+			node = newDir(d.vfs, d.f, d, item)
 		}
-		switch item := entry.(type) {
-		case fs.Object:
-			obj := item
-			// Reuse old file value if it exists
-			if file, ok := node.(*File); node != nil && ok {
-				file.setObjectNoUpdate(obj)
-			} else {
-				node = newFile(d, d.path, obj, name)
-			}
-		case fs.Directory:
-			// Reuse old dir value if it exists
-			if node == nil || !node.IsDir() {
-				node = newDir(d.vfs, d.f, d, item)
-			}
-			dir := node.(*Dir)
-			dir.mu.Lock()
-			dir.modTime = item.ModTime(context.TODO())
-			if dirTree != nil {
-				err = dir._readDirFromDirTree(dirTree, when)
-				if err != nil {
-					dir.read = time.Time{}
-				} else {
-					dir.read = when
-					dir.cleanupTimer.Reset(time.Duration(d.vfs.Opt.DirCacheTime * 2))
-				}
-			}
-			dir.mu.Unlock()
+		dir := node.(*Dir)
+		dir.mu.Lock()
+		dir.modTime = item.ModTime(context.TODO())
+		if dirTree != nil {
+			err = dir._readDirFromDirTree(dirTree, when)
 			if err != nil {
-				return err
+				dir.read = time.Time{}
+			} else {
+				dir.read = when
+				dir.cleanupTimer.Reset(time.Duration(d.vfs.Opt.DirCacheTime * 2))
 			}
-		default:
-			err = fmt.Errorf("unknown type %T", item)
-			fs.Errorf(d, "readDir error: %v", err)
+		}
+		dir.mu.Unlock()
+		if err != nil {
 			return err
 		}
-		d.items[name] = node
+	default:
+		err = fmt.Errorf("unknown type %T", item)
+		fs.Errorf(d, "readDir error: %v", err)
+		return err
+	}
+
+	d.items[name] = node
+	return nil
+}
+
+// update d.items and if dirTree is not nil update each dir in the DirTree below this one and
+// set the last read time - must be called with the lock held
+func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree, when time.Time) error {
+	mv := d._newManageVirtuals()
+	for _, entry := range entries {
+		if err := d._processEntry(entry, mv, dirTree, when); err != nil {
+			return fmt.Errorf("%s: processing entry: %s: %v", d.path, path.Base(entry.Remote()), err)
+		}
 	}
 	mv.end(d)
 	return nil
@@ -849,20 +861,78 @@ func (d *Dir) readDir() error {
 	return d._readDir()
 }
 
+// stat a single item in the directory via the stat-er interface
+//
+// returns ENOENT if not found.
+func (d *Dir) _stat(leaf string, when time.Time) (bool, error) {
+	stater, staterOk := d.f.(fs.Stater)
+	if !staterOk {
+		return false, nil
+	}
+
+	// Use statRead to cache the stat time and avoid doing it again
+	statTime, statTimeOk := d.statRead[leaf]
+	if statTimeOk {
+		age := when.Sub(statTime)
+		stale := age > time.Duration(d.vfs.Opt.DirCacheTime)
+		if !stale {
+			return true, nil
+		}
+	}
+
+	entry, err := stater.Stat(context.TODO(), d.path, leaf)
+	if err != nil {
+		switch {
+		case errors.Is(err, fs.ErrorObjectNotFound):
+			return false, ENOENT
+		case errors.Is(err, fs.ErrorNotImplemented):
+			return false, nil
+		default:
+			fs.Errorf(d, "stat error: %s: %v", leaf, err)
+		}
+	}
+
+	mv := d._newManageVirtuals()
+	if err = d._processEntry(entry, mv, nil, when); err != nil {
+		return false, fmt.Errorf("%s: processing entry: %s: %v", d.path, path.Base(entry.Remote()), err)
+	}
+
+	d.statRead[leaf] = when
+
+	return true, nil
+}
+
 // stat a single item in the directory
 //
 // returns ENOENT if not found.
 // returns a custom error if directory on a case-insensitive file system
 // contains files with names that differ only by case.
 func (d *Dir) stat(leaf string) (Node, error) {
+	var (
+		ok   bool
+		err  error
+		item Node
+	)
+
 	d.mu.Lock()
 	defer d.mu.Unlock()
-	// TODO: Define & Use the Stat-er interface to avoid reading the entire directory
-	err := d._readDir()
-	if err != nil {
-		return nil, err
+
+	when := time.Now()
+	if _, stale := d._age(when); stale {
+		ok, err = d._stat(leaf, when)
+		if err != nil {
+			return nil, err
+		}
 	}
-	item, ok := d.items[leaf]
+
+	if !ok {
+		err := d._readDir()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	item, ok = d.items[leaf]
 
 	ci := fs.GetConfig(context.TODO())
 	normUnicode := !ci.NoUnicodeNormalization
@@ -885,6 +955,7 @@ func (d *Dir) stat(leaf string) (Node, error) {
 	if !ok {
 		return nil, ENOENT
 	}
+
 	return item, nil
 }