wip: vfs,local,union: use specialized stat interface when available

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

Author: darthShadow

backend/drive/drive.go

@@ -3802,16 +3802,16 @@ Use the --interactive/-i or --dry-run flag to see what would be moved beforehand
 Usage:
 
     rclone backend query drive: query
-    
-The query syntax is documented at [Google Drive Search query terms and 
+
+The query syntax is documented at [Google Drive Search query terms and
 operators](https://developers.google.com/drive/api/guides/ref-search-terms).
 
 For example:
 
 	rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
 
 If the query contains literal ' or \ characters, these need to be escaped with
-\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a 
+\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a
 file named "foo ' \.txt":
 
 	rclone backend query drive: "name = 'foo \' \\\.txt'"

backend/local/local.go

@@ -1123,6 +1123,127 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	}
 }
 
+// Stat returns the DirEntry for the given name
+//
+// It should return fs.ErrorNotImplemented if it can't return a DirEntry
+func (f *Fs) Stat(ctx context.Context, dir string, leaf string) (entry fs.DirEntry, err error) {
+	filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx)
+
+	fsDirPath := f.localPath(dir)
+	_, err = os.Stat(fsDirPath)
+	if err != nil {
+		return nil, fs.ErrorDirNotFound
+	}
+
+	fd, err := os.Open(fsDirPath)
+	if err != nil {
+		isPerm := os.IsPermission(err)
+		err = fmt.Errorf("failed to open directory %q: %w", dir, err)
+		fs.Errorf(dir, "%v", err)
+		if isPerm {
+			_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(err))
+			err = nil // ignore error but fail sync
+		}
+		return nil, err
+	}
+	defer func() {
+		cerr := fd.Close()
+		if cerr != nil && err == nil {
+			err = fmt.Errorf("failed to close directory %q: %w", dir, cerr)
+		}
+	}()
+
+	names, err := fd.Readdirnames(-1)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory entry: %w", err)
+	}
+
+	foundLeaf := false
+	// Find the leaf in the directory
+	for _, name := range names {
+		if name == leaf {
+			foundLeaf = true
+			break
+		}
+	}
+	if !foundLeaf {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	leafPath := join.FilePathJoin(fsDirPath, leaf)
+	// Just a stat won't work without checking the entries from readdir first because of differences in unicode handling
+	fi, fierr := os.Lstat(leafPath)
+	if fierr != nil {
+		// Don't report errors on any file names that are excluded
+		if useFilter {
+			newRemote := f.cleanRemote(dir, leaf)
+			if !filter.IncludeRemote(newRemote) {
+				return nil, fs.ErrorObjectNotFound
+			}
+		}
+
+		_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
+
+		return nil, fmt.Errorf("%q: failed to get info about directory entry %q: %w", dir, leafPath, fierr)
+	}
+
+	name := fi.Name()
+	mode := fi.Mode()
+	newRemote := f.cleanRemote(dir, name)
+
+	// Follow symlinks if required
+	if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
+		localPath := join.FilePathJoin(fsDirPath, name)
+		fi, fierr = os.Stat(localPath)
+		// Quietly skip errors on excluded files and directories
+		if fierr != nil && useFilter && !filter.IncludeRemote(newRemote) {
+			return nil, fs.ErrorObjectNotFound
+		}
+
+		if fierr != nil {
+			// Check for bad symlinks and circular symlinks
+			if os.IsNotExist(fierr) || isCircularSymlinkError(fierr) {
+				_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
+			}
+			return nil, fmt.Errorf("%q: following symlink for %q: %w", dir, leafPath, fierr)
+		}
+
+		mode = fi.Mode()
+	}
+
+	if fi.IsDir() {
+		// Ignore directories which are symlinks.  These are junction points under windows which
+		// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
+		if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
+			d := f.newDirectory(newRemote, fi)
+			return d, nil
+		}
+
+	} else {
+		// Check whether this link should be translated
+		if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
+			newRemote += fs.LinkSuffix
+		}
+
+		// Don't include non directory if not included
+		// we leave directory filtering to the layer above
+		if useFilter && !filter.IncludeRemote(newRemote) {
+			return nil, fs.ErrorObjectNotFound
+		}
+
+		fso, err := f.newObjectWithInfo(newRemote, fi)
+		if err != nil {
+			return nil, err
+		}
+
+		if fso.Storable() {
+			return fso, nil
+		}
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1768,6 +1889,7 @@ var (
 	_ fs.OpenWriterAter  = &Fs{}
 	_ fs.DirSetModTimer  = &Fs{}
 	_ fs.MkdirMetadataer = &Fs{}
+	_ fs.Stater          = &Fs{}
 	_ fs.Object          = &Object{}
 	_ fs.Metadataer      = &Object{}
 	_ fs.SetMetadataer   = &Object{}

backend/onedrive/onedrive.go

@@ -356,7 +356,7 @@ block download of the file.
 
 In this case you will see a message like this
 
-    server reports this file is infected with a virus - use --onedrive-av-override to download anyway: Infected (name of virus): 403 Forbidden: 
+    server reports this file is infected with a virus - use --onedrive-av-override to download anyway: Infected (name of virus): 403 Forbidden:
 
 If you are 100% sure you want to download this file anyway then use
 the --onedrive-av-override flag, or av_override = true in the config
@@ -2988,13 +2988,13 @@ var (
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
-	_ fs.MimeTyper       = &Object{}
-	_ fs.IDer            = &Object{}
+	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 	_ fs.Metadataer      = (*Object)(nil)
 	_ fs.Metadataer      = (*Directory)(nil)
 	_ fs.SetModTimer     = (*Directory)(nil)
 	_ fs.SetMetadataer   = (*Directory)(nil)
-	_ fs.MimeTyper       = &Directory{}
+	_ fs.MimeTyper       = (*Directory)(nil)
 	_ fs.DirSetModTimer  = (*Fs)(nil)
 	_ fs.MkdirMetadataer = (*Fs)(nil)
 )

backend/union/union.go

@@ -796,6 +796,54 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	return callback(entries)
 }
 
+// Stat returns the DirEntry for the given name
+//
+// It should return fs.ErrorNotImplemented if it can't return a DirEntry
+func (f *Fs) Stat(ctx context.Context, dir string, leaf string) (fs.DirEntry, error) {
+	entriesList := make([][]upstream.Entry, len(f.upstreams))
+	errs := Errors(make([]error, len(f.upstreams)))
+
+	multithread(len(f.upstreams), func(i int) {
+		u := f.upstreams[i]
+		stater, ok := u.Fs.(fs.Stater)
+		if !ok {
+			errs[i] = fs.ErrorNotImplemented
+			return
+		}
+		entry, err := stater.Stat(ctx, dir, leaf)
+		if err != nil {
+			errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
+			return
+		}
+		uEntries := make([]upstream.Entry, 1)
+		uEntries[0], _ = u.WrapEntry(entry)
+		entriesList[i] = uEntries
+	})
+
+	if len(errs) == len(errs.FilterNil()) {
+		errs = errs.Map(func(e error) error {
+			if errors.Is(e, fs.ErrorDirNotFound) {
+				return nil
+			}
+			return e
+		})
+		if len(errs) == 0 {
+			return nil, fs.ErrorDirNotFound
+		}
+		return nil, errs.Err()
+	}
+
+	entries, err := f.mergeDirEntries(entriesList)
+	if err != nil {
+		return nil, err
+	}
+	if len(entries) != 1 {
+		return nil, fmt.Errorf("internal error: expected 1 entry but got %d", len(entries))
+	}
+
+	return entries[0], nil
+}
+
 // NewObject creates a new remote union file object
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	objs := make([]*upstream.Object, len(f.upstreams))
@@ -1093,4 +1141,5 @@ var (
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
+	_ fs.Stater          = (*Fs)(nil)
 )

fs/types.go

@@ -218,6 +218,14 @@ type Fder interface {
 	Fd(ctx context.Context, flags int) (uintptr, error)
 }
 
+// Stater is an optional interface for Directory
+type Stater interface {
+	// Stat returns the DirEntry for the given leaf
+	//
+	// It should return fs.ErrorNotImplemented if it can't return a DirEntry
+	Stat(ctx context.Context, dir string, leaf string) (DirEntry, error)
+}
+
 // SetModTimer is an optional interface for Directory.
 //
 // Object implements this as part of its requires set of interfaces.