fix(hasher): prevent double-download when syncing with auto_size

When syncing from hasher with auto_size larger than file size, files were
downloaded twice: once by fs.Fingerprint() calling Hash() which triggered
updateHashes(), and again for the actual transfer.

Root cause: checkPartial() calls Fingerprint(ctx, src, fast=true) for
generating stable partial upload suffixes. Fingerprint checks SlowHash
before calling Hash(), but hasher didn't set SlowHash even when computing
hashes requires downloading the entire file.

Fix:
- Set SlowHash=true when passHashes is empty (no fast pass-through hashes
  from base). Must be set AFTER Mask() to avoid being ANDed to false.
- Add singleflight to deduplicate concurrent updateHashes() calls for the
  same file, preventing redundant downloads from parallel Hash() requests.

When passHashes is non-empty (e.g., S3 with MD5), SlowHash remains false
and Fingerprint uses the fast pass-through hash. When passHashes is empty
(e.g., crypt/SFTP base), Fingerprint skips hashing entirely. Explicit
rclone hashsum commands still work as they call Hash() directly.

Signed-off-by: Anagh Kumar Baranwal <[email protected]>

Author: darthShadow

backend/hasher/hasher.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/kv"
+	"golang.org/x/sync/singleflight"
 )
 
 // Register with Fs
@@ -81,6 +82,8 @@ type Fs struct {
 	slowHashes hash.Set // passed to the base and then cached
 	autoHashes hash.Set // calculated in-house and cached
 	keepHashes hash.Set // checksums to keep in cache (slow + auto)
+	// concurrency control
+	hashGroup singleflight.Group // deduplicates concurrent updateHashes calls
 }
 
 var warnExperimental sync.Once
@@ -183,6 +186,11 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
+	// Set SlowHash based on whether we have fast pass-through hashes.
+	// Must be set AFTER Mask() to avoid being ANDed to false.
+	// When passHashes is empty, computing hashes requires downloading the file.
+	f.features.SlowHash = f.passHashes.Count() == 0
+
 	// Enable ListP always
 	f.features.ListP = f.ListP
 

backend/hasher/hasher_internal_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -59,6 +60,19 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, dst)
 
+	// Verify fingerprint respects SlowHash (tests the double-download fix).
+	// With SlowHash=true and fast=true, fingerprint should skip hash computation.
+	if f.Features().SlowHash {
+		hobj, err := f.NewObject(ctx, fileName)
+		require.NoError(t, err)
+		fp := fs.Fingerprint(ctx, hobj, true)
+		// Fingerprint format is "size,modtime,hash" - with SlowHash=true and fast=true,
+		// hash should be omitted, so we expect at most 2 comma-separated parts.
+		parts := strings.Split(fp, ",")
+		assert.LessOrEqual(t, len(parts), 2,
+			"fast fingerprint should skip hash when SlowHash=true, got: %s", fp)
+	}
+
 	// check that hash was created
 	if f.opt.MaxAge > 0 {
 		hash, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)

backend/hasher/object.go

@@ -105,7 +105,10 @@ func (o *Object) Hash(ctx context.Context, hashType hash.Type) (hashVal string,
 		return hashVal, err
 	}
 	if f.autoHashes.Contains(hashType) && o.Size() < int64(f.opt.AutoSize) {
-		_ = o.updateHashes(ctx)
+		// Use singleflight to prevent concurrent downloads for same file
+		_, _, _ = f.hashGroup.Do(o.Remote(), func() (interface{}, error) {
+			return nil, o.updateHashes(ctx)
+		})
 		if hashVal, err = o.getHash(ctx, hashType); err != nil {
 			fs.Debugf(o, "auto %s = %q (%v)", hashType, hashVal, err)
 			err = nil