Merge pull request #164 from anyproto/GO-6138-batch-file-add

GO-6138 batch file add

Author: cheggaaa

.gitignore

@@ -24,6 +24,7 @@ vendor
 
 # database
 # db
+N*.yml
 
 # artefacts for Intelli-J fleet
 .fleet

filenode/rpchandler.go

@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	cidSizeLimit     = 2 << 20 // 2 Mb
+	cidSizeLimit     = 4 << 20 // 4 Mb
 	fileInfoReqLimit = 1000
 )
 
@@ -95,6 +95,58 @@ func (r rpcHandler) BlockPush(ctx context.Context, req *fileproto.BlockPushReque
 	return &fileproto.Ok{}, nil
 }
 
+func (r rpcHandler) BlockPushMany(ctx context.Context, req *fileproto.BlockPushManyRequest) (resp *fileproto.Ok, err error) {
+	var (
+		st        = time.Now()
+		dataSum   int
+		cidCount  int
+		fileCount int
+	)
+	defer func() {
+		r.f.metric.RequestLog(ctx,
+			"file.blockPushMany",
+			metric.TotalDur(time.Since(st)),
+			metric.Size(dataSum),
+			zap.Int("cidCount", cidCount),
+			zap.Int("fileCount", fileCount),
+			zap.Error(err),
+		)
+	}()
+
+	for _, fileBlock := range req.FileBlocks {
+		bs := make([]blocks.Block, 0, len(fileBlock.Blocks))
+		for _, block := range fileBlock.Blocks {
+			var c cid.Cid
+			c, err = cid.Cast(block.Cid)
+			if err != nil {
+				return nil, err
+			}
+			b, err := blocks.NewBlockWithCid(block.Data, c)
+			if err != nil {
+				return nil, err
+			}
+			chkc, err := c.Prefix().Sum(block.Data)
+			if err != nil {
+				return nil, err
+			}
+			if !chkc.Equals(c) {
+				return nil, ErrWrongHash
+			}
+			dataSum += len(b.RawData())
+			if dataSum > fileInfoReqLimit {
+				return nil, fileprotoerr.ErrQuerySizeExceeded
+			}
+			bs = append(bs, b)
+			cidCount++
+		}
+		if err = r.f.Add(ctx, fileBlock.SpaceId, fileBlock.FileId, bs); err != nil {
+			return nil, err
+		}
+		fileCount++
+	}
+	return &fileproto.Ok{}, nil
+}
+
 func (r rpcHandler) BlocksCheck(ctx context.Context, req *fileproto.BlocksCheckRequest) (resp *fileproto.BlocksCheckResponse, err error) {
 	st := time.Now()
 	defer func() {

go.mod

@@ -1,13 +1,12 @@
 module github.com/anyproto/any-sync-filenode
 
-go 1.23.10
-
-toolchain go1.24.0
+go 1.24.0
 
 require (
 	github.com/OneOfOne/xxhash v1.2.8
 	github.com/ahmetb/govvv v0.3.0
-	github.com/anyproto/any-sync v0.9.6
+	github.com/anyproto/any-store v0.3.5
+	github.com/anyproto/any-sync v0.10.0
 	github.com/aws/aws-sdk-go v1.55.8
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/go-redsync/redsync/v4 v4.13.0
@@ -16,16 +15,15 @@ require (
 	github.com/ipfs/go-block-format v0.2.2
 	github.com/ipfs/go-cid v0.5.0
 	github.com/redis/go-redis/v9 v9.12.0
-	github.com/stretchr/testify v1.10.0
-	go.uber.org/mock v0.5.2
+	github.com/stretchr/testify v1.11.1
+	go.uber.org/mock v0.6.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/sync v0.16.0
+	golang.org/x/sync v0.17.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/anyproto/any-store v0.3.3 // indirect
 	github.com/anyproto/go-chash v0.1.0 // indirect
 	github.com/anyproto/go-slip10 v1.0.0 // indirect
 	github.com/anyproto/go-slip21 v1.0.0 // indirect
@@ -52,7 +50,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
-	github.com/libp2p/go-libp2p v0.42.1 // indirect
+	github.com/libp2p/go-libp2p v0.43.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
@@ -68,9 +66,9 @@ require (
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/planetscale/vtprotobuf v0.6.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.23.0 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/quic-go/quic-go v0.54.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -80,16 +78,17 @@ require (
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.40.0 // indirect
-	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect
-	golang.org/x/mod v0.26.0 // indirect
-	golang.org/x/net v0.42.0 // indirect
-	golang.org/x/sys v0.34.0 // indirect
-	golang.org/x/time v0.12.0 // indirect
-	golang.org/x/tools v0.35.0 // indirect
-	google.golang.org/protobuf v1.36.6 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/crypto v0.42.0 // indirect
+	golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b // indirect
+	golang.org/x/mod v0.27.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sys v0.36.0 // indirect
+	golang.org/x/time v0.13.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
 	lukechampine.com/blake3 v1.4.1 // indirect
-	modernc.org/libc v1.66.3 // indirect
+	modernc.org/libc v1.66.8 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 	modernc.org/sqlite v1.38.0 // indirect