feat(baidu_netdisk): support dynamical slice size for low bandwith upload case (#7965)

* 动态分片尺寸

* 补充严格测试结果

Author: ljcbaby

drivers/baidu_netdisk/driver.go

@@ -189,15 +189,15 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
 	}
 
 	streamSize := stream.GetSize()
-	sliceSize := d.getSliceSize()
+	sliceSize := d.getSliceSize(streamSize)
 	count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1))
 	lastBlockSize := streamSize % sliceSize
 	if streamSize > 0 && lastBlockSize == 0 {
 		lastBlockSize = sliceSize
 	}
 
 	//cal md5 for first 256k data
-	const SliceSize int64 = 256 * 1024
+	const SliceSize int64 = 256 * utils.KB
 	// cal md5
 	blockList := make([]string, 0, count)
 	byteSize := sliceSize

drivers/baidu_netdisk/meta.go

@@ -8,16 +8,17 @@ import (
 type Addition struct {
 	RefreshToken string `json:"refresh_token" required:"true"`
 	driver.RootPath
-	OrderBy              string `json:"order_by" type:"select" options:"name,time,size" default:"name"`
-	OrderDirection       string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
-	DownloadAPI          string `json:"download_api" type:"select" options:"official,crack" default:"official"`
-	ClientID             string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"`
-	ClientSecret         string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"`
-	CustomCrackUA        string `json:"custom_crack_ua" required:"true" default:"netdisk"`
-	AccessToken          string
-	UploadThread         string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
-	UploadAPI            string `json:"upload_api" default:"https://d.pcs.baidu.com"`
-	CustomUploadPartSize int64  `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"`
+	OrderBy               string `json:"order_by" type:"select" options:"name,time,size" default:"name"`
+	OrderDirection        string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
+	DownloadAPI           string `json:"download_api" type:"select" options:"official,crack" default:"official"`
+	ClientID              string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"`
+	ClientSecret          string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"`
+	CustomCrackUA         string `json:"custom_crack_ua" required:"true" default:"netdisk"`
+	AccessToken           string
+	UploadThread          string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
+	UploadAPI             string `json:"upload_api" default:"https://d.pcs.baidu.com"`
+	CustomUploadPartSize  int64  `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"`
+	LowBandwithUploadMode bool   `json:"low_bandwith_upload_mode" default:"false"`
 }
 
 var config = driver.Config{

drivers/baidu_netdisk/util.go

@@ -136,7 +136,7 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) {
 	return res, nil
 }
 
-func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model.Link, error) {
+func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Link, error) {
 	var resp DownloadResp
 	params := map[string]string{
 		"method": "filemetas",
@@ -164,7 +164,7 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model
 	}, nil
 }
 
-func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Link, error) {
+func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, error) {
 	var resp DownloadResp2
 	param := map[string]string{
 		"target": fmt.Sprintf("[\"%s\"]", file.GetPath()),
@@ -230,22 +230,72 @@ func joinTime(form map[string]string, ctime, mtime int64) {
 
 const (
 	DefaultSliceSize int64 = 4 * utils.MB
-	VipSliceSize           = 16 * utils.MB
-	SVipSliceSize          = 32 * utils.MB
+	VipSliceSize     int64 = 16 * utils.MB
+	SVipSliceSize    int64 = 32 * utils.MB
+
+	MaxSliceNum       = 2048 // 文档写的是 1024/没写 ，但实际测试是 2048
+	SliceStep   int64 = 1 * utils.MB
 )
 
-func (d *BaiduNetdisk) getSliceSize() int64 {
+func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 {
+	// 非会员固定为 4MB
+	if d.vipType == 0 {
+		if d.CustomUploadPartSize != 0 {
+			log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize")
+		}
+		if filesize > MaxSliceNum*DefaultSliceSize {
+			log.Warnf("File size(%d) is too large, may cause upload failure", filesize)
+		}
+
+		return DefaultSliceSize
+	}
+
 	if d.CustomUploadPartSize != 0 {
+		if d.CustomUploadPartSize < DefaultSliceSize {
+			log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize)
+			return DefaultSliceSize
+		}
+
+		if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize {
+			log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize)
+			return VipSliceSize
+		}
+
+		if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize {
+			log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize)
+			return SVipSliceSize
+		}
+
 		return d.CustomUploadPartSize
 	}
+
+	maxSliceSize := DefaultSliceSize
+
 	switch d.vipType {
 	case 1:
-		return VipSliceSize
+		maxSliceSize = VipSliceSize
 	case 2:
-		return SVipSliceSize
-	default:
-		return DefaultSliceSize
+		maxSliceSize = SVipSliceSize
+	}
+
+	// upload on low bandwidth
+	if d.LowBandwithUploadMode {
+		size := DefaultSliceSize
+
+		for size <= maxSliceSize {
+			if filesize <= MaxSliceNum*size {
+				return size
+			}
+
+			size += SliceStep
+		}
 	}
+
+	if filesize > MaxSliceNum*maxSliceSize {
+		log.Warnf("File size(%d) is too large, may cause upload failure", filesize)
+	}
+
+	return maxSliceSize
 }
 
 // func encodeURIComponent(str string) string {