Skip to content

perf: optimize IO read/write usage #8219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
j2rong4cn wants to merge 10 commits into from
Closed

perf: optimize IO read/write usage #8219

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7999f62
perf: optimize IO read/write usage
j2rong4cn Mar 24, 2025
672d797
no test
j2rong4cn Mar 24, 2025
0036e1b
.
j2rong4cn Mar 24, 2025
75636e4
fix bug
j2rong4cn Mar 25, 2025
7b8c2af
fix bug
j2rong4cn Mar 25, 2025
6702357
.
j2rong4cn Mar 25, 2025
3d5e4e5
refactor(139): reduce a network request
j2rong4cn Jan 30, 2025
b720d11
.
j2rong4cn Mar 25, 2025
64e967f
fix bug
j2rong4cn Mar 26, 2025
a53c8b1
fix bug
j2rong4cn Mar 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 13 additions & 16 deletions drivers/115/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
driver115 "github.com/SheltonZhu/115driver/pkg/driver"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
Expand Down Expand Up @@ -149,7 +150,7 @@ func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error {
return d.client.Delete(obj.GetID())
}

func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err
}
Expand All @@ -162,7 +163,7 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
if ok, err := d.client.UploadAvailable(); err != nil || !ok {
return nil, err
}
if stream.GetSize() > d.client.UploadMetaInfo.SizeLimit {
if s.GetSize() > d.client.UploadMetaInfo.SizeLimit {
return nil, driver115.ErrUploadTooLarge
}
//if digest, err = d.client.GetDigestResult(stream); err != nil {
Expand All @@ -171,10 +172,10 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr

const PreHashSize int64 = 128 * utils.KB
hashSize := PreHashSize
if stream.GetSize() < PreHashSize {
hashSize = stream.GetSize()
if s.GetSize() < PreHashSize {
hashSize = s.GetSize()
}
reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: hashSize})
reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize})
if err != nil {
return nil, err
}
Expand All @@ -183,13 +184,9 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
return nil, err
}
preHash = strings.ToUpper(preHash)
fullHash := stream.GetHash().GetHash(utils.SHA1)
if len(fullHash) <= 0 {
tmpF, err := stream.CacheFullInTempFile()
if err != nil {
return nil, err
}
fullHash, err = utils.HashFile(utils.SHA1, tmpF)
fullHash := s.GetHash().GetHash(utils.SHA1)
if len(fullHash) != utils.SHA1.Width {
_, fullHash, err = stream.CacheFullInTempFileAndHash(s, utils.SHA1)
if err != nil {
return nil, err
}
Expand All @@ -199,7 +196,7 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
// rapid-upload
// note that 115 add timeout for rapid-upload,
// and "sig invalid" err is thrown even when the hash is correct after timeout.
if fastInfo, err = d.rapidUpload(stream.GetSize(), stream.GetName(), dirID, preHash, fullHash, stream); err != nil {
if fastInfo, err = d.rapidUpload(s.GetSize(), s.GetName(), dirID, preHash, fullHash, s); err != nil {
return nil, err
}
if matched, err := fastInfo.Ok(); err != nil {
Expand All @@ -214,13 +211,13 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr

var uploadResult *UploadResult
// 闪传失败，上传
if stream.GetSize() <= 10*utils.MB { // 文件大小小于10MB，改用普通模式上传
if uploadResult, err = d.UploadByOSS(ctx, &fastInfo.UploadOSSParams, stream, dirID, up); err != nil {
if s.GetSize() <= 10*utils.MB { // 文件大小小于10MB，改用普通模式上传
if uploadResult, err = d.UploadByOSS(ctx, &fastInfo.UploadOSSParams, s, dirID, up); err != nil {
return nil, err
}
} else {
// 分片上传
if uploadResult, err = d.UploadByMultipart(ctx, &fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID, up); err != nil {
if uploadResult, err = d.UploadByMultipart(ctx, &fastInfo.UploadOSSParams, s.GetSize(), s, dirID, up); err != nil {
return nil, err
}
}
Expand Down
42 changes: 24 additions & 18 deletions drivers/115_open/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
sdk "github.com/xhofe/115-sdk-go"
Expand Down Expand Up @@ -183,33 +185,37 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error {
return nil
}

func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
tempF, err := file.CacheFullInTempFile()
if err != nil {
return err
}
func (d *Open115) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error {
// cal full sha1
sha1, err := utils.HashReader(utils.SHA1, tempF)
if err != nil {
return err
}
_, err = tempF.Seek(0, io.SeekStart)
if err != nil {
return err
sha1 := s.GetHash().GetHash(utils.SHA1)
var tempF model.File
var err error
if len(sha1) != utils.SHA1.Width {
tempF, sha1, err = stream.CacheFullInTempFileAndHash(s, utils.SHA1)
if err != nil {
return err
}
} else {
tempF, err = s.CacheFullInTempFile()
if err != nil {
return err
}
}
// pre 128k sha1
sha1128k, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, 128*1024))
const PreHashSize int64 = 128 * utils.KB
hashSize := min(s.GetSize(), PreHashSize)
reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize})
if err != nil {
return err
}
_, err = tempF.Seek(0, io.SeekStart)
sha1128k, err := utils.HashReader(utils.SHA1, reader)
if err != nil {
return err
}
// 1. Init
resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{
FileName: file.GetName(),
FileSize: file.GetSize(),
FileName: s.GetName(),
FileSize: s.GetSize(),
Target: dstDir.GetID(),
FileID: strings.ToUpper(sha1),
PreID: strings.ToUpper(sha1128k),
Expand Down Expand Up @@ -244,8 +250,8 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
return err
}
resp, err = d.client.UploadInit(ctx, &sdk.UploadInitReq{
FileName: file.GetName(),
FileSize: file.GetSize(),
FileName: s.GetName(),
FileSize: s.GetSize(),
Target: dstDir.GetID(),
FileID: strings.ToUpper(sha1),
PreID: strings.ToUpper(sha1128k),
Expand Down
23 changes: 4 additions & 19 deletions drivers/123/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,8 @@ package _123

import (
"context"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"net/http"
"net/url"
"sync"
Expand All @@ -18,6 +15,7 @@ import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
Expand Down Expand Up @@ -187,25 +185,12 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error {

func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
etag := file.GetHash().GetHash(utils.MD5)
if len(etag) < utils.MD5.Width {
// const DEFAULT int64 = 10485760
h := md5.New()
// need to calculate md5 of the full content
tempFile, err := file.CacheFullInTempFile()
var err error
if len(etag) != utils.MD5.Width {
_, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5)
if err != nil {
return err
}
defer func() {
_ = tempFile.Close()
}()
if _, err = utils.CopyWithBuffer(h, tempFile); err != nil {
return err
}
_, err = tempFile.Seek(0, io.SeekStart)
if err != nil {
return err
}
etag = hex.EncodeToString(h.Sum(nil))
}
data := base.Json{
"driveId": 0,
Expand Down
61 changes: 29 additions & 32 deletions drivers/139/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/cron"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/pkg/utils/random"
Expand Down Expand Up @@ -521,25 +522,21 @@ func (d *Yun139) getPartSize(size int64) int64 {
return 100 * MB
}

func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error {
switch d.Addition.Type {
case MetaPersonalNew:
var err error
fullHash := stream.GetHash().GetHash(utils.SHA256)
if len(fullHash) <= 0 {
tmpF, err := stream.CacheFullInTempFile()
if err != nil {
return err
}
fullHash, err = utils.HashFile(utils.SHA256, tmpF)
fullHash := s.GetHash().GetHash(utils.SHA256)
if len(fullHash) != utils.SHA256.Width {
_, fullHash, err = stream.CacheFullInTempFileAndHash(s, utils.SHA256)
if err != nil {
return err
}
}

partInfos := []PartInfo{}
var partSize = d.getPartSize(stream.GetSize())
part := (stream.GetSize() + partSize - 1) / partSize
var partSize = d.getPartSize(s.GetSize())
part := (s.GetSize() + partSize - 1) / partSize
if part == 0 {
part = 1
}
Expand All @@ -548,7 +545,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
return ctx.Err()
}
start := i * partSize
byteSize := stream.GetSize() - start
byteSize := s.GetSize() - start
Comment thread
j2rong4cn marked this conversation as resolved.
Outdated
if byteSize > partSize {
byteSize = partSize
}
Expand Down Expand Up @@ -576,9 +573,9 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
"contentType": "application/octet-stream",
"parallelUpload": false,
"partInfos": firstPartInfos,
"size": stream.GetSize(),
"size": s.GetSize(),
"parentFileId": dstDir.GetID(),
"name": stream.GetName(),
"name": s.GetName(),
"type": "file",
"fileRenameMode": "auto_rename",
}
Expand Down Expand Up @@ -629,9 +626,9 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}

// Progress
p := driver.NewProgress(stream.GetSize(), up)
p := driver.NewProgress(s.GetSize(), up)

rateLimited := driver.NewLimitedUploadStream(ctx, stream)
rateLimited := driver.NewLimitedUploadStream(ctx, s)
// 上传所有分片
for _, uploadPartInfo := range uploadPartInfos {
index := uploadPartInfo.PartNumber - 1
Expand Down Expand Up @@ -677,8 +674,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}

// 处理冲突
if resp.Data.FileName != stream.GetName() {
log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, stream.GetName())
if resp.Data.FileName != s.GetName() {
log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, s.GetName())
// 给服务器一定时间处理数据，避免无法刷新文件列表
time.Sleep(time.Millisecond * 500)
// 刷新并获取文件列表
Expand All @@ -688,10 +685,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}
// 删除旧文件
for _, file := range files {
if file.GetName() == stream.GetName() {
if file.GetName() == s.GetName() {
log.Debugf("[139] conflict: removing old: %s", file.GetName())
// 删除前重命名旧文件，避免仍旧冲突
err = d.Rename(ctx, file, stream.GetName()+random.String(4))
err = d.Rename(ctx, file, s.GetName()+random.String(4))
if err != nil {
return err
}
Expand All @@ -705,8 +702,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
// 重命名新文件
for _, file := range files {
if file.GetName() == resp.Data.FileName {
log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), stream.GetName())
err = d.Rename(ctx, file, stream.GetName())
log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), s.GetName())
err = d.Rename(ctx, file, s.GetName())
if err != nil {
return err
}
Expand All @@ -726,10 +723,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}
// 删除旧文件
for _, file := range files {
if file.GetName() == stream.GetName() {
if file.GetName() == s.GetName() {
log.Debugf("[139] conflict: removing old: %s", file.GetName())
// 删除前重命名旧文件，避免仍旧冲突
err = d.Rename(ctx, file, stream.GetName()+random.String(4))
err = d.Rename(ctx, file, s.GetName()+random.String(4))
if err != nil {
return err
}
Expand All @@ -746,7 +743,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
"fileCount": 1,
"totalSize": 0, // 去除上传大小限制
"uploadContentList": []base.Json{{
"contentName": stream.GetName(),
"contentName": s.GetName(),
"contentSize": 0, // 去除上传大小限制
// "digest": "5a3231986ce7a6b46e408612d385bafa"
}},
Expand All @@ -767,7 +764,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
"seqNo": random.String(32), //序列号不能为空
"totalSize": 0,
"uploadContentList": []base.Json{{
"contentName": stream.GetName(),
"contentName": s.GetName(),
"contentSize": 0,
// "digest": "5a3231986ce7a6b46e408612d385bafa"
}},
Expand All @@ -781,21 +778,21 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}

// Progress
p := driver.NewProgress(stream.GetSize(), up)
p := driver.NewProgress(s.GetSize(), up)

var partSize = d.getPartSize(stream.GetSize())
part := (stream.GetSize() + partSize - 1) / partSize
var partSize = d.getPartSize(s.GetSize())
part := (s.GetSize() + partSize - 1) / partSize
if part == 0 {
part = 1
}
rateLimited := driver.NewLimitedUploadStream(ctx, stream)
rateLimited := driver.NewLimitedUploadStream(ctx, s)
for i := int64(0); i < part; i++ {
if utils.IsCanceled(ctx) {
return ctx.Err()
}

start := i * partSize
byteSize := stream.GetSize() - start
byteSize := s.GetSize() - start
if byteSize > partSize {
byteSize = partSize
}
Expand All @@ -809,8 +806,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}

req = req.WithContext(ctx)
req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName()))
req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10))
req.Header.Set("Content-Type", "text/plain;name="+unicode(s.GetName()))
req.Header.Set("contentSize", strconv.FormatInt(s.GetSize(), 10))
req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1))
req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID)
req.Header.Set("rangeType", "0")
Expand Down
Loading
Loading