Skip to content

Commit 7999f62

Browse files
j2rong4cnj2rong4cn
committed
perf: optimize IO read/write usage
1 parent 32890da commit 7999f62

10 files changed

Lines changed: 237 additions & 196 deletions

File tree

‎drivers/123/driver.go‎

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@ package _123
22

33
import (
44
"context"
5-
"crypto/md5"
65
"encoding/base64"
7-
"encoding/hex"
86
"fmt"
9-
"io"
107
"net/http"
118
"net/url"
129
"sync"
@@ -18,6 +15,7 @@ import (
1815
"github.com/alist-org/alist/v3/internal/driver"
1916
"github.com/alist-org/alist/v3/internal/errs"
2017
"github.com/alist-org/alist/v3/internal/model"
18+
"github.com/alist-org/alist/v3/internal/stream"
2119
"github.com/alist-org/alist/v3/pkg/utils"
2220
"github.com/aws/aws-sdk-go/aws"
2321
"github.com/aws/aws-sdk-go/aws/credentials"
@@ -187,25 +185,12 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error {
187185

188186
func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
189187
etag := file.GetHash().GetHash(utils.MD5)
190-
if len(etag) < utils.MD5.Width {
191-
// const DEFAULT int64 = 10485760
192-
h := md5.New()
193-
// need to calculate md5 of the full content
194-
tempFile, err := file.CacheFullInTempFile()
188+
var err error
189+
if len(etag) != utils.MD5.Width {
190+
_, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5)
195191
if err != nil {
196192
return err
197193
}
198-
defer func() {
199-
_ = tempFile.Close()
200-
}()
201-
if _, err = utils.CopyWithBuffer(h, tempFile); err != nil {
202-
return err
203-
}
204-
_, err = tempFile.Seek(0, io.SeekStart)
205-
if err != nil {
206-
return err
207-
}
208-
etag = hex.EncodeToString(h.Sum(nil))
209194
}
210195
data := base.Json{
211196
"driveId": 0,

‎drivers/139/driver.go‎

Lines changed: 29 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/alist-org/alist/v3/internal/driver"
1616
"github.com/alist-org/alist/v3/internal/errs"
1717
"github.com/alist-org/alist/v3/internal/model"
18+
"github.com/alist-org/alist/v3/internal/stream"
1819
"github.com/alist-org/alist/v3/pkg/cron"
1920
"github.com/alist-org/alist/v3/pkg/utils"
2021
"github.com/alist-org/alist/v3/pkg/utils/random"
@@ -521,25 +522,21 @@ func (d *Yun139) getPartSize(size int64) int64 {
521522
return 100 * MB
522523
}
523524

524-
func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
525+
func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error {
525526
switch d.Addition.Type {
526527
case MetaPersonalNew:
527528
var err error
528-
fullHash := stream.GetHash().GetHash(utils.SHA256)
529-
if len(fullHash) <= 0 {
530-
tmpF, err := stream.CacheFullInTempFile()
531-
if err != nil {
532-
return err
533-
}
534-
fullHash, err = utils.HashFile(utils.SHA256, tmpF)
529+
fullHash := s.GetHash().GetHash(utils.SHA256)
530+
if len(fullHash) != utils.SHA256.Width {
531+
_, fullHash, err = stream.CacheFullInTempFileAndHash(s, utils.SHA256)
535532
if err != nil {
536533
return err
537534
}
538535
}
539536

540537
partInfos := []PartInfo{}
541-
var partSize = d.getPartSize(stream.GetSize())
542-
part := (stream.GetSize() + partSize - 1) / partSize
538+
var partSize = d.getPartSize(s.GetSize())
539+
part := (s.GetSize() + partSize - 1) / partSize
543540
if part == 0 {
544541
part = 1
545542
}
@@ -548,7 +545,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
548545
return ctx.Err()
549546
}
550547
start := i * partSize
551-
byteSize := stream.GetSize() - start
548+
byteSize := s.GetSize() - start
552549
if byteSize > partSize {
553550
byteSize = partSize
554551
}
@@ -576,9 +573,9 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
576573
"contentType": "application/octet-stream",
577574
"parallelUpload": false,
578575
"partInfos": firstPartInfos,
579-
"size": stream.GetSize(),
576+
"size": s.GetSize(),
580577
"parentFileId": dstDir.GetID(),
581-
"name": stream.GetName(),
578+
"name": s.GetName(),
582579
"type": "file",
583580
"fileRenameMode": "auto_rename",
584581
}
@@ -629,9 +626,9 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
629626
}
630627

631628
// Progress
632-
p := driver.NewProgress(stream.GetSize(), up)
629+
p := driver.NewProgress(s.GetSize(), up)
633630

634-
rateLimited := driver.NewLimitedUploadStream(ctx, stream)
631+
rateLimited := driver.NewLimitedUploadStream(ctx, s)
635632
// 上传所有分片
636633
for _, uploadPartInfo := range uploadPartInfos {
637634
index := uploadPartInfo.PartNumber - 1
@@ -677,8 +674,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
677674
}
678675

679676
// 处理冲突
680-
if resp.Data.FileName != stream.GetName() {
681-
log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, stream.GetName())
677+
if resp.Data.FileName != s.GetName() {
678+
log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, s.GetName())
682679
// 给服务器一定时间处理数据，避免无法刷新文件列表
683680
time.Sleep(time.Millisecond * 500)
684681
// 刷新并获取文件列表
@@ -688,10 +685,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
688685
}
689686
// 删除旧文件
690687
for _, file := range files {
691-
if file.GetName() == stream.GetName() {
688+
if file.GetName() == s.GetName() {
692689
log.Debugf("[139] conflict: removing old: %s", file.GetName())
693690
// 删除前重命名旧文件，避免仍旧冲突
694-
err = d.Rename(ctx, file, stream.GetName()+random.String(4))
691+
err = d.Rename(ctx, file, s.GetName()+random.String(4))
695692
if err != nil {
696693
return err
697694
}
@@ -705,8 +702,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
705702
// 重命名新文件
706703
for _, file := range files {
707704
if file.GetName() == resp.Data.FileName {
708-
log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), stream.GetName())
709-
err = d.Rename(ctx, file, stream.GetName())
705+
log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), s.GetName())
706+
err = d.Rename(ctx, file, s.GetName())
710707
if err != nil {
711708
return err
712709
}
@@ -726,10 +723,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
726723
}
727724
// 删除旧文件
728725
for _, file := range files {
729-
if file.GetName() == stream.GetName() {
726+
if file.GetName() == s.GetName() {
730727
log.Debugf("[139] conflict: removing old: %s", file.GetName())
731728
// 删除前重命名旧文件，避免仍旧冲突
732-
err = d.Rename(ctx, file, stream.GetName()+random.String(4))
729+
err = d.Rename(ctx, file, s.GetName()+random.String(4))
733730
if err != nil {
734731
return err
735732
}
@@ -746,7 +743,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
746743
"fileCount": 1,
747744
"totalSize": 0, // 去除上传大小限制
748745
"uploadContentList": []base.Json{{
749-
"contentName": stream.GetName(),
746+
"contentName": s.GetName(),
750747
"contentSize": 0, // 去除上传大小限制
751748
// "digest": "5a3231986ce7a6b46e408612d385bafa"
752749
}},
@@ -767,7 +764,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
767764
"seqNo": random.String(32), //序列号不能为空
768765
"totalSize": 0,
769766
"uploadContentList": []base.Json{{
770-
"contentName": stream.GetName(),
767+
"contentName": s.GetName(),
771768
"contentSize": 0,
772769
// "digest": "5a3231986ce7a6b46e408612d385bafa"
773770
}},
@@ -781,21 +778,21 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
781778
}
782779

783780
// Progress
784-
p := driver.NewProgress(stream.GetSize(), up)
781+
p := driver.NewProgress(s.GetSize(), up)
785782

786-
var partSize = d.getPartSize(stream.GetSize())
787-
part := (stream.GetSize() + partSize - 1) / partSize
783+
var partSize = d.getPartSize(s.GetSize())
784+
part := (s.GetSize() + partSize - 1) / partSize
788785
if part == 0 {
789786
part = 1
790787
}
791-
rateLimited := driver.NewLimitedUploadStream(ctx, stream)
788+
rateLimited := driver.NewLimitedUploadStream(ctx, s)
792789
for i := int64(0); i < part; i++ {
793790
if utils.IsCanceled(ctx) {
794791
return ctx.Err()
795792
}
796793

797794
start := i * partSize
798-
byteSize := stream.GetSize() - start
795+
byteSize := s.GetSize() - start
799796
if byteSize > partSize {
800797
byteSize = partSize
801798
}
@@ -809,8 +806,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
809806
}
810807

811808
req = req.WithContext(ctx)
812-
req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName()))
813-
req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10))
809+
req.Header.Set("Content-Type", "text/plain;name="+unicode(s.GetName()))
810+
req.Header.Set("contentSize", strconv.FormatInt(s.GetSize(), 10))
814811
req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1))
815812
req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID)
816813
req.Header.Set("rangeType", "0")

0 commit comments

Comments
 (0)