Skip to content

Commit 8f17d35

Browse files
j2rong4cnKirCute
j2rong4cn
and
KirCute
authored
feat(chunk): add ChunkPrefix and ChunkLargeFileOnly options (#1321)
* fix(chunk): move chunk existence check to Link method * feat(chunk): add chunk prefix configuration * feat(chunk): add chunk_large_file_only configuration * feat(chunk): concurrently list chunk folder * refactor(chunk): remove unnecessary mutex for result handling in List method --------- Co-authored-by: KirCute <[email protected]>
1 parent 89759b6 commit 8f17d35

File tree

2 files changed

+99
-75
lines changed

2 files changed

+99
-75
lines changed

‎drivers/chunk/driver.go‎

Lines changed: 87 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@ import (
1717
"github.com/OpenListTeam/OpenList/v4/internal/op"
1818
"github.com/OpenListTeam/OpenList/v4/internal/sign"
1919
"github.com/OpenListTeam/OpenList/v4/internal/stream"
20+
"github.com/OpenListTeam/OpenList/v4/pkg/errgroup"
2021
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
2122
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
2223
"github.com/OpenListTeam/OpenList/v4/server/common"
24+
"github.com/avast/retry-go"
2325
)
2426

2527
type Chunk struct {
@@ -39,6 +41,9 @@ func (d *Chunk) Init(ctx context.Context) error {
3941
if d.PartSize <= 0 {
4042
return errors.New("part size must be positive")
4143
}
44+
if len(d.ChunkPrefix) <= 0 {
45+
return errors.New("chunk folder prefix must not be empty")
46+
}
4247
d.RemotePath = utils.FixAndCleanPath(d.RemotePath)
4348
return nil
4449
}
@@ -72,13 +77,13 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) {
7277
}
7378

7479
remoteActualDir, name := stdpath.Split(remoteActualPath)
75-
chunkName := "[openlist_chunk]" + name
80+
chunkName := d.ChunkPrefix + name
7681
chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, chunkName), model.ListArgs{})
7782
if err != nil {
7883
return nil, err
7984
}
8085
var totalSize int64 = 0
81-
// 0号块必须存在
86+
// 0号块默认为-1 以支持空文件
8287
chunkSizes := []int64{-1}
8388
h := make(map[*utils.HashType]string)
8489
var first model.Obj
@@ -115,21 +120,6 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) {
115120
chunkSizes[idx] = o.GetSize()
116121
}
117122
}
118-
// 检查0号块不等于-1 以支持空文件
119-
// 如果块数量大于1 最后一块不可能为0
120-
// 只检查中间块是否有0
121-
for i, l := 0, len(chunkSizes)-2; ; i++ {
122-
if i == 0 {
123-
if chunkSizes[i] == -1 {
124-
return nil, fmt.Errorf("chunk part[%d] are missing", i)
125-
}
126-
} else if chunkSizes[i] == 0 {
127-
return nil, fmt.Errorf("chunk part[%d] are missing", i)
128-
}
129-
if i >= l {
130-
break
131-
}
132-
}
133123
reqDir, _ := stdpath.Split(path)
134124
objRes := chunkObject{
135125
Object: model.Object{
@@ -161,67 +151,76 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
161151
return nil, err
162152
}
163153
result := make([]model.Obj, 0, len(remoteObjs))
154+
listG, listCtx := errgroup.NewGroupWithContext(ctx, d.NumListWorkers, retry.Attempts(3))
164155
for _, obj := range remoteObjs {
156+
if utils.IsCanceled(listCtx) {
157+
break
158+
}
165159
rawName := obj.GetName()
166160
if obj.IsDir() {
167-
if name, ok := strings.CutPrefix(rawName, "[openlist_chunk]"); ok {
168-
chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, rawName), model.ListArgs{
169-
ReqPath: stdpath.Join(args.ReqPath, rawName),
170-
Refresh: args.Refresh,
171-
})
172-
if err != nil {
173-
return nil, err
174-
}
175-
totalSize := int64(0)
176-
h := make(map[*utils.HashType]string)
177-
first := obj
178-
for _, o := range chunkObjs {
179-
if o.IsDir() {
180-
continue
161+
if name, ok := strings.CutPrefix(rawName, d.ChunkPrefix); ok {
162+
resultIdx := len(result)
163+
result = append(result, nil)
164+
listG.Go(func(ctx context.Context) error {
165+
chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, rawName), model.ListArgs{
166+
ReqPath: stdpath.Join(args.ReqPath, rawName),
167+
Refresh: args.Refresh,
168+
})
169+
if err != nil {
170+
return err
181171
}
182-
if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok {
183-
hn, value, ok := strings.Cut(after, "_")
184-
if ok {
185-
ht, ok := utils.GetHashByName(hn)
172+
totalSize := int64(0)
173+
h := make(map[*utils.HashType]string)
174+
first := obj
175+
for _, o := range chunkObjs {
176+
if o.IsDir() {
177+
continue
178+
}
179+
if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok {
180+
hn, value, ok := strings.Cut(after, "_")
186181
if ok {
187-
h[ht] = value
182+
ht, ok := utils.GetHashByName(hn)
183+
if ok {
184+
h[ht] = value
185+
}
186+
continue
188187
}
188+
}
189+
idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt))
190+
if err != nil {
189191
continue
190192
}
193+
if idx == 0 {
194+
first = o
195+
}
196+
totalSize += o.GetSize()
191197
}
192-
idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt))
193-
if err != nil {
194-
continue
198+
objRes := model.Object{
199+
Name: name,
200+
Size: totalSize,
201+
Modified: first.ModTime(),
202+
Ctime: first.CreateTime(),
195203
}
196-
if idx == 0 {
197-
first = o
204+
if len(h) > 0 {
205+
objRes.HashInfo = utils.NewHashInfoByMap(h)
198206
}
199-
totalSize += o.GetSize()
200-
}
201-
objRes := model.Object{
202-
Name: name,
203-
Size: totalSize,
204-
Modified: first.ModTime(),
205-
Ctime: first.CreateTime(),
206-
}
207-
if len(h) > 0 {
208-
objRes.HashInfo = utils.NewHashInfoByMap(h)
209-
}
210-
if !d.Thumbnail {
211-
result = append(result, &objRes)
212-
} else {
213-
thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp")
214-
thumb := fmt.Sprintf("%s/d%s?sign=%s",
215-
common.GetApiUrl(ctx),
216-
utils.EncodePath(thumbPath, true),
217-
sign.Sign(thumbPath))
218-
result = append(result, &model.ObjThumb{
219-
Object: objRes,
220-
Thumbnail: model.Thumbnail{
221-
Thumbnail: thumb,
222-
},
223-
})
224-
}
207+
if !d.Thumbnail {
208+
result[resultIdx] = &objRes
209+
} else {
210+
thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp")
211+
thumb := fmt.Sprintf("%s/d%s?sign=%s",
212+
common.GetApiUrl(ctx),
213+
utils.EncodePath(thumbPath, true),
214+
sign.Sign(thumbPath))
215+
result[resultIdx] = &model.ObjThumb{
216+
Object: objRes,
217+
Thumbnail: model.Thumbnail{
218+
Thumbnail: thumb,
219+
},
220+
}
221+
}
222+
return nil
223+
})
225224
continue
226225
}
227226
}
@@ -248,6 +247,9 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
248247
})
249248
}
250249
}
250+
if err = listG.Wait(); err != nil {
251+
return nil, err
252+
}
251253
return result, nil
252254
}
253255

@@ -267,6 +269,21 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
267269
resultLink.SyncClosers = utils.NewSyncClosers(l)
268270
return &resultLink, nil
269271
}
272+
// 检查0号块不等于-1 以支持空文件
273+
// 如果块数量大于1 最后一块不可能为0
274+
// 只检查中间块是否有0
275+
for i, l := 0, len(chunkFile.chunkSizes)-2; ; i++ {
276+
if i == 0 {
277+
if chunkFile.chunkSizes[i] == -1 {
278+
return nil, fmt.Errorf("chunk part[%d] are missing", i)
279+
}
280+
} else if chunkFile.chunkSizes[i] == 0 {
281+
return nil, fmt.Errorf("chunk part[%d] are missing", i)
282+
}
283+
if i >= l {
284+
break
285+
}
286+
}
270287
fileSize := chunkFile.GetSize()
271288
mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
272289
start := httpRange.Start
@@ -383,7 +400,7 @@ func (d *Chunk) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
383400

384401
func (d *Chunk) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
385402
if _, ok := srcObj.(*chunkObject); ok {
386-
newName = "[openlist_chunk]" + newName
403+
newName = d.ChunkPrefix + newName
387404
}
388405
return fs.Rename(ctx, stdpath.Join(d.RemotePath, srcObj.GetPath()), newName)
389406
}
@@ -404,14 +421,14 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream
404421
if err != nil {
405422
return err
406423
}
407-
if d.Thumbnail && dstDir.GetName() == ".thumbnails" {
424+
if (d.Thumbnail && dstDir.GetName() == ".thumbnails") || (d.ChunkLargeFileOnly && file.GetSize() <= d.PartSize) {
408425
return op.Put(ctx, remoteStorage, stdpath.Join(remoteActualPath, dstDir.GetPath()), file, up)
409426
}
410427
upReader := &driver.ReaderUpdatingProgress{
411428
Reader: file,
412429
UpdateProgress: up,
413430
}
414-
dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), "[openlist_chunk]"+file.GetName())
431+
dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), d.ChunkPrefix+file.GetName())
415432
if d.StoreHash {
416433
for ht, value := range file.GetHash().All() {
417434
_ = op.Put(ctx, remoteStorage, dst, &stream.FileStream{

‎drivers/chunk/meta.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@ import (
66
)
77

88
type Addition struct {
9-
RemotePath string `json:"remote_path" required:"true"`
10-
PartSize int64 `json:"part_size" required:"true" type:"number" help:"bytes"`
11-
CustomExt string `json:"custom_ext" type:"string"`
12-
StoreHash bool `json:"store_hash" type:"bool" default:"true"`
9+
RemotePath string `json:"remote_path" required:"true"`
10+
PartSize int64 `json:"part_size" required:"true" type:"number" help:"bytes"`
11+
ChunkLargeFileOnly bool `json:"chunk_large_file_only" default:"false" help:"chunk only if file size > part_size"`
12+
ChunkPrefix string `json:"chunk_prefix" type:"string" default:"[openlist_chunk]" help:"the prefix of chunk folder"`
13+
CustomExt string `json:"custom_ext" type:"string"`
14+
StoreHash bool `json:"store_hash" type:"bool" default:"true"`
15+
NumListWorkers int `json:"num_list_workers" required:"true" type:"number" default:"5"`
1316

1417
Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"`
1518
ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"`
@@ -26,6 +29,10 @@ var config = driver.Config{
2629

2730
func init() {
2831
op.RegisterDriver(func() driver.Driver {
29-
return &Chunk{}
32+
return &Chunk{
33+
Addition: Addition{
34+
ChunkPrefix: "[openlist_chunk]",
35+
},
36+
}
3037
})
3138
}

0 commit comments

Comments
 (0)