From 26cd4dd31cfda81bb54d2d27d43d98f0e7f01db6 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 07:58:32 +0000 Subject: [PATCH 01/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=99=BE=E5=BA=A6?= =?UTF-8?q?=E4=BA=91=E7=9B=98=E6=89=80=E6=9C=89=E6=83=85=E5=86=B5=E9=83=BD?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=96=AD=E7=82=B9=E7=BB=AD=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/baidu_netdisk/driver.go | 161 +++++++++++++++++++++----------- 1 file changed, 108 insertions(+), 53 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 0fa94e885..256219447 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,11 +5,13 @@ import ( "crypto/md5" "encoding/hex" "errors" + "fmt" "io" "net/url" "os" stdpath "path" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -31,6 +33,8 @@ type BaiduNetdisk struct { vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } +var ErrUploadIDExpired = errors.New("uploadid expired") + func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -168,18 +172,15 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo if err != nil { return nil, err } - // 修复时间，具体原因见 Put 方法注释的 **注意** + // 修复时间 newFile.Ctime = stream.CreateTime().Unix() newFile.Mtime = stream.ModTime().Unix() return fileToObj(newFile), nil } -// Put -// -// **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 -// 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 +// Put 文件上传，支持断点续传和 uploadid 过期重试 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // rapid upload + // 尝试秒传 if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil } @@ -212,9 +213,8 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F lastBlockSize = sliceSize } - //cal md5 for first 256k data + // 计算 md5 const SliceSize int64 = 256 * utils.KB - // cal md5 blockList := make([]string, 0, count) byteSize := sliceSize fileMd5H := md5.New() @@ -244,7 +244,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } if tmpF != nil { if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { @@ -258,13 +258,11 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } + // 没有进度，走预上传 + params := map[string]string{"method": "precreate"} form := map[string]string{ "path": path, "size": strconv.FormatInt(streamSize, 10), @@ -276,60 +274,108 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F "slice-md5": sliceMd5, } joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) _, err = d.postForm("/xpan/file", params, form, &precreateResp) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { - //rapid upload, since got md5 match from baidu server - // 修复时间，具体原因见 Put 方法注释的 **注意** precreateResp.File.Ctime = ctime precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(1), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + cacheReaderAt, okReaderAt := cache.(io.ReaderAt) + if !okReaderAt { + return nil, fmt.Errorf("cache does not implement io.ReaderAt") } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - byteSize = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + totalParts := len(precreateResp.BlockList) + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue } - err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) - if err != nil { - return err + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消，则保存上传进度 + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 进度 + done := 0 + for _, v := range precreateResp.BlockList { + if v < 0 { + done++ + } + } + if totalParts > 0 { + up(float64(done) * 100.0 / float64(totalParts)) + } + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + // 保存进度（所有错误都会保存） + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate（所有分片都要重传） + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + joinTime(form, ctime, mtime) + var newPre PrecreateResp + _, err2 := d.postForm("/xpan/file", params, form, &newPre) + if err2 != nil { + return nil, err2 + } + if newPre.ReturnType == 2 { + newPre.File.Ctime = ctime + newPre.File.Mtime = mtime + return fileToObj(newPre.File), nil + } + precreateResp = &newPre + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } @@ -340,9 +386,10 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if err != nil { return nil, err } - // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } @@ -358,8 +405,16 @@ func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string log.Debugln(res.RawResponse.Status + res.String()) errCode := utils.Json.Get(res.Body(), "error_code").ToInt() errNo := utils.Json.Get(res.Body(), "errno").ToInt() + respStr := res.String() + lower := strings.ToLower(respStr) + if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + if strings.Contains(lower, "invalid uploadid") { + return ErrUploadIDExpired + } + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String()) } return nil } From f1f9d6497a6a96ec7becab4d147738436cd3be48 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 13:10:06 +0000 Subject: [PATCH 02/12] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=9F=E6=88=90Relea?= =?UTF-8?q?se?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release-manual.yml | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/release-manual.yml diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml new file mode 100644 index 000000000..159f89023 --- /dev/null +++ b/.github/workflows/release-manual.yml @@ -0,0 +1,29 @@ +name: Release (Manual) + +on: + workflow_dispatch: + inputs: + tag: + description: 'Release tag (e.g. v4.1.4)' + required: true + title: + description: 'Release title (optional)' + required: false + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create Release with changelog + run: | + npx changelogithub --sanitize ${{ github.event.inputs.tag }} ${{ github.event.inputs.title }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b29e090fdc139626b627c02a023c8d05464579a9 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 13:14:43 +0000 Subject: [PATCH 03/12] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=9F=E6=88=90Release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release-manual.yml | 29 ---------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/release-manual.yml diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml deleted file mode 100644 index 159f89023..000000000 --- a/.github/workflows/release-manual.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Release (Manual) - -on: - workflow_dispatch: - inputs: - tag: - description: 'Release tag (e.g. v4.1.4)' - required: true - title: - description: 'Release title (optional)' - required: false - -permissions: - contents: write - -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Create Release with changelog - run: | - npx changelogithub --sanitize ${{ github.event.inputs.tag }} ${{ github.event.inputs.title }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d018ee5fbfd6ca082972e3aa9511d2f2e523af7e Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 13:27:28 +0000 Subject: [PATCH 04/12] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E4=BA=91=E7=BC=96=E8=AF=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 125 +++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 97280bc9d..6b7aed589 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,77 +1,92 @@ -name: Release builds +name: Release Build on: - release: - types: [ published ] + push: + tags: + - "v*" + workflow_dispatch: -permissions: - contents: write +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - # Set release to prerelease first - prerelease: - name: Set Prerelease - runs-on: ubuntu-latest - steps: - - name: Prerelease - uses: irongut/EditRelease@v1.2.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - id: ${{ github.event.release.id }} - prerelease: true - - # Main release job for all platforms - release: - needs: prerelease + build: strategy: matrix: - build-type: [ 'standard', 'lite' ] - target-platform: [ '', 'android', 'freebsd', 'linux_musl', 'linux_musl_arm' ] - name: Release ${{ matrix.target-platform && format('{0} ', matrix.target-platform) || '' }}${{ matrix.build-type == 'lite' && 'Lite' || '' }} + target: + - darwin-amd64 + - darwin-arm64 + - windows-amd64 + - linux-arm64-musl + - linux-amd64-musl + - windows-arm64 + - android-arm64 + name: Build ${{ matrix.target }} runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 - - name: Free Disk Space (Ubuntu) - if: matrix.target-platform == '' - uses: jlumbroso/free-disk-space@main - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true + - uses: benjlevesque/short-sha@v3.0 + id: short-sha + + - name: Get tag name + id: vars + run: echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.24' - - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install dependencies - if: matrix.target-platform == '' - run: | - sudo snap install zig --classic --beta - docker pull crazymax/xgo:latest - go install github.com/crazy-max/xgo@latest - sudo apt install upx + go-version: "1.24.5" - - name: Build - run: | - bash build.sh release ${{ matrix.build-type == 'lite' && 'lite' || '' }} ${{ matrix.target-platform }} + - name: Setup web + run: bash build.sh dev web env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - - name: Upload assets + - name: Build + uses: OpenListTeam/cgo-actions@v1.2.2 + with: + targets: ${{ matrix.target }} + musl-target-format: $os-$musl-$arch + github-token: ${{ secrets.GITHUB_TOKEN }} + out-dir: build + x-flags: | + github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=${{ github.run_id }} + github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors + github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=${{ steps.short-sha.outputs.sha }} + github.com/OpenListTeam/OpenList/v4/internal/conf.Version=${{ steps.vars.outputs.tag }} + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + output: openlist$ext + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: openlist_${{ steps.vars.outputs.tag }}_${{ matrix.target }} + path: build/* + + release: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Get tag name + id: vars + run: echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Create Release uses: softprops/action-gh-release@v2 with: - files: build/compress/* + tag_name: ${{ steps.vars.outputs.tag }} + name: Release ${{ steps.vars.outputs.tag }} + draft: false prerelease: false - tag_name: ${{ github.event.release.tag_name }} - + files: artifacts/**/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6fd166be923841d5e561ec2fa717e269addce64e Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 13:37:13 +0000 Subject: [PATCH 05/12] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E4=BA=91=E7=BC=96=E8=AF=911?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b7aed589..e08d135ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,10 +31,6 @@ jobs: - uses: benjlevesque/short-sha@v3.0 id: short-sha - - name: Get tag name - id: vars - run: echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - name: Setup Go uses: actions/setup-go@v5 with: @@ -57,14 +53,14 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=${{ github.run_id }} github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=${{ steps.short-sha.outputs.sha }} - github.com/OpenListTeam/OpenList/v4/internal/conf.Version=${{ steps.vars.outputs.tag }} + github.com/OpenListTeam/OpenList/v4/internal/conf.Version=${{ github.ref_name }} github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling output: openlist$ext - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: openlist_${{ steps.vars.outputs.tag }}_${{ matrix.target }} + name: openlist_${{ matrix.target }} path: build/* release: @@ -76,17 +72,13 @@ jobs: with: path: artifacts - - name: Get tag name - id: vars - run: echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - name: Create Release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ steps.vars.outputs.tag }} - name: Release ${{ steps.vars.outputs.tag }} + tag_name: ${{ github.ref_name }} + name: ${{ github.ref_name }} # 这里保证 release 名称就是 v1.1.1 draft: false prerelease: false - files: artifacts/**/* + files: artifacts/**/* # 明确包含所有 target 的文件 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d9c4a5707e954fb2326de16316d0028f644af3b3 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 13:45:23 +0000 Subject: [PATCH 06/12] =?UTF-8?q?=E6=81=A2=E5=A4=8DRelease.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 116 ++++++++++++++++------------------ 1 file changed, 54 insertions(+), 62 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e08d135ac..dac3f60a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,84 +1,76 @@ -name: Release Build +name: Release builds on: - push: - tags: - - "v*" - workflow_dispatch: + release: + types: [ published ] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +permissions: + contents: write jobs: - build: + # Set release to prerelease first + prerelease: + name: Set Prerelease + runs-on: ubuntu-latest + steps: + - name: Prerelease + uses: irongut/EditRelease@v1.2.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + id: ${{ github.event.release.id }} + prerelease: true + + # Main release job for all platforms + release: + needs: prerelease strategy: matrix: - target: - - darwin-amd64 - - darwin-arm64 - - windows-amd64 - - linux-arm64-musl - - linux-amd64-musl - - windows-arm64 - - android-arm64 - name: Build ${{ matrix.target }} + build-type: [ 'standard', 'lite' ] + target-platform: [ '', 'android', 'freebsd', 'linux_musl', 'linux_musl_arm' ] + name: Release ${{ matrix.target-platform && format('{0} ', matrix.target-platform) || '' }}${{ matrix.build-type == 'lite' && 'Lite' || '' }} runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: benjlevesque/short-sha@v3.0 - id: short-sha + - name: Free Disk Space (Ubuntu) + if: matrix.target-platform == '' + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true - name: Setup Go uses: actions/setup-go@v5 with: - go-version: "1.24.5" + go-version: '1.24' - - name: Setup web - run: bash build.sh dev web - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - - - name: Build - uses: OpenListTeam/cgo-actions@v1.2.2 + - name: Checkout + uses: actions/checkout@v4 with: - targets: ${{ matrix.target }} - musl-target-format: $os-$musl-$arch - github-token: ${{ secrets.GITHUB_TOKEN }} - out-dir: build - x-flags: | - github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=${{ github.run_id }} - github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors - github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=${{ steps.short-sha.outputs.sha }} - github.com/OpenListTeam/OpenList/v4/internal/conf.Version=${{ github.ref_name }} - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling - output: openlist$ext + fetch-depth: 0 - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: openlist_${{ matrix.target }} - path: build/* + - name: Install dependencies + if: matrix.target-platform == '' + run: | + sudo snap install zig --classic --beta + docker pull crazymax/xgo:latest + go install github.com/crazy-max/xgo@latest + sudo apt install upx - release: - needs: build - runs-on: ubuntu-latest - steps: - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts + - name: Build + run: | + bash build.sh release ${{ matrix.build-type == 'lite' && 'lite' || '' }} ${{ matrix.target-platform }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - - name: Create Release + - name: Upload assets uses: softprops/action-gh-release@v2 with: - tag_name: ${{ github.ref_name }} - name: ${{ github.ref_name }} # 这里保证 release 名称就是 v1.1.1 - draft: false + files: build/compress/* prerelease: false - files: artifacts/**/* # 明确包含所有 target 的文件 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag_name: ${{ github.event.release.tag_name }} From 539c7eaaa41fcc7e0f66d8962b99b5c1ad43d49b Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 22:53:52 +0800 Subject: [PATCH 07/12] REUpdate driver.go Signed-off-by: jenfonro <799170122@qq.com> --- drivers/baidu_netdisk/driver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 256219447..e2149f796 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -420,3 +420,4 @@ func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string } var _ driver.Driver = (*BaiduNetdisk)(nil) + From 272036a3864ee6f075d01dc391e11d03e8e69ae5 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 14 Sep 2025 15:02:12 +0000 Subject: [PATCH 08/12] Restore file(s) from upstream --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dac3f60a1..97280bc9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,3 +74,4 @@ jobs: files: build/compress/* prerelease: false tag_name: ${{ github.event.release.tag_name }} + From 7ac4494341820d6b6570f223e4ec1c28cc666612 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Mon, 15 Sep 2025 01:10:15 +0000 Subject: [PATCH 09/12] restore notes --- drivers/baidu_netdisk/driver.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index e2149f796..bbc29dc1a 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -172,15 +172,18 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo if err != nil { return nil, err } - // 修复时间 + // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = stream.CreateTime().Unix() newFile.Mtime = stream.ModTime().Unix() return fileToObj(newFile), nil } // Put 文件上传，支持断点续传和 uploadid 过期重试 +// +// **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 +// 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 尝试秒传 + // rapid upload 尝试秒传 if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil } @@ -213,7 +216,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F lastBlockSize = sliceSize } - // 计算 md5 + //cal md5 for first 256k data 计算 md5 const SliceSize int64 = 256 * utils.KB blockList := make([]string, 0, count) byteSize := sliceSize @@ -279,6 +282,8 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return nil, err } if precreateResp.ReturnType == 2 { + //rapid upload, since got md5 match from baidu server + // 修复时间，具体原因见 Put 方法注释的 **注意** precreateResp.File.Ctime = ctime precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil @@ -386,6 +391,7 @@ uploadLoop: if err != nil { return nil, err } + // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime // 上传成功清理进度 From 0835babf269f754020406efaf7e07afb1c759dc1 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Mon, 15 Sep 2025 01:21:03 +0000 Subject: [PATCH 10/12] Duplicate logical merge and new counter --- drivers/baidu_netdisk/driver.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index bbc29dc1a..71b0a526f 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -304,6 +304,8 @@ uploadLoop: } totalParts := len(precreateResp.BlockList) + doneParts := 0 // 新增计数器 + for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) || partseq < 0 { continue @@ -328,15 +330,9 @@ uploadLoop: return err } precreateResp.BlockList[i] = -1 - // 进度 - done := 0 - for _, v := range precreateResp.BlockList { - if v < 0 { - done++ - } - } + doneParts++ // 直接递增计数器 if totalParts > 0 { - up(float64(done) * 100.0 / float64(totalParts)) + up(float64(doneParts) * 100.0 / float64(totalParts)) } return nil }) @@ -413,13 +409,13 @@ func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string errNo := utils.Json.Get(res.Body(), "errno").ToInt() respStr := res.String() lower := strings.ToLower(respStr) - if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { +// 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { return ErrUploadIDExpired } + if errCode != 0 || errNo != 0 { - if strings.Contains(lower, "invalid uploadid") { - return ErrUploadIDExpired - } return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String()) } return nil From 8e03f6e68845b0116b5cc0db0fba702534081425 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Mon, 15 Sep 2025 01:34:57 +0000 Subject: [PATCH 11/12] error message with detailed interface --- drivers/baidu_netdisk/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 71b0a526f..bf4a056ee 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -300,7 +300,7 @@ uploadLoop: cacheReaderAt, okReaderAt := cache.(io.ReaderAt) if !okReaderAt { - return nil, fmt.Errorf("cache does not implement io.ReaderAt") + return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations") } totalParts := len(precreateResp.BlockList) From e270f166daa698d7ccc21db9c283c464fd2646ae Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Tue, 23 Sep 2025 00:00:59 +0800 Subject: [PATCH 12/12] add precreate function --- drivers/baidu_netdisk/driver.go | 72 ++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index f528d5cd7..0d0946933 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -265,27 +265,12 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 - params := map[string]string{"method": "precreate"} - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } if precreateResp.ReturnType == 2 { //rapid upload, since got md5 match from baidu server - // 修复时间，具体原因见 Put 方法注释的 **注意** - precreateResp.File.Ctime = ctime - precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } @@ -353,27 +338,14 @@ uploadLoop: if errors.Is(err, ErrUploadIDExpired) { log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") // 重新 precreate（所有分片都要重传） - params := map[string]string{"method": "precreate"} - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - } - joinTime(form, ctime, mtime) - var newPre PrecreateResp - _, err2 := d.postForm("/xpan/file", params, form, &newPre) + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) if err2 != nil { return nil, err2 } if newPre.ReturnType == 2 { - newPre.File.Ctime = ctime - newPre.File.Mtime = mtime return fileToObj(newPre.File), nil } - precreateResp = &newPre + precreateResp = newPre // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) continue uploadLoop @@ -395,6 +367,41 @@ uploadLoop: return fileToObj(newFile), nil } +// precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间，具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { res, err := base.RestyClient.R(). SetContext(ctx). @@ -409,7 +416,7 @@ func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string errNo := utils.Json.Get(res.Body(), "errno").ToInt() respStr := res.String() lower := strings.ToLower(respStr) -// 合并 uploadid 过期检测逻辑 + // 合并 uploadid 过期检测逻辑 if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { return ErrUploadIDExpired @@ -430,4 +437,3 @@ func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, e } var _ driver.Driver = (*BaiduNetdisk)(nil) -