Stream repo zip/tar.gz/bundle achives by default

Author: ChristopherHX

custom/conf/app.example.ini

@@ -1083,6 +1083,9 @@ LEVEL = Info
 ;; This feature is experimental, not fully tested, and may be changed in the future
 ;ALLOW_FORK_INTO_SAME_OWNER = false
 
+;; Repository archives (tar.gz, zip and bundle) are streamed when this is enabled instead of being cached
+;STREAM_ARCHIVES = true
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;[repository.editor]

modules/setting/repository.go

@@ -53,6 +53,7 @@ var (
 		DisableDownloadSourceArchives           bool
 		AllowForkWithoutMaximumLimit            bool
 		AllowForkIntoSameOwner                  bool
+		StreamArchives                          bool
 
 		// Repository editor settings
 		Editor struct {
@@ -167,6 +168,7 @@ var (
 		DisableStars:                            false,
 		DefaultBranch:                           "main",
 		AllowForkWithoutMaximumLimit:            true,
+		StreamArchives:                          true,
 
 		// Repository editor settings
 		Editor: struct {

routers/api/v1/repo/file.go

@@ -307,6 +307,29 @@ func archiveDownload(ctx *context.APIContext) {
 		return
 	}
 
+	if setting.Repository.StreamArchives {
+
+		downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
+
+		// Add nix format link header so tarballs lock correctly:
+		// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
+		ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
+			ctx.Repo.Repository.APIURL(),
+			aReq.CommitID,
+			aReq.Type.String(),
+			aReq.CommitID,
+		))
+
+		ctx.SetServeHeaders(&context.ServeHeaderOptions{
+			Filename: downloadName,
+		})
+
+		if err := aReq.Stream(ctx, ctx.Resp); err != nil && !ctx.Written() {
+			ctx.APIErrorInternal(err)
+		}
+		return
+	}
+
 	archiver, err := aReq.Await(ctx)
 	if err != nil {
 		ctx.APIErrorInternal(err)

routers/web/repo/repo.go

@@ -377,6 +377,28 @@ func Download(ctx *context.Context) {
 		return
 	}
 
+	if setting.Repository.StreamArchives {
+		downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
+
+		// Add nix format link header so tarballs lock correctly:
+		// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
+		ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
+			ctx.Repo.Repository.APIURL(),
+			aReq.CommitID,
+			aReq.Type.String(),
+			aReq.CommitID,
+		))
+
+		ctx.SetServeHeaders(&context.ServeHeaderOptions{
+			Filename: downloadName,
+		})
+
+		if err := aReq.Stream(ctx, ctx.Resp); err != nil && !ctx.Written() {
+			ctx.ServerError("archiver.StreamArchive", err)
+		}
+		return
+	}
+
 	archiver, err := aReq.Await(ctx)
 	if err != nil {
 		ctx.ServerError("archiver.Await", err)
@@ -423,6 +445,12 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep
 // a request that's already in-progress, but the archiver service will just
 // kind of drop it on the floor if this is the case.
 func InitiateDownload(ctx *context.Context) {
+	if setting.Repository.StreamArchives {
+		ctx.JSON(http.StatusOK, map[string]any{
+			"complete": true,
+		})
+		return
+	}
 	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, ctx.PathParam("*"))
 	if err != nil {
 		ctx.HTTPError(http.StatusBadRequest, "invalid archive request")

services/repository/archiver/archiver.go

@@ -138,6 +138,42 @@ func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver
 	}
 }
 
+// Stream satisfies the ArchiveRequest being passed in.  Processing
+// will occur directly in this routine.
+func (aReq *ArchiveRequest) Stream(ctx context.Context, w io.Writer) error {
+	ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("ArchiveRequest[%d]: %s", aReq.RepoID, aReq.GetArchiveName()))
+	defer finished()
+
+	repo, err := repo_model.GetRepositoryByID(ctx, aReq.RepoID)
+	if err != nil {
+		return fmt.Errorf("archiver.LoadRepo failed: %w", err)
+	}
+
+	gitRepo, err := gitrepo.OpenRepository(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer gitRepo.Close()
+
+	if aReq.Type == git.ArchiveBundle {
+		err = gitRepo.CreateBundle(
+			ctx,
+			aReq.CommitID,
+			w,
+		)
+	} else {
+		err = gitRepo.CreateArchive(
+			ctx,
+			aReq.Type,
+			w,
+			setting.Repository.PrefixArchiveFiles,
+			aReq.CommitID,
+		)
+	}
+
+	return err
+}
+
 // doArchive satisfies the ArchiveRequest being passed in.  Processing
 // will occur in a separate goroutine, as this phase may take a while to
 // complete.  If the archive already exists, doArchive will not do