Stream repo zip/tar.gz/bundle achives by default

custom/conf/app.example.ini

@@ -1083,6 +1083,9 @@ LEVEL = Info
 ;; This feature is experimental, not fully tested, and may be changed in the future
 ;ALLOW_FORK_INTO_SAME_OWNER = false
 
+;; Repository archives (tar.gz, zip and bundle) are streamed when this is enabled instead of being cached
+;STREAM_ARCHIVES = true
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;[repository.editor]

modules/setting/repository.go

@@ -53,6 +53,7 @@ var (
 		DisableDownloadSourceArchives           bool
 		AllowForkWithoutMaximumLimit            bool
 		AllowForkIntoSameOwner                  bool
+		StreamArchives                          bool
 
 		// Repository editor settings
 		Editor struct {
@@ -167,6 +168,7 @@ var (
 		DisableStars:                            false,
 		DefaultBranch:                           "main",
 		AllowForkWithoutMaximumLimit:            true,
+		StreamArchives:                          true,
 
 		// Repository editor settings
 		Editor: struct {

modules/templates/helper.go

@@ -127,6 +127,9 @@ func NewFuncMap() template.FuncMap {
 		"EnableTimetracking": func() bool {
 			return setting.Service.EnableTimetracking
 		},
+		"StreamArchives": func() bool {
+			return setting.Repository.StreamArchives
+		},
 		"DisableWebhooks": func() bool {
 			return setting.DisableWebhooks
 		},

routers/api/v1/repo/file.go

@@ -307,6 +307,28 @@ func archiveDownload(ctx *context.APIContext) {
 		return
 	}
 
+	// Add nix format link header so tarballs lock correctly:
+	// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
+	ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
+		ctx.Repo.Repository.APIURL(),
+		aReq.CommitID,
+		aReq.Type.String(),
+		aReq.CommitID,
+	))
+
+	if setting.Repository.StreamArchives {
+		downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
+
+		ctx.SetServeHeaders(&context.ServeHeaderOptions{
+			Filename: downloadName,
+		})
+
+		if err := aReq.Stream(ctx, ctx.Repo.GitRepo, ctx.Resp); err != nil && !ctx.Written() {
+			ctx.APIErrorInternal(err)
+		}
+		return
+	}
+
 	archiver, err := aReq.Await(ctx)
 	if err != nil {
 		ctx.APIErrorInternal(err)
@@ -319,15 +341,6 @@ func archiveDownload(ctx *context.APIContext) {
 func download(ctx *context.APIContext, archiveName string, archiver *repo_model.RepoArchiver) {
 	downloadName := ctx.Repo.Repository.Name + "-" + archiveName
 
-	// Add nix format link header so tarballs lock correctly:
-	// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
-	ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
-		ctx.Repo.Repository.APIURL(),
-		archiver.CommitID,
-		archiver.Type.String(),
-		archiver.CommitID,
-	))
-
 	rPath := archiver.RelativePath()
 	if setting.RepoArchive.Storage.ServeDirect() {
 		// If we have a signed url (S3, object storage), redirect to this directly.

routers/web/repo/repo.go

@@ -377,6 +377,28 @@ func Download(ctx *context.Context) {
 		return
 	}
 
+	// Add nix format link header so tarballs lock correctly:
+	// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
+	ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
+		ctx.Repo.Repository.APIURL(),
+		aReq.CommitID,
+		aReq.Type.String(),
+		aReq.CommitID,
+	))
+
+	if setting.Repository.StreamArchives {
+		downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
+
+		ctx.SetServeHeaders(&context.ServeHeaderOptions{
+			Filename: downloadName,
+		})
+
+		if err := aReq.Stream(ctx, ctx.Repo.GitRepo, ctx.Resp); err != nil && !ctx.Written() {
+			ctx.ServerError("archiver.StreamArchive", err)
+		}
+		return
+	}
+
 	archiver, err := aReq.Await(ctx)
 	if err != nil {
 		ctx.ServerError("archiver.Await", err)
@@ -389,12 +411,6 @@ func Download(ctx *context.Context) {
 func download(ctx *context.Context, archiveName string, archiver *repo_model.RepoArchiver) {
 	downloadName := ctx.Repo.Repository.Name + "-" + archiveName
 
-	// Add nix format link header so tarballs lock correctly:
-	// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
-	ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.tar.gz?rev=%s>; rel="immutable"`,
-		ctx.Repo.Repository.APIURL(),
-		archiver.CommitID, archiver.CommitID))
-
 	rPath := archiver.RelativePath()
 	if setting.RepoArchive.Storage.ServeDirect() {
 		// If we have a signed url (S3, object storage), redirect to this directly.
@@ -423,6 +439,12 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep
 // a request that's already in-progress, but the archiver service will just
 // kind of drop it on the floor if this is the case.
 func InitiateDownload(ctx *context.Context) {
+	if setting.Repository.StreamArchives {
+		ctx.JSON(http.StatusOK, map[string]any{
+			"complete": true,
+		})
+		return
+	}
 	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, ctx.PathParam("*"))
 	if err != nil {
 		ctx.HTTPError(http.StatusBadRequest, "invalid archive request")

services/repository/archiver/archiver.go

@@ -138,6 +138,25 @@
 	}
 }
 
+// Stream satisfies the ArchiveRequest being passed in.  Processing
+// will occur directly in this routine.
+func (aReq *ArchiveRequest) Stream(ctx context.Context, gitRepo *git.Repository, w io.Writer) error {
+	if aReq.Type == git.ArchiveBundle {
+		return gitRepo.CreateBundle(
+			ctx,
+			aReq.CommitID,
+			w,
+		)
+	}
+	return gitRepo.CreateArchive(
+		ctx,
+		aReq.Type,
+		w,
+		setting.Repository.PrefixArchiveFiles,
+		aReq.CommitID,
+	)
+}
+
 // doArchive satisfies the ArchiveRequest being passed in.  Processing
 // will occur in a separate goroutine, as this phase may take a while to
 // complete.  If the archive already exists, doArchive will not do
@@ -204,28 +223,14 @@
 	}
 	defer gitRepo.Close()
 
 	go func(done chan error, w *io.PipeWriter, archiver *repo_model.RepoArchiver, gitRepo *git.Repository) {
 		defer func() {
 			if r := recover(); r != nil {
 				done <- fmt.Errorf("%v", r)
 			}
 		}()
 
-		if archiver.Type == git.ArchiveBundle {
-			err = gitRepo.CreateBundle(
-				ctx,
-				archiver.CommitID,
-				w,
-			)
-		} else {
-			err = gitRepo.CreateArchive(
-				ctx,
-				archiver.Type,
-				w,
-				setting.Repository.PrefixArchiveFiles,
-				archiver.CommitID,
-			)
-		}
+		err = r.Stream(ctx, gitRepo, w)
 		_ = w.CloseWithError(err)
 		done <- err
 	}(done, w, archiver, gitRepo)

templates/base/head_script.tmpl

@@ -17,6 +17,7 @@ If you introduce mistakes in it, Gitea JavaScript code wouldn't run correctly.
 		pageData: {{.PageData}},
 		notificationSettings: {{NotificationSettings}}, {{/*a map provided by NewFuncMap in helper.go*/}}
 		enableTimeTracking: {{EnableTimetracking}},
+		streamArchives: {{StreamArchives}},
 		{{if or .Participants .Assignees .MentionableTeams}}
 		mentionValues: Array.from(new Map([
 			{{- range .Participants -}}

web_src/js/features/repo-common.ts

@@ -32,7 +32,10 @@ async function onDownloadArchive(e: DOMEvent<MouseEvent>) {
 }
 
 export function initRepoArchiveLinks() {
-  queryElems(document, 'a.archive-link[href]', (el) => el.addEventListener('click', onDownloadArchive));
+  // when archive streaming is enabled, the links will work natively without JS
+  if (!window.config.streamArchives) {
+    queryElems(document, 'a.archive-link[href]', (el) => el.addEventListener('click', onDownloadArchive));
+  }
 }
 
 export function initRepoActivityTopAuthorsChart() {

web_src/js/types.ts

@@ -20,6 +20,7 @@ export type Config = {
   mentionValues?: MentionValue[],
   mermaidMaxSourceCharacters: number,
   i18n: Record<string, string>,
+  streamArchives: boolean,
 };
 
 export type IntervalId = ReturnType<typeof setInterval>;

web_src/js/vitest.setup.ts

@@ -1,3 +1,5 @@
+import type {Config} from './types.ts';
+
 window.__webpack_public_path__ = '';
 
 window.config = {
@@ -22,4 +24,5 @@ window.config = {
   ],
   mermaidMaxSourceCharacters: 5000,
   i18n: {},
-};
+  streamArchives: true,
+} satisfies Config;