Refactor cache-control (go-gitea#33861)

And fix go-gitea#21391

Author: wxiaoguang

modules/httpcache/httpcache.go

@@ -4,40 +4,60 @@
 package httpcache
 
 import (
-	"io"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
 )
 
+type CacheControlOptions struct {
+	IsPublic    bool
+	MaxAge      time.Duration
+	NoTransform bool
+}
+
 // SetCacheControlInHeader sets suitable cache-control headers in the response
-func SetCacheControlInHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) {
-	directives := make([]string, 0, 2+len(additionalDirectives))
+func SetCacheControlInHeader(h http.Header, opts *CacheControlOptions) {
+	directives := make([]string, 0, 4)
 
 	// "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store"
 	// because browsers may restore some input fields after navigate-back / reload a page.
+	publicPrivate := util.Iif(opts.IsPublic, "public", "private")
 	if setting.IsProd {
-		if maxAge == 0 {
+		if opts.MaxAge == 0 {
 			directives = append(directives, "max-age=0", "private", "must-revalidate")
 		} else {
-			directives = append(directives, "private", "max-age="+strconv.Itoa(int(maxAge.Seconds())))
+			directives = append(directives, publicPrivate, "max-age="+strconv.Itoa(int(opts.MaxAge.Seconds())))
 		}
 	} else {
-		directives = append(directives, "max-age=0", "private", "must-revalidate")
+		// use dev-related controls, and remind users they are using non-prod setting.
+		directives = append(directives, "max-age=0", publicPrivate, "must-revalidate")
+		h.Set("X-Gitea-Debug", fmt.Sprintf("RUN_MODE=%v, MaxAge=%s", setting.RunMode, opts.MaxAge))
+	}
 
-		// to remind users they are using non-prod setting.
-		h.Set("X-Gitea-Debug", "RUN_MODE="+setting.RunMode)
+	if opts.NoTransform {
+		directives = append(directives, "no-transform")
 	}
+	h.Set("Cache-Control", strings.Join(directives, ", "))
+}
 
-	h.Set("Cache-Control", strings.Join(append(directives, additionalDirectives...), ", "))
+func CacheControlForPublicStatic() *CacheControlOptions {
+	return &CacheControlOptions{
+		IsPublic:    true,
+		MaxAge:      setting.StaticCacheTime,
+		NoTransform: true,
+	}
 }
 
-func ServeContentWithCacheControl(w http.ResponseWriter, req *http.Request, name string, modTime time.Time, content io.ReadSeeker) {
-	SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
-	http.ServeContent(w, req, name, modTime, content)
+func CacheControlForPrivateStatic() *CacheControlOptions {
+	return &CacheControlOptions{
+		MaxAge:      setting.StaticCacheTime,
+		NoTransform: true,
+	}
 }
 
 // HandleGenericETagCache handles ETag-based caching for a HTTP request.
@@ -50,7 +70,8 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin
 			return true
 		}
 	}
-	SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
+	// not sure whether it is a public content, so just use "private" (old behavior)
+	SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic())
 	return false
 }
 
@@ -95,6 +116,8 @@ func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag s
 			}
 		}
 	}
-	SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
+
+	// not sure whether it is a public content, so just use "private" (old behavior)
+	SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic())
 	return false
 }

modules/httplib/serve.go

@@ -33,6 +33,7 @@ type ServeHeaderOptions struct {
 	ContentLength      *int64
 	Disposition        string // defaults to "attachment"
 	Filename           string
+	CacheIsPublic      bool
 	CacheDuration      time.Duration // defaults to 5 minutes
 	LastModified       time.Time
 }
@@ -72,11 +73,11 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
 		header.Set("Access-Control-Expose-Headers", "Content-Disposition")
 	}
 
-	duration := opts.CacheDuration
-	if duration == 0 {
-		duration = 5 * time.Minute
-	}
-	httpcache.SetCacheControlInHeader(header, duration)
+	httpcache.SetCacheControlInHeader(header, &httpcache.CacheControlOptions{
+		IsPublic:    opts.CacheIsPublic,
+		MaxAge:      opts.CacheDuration,
+		NoTransform: true,
+	})
 
 	if !opts.LastModified.IsZero() {
 		// http.TimeFormat required a UTC time, refer to https://pkg.go.dev/net/http#TimeFormat
@@ -85,19 +86,15 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
 }
 
 // ServeData download file from io.Reader
-func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) {
+func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byte, opts *ServeHeaderOptions) {
 	// do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests
-	opts := &ServeHeaderOptions{
-		Filename: path.Base(filePath),
-	}
-
 	sniffedType := typesniffer.DetectContentType(mineBuf)
 
 	// the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later
 	isPlain := sniffedType.IsText() || r.FormValue("render") != ""
 
 	if setting.MimeTypeMap.Enabled {
-		fileExtension := strings.ToLower(filepath.Ext(filePath))
+		fileExtension := strings.ToLower(filepath.Ext(opts.Filename))
 		opts.ContentType = setting.MimeTypeMap.Map[fileExtension]
 	}
 
@@ -114,7 +111,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri
 	if isPlain {
 		charset, err := charsetModule.DetectEncoding(mineBuf)
 		if err != nil {
-			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err)
+			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err)
 			charset = "utf-8"
 		}
 		opts.ContentTypeCharset = strings.ToLower(charset)
@@ -142,7 +139,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri
 
 const mimeDetectionBufferLen = 1024
 
-func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) {
+func ServeContentByReader(r *http.Request, w http.ResponseWriter, size int64, reader io.Reader, opts *ServeHeaderOptions) {
 	buf := make([]byte, mimeDetectionBufferLen)
 	n, err := util.ReadAtMost(reader, buf)
 	if err != nil {
@@ -152,7 +149,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin
 	if n >= 0 {
 		buf = buf[:n]
 	}
-	setServeHeadersByFile(r, w, filePath, buf)
+	setServeHeadersByFile(r, w, buf, opts)
 
 	// reset the reader to the beginning
 	reader = io.MultiReader(bytes.NewReader(buf), reader)
@@ -215,7 +212,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin
 	_, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error
 }
 
-func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime *time.Time, reader io.ReadSeeker) {
+func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, modTime *time.Time, reader io.ReadSeeker, opts *ServeHeaderOptions) {
 	buf := make([]byte, mimeDetectionBufferLen)
 	n, err := util.ReadAtMost(reader, buf)
 	if err != nil {
@@ -229,9 +226,9 @@ func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath s
 	if n >= 0 {
 		buf = buf[:n]
 	}
-	setServeHeadersByFile(r, w, filePath, buf)
+	setServeHeadersByFile(r, w, buf, opts)
 	if modTime == nil {
 		modTime = &time.Time{}
 	}
-	http.ServeContent(w, r, path.Base(filePath), *modTime, reader)
+	http.ServeContent(w, r, opts.Filename, *modTime, reader)
 }

modules/httplib/serve_test.go

@@ -27,7 +27,7 @@ func TestServeContentByReader(t *testing.T) {
 		}
 		reader := strings.NewReader(data)
 		w := httptest.NewRecorder()
-		ServeContentByReader(r, w, "test", int64(len(data)), reader)
+		ServeContentByReader(r, w, int64(len(data)), reader, &ServeHeaderOptions{})
 		assert.Equal(t, expectedStatusCode, w.Code)
 		if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK {
 			assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length"))
@@ -76,7 +76,7 @@ func TestServeContentByReadSeeker(t *testing.T) {
 		defer seekReader.Close()
 
 		w := httptest.NewRecorder()
-		ServeContentByReadSeeker(r, w, "test", nil, seekReader)
+		ServeContentByReadSeeker(r, w, nil, seekReader, &ServeHeaderOptions{})
 		assert.Equal(t, expectedStatusCode, w.Code)
 		if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK {
 			assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length"))

modules/public/public.go

@@ -86,17 +86,17 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem,
 		return
 	}
 
-	serveContent(w, req, fi, fi.ModTime(), f)
+	servePublicAsset(w, req, fi, fi.ModTime(), f)
 }
 
 type GzipBytesProvider interface {
 	GzipBytes() []byte
 }
 
-// serveContent serve http content
-func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
+// servePublicAsset serve http content
+func servePublicAsset(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
 	setWellKnownContentType(w, fi.Name())
-
+	httpcache.SetCacheControlInHeader(w.Header(), httpcache.CacheControlForPublicStatic())
 	encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding"))
 	if encodings.Contains("gzip") {
 		// try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo)
@@ -108,11 +108,10 @@ func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modt
 				w.Header().Set("Content-Type", "application/octet-stream")
 			}
 			w.Header().Set("Content-Encoding", "gzip")
-			httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip)
+			http.ServeContent(w, req, fi.Name(), modtime, rdGzip)
 			return
 		}
 	}
-
-	httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, content)
+	http.ServeContent(w, req, fi.Name(), modtime, content)
 	return
 }

routers/api/v1/repo/file.go

@@ -83,7 +83,7 @@ func GetRawFile(ctx *context.APIContext) {
 
 	ctx.RespHeader().Set(giteaObjectTypeHeader, string(files_service.GetObjectTypeFromTreeEntry(entry)))
 
-	if err := common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified); err != nil {
+	if err := common.ServeBlob(ctx.Base, ctx.Repo.Repository, ctx.Repo.TreePath, blob, lastModified); err != nil {
 		ctx.APIErrorInternal(err)
 	}
 }
@@ -144,7 +144,7 @@ func GetRawFileOrLFS(ctx *context.APIContext) {
 		}
 
 		// OK not cached - serve!
-		if err := common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified); err != nil {
+		if err := common.ServeBlob(ctx.Base, ctx.Repo.Repository, ctx.Repo.TreePath, blob, lastModified); err != nil {
 			ctx.APIErrorInternal(err)
 		}
 		return

routers/common/errpage.go

@@ -32,7 +32,7 @@ func RenderPanicErrorPage(w http.ResponseWriter, req *http.Request, err any) {
 
 	routing.UpdatePanicError(req.Context(), err)
 
-	httpcache.SetCacheControlInHeader(w.Header(), 0, "no-transform")
+	httpcache.SetCacheControlInHeader(w.Header(), &httpcache.CacheControlOptions{NoTransform: true})
 	w.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions)
 
 	tmplCtx := context.TemplateContext{}

routers/common/serve.go

@@ -5,17 +5,21 @@ package common
 
 import (
 	"io"
+	"path"
 	"time"
 
+	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/httpcache"
 	"code.gitea.io/gitea/modules/httplib"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/services/context"
 )
 
 // ServeBlob download a git.Blob
-func ServeBlob(ctx *context.Base, filePath string, blob *git.Blob, lastModified *time.Time) error {
+func ServeBlob(ctx *context.Base, repo *repo_model.Repository, filePath string, blob *git.Blob, lastModified *time.Time) error {
 	if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) {
 		return nil
 	}
@@ -30,14 +34,19 @@ func ServeBlob(ctx *context.Base, filePath string, blob *git.Blob, lastModified
 		}
 	}()
 
-	httplib.ServeContentByReader(ctx.Req, ctx.Resp, filePath, blob.Size(), dataRc)
+	_ = repo.LoadOwner(ctx)
+	httplib.ServeContentByReader(ctx.Req, ctx.Resp, blob.Size(), dataRc, &httplib.ServeHeaderOptions{
+		Filename:      path.Base(filePath),
+		CacheIsPublic: !repo.IsPrivate && repo.Owner != nil && repo.Owner.Visibility == structs.VisibleTypePublic,
+		CacheDuration: setting.StaticCacheTime,
+	})
 	return nil
 }
 
 func ServeContentByReader(ctx *context.Base, filePath string, size int64, reader io.Reader) {
-	httplib.ServeContentByReader(ctx.Req, ctx.Resp, filePath, size, reader)
+	httplib.ServeContentByReader(ctx.Req, ctx.Resp, size, reader, &httplib.ServeHeaderOptions{Filename: path.Base(filePath)})
 }
 
 func ServeContentByReadSeeker(ctx *context.Base, filePath string, modTime *time.Time, reader io.ReadSeeker) {
-	httplib.ServeContentByReadSeeker(ctx.Req, ctx.Resp, filePath, modTime, reader)
+	httplib.ServeContentByReadSeeker(ctx.Req, ctx.Resp, modTime, reader, &httplib.ServeHeaderOptions{Filename: path.Base(filePath)})
 }

routers/web/base.go

@@ -19,12 +19,12 @@ import (
 	"code.gitea.io/gitea/modules/web/routing"
 )
 
-func storageHandler(storageSetting *setting.Storage, prefix string, objStore storage.ObjectStorage) http.HandlerFunc {
+func avatarStorageHandler(storageSetting *setting.Storage, prefix string, objStore storage.ObjectStorage) http.HandlerFunc {
 	prefix = strings.Trim(prefix, "/")
-	funcInfo := routing.GetFuncInfo(storageHandler, prefix)
+	funcInfo := routing.GetFuncInfo(avatarStorageHandler, prefix)
 
 	if storageSetting.ServeDirect() {
-		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		return func(w http.ResponseWriter, req *http.Request) {
 			if req.Method != "GET" && req.Method != "HEAD" {
 				http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
 				return
@@ -52,10 +52,10 @@ func storageHandler(storageSetting *setting.Storage, prefix string, objStore sto
 			}
 
 			http.Redirect(w, req, u.String(), http.StatusTemporaryRedirect)
-		})
+		}
 	}
 
-	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
 		if req.Method != "GET" && req.Method != "HEAD" {
 			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
 			return
@@ -93,6 +93,8 @@ func storageHandler(storageSetting *setting.Storage, prefix string, objStore sto
 			return
 		}
 		defer fr.Close()
-		httpcache.ServeContentWithCacheControl(w, req, path.Base(rPath), fi.ModTime(), fr)
-	})
+
+		httpcache.SetCacheControlInHeader(w.Header(), httpcache.CacheControlForPublicStatic())
+		http.ServeContent(w, req, path.Base(rPath), fi.ModTime(), fr)
+	}
 }

routers/web/misc/misc.go

@@ -38,7 +38,7 @@ func RobotsTxt(w http.ResponseWriter, req *http.Request) {
 	if ok, _ := util.IsExist(robotsTxt); !ok {
 		robotsTxt = util.FilePathJoinAbs(setting.CustomPath, "robots.txt") // the legacy "robots.txt"
 	}
-	httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
+	httpcache.SetCacheControlInHeader(w.Header(), httpcache.CacheControlForPublicStatic())
 	http.ServeFile(w, req, robotsTxt)
 }
 

routers/web/repo/download.go

@@ -46,7 +46,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim
 				log.Error("ServeBlobOrLFS: Close: %v", err)
 			}
 			closed = true
-			return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified)
+			return common.ServeBlob(ctx.Base, ctx.Repo.Repository, ctx.Repo.TreePath, blob, lastModified)
 		}
 		if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+pointer.Oid+`"`) {
 			return nil
@@ -78,7 +78,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim
 	}
 	closed = true
 
-	return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified)
+	return common.ServeBlob(ctx.Base, ctx.Repo.Repository, ctx.Repo.TreePath, blob, lastModified)
 }
 
 func getBlobForEntry(ctx *context.Context) (*git.Blob, *time.Time) {
@@ -114,7 +114,7 @@ func SingleDownload(ctx *context.Context) {
 		return
 	}
 
-	if err := common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified); err != nil {
+	if err := common.ServeBlob(ctx.Base, ctx.Repo.Repository, ctx.Repo.TreePath, blob, lastModified); err != nil {
 		ctx.ServerError("ServeBlob", err)
 	}
 }
@@ -142,7 +142,7 @@ func DownloadByID(ctx *context.Context) {
 		}
 		return
 	}
-	if err = common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, nil); err != nil {
+	if err = common.ServeBlob(ctx.Base, ctx.Repo.Repository, ctx.Repo.TreePath, blob, nil); err != nil {
 		ctx.ServerError("ServeBlob", err)
 	}
 }