Merge branch 'main' into lunny/actions_log_api

Author: lunny

modules/git/grep.go

@@ -23,11 +23,19 @@ type GrepResult struct {
 	LineCodes   []string
 }
 
+type GrepModeType string
+
+const (
+	GrepModeExact  GrepModeType = "exact"
+	GrepModeWords  GrepModeType = "words"
+	GrepModeRegexp GrepModeType = "regexp"
+)
+
 type GrepOptions struct {
 	RefName           string
 	MaxResultLimit    int
 	ContextLineNumber int
-	IsFuzzy           bool
+	GrepMode          GrepModeType
 	MaxLineLength     int // the maximum length of a line to parse, exceeding chars will be truncated
 	PathspecList      []string
 }
@@ -52,15 +60,23 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
 	 2^@repo: go-gitea/gitea
 	*/
 	var results []*GrepResult
-	cmd := NewCommand("grep", "--null", "--break", "--heading", "--fixed-strings", "--line-number", "--ignore-case", "--full-name")
+	cmd := NewCommand("grep", "--null", "--break", "--heading", "--line-number", "--full-name")
 	cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber))
-	if opts.IsFuzzy {
+	if opts.GrepMode == GrepModeExact {
+		cmd.AddArguments("--fixed-strings")
+		cmd.AddOptionValues("-e", strings.TrimLeft(search, "-"))
+	} else if opts.GrepMode == GrepModeRegexp {
+		cmd.AddArguments("--perl-regexp")
+		cmd.AddOptionValues("-e", strings.TrimLeft(search, "-"))
+	} else /* words */ {
 		words := strings.Fields(search)
-		for _, word := range words {
+		cmd.AddArguments("--fixed-strings", "--ignore-case")
+		for i, word := range words {
 			cmd.AddOptionValues("-e", strings.TrimLeft(word, "-"))
+			if i < len(words)-1 {
+				cmd.AddOptionValues("--and")
+			}
 		}
-	} else {
-		cmd.AddOptionValues("-e", strings.TrimLeft(search, "-"))
 	}
 	cmd.AddDynamicArguments(util.IfZero(opts.RefName, "HEAD"))
 	cmd.AddDashesAndList(opts.PathspecList...)

modules/httpcache/httpcache.go

@@ -4,40 +4,60 @@
 package httpcache
 
 import (
-	"io"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
 )
 
+type CacheControlOptions struct {
+	IsPublic    bool
+	MaxAge      time.Duration
+	NoTransform bool
+}
+
 // SetCacheControlInHeader sets suitable cache-control headers in the response
-func SetCacheControlInHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) {
-	directives := make([]string, 0, 2+len(additionalDirectives))
+func SetCacheControlInHeader(h http.Header, opts *CacheControlOptions) {
+	directives := make([]string, 0, 4)
 
 	// "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store"
 	// because browsers may restore some input fields after navigate-back / reload a page.
+	publicPrivate := util.Iif(opts.IsPublic, "public", "private")
 	if setting.IsProd {
-		if maxAge == 0 {
+		if opts.MaxAge == 0 {
 			directives = append(directives, "max-age=0", "private", "must-revalidate")
 		} else {
-			directives = append(directives, "private", "max-age="+strconv.Itoa(int(maxAge.Seconds())))
+			directives = append(directives, publicPrivate, "max-age="+strconv.Itoa(int(opts.MaxAge.Seconds())))
 		}
 	} else {
-		directives = append(directives, "max-age=0", "private", "must-revalidate")
+		// use dev-related controls, and remind users they are using non-prod setting.
+		directives = append(directives, "max-age=0", publicPrivate, "must-revalidate")
+		h.Set("X-Gitea-Debug", fmt.Sprintf("RUN_MODE=%v, MaxAge=%s", setting.RunMode, opts.MaxAge))
+	}
 
-		// to remind users they are using non-prod setting.
-		h.Set("X-Gitea-Debug", "RUN_MODE="+setting.RunMode)
+	if opts.NoTransform {
+		directives = append(directives, "no-transform")
 	}
+	h.Set("Cache-Control", strings.Join(directives, ", "))
+}
 
-	h.Set("Cache-Control", strings.Join(append(directives, additionalDirectives...), ", "))
+func CacheControlForPublicStatic() *CacheControlOptions {
+	return &CacheControlOptions{
+		IsPublic:    true,
+		MaxAge:      setting.StaticCacheTime,
+		NoTransform: true,
+	}
 }
 
-func ServeContentWithCacheControl(w http.ResponseWriter, req *http.Request, name string, modTime time.Time, content io.ReadSeeker) {
-	SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
-	http.ServeContent(w, req, name, modTime, content)
+func CacheControlForPrivateStatic() *CacheControlOptions {
+	return &CacheControlOptions{
+		MaxAge:      setting.StaticCacheTime,
+		NoTransform: true,
+	}
 }
 
 // HandleGenericETagCache handles ETag-based caching for a HTTP request.
@@ -50,7 +70,8 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin
 			return true
 		}
 	}
-	SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
+	// not sure whether it is a public content, so just use "private" (old behavior)
+	SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic())
 	return false
 }
 
@@ -95,6 +116,8 @@ func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag s
 			}
 		}
 	}
-	SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
+
+	// not sure whether it is a public content, so just use "private" (old behavior)
+	SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic())
 	return false
 }

modules/httplib/serve.go

@@ -33,6 +33,7 @@ type ServeHeaderOptions struct {
 	ContentLength      *int64
 	Disposition        string // defaults to "attachment"
 	Filename           string
+	CacheIsPublic      bool
 	CacheDuration      time.Duration // defaults to 5 minutes
 	LastModified       time.Time
 }
@@ -72,11 +73,11 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
 		header.Set("Access-Control-Expose-Headers", "Content-Disposition")
 	}
 
-	duration := opts.CacheDuration
-	if duration == 0 {
-		duration = 5 * time.Minute
-	}
-	httpcache.SetCacheControlInHeader(header, duration)
+	httpcache.SetCacheControlInHeader(header, &httpcache.CacheControlOptions{
+		IsPublic:    opts.CacheIsPublic,
+		MaxAge:      opts.CacheDuration,
+		NoTransform: true,
+	})
 
 	if !opts.LastModified.IsZero() {
 		// http.TimeFormat required a UTC time, refer to https://pkg.go.dev/net/http#TimeFormat
@@ -85,19 +86,15 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
 }
 
 // ServeData download file from io.Reader
-func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) {
+func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byte, opts *ServeHeaderOptions) {
 	// do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests
-	opts := &ServeHeaderOptions{
-		Filename: path.Base(filePath),
-	}
-
 	sniffedType := typesniffer.DetectContentType(mineBuf)
 
 	// the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later
 	isPlain := sniffedType.IsText() || r.FormValue("render") != ""
 
 	if setting.MimeTypeMap.Enabled {
-		fileExtension := strings.ToLower(filepath.Ext(filePath))
+		fileExtension := strings.ToLower(filepath.Ext(opts.Filename))
 		opts.ContentType = setting.MimeTypeMap.Map[fileExtension]
 	}
 
@@ -114,7 +111,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri
 	if isPlain {
 		charset, err := charsetModule.DetectEncoding(mineBuf)
 		if err != nil {
-			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err)
+			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err)
 			charset = "utf-8"
 		}
 		opts.ContentTypeCharset = strings.ToLower(charset)
@@ -142,7 +139,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri
 
 const mimeDetectionBufferLen = 1024
 
-func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) {
+func ServeContentByReader(r *http.Request, w http.ResponseWriter, size int64, reader io.Reader, opts *ServeHeaderOptions) {
 	buf := make([]byte, mimeDetectionBufferLen)
 	n, err := util.ReadAtMost(reader, buf)
 	if err != nil {
@@ -152,7 +149,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin
 	if n >= 0 {
 		buf = buf[:n]
 	}
-	setServeHeadersByFile(r, w, filePath, buf)
+	setServeHeadersByFile(r, w, buf, opts)
 
 	// reset the reader to the beginning
 	reader = io.MultiReader(bytes.NewReader(buf), reader)
@@ -215,7 +212,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin
 	_, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error
 }
 
-func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime *time.Time, reader io.ReadSeeker) {
+func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, modTime *time.Time, reader io.ReadSeeker, opts *ServeHeaderOptions) {
 	buf := make([]byte, mimeDetectionBufferLen)
 	n, err := util.ReadAtMost(reader, buf)
 	if err != nil {
@@ -229,9 +226,9 @@ func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath s
 	if n >= 0 {
 		buf = buf[:n]
 	}
-	setServeHeadersByFile(r, w, filePath, buf)
+	setServeHeadersByFile(r, w, buf, opts)
 	if modTime == nil {
 		modTime = &time.Time{}
 	}
-	http.ServeContent(w, r, path.Base(filePath), *modTime, reader)
+	http.ServeContent(w, r, opts.Filename, *modTime, reader)
 }

modules/httplib/serve_test.go

@@ -27,7 +27,7 @@ func TestServeContentByReader(t *testing.T) {
 		}
 		reader := strings.NewReader(data)
 		w := httptest.NewRecorder()
-		ServeContentByReader(r, w, "test", int64(len(data)), reader)
+		ServeContentByReader(r, w, int64(len(data)), reader, &ServeHeaderOptions{})
 		assert.Equal(t, expectedStatusCode, w.Code)
 		if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK {
 			assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length"))
@@ -76,7 +76,7 @@ func TestServeContentByReadSeeker(t *testing.T) {
 		defer seekReader.Close()
 
 		w := httptest.NewRecorder()
-		ServeContentByReadSeeker(r, w, "test", nil, seekReader)
+		ServeContentByReadSeeker(r, w, nil, seekReader, &ServeHeaderOptions{})
 		assert.Equal(t, expectedStatusCode, w.Code)
 		if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK {
 			assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length"))

modules/indexer/code/bleve/bleve.go

@@ -17,6 +17,7 @@ import (
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/indexer"
 	path_filter "code.gitea.io/gitea/modules/indexer/code/bleve/token/path"
 	"code.gitea.io/gitea/modules/indexer/code/internal"
 	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
@@ -136,6 +137,10 @@ type Indexer struct {
 	indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much
 }
 
+func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
+	return indexer.SearchModesExactWords()
+}
+
 // NewIndexer creates a new bleve local indexer
 func NewIndexer(indexDir string) *Indexer {
 	inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping)
@@ -267,19 +272,18 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	pathQuery.FieldVal = "Filename"
 	pathQuery.SetBoost(10)
 
-	keywordAsPhrase, isPhrase := internal.ParseKeywordAsPhrase(opts.Keyword)
-	if isPhrase {
-		q := bleve.NewMatchPhraseQuery(keywordAsPhrase)
+	if opts.SearchMode == indexer.SearchModeExact {
+		q := bleve.NewMatchPhraseQuery(opts.Keyword)
 		q.FieldVal = "Content"
-		if opts.IsKeywordFuzzy {
-			q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(keywordAsPhrase)
-		}
 		contentQuery = q
-	} else {
+	} else /* words */ {
 		q := bleve.NewMatchQuery(opts.Keyword)
 		q.FieldVal = "Content"
-		if opts.IsKeywordFuzzy {
+		if opts.SearchMode == indexer.SearchModeFuzzy {
+			// this logic doesn't seem right, it is only used to pass the test-case `Keyword:    "dESCRIPTION"`, which doesn't seem to be a real-life use-case.
 			q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
+		} else {
+			q.Operator = query.MatchQueryOperatorAnd
 		}
 		contentQuery = q
 	}

modules/indexer/code/elasticsearch/elasticsearch.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/indexer/code/internal"
 	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
 	inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
@@ -24,7 +25,6 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/typesniffer"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/go-enry/go-enry/v2"
 	"github.com/olivere/elastic/v7"
@@ -46,6 +46,10 @@ type Indexer struct {
 	indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
 }
 
+func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
+	return indexer.SearchModesExactWords()
+}
+
 // NewIndexer creates a new elasticsearch indexer
 func NewIndexer(url, indexerName string) *Indexer {
 	inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping)
@@ -361,15 +365,10 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan
 // Search searches for codes and language stats by given conditions.
 func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	var contentQuery elastic.Query
-	keywordAsPhrase, isPhrase := internal.ParseKeywordAsPhrase(opts.Keyword)
-	if isPhrase {
-		contentQuery = elastic.NewMatchPhraseQuery("content", keywordAsPhrase)
-	} else {
-		// TODO: this is the old logic, but not really using "fuzziness"
-		// * IsKeywordFuzzy=true: "best_fields"
-		// * IsKeywordFuzzy=false: "phrase_prefix"
-		contentQuery = elastic.NewMultiMatchQuery("content", opts.Keyword).
-			Type(util.Iif(opts.IsKeywordFuzzy, esMultiMatchTypeBestFields, esMultiMatchTypePhrasePrefix))
+	if opts.SearchMode == indexer.SearchModeExact {
+		contentQuery = elastic.NewMatchPhraseQuery("content", opts.Keyword)
+	} else /* words */ {
+		contentQuery = elastic.NewMultiMatchQuery("content", opts.Keyword).Type(esMultiMatchTypeBestFields).Operator("and")
 	}
 	kwQuery := elastic.NewBoolQuery().Should(
 		contentQuery,

modules/indexer/code/gitgrep/gitgrep.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/indexer"
 	code_indexer "code.gitea.io/gitea/modules/indexer/code"
 	"code.gitea.io/gitea/modules/setting"
 )
@@ -23,11 +24,16 @@ func indexSettingToGitGrepPathspecList() (list []string) {
 	return list
 }
 
-func PerformSearch(ctx context.Context, page int, repoID int64, gitRepo *git.Repository, ref git.RefName, keyword string, isFuzzy bool) (searchResults []*code_indexer.Result, total int, err error) {
-	// TODO: it should also respect ParseKeywordAsPhrase and clarify the "fuzzy" behavior
+func PerformSearch(ctx context.Context, page int, repoID int64, gitRepo *git.Repository, ref git.RefName, keyword string, searchMode indexer.SearchModeType) (searchResults []*code_indexer.Result, total int, err error) {
+	grepMode := git.GrepModeWords
+	if searchMode == indexer.SearchModeExact {
+		grepMode = git.GrepModeExact
+	} else if searchMode == indexer.SearchModeRegexp {
+		grepMode = git.GrepModeRegexp
+	}
 	res, err := git.GrepSearch(ctx, gitRepo, keyword, git.GrepOptions{
 		ContextLineNumber: 1,
-		IsFuzzy:           isFuzzy,
+		GrepMode:          grepMode,
 		RefName:           ref.String(),
 		PathspecList:      indexSettingToGitGrepPathspecList(),
 	})

modules/indexer/code/indexer.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/indexer/code/bleve"
 	"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
 	"code.gitea.io/gitea/modules/indexer/code/internal"
@@ -302,3 +303,11 @@ func populateRepoIndexer(ctx context.Context) {
 	}
 	log.Info("Done (re)populating the repo indexer with existing repositories")
 }
+
+func SupportedSearchModes() []indexer.SearchMode {
+	gi := globalIndexer.Load()
+	if gi == nil {
+		return nil
+	}
+	return (*gi).SupportedSearchModes()
+}