Merge branch 'main' into runners-rest-api

ChristopherHX · web-flow · commit e95db5abfbb0 · 2025-03-14T20:30:47.000+01:00
diff --git a/models/actions/runner.go b/models/actions/runner.go
@@ -57,6 +57,8 @@ type ActionRunner struct {
 
 	// Store labels defined in state file (default: .runner file) of `act_runner`
 	AgentLabels []string `xorm:"TEXT"`
+	// Store if this is a runner that only ever get one single job assigned
+	Ephemeral bool `xorm:"ephemeral NOT NULL DEFAULT false"`
 
 	Created timeutil.TimeStamp `xorm:"created"`
 	Updated timeutil.TimeStamp `xorm:"updated"`
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
@@ -375,6 +375,7 @@ func prepareMigrationTasks() []*migration {
 		newMigration(312, "Add DeleteBranchAfterMerge to AutoMerge", v1_24.AddDeleteBranchAfterMergeForAutoMerge),
 		newMigration(313, "Move PinOrder from issue table to a new table issue_pin", v1_24.MovePinOrderToTableIssuePin),
 		newMigration(314, "Update OwnerID as zero for repository level action tables", v1_24.UpdateOwnerIDOfRepoLevelActionsTables),
+		newMigration(315, "Add Ephemeral to ActionRunner", v1_24.AddEphemeralToActionRunner),
 	}
 	return preparedMigrations
 }
diff --git a/models/migrations/v1_24/v315.go b/models/migrations/v1_24/v315.go
@@ -0,0 +1,16 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_24 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddEphemeralToActionRunner(x *xorm.Engine) error {
+	type ActionRunner struct {
+		Ephemeral bool `xorm:"ephemeral NOT NULL DEFAULT false"`
+	}
+
+	return x.Sync(new(ActionRunner))
+}
diff --git a/modules/gitrepo/gitrepo.go b/modules/gitrepo/gitrepo.go
@@ -5,6 +5,7 @@ package gitrepo
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"path/filepath"
 	"strings"
@@ -20,8 +21,12 @@ type Repository interface {
 	GetOwnerName() string
 }
 
+func absPath(owner, name string) string {
+	return filepath.Join(setting.RepoRootPath, strings.ToLower(owner), strings.ToLower(name)+".git")
+}
+
 func repoPath(repo Repository) string {
-	return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".git")
+	return absPath(repo.GetOwnerName(), repo.GetName())
 }
 
 func wikiPath(repo Repository) string {
@@ -74,3 +79,17 @@ func RepositoryFromRequestContextOrOpen(ctx reqctx.RequestContext, repo Reposito
 func IsRepositoryExist(ctx context.Context, repo Repository) (bool, error) {
 	return util.IsExist(repoPath(repo))
 }
+
+// DeleteRepository deletes the repository directory from the disk
+func DeleteRepository(ctx context.Context, repo Repository) error {
+	return util.RemoveAll(repoPath(repo))
+}
+
+// RenameRepository renames a repository's name on disk
+func RenameRepository(ctx context.Context, repo Repository, newName string) error {
+	newRepoPath := absPath(repo.GetOwnerName(), newName)
+	if err := util.Rename(repoPath(repo), newRepoPath); err != nil {
+		return fmt.Errorf("rename repository directory: %w", err)
+	}
+	return nil
+}
diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go
@@ -25,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/util"
 
 	"github.com/blevesearch/bleve/v2"
 	analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -272,14 +273,18 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	pathQuery.FieldVal = "Filename"
 	pathQuery.SetBoost(10)
 
-	if opts.SearchMode == indexer.SearchModeExact {
+	searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
+	if searchMode == indexer.SearchModeExact {
+		// 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery
 		q := bleve.NewMatchPhraseQuery(opts.Keyword)
+		q.Analyzer = repoIndexerAnalyzer
 		q.FieldVal = "Content"
 		contentQuery = q
 	} else /* words */ {
 		q := bleve.NewMatchQuery(opts.Keyword)
 		q.FieldVal = "Content"
-		if opts.SearchMode == indexer.SearchModeFuzzy {
+		q.Analyzer = repoIndexerAnalyzer
+		if searchMode == indexer.SearchModeFuzzy {
 			// this logic doesn't seem right, it is only used to pass the test-case `Keyword:    "dESCRIPTION"`, which doesn't seem to be a real-life use-case.
 			q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
 		} else {
diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go
@@ -25,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/util"
 
 	"github.com/go-enry/go-enry/v2"
 	"github.com/olivere/elastic/v7"
@@ -365,7 +366,9 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan
 // Search searches for codes and language stats by given conditions.
 func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	var contentQuery elastic.Query
-	if opts.SearchMode == indexer.SearchModeExact {
+	searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
+	if searchMode == indexer.SearchModeExact {
+		// 1.21 used NewMultiMatchQuery().Type(esMultiMatchTypePhrasePrefix), but later releases changed to NewMatchPhraseQuery
 		contentQuery = elastic.NewMatchPhraseQuery("content", opts.Keyword)
 	} else /* words */ {
 		contentQuery = elastic.NewMultiMatchQuery("content", opts.Keyword).Type(esMultiMatchTypeBestFields).Operator("and")
diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go
@@ -17,6 +17,7 @@ import (
 	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/test"
+	"code.gitea.io/gitea/modules/util"
 
 	_ "code.gitea.io/gitea/models"
 	_ "code.gitea.io/gitea/models/actions"
@@ -240,7 +241,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 				total, res, langs, err := indexer.Search(t.Context(), &internal.SearchOptions{
 					RepoIDs:    kw.RepoIDs,
 					Keyword:    kw.Keyword,
-					SearchMode: kw.SearchMode,
+					SearchMode: util.IfZero(kw.SearchMode, indexer_module.SearchModeWords),
 					Paginator: &db.ListOptions{
 						Page:     1,
 						PageSize: 10,
diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go
@@ -10,6 +10,7 @@ import (
 	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
 	inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
 	"code.gitea.io/gitea/modules/indexer/issues/internal"
+	"code.gitea.io/gitea/modules/util"
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -162,9 +163,10 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 	var queries []query.Query
 
 	if options.Keyword != "" {
-		if options.SearchMode == indexer.SearchModeWords || options.SearchMode == indexer.SearchModeFuzzy {
+		searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue)
+		if searchMode == indexer.SearchModeWords || searchMode == indexer.SearchModeFuzzy {
 			fuzziness := 0
-			if options.SearchMode == indexer.SearchModeFuzzy {
+			if searchMode == indexer.SearchModeFuzzy {
 				fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword)
 			}
 			queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go
@@ -13,6 +13,7 @@ import (
 	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
 	inner_db "code.gitea.io/gitea/modules/indexer/internal/db"
 	"code.gitea.io/gitea/modules/indexer/issues/internal"
+	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/builder"
 )
@@ -46,7 +47,7 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error {
 
 func buildMatchQuery(mode indexer.SearchModeType, colName, keyword string) builder.Cond {
 	if mode == indexer.SearchModeExact {
-		return db.BuildCaseInsensitiveLike("issue.name", keyword)
+		return db.BuildCaseInsensitiveLike(colName, keyword)
 	}
 
 	// match words
@@ -84,16 +85,16 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 			repoCond = builder.Eq{"repo_id": options.RepoIDs[0]}
 		}
 		subQuery := builder.Select("id").From("issue").Where(repoCond)
-
+		searchMode := util.IfZero(options.SearchMode, i.SupportedSearchModes()[0].ModeValue)
 		cond = builder.Or(
-			buildMatchQuery(options.SearchMode, "issue.name", options.Keyword),
-			buildMatchQuery(options.SearchMode, "issue.content", options.Keyword),
+			buildMatchQuery(searchMode, "issue.name", options.Keyword),
+			buildMatchQuery(searchMode, "issue.content", options.Keyword),
 			builder.In("issue.id", builder.Select("issue_id").
 				From("comment").
 				Where(builder.And(
 					builder.Eq{"type": issue_model.CommentTypeComment},
 					builder.In("issue_id", subQuery),
-					buildMatchQuery(options.SearchMode, "content", options.Keyword),
+					buildMatchQuery(searchMode, "content", options.Keyword),
 				)),
 			),
 		)
diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go
@@ -14,6 +14,7 @@ import (
 	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
 	inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
 	"code.gitea.io/gitea/modules/indexer/issues/internal"
+	"code.gitea.io/gitea/modules/util"
 
 	"github.com/olivere/elastic/v7"
 )
@@ -152,7 +153,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 	query := elastic.NewBoolQuery()
 
 	if options.Keyword != "" {
-		if options.SearchMode == indexer.SearchModeExact {
+		searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue)
+		if searchMode == indexer.SearchModeExact {
 			query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypePhrasePrefix))
 		} else /* words */ {
 			query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypeBestFields).Operator("and"))
diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go
@@ -282,7 +282,7 @@ const (
 
 // SearchIssues search issues by options.
 func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) {
-	indexer := *globalIndexer.Load()
+	ix := *globalIndexer.Load()
 
 	if opts.Keyword == "" || opts.IsKeywordNumeric() {
 		// This is a conservative shortcut.
@@ -291,10 +291,9 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err
 		// So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue.
 		// Even worse, the external indexer like elastic search may not be available for a while,
 		// and the user may not be able to list issues completely until it is available again.
-		indexer = db.NewIndexer()
+		ix = db.NewIndexer()
 	}
-
-	result, err := indexer.Search(ctx, opts)
+	result, err := ix.Search(ctx, opts)
 	if err != nil {
 		return nil, 0, err
 	}
diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go
@@ -82,9 +82,11 @@ func searchIssueWithKeyword(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		issueIDs, _, err := SearchIssues(t.Context(), &test.opts)
-		require.NoError(t, err)
-		assert.Equal(t, test.expectedIDs, issueIDs)
+		t.Run(test.opts.Keyword, func(t *testing.T) {
+			issueIDs, _, err := SearchIssues(t.Context(), &test.opts)
+			require.NoError(t, err)
+			assert.Equal(t, test.expectedIDs, issueIDs)
+		})
 	}
 }
 
diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go
@@ -78,6 +78,7 @@ func (s *Service) Register(
 		RepoID:      runnerToken.RepoID,
 		Version:     req.Msg.Version,
 		AgentLabels: labels,
+		Ephemeral:   req.Msg.Ephemeral,
 	}
 	if err := runner.GenerateToken(); err != nil {
 		return nil, errors.New("can't generate token")
@@ -96,12 +97,13 @@ func (s *Service) Register(
 
 	res := connect.NewResponse(&runnerv1.RegisterResponse{
 		Runner: &runnerv1.Runner{
-			Id:      runner.ID,
-			Uuid:    runner.UUID,
-			Token:   runner.Token,
-			Name:    runner.Name,
-			Version: runner.Version,
-			Labels:  runner.AgentLabels,
+			Id:        runner.ID,
+			Uuid:      runner.UUID,
+			Token:     runner.Token,
+			Name:      runner.Name,
+			Version:   runner.Version,
+			Labels:    runner.AgentLabels,
+			Ephemeral: runner.Ephemeral,
 		},
 	})
 
diff --git a/routers/web/repo/blame.go b/routers/web/repo/blame.go
@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
@@ -105,7 +106,7 @@ func RefBlame(ctx *context.Context) {
 
 	bypassBlameIgnore, _ := strconv.ParseBool(ctx.FormString("bypass-blame-ignore"))
 
-	result, err := performBlame(ctx, ctx.Repo.Repository.RepoPath(), ctx.Repo.Commit, fileName, bypassBlameIgnore)
+	result, err := performBlame(ctx, ctx.Repo.Repository, ctx.Repo.Commit, fileName, bypassBlameIgnore)
 	if err != nil {
 		ctx.NotFound(err)
 		return
@@ -130,10 +131,10 @@ type blameResult struct {
 	FaultyIgnoreRevsFile bool
 }
 
-func performBlame(ctx *context.Context, repoPath string, commit *git.Commit, file string, bypassBlameIgnore bool) (*blameResult, error) {
+func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git.Commit, file string, bypassBlameIgnore bool) (*blameResult, error) {
 	objectFormat := ctx.Repo.GetObjectFormat()
 
-	blameReader, err := git.CreateBlameReader(ctx, objectFormat, repoPath, commit, file, bypassBlameIgnore)
+	blameReader, err := git.CreateBlameReader(ctx, objectFormat, repo.RepoPath(), commit, file, bypassBlameIgnore)
 	if err != nil {
 		return nil, err
 	}
@@ -149,7 +150,7 @@ func performBlame(ctx *context.Context, repoPath string, commit *git.Commit, fil
 		if len(r.Parts) == 0 && r.UsesIgnoreRevs {
 			// try again without ignored revs
 
-			blameReader, err = git.CreateBlameReader(ctx, objectFormat, repoPath, commit, file, true)
+			blameReader, err = git.CreateBlameReader(ctx, objectFormat, repo.RepoPath(), commit, file, true)
 			if err != nil {
 				return nil, err
 			}
diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go
@@ -9,14 +9,17 @@ import (
 	"time"
 
 	actions_model "code.gitea.io/gitea/models/actions"
+	"code.gitea.io/gitea/models/db"
 	actions_module "code.gitea.io/gitea/modules/actions"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/builder"
 )
 
-// Cleanup removes expired actions logs, data and artifacts
+// Cleanup removes expired actions logs, data, artifacts and used ephemeral runners
 func Cleanup(ctx context.Context) error {
 	// clean up expired artifacts
 	if err := CleanupArtifacts(ctx); err != nil {
@@ -28,6 +31,11 @@ func Cleanup(ctx context.Context) error {
 		return fmt.Errorf("cleanup logs: %w", err)
 	}
 
+	// clean up old ephemeral runners
+	if err := CleanupEphemeralRunners(ctx); err != nil {
+		return fmt.Errorf("cleanup old ephemeral runners: %w", err)
+	}
+
 	return nil
 }
 
@@ -123,3 +131,20 @@ func CleanupLogs(ctx context.Context) error {
 	log.Info("Removed %d logs", count)
 	return nil
 }
+
+// CleanupEphemeralRunners removes used ephemeral runners which are no longer able to process jobs
+func CleanupEphemeralRunners(ctx context.Context) error {
+	subQuery := builder.Select("`action_runner`.id").
+		From(builder.Select("*").From("`action_runner`"), "`action_runner`"). // mysql needs this redundant subquery
+		Join("INNER", "`action_task`", "`action_task`.`runner_id` = `action_runner`.`id`").
+		Where(builder.Eq{"`action_runner`.`ephemeral`": true}).
+		And(builder.NotIn("`action_task`.`status`", actions_model.StatusWaiting, actions_model.StatusRunning, actions_model.StatusBlocked))
+	b := builder.Delete(builder.In("id", subQuery)).From("`action_runner`")
+	res, err := db.GetEngine(ctx).Exec(b)
+	if err != nil {
+		return fmt.Errorf("find runners: %w", err)
+	}
+	affected, _ := res.RowsAffected()
+	log.Info("Removed %d runners", affected)
+	return nil
+}
diff --git a/services/actions/task.go b/services/actions/task.go
@@ -23,6 +23,26 @@ func PickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv
 		actionTask *actions_model.ActionTask
 	)
 
+	if runner.Ephemeral {
+		var task actions_model.ActionTask
+		has, err := db.GetEngine(ctx).Where("runner_id = ?", runner.ID).Get(&task)
+		// Let the runner retry the request, do not allow to proceed
+		if err != nil {
+			return nil, false, err
+		}
+		if has {
+			if task.Status == actions_model.StatusWaiting || task.Status == actions_model.StatusRunning || task.Status == actions_model.StatusBlocked {
+				return nil, false, nil
+			}
+			// task has been finished, remove it
+			_, err = db.DeleteByID[actions_model.ActionRunner](ctx, runner.ID)
+			if err != nil {
+				return nil, false, err
+			}
+			return nil, false, fmt.Errorf("runner has been removed")
+		}
+	}
+
 	if err := db.WithTx(ctx, func(ctx context.Context) error {
 		t, ok, err := actions_model.CreateTaskForRunner(ctx, runner)
 		if err != nil {
diff --git a/services/repository/create.go b/services/repository/create.go
diff --git a/services/repository/delete.go b/services/repository/delete.go
diff --git a/services/repository/fork.go b/services/repository/fork.go
diff --git a/services/repository/transfer.go b/services/repository/transfer.go
diff --git a/tests/integration/actions_job_test.go b/tests/integration/actions_job_test.go
diff --git a/tests/integration/actions_log_test.go b/tests/integration/actions_log_test.go
diff --git a/tests/integration/actions_runner_test.go b/tests/integration/actions_runner_test.go
diff --git a/tests/integration/repo_webhook_test.go b/tests/integration/repo_webhook_test.go

Original file line number	Diff line number	Diff line change
`@@ -375,6 +375,7 @@ func prepareMigrationTasks() []*migration {`
`375`	`375`	`newMigration(312, "Add DeleteBranchAfterMerge to AutoMerge", v1_24.AddDeleteBranchAfterMergeForAutoMerge),`
`376`	`376`	`newMigration(313, "Move PinOrder from issue table to a new table issue_pin", v1_24.MovePinOrderToTableIssuePin),`
`377`	`377`	`newMigration(314, "Update OwnerID as zero for repository level action tables", v1_24.UpdateOwnerIDOfRepoLevelActionsTables),`
	`378`	`+ newMigration(315, "Add Ephemeral to ActionRunner", v1_24.AddEphemeralToActionRunner),`
`378`	`379`	`}`
`379`	`380`	`return preparedMigrations`
`380`	`381`	`}`