Clear up old Actions logs (#31735)

Part of #24256.

Clear up old action logs to free up storage space.

Users will see a message indicating that the log has been cleared if
they view old tasks.

<img width="1361" alt="image"
src="https://github.com/user-attachments/assets/9f0f3a3a-bc5a-402f-90ca-49282d196c22">

Docs: https://gitea.com/gitea/docs/pulls/40

---------

Co-authored-by: silverwind <[email protected]>

Author: wolfogre

custom/conf/app.example.ini

@@ -2684,6 +2684,8 @@ LEVEL = Info
 ;;
 ;; Default platform to get action plugins, `github` for `https://github.com`, `self` for the current Gitea instance.
 ;DEFAULT_ACTIONS_URL = github
+;; Logs retention time in days. Old logs will be deleted after this period.
+;LOG_RETENTION_DAYS = 365
 ;; Default artifact retention time in days. Artifacts could have their own retention periods by setting the `retention-days` option in `actions/upload-artifact` step.
 ;ARTIFACT_RETENTION_DAYS = 90
 ;; Timeout to stop the task which have running status, but haven't been updated for a long time

models/actions/task.go

@@ -35,7 +35,7 @@ type ActionTask struct {
 	RunnerID int64              `xorm:"index"`
 	Status   Status             `xorm:"index"`
 	Started  timeutil.TimeStamp `xorm:"index"`
-	Stopped  timeutil.TimeStamp
+	Stopped  timeutil.TimeStamp `xorm:"index(stopped_log_expired)"`
 
 	RepoID            int64  `xorm:"index"`
 	OwnerID           int64  `xorm:"index"`
@@ -51,8 +51,8 @@ type ActionTask struct {
 	LogInStorage bool       // read log from database or from storage
 	LogLength    int64      // lines count
 	LogSize      int64      // blob size
-	LogIndexes   LogIndexes `xorm:"LONGBLOB"` // line number to offset
-	LogExpired   bool       // files that are too old will be deleted
+	LogIndexes   LogIndexes `xorm:"LONGBLOB"`                   // line number to offset
+	LogExpired   bool       `xorm:"index(stopped_log_expired)"` // files that are too old will be deleted
 
 	Created timeutil.TimeStamp `xorm:"created"`
 	Updated timeutil.TimeStamp `xorm:"updated index"`
@@ -470,6 +470,16 @@ func StopTask(ctx context.Context, taskID int64, status Status) error {
 	return nil
 }
 
+func FindOldTasksToExpire(ctx context.Context, olderThan timeutil.TimeStamp, limit int) ([]*ActionTask, error) {
+	e := db.GetEngine(ctx)
+
+	tasks := make([]*ActionTask, 0, limit)
+	// Check "stopped > 0" to avoid deleting tasks that are still running
+	return tasks, e.Where("stopped > 0 AND stopped < ? AND log_expired = ?", olderThan, false).
+		Limit(limit).
+		Find(&tasks)
+}
+
 func isSubset(set, subset []string) bool {
 	m := make(container.Set[string], len(set))
 	for _, v := range set {

models/migrations/migrations.go

@@ -595,6 +595,8 @@ var migrations = []Migration{
 	NewMigration("Add force-push branch protection support", v1_23.AddForcePushBranchProtection),
 	// v301 -> v302
 	NewMigration("Add skip_secondary_authorization option to oauth2 application table", v1_23.AddSkipSecondaryAuthColumnToOAuth2ApplicationTable),
+	// v302 -> v303
+	NewMigration("Add index to action_task stopped log_expired", v1_23.AddIndexToActionTaskStoppedLogExpired),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_23/v302.go

@@ -0,0 +1,18 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_23 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func AddIndexToActionTaskStoppedLogExpired(x *xorm.Engine) error {
+	type ActionTask struct {
+		Stopped    timeutil.TimeStamp `xorm:"index(stopped_log_expired)"`
+		LogExpired bool               `xorm:"index(stopped_log_expired)"`
+	}
+	return x.Sync(new(ActionTask))
+}

modules/setting/actions.go

@@ -14,10 +14,11 @@ import (
 // Actions settings
 var (
 	Actions = struct {
-		LogStorage            *Storage // how the created logs should be stored
-		ArtifactStorage       *Storage // how the created artifacts should be stored
-		ArtifactRetentionDays int64    `ini:"ARTIFACT_RETENTION_DAYS"`
 		Enabled               bool
+		LogStorage            *Storage          // how the created logs should be stored
+		LogRetentionDays      int64             `ini:"LOG_RETENTION_DAYS"`
+		ArtifactStorage       *Storage          // how the created artifacts should be stored
+		ArtifactRetentionDays int64             `ini:"ARTIFACT_RETENTION_DAYS"`
 		DefaultActionsURL     defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
 		ZombieTaskTimeout     time.Duration     `ini:"ZOMBIE_TASK_TIMEOUT"`
 		EndlessTaskTimeout    time.Duration     `ini:"ENDLESS_TASK_TIMEOUT"`
@@ -78,10 +79,17 @@ func loadActionsFrom(rootCfg ConfigProvider) error {
 	if err != nil {
 		return err
 	}
+	// default to 1 year
+	if Actions.LogRetentionDays <= 0 {
+		Actions.LogRetentionDays = 365
+	}
 
 	actionsSec, _ := rootCfg.GetSection("actions.artifacts")
 
 	Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)
+	if err != nil {
+		return err
+	}
 
 	// default to 90 days in Github Actions
 	if Actions.ArtifactRetentionDays <= 0 {
@@ -92,5 +100,5 @@ func loadActionsFrom(rootCfg ConfigProvider) error {
 	Actions.EndlessTaskTimeout = sec.Key("ENDLESS_TASK_TIMEOUT").MustDuration(3 * time.Hour)
 	Actions.AbandonedJobTimeout = sec.Key("ABANDONED_JOB_TIMEOUT").MustDuration(24 * time.Hour)
 
-	return err
+	return nil
 }

options/locale/locale_en-US.ini

@@ -3679,6 +3679,7 @@ runs.no_workflows.quick_start = Don't know how to start with Gitea Actions? See
 runs.no_workflows.documentation = For more information on Gitea Actions, see <a target="_blank" rel="noopener noreferrer" href="%s">the documentation</a>.
 runs.no_runs = The workflow has no runs yet.
 runs.empty_commit_message = (empty commit message)
+runs.expire_log_message = Logs have been purged because they were too old.
 
 workflow.disable = Disable Workflow
 workflow.disable_success = Workflow '%s' disabled successfully.

routers/web/repo/actions/view.go

@@ -222,6 +222,27 @@ func ViewPost(ctx *context_module.Context) {
 
 			step := steps[cursor.Step]
 
+			// if task log is expired, return a consistent log line
+			if task.LogExpired {
+				if cursor.Cursor == 0 {
+					resp.Logs.StepsLog = append(resp.Logs.StepsLog, &ViewStepLog{
+						Step:   cursor.Step,
+						Cursor: 1,
+						Lines: []*ViewStepLogLine{
+							{
+								Index:   1,
+								Message: ctx.Locale.TrString("actions.runs.expire_log_message"),
+								// Timestamp doesn't mean anything when the log is expired.
+								// Set it to the task's updated time since it's probably the time when the log has expired.
+								Timestamp: float64(task.Updated.AsTime().UnixNano()) / float64(time.Second),
+							},
+						},
+						Started: int64(step.Started),
+					})
+				}
+				continue
+			}
+
 			logLines := make([]*ViewStepLogLine, 0) // marshal to '[]' instead fo 'null' in json
 
 			index := step.LogIndex + cursor.Cursor

services/actions/cleanup.go

@@ -5,18 +5,30 @@ package actions
 
 import (
 	"context"
+	"fmt"
+	"time"
 
-	"code.gitea.io/gitea/models/actions"
+	actions_model "code.gitea.io/gitea/models/actions"
+	actions_module "code.gitea.io/gitea/modules/actions"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/timeutil"
 )
 
 // Cleanup removes expired actions logs, data and artifacts
-func Cleanup(taskCtx context.Context) error {
-	// TODO: clean up expired actions logs
-
+func Cleanup(ctx context.Context) error {
 	// clean up expired artifacts
-	return CleanupArtifacts(taskCtx)
+	if err := CleanupArtifacts(ctx); err != nil {
+		return fmt.Errorf("cleanup artifacts: %w", err)
+	}
+
+	// clean up old logs
+	if err := CleanupLogs(ctx); err != nil {
+		return fmt.Errorf("cleanup logs: %w", err)
+	}
+
+	return nil
 }
 
 // CleanupArtifacts removes expired add need-deleted artifacts and set records expired status
@@ -28,13 +40,13 @@ func CleanupArtifacts(taskCtx context.Context) error {
 }
 
 func cleanExpiredArtifacts(taskCtx context.Context) error {
-	artifacts, err := actions.ListNeedExpiredArtifacts(taskCtx)
+	artifacts, err := actions_model.ListNeedExpiredArtifacts(taskCtx)
 	if err != nil {
 		return err
 	}
 	log.Info("Found %d expired artifacts", len(artifacts))
 	for _, artifact := range artifacts {
-		if err := actions.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
+		if err := actions_model.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
 			log.Error("Cannot set artifact %d expired: %v", artifact.ID, err)
 			continue
 		}
@@ -52,13 +64,13 @@ const deleteArtifactBatchSize = 100
 
 func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
 	for {
-		artifacts, err := actions.ListPendingDeleteArtifacts(taskCtx, deleteArtifactBatchSize)
+		artifacts, err := actions_model.ListPendingDeleteArtifacts(taskCtx, deleteArtifactBatchSize)
 		if err != nil {
 			return err
 		}
 		log.Info("Found %d artifacts pending deletion", len(artifacts))
 		for _, artifact := range artifacts {
-			if err := actions.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
+			if err := actions_model.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
 				log.Error("Cannot set artifact %d deleted: %v", artifact.ID, err)
 				continue
 			}
@@ -75,3 +87,40 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
 	}
 	return nil
 }
+
+const deleteLogBatchSize = 100
+
+// CleanupLogs removes logs which are older than the configured retention time
+func CleanupLogs(ctx context.Context) error {
+	olderThan := timeutil.TimeStampNow().AddDuration(-time.Duration(setting.Actions.LogRetentionDays) * 24 * time.Hour)
+
+	count := 0
+	for {
+		tasks, err := actions_model.FindOldTasksToExpire(ctx, olderThan, deleteLogBatchSize)
+		if err != nil {
+			return fmt.Errorf("find old tasks: %w", err)
+		}
+		for _, task := range tasks {
+			if err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename); err != nil {
+				log.Error("Failed to remove log %s (in storage %v) of task %v: %v", task.LogFilename, task.LogInStorage, task.ID, err)
+				// do not return error here, continue to next task
+				continue
+			}
+			task.LogIndexes = nil // clear log indexes since it's a heavy field
+			task.LogExpired = true
+			if err := actions_model.UpdateTask(ctx, task, "log_indexes", "log_expired"); err != nil {
+				log.Error("Failed to update task %v: %v", task.ID, err)
+				// do not return error here, continue to next task
+				continue
+			}
+			count++
+			log.Trace("Removed log %s of task %v", task.LogFilename, task.ID)
+		}
+		if len(tasks) < deleteLogBatchSize {
+			break
+		}
+	}
+
+	log.Info("Removed %d logs", count)
+	return nil
+}

services/cron/tasks_actions.go

@@ -68,7 +68,7 @@ func registerScheduleTasks() {
 func registerActionsCleanup() {
 	RegisterTaskFatal("cleanup_actions", &BaseConfig{
 		Enabled:    true,
-		RunAtStart: true,
+		RunAtStart: false,
 		Schedule:   "@midnight",
 	}, func(ctx context.Context, _ *user_model.User, _ Config) error {
 		return actions_service.Cleanup(ctx)