Fix starter_rescued_runs metric and rename from recovered to rescued

- Rename CountRecovered to CountRescued to match naming convention
- Rename starter_recovered_runs metric to starter_rescued_runs
- Add logic to increment counter when rescue jobs are enqueued
- Fix issue where metric was always zero

Author: whywaita

pkg/metric/scrape_memory.go

@@ -33,9 +33,9 @@ var (
 		"waiting queue in starter",
 		[]string{"starter"}, nil,
 	)
-	memoryStarterRecoveredRuns = prometheus.NewDesc(
-		prometheus.BuildFQName(namespace, memoryName, "starter_recovered_runs"),
-		"recovered runs in starter",
+	memoryStarterRescuedRuns = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, memoryName, "starter_rescued_runs"),
+		"rescued runs in starter",
 		[]string{"starter", "target"}, nil,
 	)
 	memoryGitHubRateLimitRemaining = prometheus.NewDesc(
@@ -125,9 +125,9 @@ func scrapeStarterValues(ch chan<- prometheus.Metric) error {
 	ch <- prometheus.MustNewConstMetric(
 		memoryStarterQueueWaiting, prometheus.GaugeValue, float64(countWaiting), labelStarter)
 
-	starter.CountRecovered.Range(func(key, value interface{}) bool {
+	starter.CountRescued.Range(func(key, value interface{}) bool {
 		ch <- prometheus.MustNewConstMetric(
-			memoryStarterRecoveredRuns, prometheus.GaugeValue, float64(value.(int)), labelStarter, key.(string),
+			memoryStarterRescuedRuns, prometheus.GaugeValue, float64(value.(int)), labelStarter, key.(string),
 		)
 		return true
 	})

pkg/starter/starter.go

@@ -35,8 +35,8 @@ var (
 	// CountWaiting is count of waiting job
 	CountWaiting atomic.Int64
 
-	// CountRecovered is count of recovered job per target
-	CountRecovered = sync.Map{}
+	// CountRescued is count of rescued job per target
+	CountRescued = sync.Map{}
 
 	inProgress = sync.Map{}
 
@@ -201,7 +201,7 @@ func (s *Starter) ProcessJob(ctx context.Context, job datastore.Job) error {
 		return fmt.Errorf("failed to retrieve relational target: (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err)
 	}
 
-	CountRecovered.LoadOrStore(target.Scope, 0)
+	CountRescued.LoadOrStore(target.Scope, 0)
 
 	cctx, cancel := context.WithTimeout(ctx, runner.MustRunningTime)
 	defer cancel()
@@ -445,7 +445,7 @@ func enqueueRescueRun(ctx context.Context, pendingRun datastore.PendingWorkflowR
 		// Get full installation data from cache
 		installation, err := gh.GetInstallationByID(ctx, installationID)
 		if err != nil {
-      logger.Logf(false, "failed to get installation from cache (installationID: %d), using minimal data: %+v", installationID, err)
+			logger.Logf(false, "failed to get installation from cache (installationID: %d), using minimal data: %+v", installationID, err)
 			// Fallback to minimal installation data
 			installation = &github.Installation{
 				ID: &installationID,
@@ -461,7 +461,7 @@ func enqueueRescueRun(ctx context.Context, pendingRun datastore.PendingWorkflowR
 				Name:  owner.Name,
 			}
 		}
-		
+
 		event := &github.WorkflowJobEvent{
 			WorkflowJob:  job,
 			Action:       github.String("queued"),
@@ -524,5 +524,12 @@ func enqueueRescueJob(ctx context.Context, workflowJob *github.WorkflowJobEvent,
 		return fmt.Errorf("failed to enqueue job: %w", err)
 	}
 
+	// Increment rescued runs counter
+	if count, ok := CountRescued.Load(target.Scope); ok {
+		CountRescued.Store(target.Scope, count.(int)+1)
+	} else {
+		CountRescued.Store(target.Scope, 1)
+	}
+
 	return nil
 }