Fix starter_rescued_runs metric and rename from recovered to rescued

- Rename CountRecovered to CountRescued to match naming convention
- Rename starter_recovered_runs metric to starter_rescued_runs
- Add logic to increment counter when rescue jobs are enqueued
- Fix issue where metric was always zero

Author: whywaita

pkg/metric/scrape_memory.go

@@ -33,9 +33,9 @@ var (
 		"waiting queue in starter",
 		[]string{"starter"}, nil,
 	)
-	memoryStarterRecoveredRuns = prometheus.NewDesc(
-		prometheus.BuildFQName(namespace, memoryName, "starter_recovered_runs"),
-		"recovered runs in starter",
+	memoryStarterRescuedRuns = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, memoryName, "starter_rescued_runs"),
+		"rescued runs in starter",
 		[]string{"starter", "target"}, nil,
 	)
 	memoryGitHubRateLimitRemaining = prometheus.NewDesc(
@@ -125,9 +125,9 @@ func scrapeStarterValues(ch chan<- prometheus.Metric) error {
 	ch <- prometheus.MustNewConstMetric(
 		memoryStarterQueueWaiting, prometheus.GaugeValue, float64(countWaiting), labelStarter)
 
-	starter.CountRecovered.Range(func(key, value interface{}) bool {
+	starter.CountRescued.Range(func(key, value interface{}) bool {
 		ch <- prometheus.MustNewConstMetric(
-			memoryStarterRecoveredRuns, prometheus.GaugeValue, float64(value.(int)), labelStarter, key.(string),
+			memoryStarterRescuedRuns, prometheus.GaugeValue, float64(value.(int)), labelStarter, key.(string),
 		)
 		return true
 	})

pkg/starter/starter.go

@@ -35,8 +35,8 @@ var (
 	// CountWaiting is count of waiting job
 	CountWaiting atomic.Int64
 
-	// CountRecovered is count of recovered job per target
-	CountRecovered = sync.Map{}
+	// CountRescued is count of rescued job per target
+	CountRescued = sync.Map{}
 
 	inProgress = sync.Map{}
 
@@ -230,7 +230,7 @@ func (s *Starter) ProcessJob(ctx context.Context, job datastore.Job) error {
 		return fmt.Errorf("failed to retrieve relational target: (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err)
 	}
 
-	CountRecovered.LoadOrStore(target.Scope, 0)
+	CountRescued.LoadOrStore(target.Scope, 0)
 
 	cctx, cancel := context.WithTimeout(ctx, runner.MustRunningTime)
 	defer cancel()
@@ -571,5 +571,12 @@ func enqueueRescueJob(ctx context.Context, workflowJob *github.WorkflowJobEvent,
 		return fmt.Errorf("failed to enqueue job: %w", err)
 	}
 
+	// Increment rescued runs counter
+	if count, ok := CountRescued.Load(target.Scope); ok {
+		CountRescued.Store(target.Scope, count.(int)+1)
+	} else {
+		CountRescued.Store(target.Scope, 1)
+	}
+
 	return nil
 }