Fixing #87

Author: Vitaliy Zakaznikov

testflows/github/hetzner/runners/dashboard/metrics/servers.py

@@ -28,7 +28,7 @@
 tracker.track("github_hetzner_runners_servers_total", states=states)
 tracker.track("github_hetzner_runners_zombie_servers_total_count")
 tracker.track("github_hetzner_runners_unused_runners_total_count")
-tracker.track("github_hetzner_runners_recycled_servers_total")
+tracker.track("github_hetzner_runners_recycled_servers_total_count")
 # Register individual standby server status metrics
 for status in states:
     tracker.track(
@@ -157,7 +157,9 @@ def recycled_total_count():
     Returns:
         int: Number of recycled servers, or 0 if metric not available
     """
-    return int(recycled_summary()["total"] or 0)
+    return int(
+        get.metric_value("github_hetzner_runners_recycled_servers_total_count") or 0
+    )
 
 
 def labels_info():
@@ -402,7 +404,7 @@ def health_history(cutoff_minutes=15):
 
     # Get recycled servers total count history
     recycled_timestamps, recycled_values = history.data(
-        "github_hetzner_runners_recycled_servers_total",
+        "github_hetzner_runners_recycled_servers_total_count",
         cutoff_minutes=cutoff_minutes,
     )
     health_metrics["recycled"] = {

testflows/github/hetzner/runners/metrics.py

@@ -23,6 +23,7 @@
 from prometheus_client import Counter, Gauge, Histogram, Info
 from .estimate import get_server_price
 from .constants import standby_server_name_prefix
+from .constants import recycle_server_name_prefix
 from .server import get_runner_server_name
 
 
@@ -257,6 +258,24 @@
     ],
 )
 
+# Recycled servers metrics
+RECYCLED_SERVERS_TOTAL = Gauge(
+    "github_hetzner_runners_recycled_servers_total",
+    "Total number of recycled servers",
+    ["server_type", "location"],
+)
+
+RECYCLED_SERVERS_TOTAL_COUNT = Gauge(
+    "github_hetzner_runners_recycled_servers_total_count",
+    "Total number of recycled servers across all types and locations",
+)
+
+RECYCLED_SERVER_INFO = Gauge(
+    "github_hetzner_runners_recycled_server",
+    "Recycled server information including age in seconds",
+    ["server_id", "server_name", "server_type", "location", "status", "created"],
+)
+
 # Runner pool metrics
 RUNNER_POOL_STATUS = Gauge(
     "github_hetzner_runners_pool_status",
@@ -1201,6 +1220,55 @@ def update_unused_runners(unused_runners_dict):
         )
 
 
+def update_recycled_servers(servers):
+    """Update recycled server metrics.
+
+    Args:
+        servers: List of server objects to check for recycled servers
+    """
+    # Clear existing recycled server metrics
+    RECYCLED_SERVERS_TOTAL._metrics.clear()
+    RECYCLED_SERVER_INFO._metrics.clear()
+    RECYCLED_SERVERS_TOTAL_COUNT.set(0)
+
+    total_recycled_servers = 0
+    recycled_counts = {}
+    current_time = time.time()
+
+    for server in servers:
+        # Check if this is a recycled server by name prefix
+        if server.name.startswith(recycle_server_name_prefix):
+            server_type = server.server_type.name
+            location = server.datacenter.location.name
+            key = (server_type, location)
+
+            # Count by type and location
+            recycled_counts[key] = recycled_counts.get(key, 0) + 1
+            total_recycled_servers += 1
+
+            # Track recycled server age
+            server_age = (
+                current_time - server.created.timestamp() if server.created else 0
+            )
+            RECYCLED_SERVER_INFO.labels(
+                server_id=str(server.id),
+                server_name=server.name,
+                server_type=server_type,
+                location=location,
+                status=server.status,
+                created=server.created.isoformat() if server.created else "",
+            ).set(server_age)
+
+    # Set total count
+    RECYCLED_SERVERS_TOTAL_COUNT.set(total_recycled_servers)
+
+    # Set counts by type and location
+    for (server_type, location), count in recycled_counts.items():
+        RECYCLED_SERVERS_TOTAL.labels(server_type=server_type, location=location).set(
+            count
+        )
+
+
 def record_server_creation(server_type: str, location: str, creation_time: float):
     """Record metrics for a server creation.
 

testflows/github/hetzner/runners/scale_down.py

@@ -438,14 +438,15 @@ def scale_down(
                                 current_interval
                             )
 
-            # Update zombie and unused runner metrics
+            # Update zombie, unused runner, and recycled server metrics
             with Action(
-                "Updating zombie and unused runner metrics",
+                "Updating zombie, unused runner, and recycled server metrics",
                 level=logging.DEBUG,
                 interval=interval,
             ):
                 metrics.update_zombie_servers(zombie_servers)
                 metrics.update_unused_runners(unused_runners)
+                metrics.update_recycled_servers(servers)
 
             with Action(
                 "Checking for scale up failures", level=logging.DEBUG, interval=interval