Merge pull request #38 from zachyzissou/codex/metrics-endpoint-observability

[codex] add /metrics endpoint for operational alerting

Author: zachyzissou

README.md

@@ -93,6 +93,11 @@ docker run -d --name toonami-downlink -p 7004:7004 -v ./data:/data toonami-downl
 curl http://localhost:7004/health
 ```
 
+```bash
+# Scrape Prometheus-style metrics
+curl http://localhost:7004/metrics
+```
+
 ```text
 {"status":"ok","version":"1.0.0","artifacts":"m3u xml"}
 ```
@@ -139,6 +144,7 @@ npm run lint:js
 ## Observability
 
 - Health and status endpoints are the primary runtime checks.
+- `/metrics` exposes Prometheus-style gauges for freshness/scheduler/cron state.
 - Runtime logs expose refresh cycles and endpoint generation outcomes.
 - `TROUBLESHOOTING.md` and `AUDIT.md` contain operational notes and history.
 - CI publishes test and security artifacts where configured.

app/server.py

@@ -1104,6 +1104,49 @@ async def status():
     }
 
 
+@app.get("/metrics")
+async def metrics():
+    """Prometheus-style runtime metrics for monitoring/alerting."""
+    now = datetime.now(UTC)
+    state = read_state()
+    freshness = _guide_freshness_snapshot(now)
+    cron_supported, _ = _cron_support_status(os.environ.get("CRON_SCHEDULE", CRON_SCHEDULE))
+
+    last_update_value = state.get("last_update")
+    last_update_dt = (
+        _parse_iso_datetime(last_update_value) if isinstance(last_update_value, str) else None
+    )
+    last_update_age_seconds = (
+        max(0.0, (now - last_update_dt).total_seconds()) if last_update_dt else -1.0
+    )
+
+    try:
+        consecutive_failures = int(state.get("consecutive_failures", 0) or 0)
+    except (TypeError, ValueError):
+        consecutive_failures = 0
+
+    metrics_text = "\n".join(
+        [
+            "# HELP downlink_last_update_age_seconds Age in seconds since last successful update (-1 if unknown).",
+            "# TYPE downlink_last_update_age_seconds gauge",
+            f"downlink_last_update_age_seconds {last_update_age_seconds:.3f}",
+            "# HELP downlink_guide_stale 1 when guide data is considered stale, 0 otherwise.",
+            "# TYPE downlink_guide_stale gauge",
+            f"downlink_guide_stale {1 if freshness['is_stale'] else 0}",
+            "# HELP downlink_scheduler_consecutive_failures Current scheduler consecutive failure count.",
+            "# TYPE downlink_scheduler_consecutive_failures gauge",
+            f"downlink_scheduler_consecutive_failures {consecutive_failures}",
+            "# HELP downlink_cron_supported 1 when configured CRON_SCHEDULE is supported, 0 otherwise.",
+            "# TYPE downlink_cron_supported gauge",
+            f"downlink_cron_supported {1 if cron_supported else 0}",
+        ]
+    )
+    return Response(
+        content=f"{metrics_text}\n",
+        media_type="text/plain; version=0.0.4; charset=utf-8",
+    )
+
+
 @app.get("/m3u")
 async def get_m3u():
     """Get the M3U playlist file without stream codes."""

test_integration.py

@@ -291,6 +291,60 @@ async def noop_scheduler_loop():
             os.environ["CRON_SCHEDULE"] = previous_cron
 
 
+def test_metrics_endpoint_prometheus_gauges():
+    """Metrics endpoint should expose Prometheus gauges for guide/scheduler state."""
+    from app import server
+
+    previous_cron = os.environ.get("CRON_SCHEDULE")
+    original_scheduler_loop = server.scheduler_loop
+
+    async def noop_scheduler_loop():
+        return None
+
+    try:
+        os.environ["CRON_SCHEDULE"] = "0 3 * * *"
+        server.scheduler_loop = noop_scheduler_loop
+
+        server.M3U_PATH.parent.mkdir(parents=True, exist_ok=True)
+        server.M3U_PATH.write_text("#EXTM3U\n#EXTINF:-1,Toonami\nhttp://example.com/stream\n")
+        server.XML_PATH.write_text("<tv></tv>\n")
+        now_ts = time.time()
+        os.utime(server.M3U_PATH, (now_ts, now_ts))
+        os.utime(server.XML_PATH, (now_ts, now_ts))
+        server.write_state(
+            {
+                "last_update": datetime.now(UTC).isoformat(),
+                "consecutive_failures": 2,
+            }
+        )
+
+        with TestClient(app) as client:
+            response = client.get("/metrics")
+            assert response.status_code == 200
+            assert "text/plain" in response.headers.get("content-type", "")
+
+            metric_values = {}
+            for line in response.text.splitlines():
+                if not line or line.startswith("#"):
+                    continue
+                name, value = line.split(" ", 1)
+                metric_values[name] = float(value.strip())
+
+            assert "downlink_last_update_age_seconds" in metric_values
+            assert metric_values["downlink_last_update_age_seconds"] >= 0.0
+            assert metric_values["downlink_last_update_age_seconds"] < 600.0
+            assert metric_values["downlink_guide_stale"] == 0.0
+            assert metric_values["downlink_scheduler_consecutive_failures"] == 2.0
+            assert metric_values["downlink_cron_supported"] == 1.0
+    finally:
+        server.scheduler_loop = original_scheduler_loop
+        if previous_cron is None:
+            os.environ.pop("CRON_SCHEDULE", None)
+        else:
+            os.environ["CRON_SCHEDULE"] = previous_cron
+    print("✅ Metrics endpoint exposes expected Prometheus gauges")
+
+
 def test_record_generation_failure_updates_state():
     """Failure recorder should persist normalized error metadata."""
     from app import server
@@ -420,6 +474,7 @@ def main():
         test_lan_refresh_host_detection()
         test_cron_next_respects_dom_mon_dow()
         test_status_reports_cron_and_failure_diagnostics()
+        test_metrics_endpoint_prometheus_gauges()
         test_record_generation_failure_updates_state()
         test_health_reports_scheduler_failure_state()
         test_health_reports_stale_freshness()