Add observability dashboards and alerting config

backend/api/main.py

@@ -1,12 +1,14 @@
 """Main module for the FastAPI application."""
 
 import logging
+from time import perf_counter
 from typing import Dict
 
 from fastapi import FastAPI, Response, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from starlette.middleware.cors import CORSMiddleware
+from starlette.middleware.base import BaseHTTPMiddleware
 
 from api.config import get_settings
 from api.routes import router as api_router
@@ -15,7 +17,9 @@
 from api.routes_quote import router as quote_router
 from api.routes_cnc import router as cnc_router
 from api.routes_sync import router as sync_router
+from api.routes_observability import router as observability_router
 from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
+from api.metrics import observe_http_request
 
 # Configure logging
 logging.basicConfig(
@@ -51,13 +55,54 @@ async def _ensure_sqlite_tables() -> None:
     allow_headers=["*"],
 )
 
+
+def _resolve_route_label(request: Request) -> str:
+    route = request.scope.get("route")
+    if route and hasattr(route, "path") and route.path:
+        return route.path
+    return request.url.path or "unknown"
+
+
+class PrometheusInstrumentationMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):  # type: ignore[override]
+        if request.url.path == "/metrics":
+            return await call_next(request)
+
+        start = perf_counter()
+        try:
+            response = await call_next(request)
+        except Exception:
+            duration = perf_counter() - start
+            observe_http_request(
+                service="backend",
+                route=_resolve_route_label(request),
+                method=request.method,
+                status="500",
+                duration_seconds=duration,
+            )
+            raise
+
+        duration = perf_counter() - start
+        observe_http_request(
+            service="backend",
+            route=_resolve_route_label(request),
+            method=request.method,
+            status=str(response.status_code),
+            duration_seconds=duration,
+        )
+        return response
+
+
+app.add_middleware(PrometheusInstrumentationMiddleware)
+
 # Include API router
 app.include_router(api_router)
 app.include_router(materials_router)
 app.include_router(modules_router)
 app.include_router(quote_router)
 app.include_router(cnc_router)
 app.include_router(sync_router)
+app.include_router(observability_router)
 
 
 @app.exception_handler(RequestValidationError)

backend/api/metrics.py

@@ -1,5 +1,20 @@
 from __future__ import annotations
-from prometheus_client import Counter
+from prometheus_client import Counter, Histogram
+
+REQUEST_LATENCY_BUCKETS = (
+    0.005,
+    0.01,
+    0.025,
+    0.05,
+    0.075,
+    0.1,
+    0.25,
+    0.5,
+    0.75,
+    1.0,
+    2.5,
+    5.0,
+)
 
 # Hygraph sync counters
 sync_success_total = Counter(
@@ -18,4 +33,49 @@
     "sync_records_upserted_total",
     "Records upserted during Hygraph sync",
     labelnames=("type",),
-)
+)
+
+http_requests_total = Counter(
+    "http_requests_total",
+    "Total HTTP requests processed by the backend",
+    labelnames=("service", "route", "method", "status"),
+)
+
+http_request_duration_seconds = Histogram(
+    "http_request_duration_seconds",
+    "Latency of HTTP requests handled by the backend",
+    labelnames=("service", "route", "method", "status"),
+    buckets=REQUEST_LATENCY_BUCKETS,
+)
+
+web_vitals_lcp = Histogram(
+    "web_vitals_lcp",
+    "Largest Contentful Paint reported from the frontend (seconds)",
+    labelnames=("app",),
+    buckets=(1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10),
+)
+
+
+def observe_http_request(
+    *,
+    service: str,
+    route: str,
+    method: str,
+    status: str,
+    duration_seconds: float,
+) -> None:
+    """Record a single HTTP request observation."""
+
+    http_requests_total.labels(
+        service=service, route=route, method=method, status=status
+    ).inc()
+    http_request_duration_seconds.labels(
+        service=service, route=route, method=method, status=status
+    ).observe(duration_seconds)
+
+
+def observe_lcp(*, app: str, seconds: float) -> None:
+    """Record a Largest Contentful Paint measurement in seconds."""
+
+    web_vitals_lcp.labels(app=app).observe(seconds)
+

backend/api/routes_observability.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from fastapi import APIRouter, HTTPException, status
+from pydantic import BaseModel, Field
+
+from api.metrics import observe_lcp
+
+router = APIRouter(prefix="/observability", tags=["observability"])
+
+
+class WebVitalPayload(BaseModel):
+    name: Literal["LCP", "FCP", "CLS", "FID", "INP", "TTFB"] = Field(
+        ..., description="Name of the reported Web Vital metric"
+    )
+    value: float = Field(..., description="Value of the metric as reported by web-vitals")
+    app: str = Field("frontend", description="Logical application label for the metric")
+    id: str | None = Field(None, description="Unique identifier assigned by the web-vitals reporter")
+
+
+@router.post("/web-vitals", status_code=status.HTTP_202_ACCEPTED)
+async def ingest_web_vitals(payload: WebVitalPayload) -> dict[str, bool]:
+    """Ingest Web Vital measurements emitted by the frontend."""
+
+    if payload.name != "LCP":
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Only LCP metrics are supported at this time",
+        )
+
+    # web-vitals reports timings in milliseconds; convert to seconds for Prometheus
+    seconds = payload.value / 1000.0
+    observe_lcp(app=payload.app, seconds=seconds)
+    return {"ok": True}

backend/tests/test_observability_metrics.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from fastapi.testclient import TestClient
+from prometheus_client import REGISTRY
+
+from api.main import app
+
+client = TestClient(app)
+
+
+def _get_sample(name: str, labels: dict[str, str]) -> float:
+    value = REGISTRY.get_sample_value(name, labels)
+    return float(value) if value is not None else 0.0
+
+
+def test_lcp_ingest_records_histogram() -> None:
+    before = _get_sample("web_vitals_lcp_count", {"app": "frontend"})
+
+    response = client.post(
+        "/observability/web-vitals",
+        json={"name": "LCP", "value": 2400, "app": "frontend"},
+    )
+
+    assert response.status_code == 202
+    after = _get_sample("web_vitals_lcp_count", {"app": "frontend"})
+    assert after == before + 1
+
+
+def test_lcp_ingest_rejects_unsupported_metrics() -> None:
+    response = client.post(
+        "/observability/web-vitals",
+        json={"name": "CLS", "value": 0.04},
+    )
+
+    assert response.status_code == 400
+    payload = response.json()
+    assert payload["detail"] == "Only LCP metrics are supported at this time"
+
+
+def test_http_metrics_recorded_for_requests() -> None:
+    route_labels = {
+        "service": "backend",
+        "route": "/healthcheck",
+        "method": "GET",
+        "status": "200",
+    }
+    before = _get_sample("http_requests_total", route_labels)
+
+    resp = client.get("/healthcheck")
+
+    assert resp.status_code == 200
+    after = _get_sample("http_requests_total", route_labels)
+    assert after == before + 1

dashboards/dashboards.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+providers:
+  - name: Observability Dashboards
+    orgId: 1
+    folder: Platform Observability
+    type: file
+    disableDeletion: false
+    allowUiUpdates: true
+    updateIntervalSeconds: 30
+    options:
+      path: ops/grafana/provisioning/dashboards
+      foldersFromFilesStructure: false

frontend/package.json

@@ -20,15 +20,16 @@
     "@chakra-ui/react": "^2.8.2",
     "@emotion/react": "^11.13.3",
     "@emotion/styled": "^11.13.0",
+    "@react-three/drei": "^9.120.5",
+    "@react-three/fiber": "^8.17.10",
     "framer-motion": "^11.5.4",
     "next": "^14.2.28",
     "react": "^18",
     "react-dom": "^18",
     "react-icons": "^5.3.0",
-    "zustand": "^4.5.0",
     "three": "^0.171.0",
-    "@react-three/fiber": "^8.17.10",
-    "@react-three/drei": "^9.120.5"
+    "web-vitals": "^4.2.4",
+    "zustand": "^4.5.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.56.0",
@@ -49,12 +50,9 @@
     "postcss": "^8",
     "tailwindcss": "^3.3.0",
     "ts-jest": "^29.2.5",
+    "ts-node": "^10.9.2",
     "typescript": "^5",
-    "gltfpack": "0.25.0",
-    "meshoptimizer": "0.25.0",
-    "vitest": "^1.6.0",
-    "@playwright/test": "^1.56.0",
-    "ts-node": "^10.9.2"
+    "vitest": "^1.6.0"
   },
   "jest": {
     "setupFilesAfterEnv": [

frontend/src/app/reportWebVitals.ts

@@ -0,0 +1,55 @@
+import type { Metric } from 'next/app';
+
+const DEFAULT_ENDPOINT = '/observability/web-vitals';
+const APP_LABEL = process.env.NEXT_PUBLIC_WEB_VITALS_APP ?? 'frontend';
+
+function buildEndpoint(): string {
+  if (typeof window === 'undefined') {
+    return DEFAULT_ENDPOINT;
+  }
+
+  const protocol = process.env.NEXT_PUBLIC_BACKEND_PROTOCOL ?? window.location.protocol ?? 'http:';
+  const host = process.env.NEXT_PUBLIC_BACKEND_HOST ?? window.location.hostname;
+  const port = process.env.NEXT_PUBLIC_BACKEND_PORT ?? '';
+  const endpoint = process.env.NEXT_PUBLIC_WEB_VITALS_ENDPOINT ?? DEFAULT_ENDPOINT;
+
+  const portSegment = port ? `:${port}` : '';
+  return `${protocol}//${host}${portSegment}${endpoint}`;
+}
+
+function sendMetric(endpoint: string, body: string) {
+  if (typeof navigator !== 'undefined' && 'sendBeacon' in navigator) {
+    const blob = new Blob([body], { type: 'application/json' });
+    const success = navigator.sendBeacon(endpoint, blob);
+    if (success) {
+      return;
+    }
+  }
+
+  fetch(endpoint, {
+    method: 'POST',
+    body,
+    keepalive: true,
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  }).catch(() => {
+    // Swallow network errors; metrics reporting should be fire-and-forget
+  });
+}
+
+export function reportWebVitals(metric: Metric) {
+  if (metric.name !== 'LCP') {
+    return;
+  }
+
+  const endpoint = buildEndpoint();
+  const body = JSON.stringify({
+    name: metric.name,
+    value: metric.value,
+    app: APP_LABEL,
+    id: metric.id,
+  });
+
+  sendMetric(endpoint, body);
+}