Add OpenTelemetry instrumentation for API (#82)

* Add OpenTelemetry instrumentation for API

* Normalize http.route in telemetry middleware

Author: shayancoin

backend/api/config.py

@@ -1,7 +1,7 @@
 """Configuration settings for the FastAPI application."""
 
-from functools import lru_cache
-from typing import Any, Dict
+import os
+from typing import Any, Dict, Optional
 
 from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -62,11 +62,20 @@ def _validate_hygraph_webhook_secret(cls, value: str) -> str:
         return value.strip()
 
 
-@lru_cache()
+_SETTINGS_CACHE: Optional[Settings] = None
+_LAST_SECRET: Optional[str] = None
+
+
 def get_settings() -> Settings:
-    """Return a cached instance of :class:`Settings`."""
+    """Return a cached instance of :class:`Settings`, refreshing on secret changes."""
 
-    return Settings()
+    global _SETTINGS_CACHE, _LAST_SECRET
+    current_secret = os.getenv("HYGRAPH_WEBHOOK_SECRET")
+    if _SETTINGS_CACHE is None or _LAST_SECRET != current_secret:
+        settings = Settings()
+        _SETTINGS_CACHE = settings
+        _LAST_SECRET = settings.hygraph_webhook_secret
+    return _SETTINGS_CACHE
 
 
 def get_fastapi_settings() -> Dict[str, Any]:

backend/api/instrumentation_boot.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any, Dict
+
+from opentelemetry import metrics, trace
+from opentelemetry.propagate import set_global_textmap
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+from api.config import get_settings
+
+_OTEL_BOOTSTRAPPED = False
+
+
+class JsonLogFormatter(logging.Formatter):
+    """Formatter that outputs JSON with tracing metadata."""
+
+    def __init__(self, service_fields: Dict[str, str]) -> None:
+        super().__init__()
+        self._service_fields = service_fields
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: Dict[str, Any] = {
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "service": self._service_fields.get("service.name"),
+            "environment": self._service_fields.get("deployment.environment"),
+        }
+        if record.exc_info:
+            payload["exception"] = self.formatException(record.exc_info)
+        for key, value in self._service_fields.items():
+            payload.setdefault(key.replace(".", "_"), value)
+
+        span = trace.get_current_span()
+        ctx = span.get_span_context()
+        if ctx.trace_id:
+            payload["trace_id"] = format(ctx.trace_id, "032x")
+        if ctx.span_id:
+            payload["span_id"] = format(ctx.span_id, "016x")
+
+        for attr in ("event_id", "dedup", "counts", "elapsed_ms", "type", "error"):
+            if hasattr(record, attr):
+                payload[attr] = getattr(record, attr)
+
+        return json.dumps(payload, default=str)
+
+
+def _configure_logging(service_fields: Dict[str, str]) -> None:
+    root = logging.getLogger()
+    log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+    root.setLevel(log_level)
+
+    formatter = JsonLogFormatter(service_fields)
+    if not root.handlers:
+        handler = logging.StreamHandler()
+        handler.setFormatter(formatter)
+        root.addHandler(handler)
+    else:
+        for handler in root.handlers:
+            handler.setFormatter(formatter)
+            handler.setLevel(log_level)
+
+
+def setup_instrumentation() -> None:
+    global _OTEL_BOOTSTRAPPED
+    if _OTEL_BOOTSTRAPPED:
+        return
+
+    settings = get_settings()
+    service_name = os.getenv("OTEL_SERVICE_NAME", "paform-backend-api")
+    resource = Resource.create(
+        {
+            "service.name": service_name,
+            "service.namespace": "paform",
+            "deployment.environment": settings.environment,
+        }
+    )
+
+    set_global_textmap(TraceContextTextMapPropagator())
+
+    tracer_provider = TracerProvider(resource=resource)
+    span_exporter = OTLPSpanExporter(endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "otel-collector:4317"), insecure=True)
+    tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
+    trace.set_tracer_provider(tracer_provider)
+
+    metric_exporter = OTLPMetricExporter(
+        endpoint=os.getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "otel-collector:4317"),
+        insecure=True,
+    )
+    reader = PeriodicExportingMetricReader(metric_exporter)
+    meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
+    metrics.set_meter_provider(meter_provider)
+
+    service_fields: Dict[str, str] = {
+        "service.name": service_name,
+        "deployment.environment": settings.environment,
+    }
+    _configure_logging(service_fields)
+
+    _OTEL_BOOTSTRAPPED = True
+
+
+setup_instrumentation()

backend/api/main.py

@@ -4,13 +4,18 @@
 from time import perf_counter
 from typing import Dict
 
-from fastapi import FastAPI, Response, Request
+from fastapi import FastAPI, Request, Response
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
+from opentelemetry import trace
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.trace import Status, StatusCode
 from starlette.middleware.cors import CORSMiddleware
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from api.config import get_settings
+from api.instrumentation_boot import setup_instrumentation
+from api.metrics import record_http_request
 from api.routes import router as api_router
 from api.routes_materials import router as materials_router
 from api.routes_modules import router as modules_router
@@ -21,11 +26,7 @@
 from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
 from api.metrics import observe_http_request
 
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-)
-logger = logging.getLogger(__name__)
+setup_instrumentation()
 
 # Initialize settings
 settings = get_settings()
@@ -107,6 +108,45 @@ async def dispatch(self, request: Request, call_next):  # type: ignore[override]
 app.include_router(sync_router)
 app.include_router(observability_router)
 
+_http_tracer = trace.get_tracer("paform.api.http")
+
+
+@app.middleware("http")
+async def telemetry_middleware(request: Request, call_next):
+    method = request.method.upper()
+    path_template = request.url.path
+    start = time.perf_counter()
+
+    current_span = trace.get_current_span()
+    if current_span.get_span_context().is_valid:
+        span_context = nullcontext(current_span)
+    else:
+        span_context = _http_tracer.start_as_current_span(f"HTTP {method}")
+
+    status_code = 500
+    response: Optional[Response] = None
+    with span_context as span:
+        span.set_attribute("http.method", method)
+        span.set_attribute("deployment.environment", settings.environment)
+        try:
+            response = await call_next(request)
+            status_code = response.status_code
+            span.set_attribute("http.status_code", status_code)
+            if status_code >= 500:
+                span.set_status(Status(StatusCode.ERROR))
+        except Exception as exc:  # noqa: BLE001
+            span.record_exception(exc)
+            span.set_status(Status(StatusCode.ERROR))
+            raise
+        finally:
+            matched_route = request.scope.get("route")
+            path_template = getattr(matched_route, "path", path_template)
+            span.set_attribute("http.route", path_template)
+            duration_ms = (time.perf_counter() - start) * 1000
+            record_http_request(method, path_template, status_code, duration_ms)
+
+    return response if response is not None else Response(status_code=status_code)
+
 
 @app.exception_handler(RequestValidationError)
 async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
@@ -150,6 +190,9 @@ async def healthcheck() -> Dict[str, str]:
 
 
 @app.get("/metrics")
-async def metrics() -> Response:
-    # Expose Prometheus metrics including default process/python collectors
-    return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)
+async def metrics() -> Dict[str, str]:
+    """Compatibility endpoint for legacy Prometheus scrapes."""
+
+    return {
+        "detail": "Metrics are exported via OpenTelemetry OTLP; no local payload is available.",
+    }

backend/api/metrics.py

@@ -16,17 +16,42 @@
     5.0,
 )
 
-# Hygraph sync counters
-sync_success_total = Counter(
-    "sync_success_total",
-    "Successful Hygraph sync operations",
-    labelnames=("type",),
+from typing import Dict
+
+from opentelemetry import metrics
+
+_meter = metrics.get_meter("paform.api.metrics")
+
+_sync_success_counter = _meter.create_counter(
+    name="sync_success_total",
+    description="Successful Hygraph sync operations",
 )
 
-sync_failure_total = Counter(
-    "sync_failure_total",
-    "Failed Hygraph sync operations",
-    labelnames=("type",),
+_sync_failure_counter = _meter.create_counter(
+    name="sync_failure_total",
+    description="Failed Hygraph sync operations",
+)
+
+_sync_records_counter = _meter.create_counter(
+    name="sync_records_upserted_total",
+    description="Records upserted during Hygraph sync",
+)
+
+_sync_duration_histogram = _meter.create_histogram(
+    name="sync_duration_ms",
+    unit="ms",
+    description="Duration of Hygraph sync operations in milliseconds",
+)
+
+_http_request_counter = _meter.create_counter(
+    name="http_server_requests_total",
+    description="Total number of HTTP requests handled",
+)
+
+_http_request_duration_histogram = _meter.create_histogram(
+    name="http_server_request_duration_ms",
+    unit="ms",
+    description="HTTP server request duration in milliseconds",
 )
 
 sync_records_upserted_total = Counter(