handle exceptions in fastapi prometheus middleware

Author: bisgaard-itis

packages/service-library/src/servicelib/fastapi/monitoring.py

@@ -1,10 +1,11 @@
 # pylint: disable=protected-access
 
+import logging
 from collections.abc import AsyncIterator
 from typing import Final
 
 import prometheus_client
-from fastapi import FastAPI, Request, Response
+from fastapi import FastAPI, Request, Response, status
 from fastapi_lifespan_manager import State
 from prometheus_client import CollectorRegistry
 from servicelib.prometheus_metrics import (
@@ -21,6 +22,7 @@
     X_SIMCORE_USER_AGENT,
 )
 
+_logger = logging.getLogger(__name__)
 kPROMETHEUS_METRICS = "prometheus_metrics"
 
 
@@ -45,13 +47,19 @@ async def dispatch(
                 user_agent=user_agent,
             ):
                 response = await call_next(request)
+                status_code = response.status_code
+        except Exception:  # pylint: disable=broad-except
+            # NOTE: The prometheus metrics middleware should be "outside" exception handling
+            # middleware so at this point starlette should turn an exception into a 500
+            status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+            raise
         finally:
             record_response_metrics(
                 metrics=self.metrics,
                 method=request.method,
                 endpoint=canonical_endpoint,
                 user_agent=user_agent,
-                http_status=response.status_code,
+                http_status=status_code,
             )
 
         return response

services/api-server/src/simcore_service_api_server/core/application.py

@@ -115,16 +115,16 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
     app.add_event_handler("startup", on_startup)
     app.add_event_handler("shutdown", on_shutdown)
 
-    exceptions.setup_exception_handlers(
-        app, is_debug=settings.SC_BOOT_MODE == BootModeEnum.DEBUG
-    )
-
     if settings.API_SERVER_PROFILING:
         initialize_profiler(app)
 
     if app.state.settings.API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED:
         setup_prometheus_instrumentation(app)
 
+    exceptions.setup_exception_handlers(
+        app, is_debug=settings.SC_BOOT_MODE == BootModeEnum.DEBUG
+    )
+
     # routing
 
     # healthcheck at / and at /VTAG/