propagate tracing-data throughout fastapi apps

bisgaard-itis · bisgaard-itis · commit 33e2023c8ecb · 2025-09-29T14:23:53.000+02:00
diff --git a/packages/service-library/src/servicelib/fastapi/client_session.py b/packages/service-library/src/servicelib/fastapi/client_session.py
@@ -2,6 +2,7 @@
 
 import httpx
 from fastapi import FastAPI
+from servicelib.tracing import TracingData
 from settings_library.tracing import TracingSettings
 
 from .tracing import setup_httpx_client_tracing
@@ -13,15 +14,16 @@ def setup_client_session(
     default_timeout: datetime.timedelta = datetime.timedelta(seconds=20),
     max_keepalive_connections: int = 20,
     tracing_settings: TracingSettings | None,
+    tracing_data: TracingData | None
 ) -> None:
     async def on_startup() -> None:
         session = httpx.AsyncClient(
             transport=httpx.AsyncHTTPTransport(http2=True),
             limits=httpx.Limits(max_keepalive_connections=max_keepalive_connections),
             timeout=default_timeout.total_seconds(),
         )
-        if tracing_settings:
-            setup_httpx_client_tracing(session)
+        if tracing_settings and tracing_data:
+            setup_httpx_client_tracing(session, tracing_data=tracing_data)
         app.state.aiohttp_client_session = session
 
     async def on_shutdown() -> None:
diff --git a/packages/service-library/src/servicelib/fastapi/http_client_thin.py b/packages/service-library/src/servicelib/fastapi/http_client_thin.py
@@ -8,6 +8,7 @@
 from common_library.errors_classes import OsparcErrorMixin
 from httpx import AsyncClient, ConnectError, HTTPError, PoolTimeout, Response
 from httpx._types import TimeoutTypes, URLTypes
+from servicelib.tracing import TracingData
 from settings_library.tracing import TracingSettings
 from tenacity import RetryCallState
 from tenacity.asyncio import AsyncRetrying
@@ -201,6 +202,7 @@ def __init__(
         *,
         total_retry_interval: float,
         tracing_settings: TracingSettings | None,
+        tracing_data: TracingData | None,
         base_url: URLTypes | None = None,
         default_http_client_timeout: TimeoutTypes | None = None,
         extra_allowed_method_names: set[str] | None = None,
@@ -224,8 +226,8 @@ def __init__(
             client_args["timeout"] = default_http_client_timeout
 
         client = AsyncClient(**client_args)
-        if tracing_settings:
-            setup_httpx_client_tracing(client)
+        if tracing_settings and tracing_data:
+            setup_httpx_client_tracing(client, tracing_data=tracing_data)
         super().__init__(client=client)
 
     async def __aenter__(self):
diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py
@@ -2,6 +2,7 @@
 
 import logging
 from collections.abc import AsyncIterator
+from typing import overload
 
 from fastapi import FastAPI, Request
 from fastapi_lifespan_manager import State
@@ -215,8 +216,12 @@ def initialize_fastapi_app_tracing(
     )
 
 
-def setup_httpx_client_tracing(client: AsyncClient | Client):
-    HTTPXClientInstrumentor.instrument_client(client)
+def setup_httpx_client_tracing(
+    client: AsyncClient | Client, tracing_data: TracingData
+) -> None:
+    HTTPXClientInstrumentor.instrument_client(
+        client, tracer_provider=tracing_data.tracer_provider
+    )
 
 
 def setup_tracing(
@@ -228,10 +233,10 @@ def setup_tracing(
         tracing_settings=tracing_settings, service_name=service_name
     )
     app.state.tracing_data = tracing_data
-    _startup(tracing_settings=tracing_settings, tracing_data=get_tracing_data(app))
+    _startup(tracing_settings=tracing_settings, tracing_data=tracing_data)
 
     def _on_shutdown() -> None:
-        _shutdown(tracing_data=get_tracing_data(app))
+        _shutdown(tracing_data=tracing_data)
 
     app.add_event_handler("shutdown", _on_shutdown)
 
@@ -264,6 +269,21 @@ async def dispatch(self, request: Request, call_next):
         return response
 
 
-def get_tracing_data(app: FastAPI) -> TracingData:
+@overload
+def get_tracing_data(
+    app: FastAPI, tracing_settings: TracingSettings
+) -> TracingData: ...
+
+
+@overload
+def get_tracing_data(app: FastAPI, tracing_settings: None) -> None: ...
+
+
+def get_tracing_data(
+    app: FastAPI, tracing_settings: TracingSettings | None
+) -> TracingData | None:
+    if tracing_settings is None:
+        return None
     assert hasattr(app.state, "tracing_data"), "Tracing not setup for this app"  # nosec
+    assert isinstance(app.state.tracing_data, TracingData)
     return app.state.tracing_data
diff --git a/packages/service-library/tests/fastapi/test_http_client_thin.py b/packages/service-library/tests/fastapi/test_http_client_thin.py
@@ -100,7 +100,7 @@ async def test_retry_on_errors(
     caplog_info_level: pytest.LogCaptureFixture,
 ) -> None:
     client = FakeThickClient(
-        total_retry_interval=request_timeout, tracing_settings=None
+        total_retry_interval=request_timeout, tracing_settings=None, tracing_data=None
     )
 
     with pytest.raises(ClientHttpError):
@@ -125,7 +125,9 @@ async def raises_request_error(self) -> Response:
                 request=Request(method="GET", url=test_url),
             )
 
-    client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None)
+    client = ATestClient(
+        total_retry_interval=request_timeout, tracing_settings=None, tracing_data=None
+    )
 
     with pytest.raises(ClientHttpError):
         await client.raises_request_error()
@@ -151,7 +153,9 @@ async def raises_http_error(self) -> Response:
             msg = "mock_http_error"
             raise HTTPError(msg)
 
-    client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None)
+    client = ATestClient(
+        total_retry_interval=request_timeout, tracing_settings=None, tracing_data=None
+    )
 
     with pytest.raises(ClientHttpError):
         await client.raises_http_error()
@@ -165,15 +169,19 @@ async def public_method_ok(self) -> Response:  # type: ignore
             """this method will be ok even if no code is used"""
 
     # OK
-    OKTestClient(total_retry_interval=request_timeout, tracing_settings=None)
+    OKTestClient(
+        total_retry_interval=request_timeout, tracing_settings=None, tracing_data=None
+    )
 
     class FailWrongAnnotationTestClient(BaseThinClient):
         async def public_method_wrong_annotation(self) -> None:
             """this method will raise an error"""
 
     with pytest.raises(AssertionError, match="should return an instance"):
         FailWrongAnnotationTestClient(
-            total_retry_interval=request_timeout, tracing_settings=None
+            total_retry_interval=request_timeout,
+            tracing_settings=None,
+            tracing_data=None,
         )
 
     class FailNoAnnotationTestClient(BaseThinClient):
@@ -182,7 +190,9 @@ async def public_method_no_annotation(self):
 
     with pytest.raises(AssertionError, match="should return an instance"):
         FailNoAnnotationTestClient(
-            total_retry_interval=request_timeout, tracing_settings=None
+            total_retry_interval=request_timeout,
+            tracing_settings=None,
+            tracing_data=None,
         )
 
 
@@ -208,7 +218,7 @@ async def get_wrong_state(self) -> Response:
     respx_mock.get(get_wrong_state).mock(return_value=Response(codes.OK))
 
     test_client = ATestClient(
-        total_retry_interval=request_timeout, tracing_settings=None
+        total_retry_interval=request_timeout, tracing_settings=None, tracing_data=None
     )
 
     # OK
@@ -231,7 +241,7 @@ async def test_retry_timeout_overwrite(
     caplog_info_level: pytest.LogCaptureFixture,
 ) -> None:
     client = FakeThickClient(
-        total_retry_interval=request_timeout, tracing_settings=None
+        total_retry_interval=request_timeout, tracing_settings=None, tracing_data=None
     )
 
     caplog_info_level.clear()
diff --git a/services/agent/src/simcore_service_agent/core/application.py b/services/agent/src/simcore_service_agent/core/application.py
@@ -63,7 +63,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
     setup_rpc_api_routes(app)
 
     if settings.AGENT_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.AGENT_TRACING)
+        )
 
     async def _on_startup() -> None:
         print(APP_STARTED_BANNER_MSG, flush=True)  # noqa: T201
diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py
@@ -98,7 +98,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
 
     if settings.API_SERVER_TRACING:
         initialize_fastapi_app_tracing(
-            app, tracing_data=get_tracing_data(app), add_response_trace_id_header=True
+            app,
+            tracing_data=get_tracing_data(app, settings.API_SERVER_TRACING),
+            add_response_trace_id_header=True,
         )
 
     if settings.API_SERVER_WEBSERVER:
diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/application.py b/services/autoscaling/src/simcore_service_autoscaling/core/application.py
@@ -61,8 +61,10 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     setup_ssm(app)
     setup_redis(app)
 
-    if app.state.settings.AUTOSCALING_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+    if settings.AUTOSCALING_TRACING:
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.AUTOSCALING_TRACING)
+        )
 
     setup_auto_scaler_background_task(app)
     setup_warm_buffer_machines_pool_task(app)
diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py
@@ -66,7 +66,9 @@ def create_app(
         setup_prometheus_instrumentation(app)
 
     if settings.CATALOG_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.CATALOG_TRACING)
+        )
 
     if settings.SC_BOOT_MODE != BootModeEnum.PRODUCTION:
         # middleware to time requests (ONLY for development)
diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py
@@ -65,7 +65,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     setup_clusters_management(app)
 
     if app.state.settings.CLUSTERS_KEEPER_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.CLUSTERS_KEEPER_TRACING)
+        )
     # ERROR HANDLERS
 
     # EVENTS
diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py
@@ -63,8 +63,10 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
         )
     app.add_middleware(GZipMiddleware)
 
-    if app.state.settings.DATCORE_ADAPTER_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+    if settings.DATCORE_ADAPTER_TRACING:
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.DATCORE_ADAPTER_TRACING)
+        )
 
     # events
     app.add_event_handler("startup", on_startup)
diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_client.py b/services/director-v2/src/simcore_service_director_v2/cli/_client.py
@@ -15,6 +15,7 @@ def __init__(self):
             total_retry_interval=10,
             default_http_client_timeout=Timeout(5),
             tracing_settings=None,
+            tracing_data=None,
         )
 
     def _get_url(self, postfix: str) -> str:
diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py
@@ -200,7 +200,9 @@ def create_app(  # noqa: C901, PLR0912
     db.setup(app, settings.POSTGRES)
 
     if settings.DIRECTOR_V2_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.DIRECTOR_V2_TRACING)
+        )
 
     if settings.DYNAMIC_SERVICES.DIRECTOR_V2_DYNAMIC_SERVICES_ENABLED:
         dynamic_services.setup(app, tracing_settings=settings.DIRECTOR_V2_TRACING)
diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py
@@ -12,6 +12,8 @@
     expect_status,
     retry_on_errors,
 )
+from servicelib.fastapi.tracing import get_tracing_data
+from servicelib.tracing import TracingData
 from settings_library.tracing import TracingSettings
 
 from ....core.dynamic_services_settings.scheduler import (
@@ -35,6 +37,9 @@ def __init__(self, app: FastAPI):
         tracing_settings: TracingSettings | None = (
             app.state.settings.DIRECTOR_V2_TRACING
         )
+        tracing_data: TracingData | None = None
+        if tracing_settings:
+            tracing_data = get_tracing_data(app, settings.DIRECTOR_V2_TRACING)
 
         # timeouts
         self._health_request_timeout = Timeout(1.0, connect=1.0)
@@ -58,6 +63,7 @@ def __init__(self, app: FastAPI):
                 connect=scheduler_settings.DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT,
             ),
             tracing_settings=tracing_settings,
+            tracing_data=tracing_data,
         )
 
     def _get_url(
diff --git a/services/director/src/simcore_service_director/core/application.py b/services/director/src/simcore_service_director/core/application.py
@@ -52,11 +52,14 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
         max_keepalive_connections=settings.DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS,
         default_timeout=settings.DIRECTOR_REGISTRY_CLIENT_TIMEOUT,
         tracing_settings=settings.DIRECTOR_TRACING,
+        tracing_data=get_tracing_data(app, settings.DIRECTOR_TRACING),
     )
     setup_registry(app)
 
-    if app.state.settings.DIRECTOR_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+    if settings.DIRECTOR_TRACING:
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.DIRECTOR_TRACING)
+        )
 
     # ERROR HANDLERS
     set_app_default_http_error_handlers(app)
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/catalog/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/catalog/_thin_client.py
@@ -11,6 +11,7 @@
     expect_status,
     retry_on_errors,
 )
+from servicelib.fastapi.tracing import get_tracing_data
 from yarl import URL
 
 from ...core.settings import ApplicationSettings
@@ -35,6 +36,7 @@ def __init__(self, app: FastAPI) -> None:
             },
             base_url=settings.DYNAMIC_SCHEDULER_CATALOG_SETTINGS.api_base_url,
             tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING,
+            tracing_data=get_tracing_data(app, settings.DYNAMIC_SCHEDULER_TRACING),
         )
 
     @retry_on_errors()
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v0/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v0/_thin_client.py
@@ -11,6 +11,7 @@
     expect_status,
     retry_on_errors,
 )
+from servicelib.fastapi.tracing import get_tracing_data
 from yarl import URL
 
 from ...core.settings import ApplicationSettings
@@ -37,6 +38,7 @@ def __init__(self, app: FastAPI) -> None:
             },
             base_url=settings.DYNAMIC_SCHEDULER_DIRECTOR_V0_SETTINGS.endpoint,
             tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING,
+            tracing_data=get_tracing_data(app, settings.DYNAMIC_SCHEDULER_TRACING),
         )
 
     @retry_on_errors()
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py
@@ -24,6 +24,7 @@
     expect_status,
     retry_on_errors,
 )
+from servicelib.fastapi.tracing import get_tracing_data
 from servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services import (
     DEFAULT_LEGACY_WB_TO_DV2_HTTP_REQUESTS_TIMEOUT_S,
 )
@@ -42,6 +43,7 @@ def __init__(self, app: FastAPI) -> None:
             ),
             extra_allowed_method_names={"attach_lifespan_to"},
             tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING,
+            tracing_data=get_tracing_data(app, settings.DYNAMIC_SCHEDULER_TRACING),
         )
 
     @retry_on_errors()
diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py
@@ -196,7 +196,9 @@ def create_app() -> FastAPI:
         setup_prometheus_metrics(app)
 
     if application_settings.DYNAMIC_SIDECAR_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.DYNAMIC_SIDECAR_TRACING)
+        )
 
     # ERROR HANDLERS  ------------
     app.add_exception_handler(
diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py
@@ -62,7 +62,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
     setup_fire_and_forget(app)
 
     if app.state.settings.EFS_GUARDIAN_TRACING:
-        initialize_fastapi_app_tracing(app, tracing_data=get_tracing_data(app))
+        initialize_fastapi_app_tracing(
+            app, tracing_data=get_tracing_data(app, settings.EFS_GUARDIAN_TRACING)
+        )
 
     # EVENTS
     async def _on_startup() -> None:
diff --git a/services/invitations/src/simcore_service_invitations/core/application.py b/services/invitations/src/simcore_service_invitations/core/application.py
diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py
diff --git a/services/payments/src/simcore_service_payments/core/application.py b/services/payments/src/simcore_service_payments/core/application.py
diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py
diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py

Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,9 @@ def create_app(settings: ApplicationSettings \| None = None) -> FastAPI:`
`98`	`98`
`99`	`99`	`if settings.API_SERVER_TRACING:`
`100`	`100`	`initialize_fastapi_app_tracing(`
`101`		`- app, tracing_data=get_tracing_data(app), add_response_trace_id_header=True`
	`101`	`+ app,`
	`102`	`+ tracing_data=get_tracing_data(app, settings.API_SERVER_TRACING),`
	`103`	`+ add_response_trace_id_header=True,`
`102`	`104`	`)`
`103`	`105`
`104`	`106`	`if settings.API_SERVER_WEBSERVER:`