🐛 Fix tracing exemplars

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -101,7 +101,6 @@ def _startup(tracing_settings: TracingSettings, service_name: str) -> None:
     span_processor = BatchSpanProcessor(otlp_exporter)
     global_tracer_provider.add_span_processor(span_processor)
 
-    FastAPIInstrumentor().instrument()
     if HAS_AIOPG:
         with log_context(
             _logger,
@@ -181,14 +180,18 @@ def _shutdown() -> None:
             _logger.exception("Failed to uninstrument RequestsInstrumentor")
 
 
+def setup_fastapi_app_tracing(app: FastAPI):
+    FastAPIInstrumentor.instrument_app(app)
+
+
 def setup_httpx_client_tracing(client: AsyncClient | Client):
     HTTPXClientInstrumentor.instrument_client(client)
 
 
-def setup_tracing(
+def tracing_instrument_tooling(
     app: FastAPI, tracing_settings: TracingSettings, service_name: str
 ) -> None:
-
+    # NOTE: This does not instrument the app itself. Call setup_fastapi_app_tracing to do that.
     _startup(tracing_settings=tracing_settings, service_name=service_name)
 
     def _on_shutdown() -> None:
@@ -200,7 +203,7 @@ def _on_shutdown() -> None:
 def get_tracing_instrumentation_lifespan(
     tracing_settings: TracingSettings, service_name: str
 ):
-
+    # NOTE: This lifespan does not instrument the app itself. Call setup_fastapi_app_tracing to do that.
     _startup(tracing_settings=tracing_settings, service_name=service_name)
 
     async def tracing_instrumentation_lifespan(

services/agent/src/simcore_service_agent/core/application.py

@@ -5,7 +5,10 @@
     get_common_oas_options,
     override_fastapi_openapi_method,
 )
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 from servicelib.logging_utils import config_all_loggers
 
 from .._meta import (
@@ -56,7 +59,7 @@ def create_app() -> FastAPI:
     app.state.settings = settings
 
     if settings.AGENT_TRACING:
-        setup_tracing(app, settings.AGENT_TRACING, APP_NAME)
+        tracing_instrument_tooling(app, settings.AGENT_TRACING, APP_NAME)
 
     setup_instrumentation(app)
 
@@ -66,6 +69,9 @@ def create_app() -> FastAPI:
     setup_rest_api(app)
     setup_rpc_api_routes(app)
 
+    if settings.AGENT_TRACING:
+        setup_fastapi_app_tracing(app)
+
     async def _on_startup() -> None:
         print(APP_STARTED_BANNER_MSG, flush=True)  # noqa: T201
 

services/api-server/src/simcore_service_api_server/core/application.py

@@ -5,7 +5,10 @@
 from models_library.basic_types import BootModeEnum
 from packaging.version import Version
 from servicelib.fastapi.profiler import initialize_profiler
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 from servicelib.logging_utils import config_all_loggers
 
 from .. import exceptions
@@ -83,7 +86,7 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
     app.state.settings = settings
 
     if settings.API_SERVER_TRACING:
-        setup_tracing(app, settings.API_SERVER_TRACING, APP_NAME)
+        tracing_instrument_tooling(app, settings.API_SERVER_TRACING, APP_NAME)
 
     if settings.API_SERVER_POSTGRES:
         setup_postgres(app)
@@ -93,6 +96,9 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
     if app.state.settings.API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED:
         setup_prometheus_instrumentation(app)
 
+    if settings.API_SERVER_TRACING:
+        setup_fastapi_app_tracing(app)
+
     if settings.API_SERVER_WEBSERVER:
         webserver.setup(
             app,

services/autoscaling/src/simcore_service_autoscaling/core/application.py

@@ -1,7 +1,10 @@
 import logging
 
 from fastapi import FastAPI
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 
 from .._meta import (
     API_VERSION,
@@ -61,7 +64,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
 
     # PLUGINS SETUP
     if app.state.settings.AUTOSCALING_TRACING:
-        setup_tracing(app, app.state.settings.AUTOSCALING_TRACING, APP_NAME)
+        tracing_instrument_tooling(
+            app, app.state.settings.AUTOSCALING_TRACING, APP_NAME
+        )
 
     setup_instrumentation(app)
     setup_api_routes(app)
@@ -71,6 +76,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     setup_ssm(app)
     setup_redis(app)
 
+    if app.state.settings.AUTOSCALING_TRACING:
+        setup_fastapi_app_tracing(app)
+
     setup_auto_scaler_background_task(app)
     setup_buffer_machines_pool_task(app)
 

services/catalog/src/simcore_service_catalog/core/application.py

@@ -8,7 +8,10 @@
     setup_prometheus_instrumentation,
 )
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from .._meta import (
@@ -64,10 +67,13 @@ def create_app() -> FastAPI:
 
     # MIDDLEWARES
     if settings.CATALOG_TRACING:
-        setup_tracing(app, settings.CATALOG_TRACING, APP_NAME)
+        tracing_instrument_tooling(app, settings.CATALOG_TRACING, APP_NAME)
     if settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED:
         setup_prometheus_instrumentation(app)
 
+    if settings.CATALOG_TRACING:
+        setup_fastapi_app_tracing(app)
+
     if settings.SC_BOOT_MODE != BootModeEnum.PRODUCTION:
         # middleware to time requests (ONLY for development)
         app.add_middleware(

services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py

@@ -4,7 +4,10 @@
 from servicelib.fastapi.monitoring import (
     setup_prometheus_instrumentation,
 )
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 
 from .._meta import (
     API_VERSION,
@@ -59,7 +62,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     assert app.state.settings.API_VERSION == API_VERSION  # nosec
 
     if app.state.settings.CLUSTERS_KEEPER_TRACING:
-        setup_tracing(
+        tracing_instrument_tooling(
             app,
             app.state.settings.CLUSTERS_KEEPER_TRACING,
             APP_NAME,
@@ -76,6 +79,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     setup_redis(app)
     setup_clusters_management(app)
 
+    if app.state.settings.CLUSTERS_KEEPER_TRACING:
+        setup_fastapi_app_tracing(app)
     # ERROR HANDLERS
 
     # EVENTS

services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py

@@ -10,7 +10,10 @@
     setup_prometheus_instrumentation,
 )
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from .._meta import API_VERSION, API_VTAG, APP_NAME
@@ -61,7 +64,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     app.state.settings = settings
 
     if app.state.settings.DATCORE_ADAPTER_TRACING:
-        setup_tracing(
+        tracing_instrument_tooling(
             app,
             app.state.settings.DATCORE_ADAPTER_TRACING,
             APP_NAME,
@@ -76,6 +79,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
         )
     app.add_middleware(GZipMiddleware)
 
+    if app.state.settings.DATCORE_ADAPTER_TRACING:
+        setup_fastapi_app_tracing(app)
+
     # events
     app.add_event_handler("startup", on_startup)
     app.add_event_handler("startup", create_start_app_handler(app))

services/director-v2/src/simcore_service_director_v2/core/application.py

@@ -7,7 +7,10 @@
     override_fastapi_openapi_method,
 )
 from servicelib.fastapi.profiler import initialize_profiler
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 from servicelib.logging_utils import config_all_loggers
 
 from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY
@@ -146,7 +149,7 @@ def init_app(settings: AppSettings | None = None) -> FastAPI:
     substitutions.setup(app)
 
     if settings.DIRECTOR_V2_TRACING:
-        setup_tracing(app, settings.DIRECTOR_V2_TRACING, APP_NAME)
+        tracing_instrument_tooling(app, settings.DIRECTOR_V2_TRACING, APP_NAME)
 
     if settings.DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED:
         instrumentation.setup(app)
@@ -174,6 +177,9 @@ def init_app(settings: AppSettings | None = None) -> FastAPI:
 
     db.setup(app, settings.POSTGRES)
 
+    if settings.DIRECTOR_V2_TRACING:
+        setup_fastapi_app_tracing(app)
+
     if settings.DYNAMIC_SERVICES.DIRECTOR_V2_DYNAMIC_SERVICES_ENABLED:
         dynamic_services.setup(app, tracing_settings=settings.DIRECTOR_V2_TRACING)
 

services/director/src/simcore_service_director/core/application.py

@@ -4,7 +4,10 @@
 from fastapi import FastAPI
 from servicelib.async_utils import cancel_sequential_workers
 from servicelib.fastapi.client_session import setup_client_session
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 
 from .._meta import (
     API_VERSION,
@@ -49,7 +52,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
 
     # PLUGINS SETUP
     if app.state.settings.DIRECTOR_TRACING:
-        setup_tracing(app, app.state.settings.DIRECTOR_TRACING, APP_NAME)
+        tracing_instrument_tooling(app, app.state.settings.DIRECTOR_TRACING, APP_NAME)
 
     setup_api_routes(app)
 
@@ -62,6 +65,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     )
     setup_registry(app)
 
+    if app.state.settings.DIRECTOR_TRACING:
+        setup_fastapi_app_tracing(app)
+
     # ERROR HANDLERS
 
     # EVENTS

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py

@@ -4,6 +4,7 @@
 )
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
 from servicelib.fastapi.profiler import initialize_profiler
+from servicelib.fastapi.tracing import setup_fastapi_app_tracing
 
 from .._meta import API_VERSION, API_VTAG, PROJECT_NAME, SUMMARY
 from ..api.frontend import initialize_frontend
@@ -42,4 +43,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
     if app_settings.DYNAMIC_SCHEDULER_PROFILING:
         initialize_profiler(app)
 
+    if app_settings.DYNAMIC_SCHEDULER_TRACING:
+        setup_fastapi_app_tracing(app)
+
     return app

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py

@@ -9,7 +9,10 @@
     get_common_oas_options,
     override_fastapi_openapi_method,
 )
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 from servicelib.logging_utils import config_all_loggers
 from simcore_sdk.node_ports_common.exceptions import NodeNotFound
 
@@ -176,7 +179,9 @@ def create_app():
     application_settings: ApplicationSettings = app.state.settings
 
     if application_settings.DYNAMIC_SIDECAR_TRACING:
-        setup_tracing(app, application_settings.DYNAMIC_SIDECAR_TRACING, PROJECT_NAME)
+        tracing_instrument_tooling(
+            app, application_settings.DYNAMIC_SIDECAR_TRACING, PROJECT_NAME
+        )
 
     setup_rabbitmq(app)
     setup_rpc_api_routes(app)
@@ -196,6 +201,9 @@ def create_app():
     if application_settings.are_prometheus_metrics_enabled:
         setup_prometheus_metrics(app)
 
+    if application_settings.DYNAMIC_SIDECAR_TRACING:
+        setup_fastapi_app_tracing(app)
+
     # ERROR HANDLERS  ------------
     app.add_exception_handler(
         NodeNotFound,

services/efs-guardian/src/simcore_service_efs_guardian/core/application.py

@@ -1,7 +1,10 @@
 import logging
 
 from fastapi import FastAPI
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 
 from .._meta import (
     API_VERSION,
@@ -43,7 +46,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
     app.state.settings = app_settings
     assert app.state.settings.API_VERSION == API_VERSION  # nosec
     if app.state.settings.EFS_GUARDIAN_TRACING:
-        setup_tracing(app, app.state.settings.EFS_GUARDIAN_TRACING, APP_NAME)
+        tracing_instrument_tooling(
+            app, app.state.settings.EFS_GUARDIAN_TRACING, APP_NAME
+        )
 
     # PLUGINS SETUP
     setup_rabbitmq(app)
@@ -59,6 +64,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
 
     setup_fire_and_forget(app)
 
+    if app.state.settings.EFS_GUARDIAN_TRACING:
+        setup_fastapi_app_tracing(app)
+
     # EVENTS
     async def _on_startup() -> None:
         print(APP_STARTED_BANNER_MSG, flush=True)  # noqa: T201

services/invitations/src/simcore_service_invitations/core/application.py

@@ -3,7 +3,10 @@
     setup_prometheus_instrumentation,
 )
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_fastapi_app_tracing,
+    tracing_instrument_tooling,
+)
 
 from .._meta import (
     API_VERSION,
@@ -35,14 +38,20 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI:
     app.state.settings = settings or ApplicationSettings()  # type: ignore[call-arg]
     assert app.state.settings.API_VERSION == API_VERSION  # nosec
 
+    if app.state.settings.INVITATIONS_TRACING:
+        tracing_instrument_tooling(
+            app, app.state.settings.INVITATIONS_TRACING, APP_NAME
+        )
+
     # PLUGINS SETUP
     setup_api_routes(app)
 
-    if app.state.settings.INVITATIONS_TRACING:
-        setup_tracing(app, app.state.settings.INVITATIONS_TRACING, APP_NAME)
     if app.state.settings.INVITATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED:
         setup_prometheus_instrumentation(app)
 
+    if app.state.settings.INVITATIONS_TRACING:
+        setup_fastapi_app_tracing(app)
+
     # ERROR HANDLERS
     exceptions_handlers.setup(app)