instrument httpx clients in api-server

Author: bisgaard-itis

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -5,6 +5,7 @@
 import logging
 
 from fastapi import FastAPI
+from httpx import AsyncClient, Client
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
     OTLPSpanExporter as OTLPSpanExporterHTTP,
@@ -84,9 +85,8 @@ def setup_tracing(
     otlp_exporter = OTLPSpanExporterHTTP(endpoint=tracing_destination)
     span_processor = BatchSpanProcessor(otlp_exporter)
     global_tracer_provider.add_span_processor(span_processor)
-    # Instrument FastAPI and all httpx clients
+    # Instrument FastAPI
     FastAPIInstrumentor().instrument_app(app)
-    HTTPXClientInstrumentor().instrument()
 
     if HAS_AIOPG:
         with log_context(
@@ -123,3 +123,7 @@ def setup_tracing(
             msg="Attempting to add requests opentelemetry autoinstrumentation...",
         ):
             RequestsInstrumentor().instrument()
+
+
+def setup_httpx_client_tracing(client: AsyncClient | Client):
+    HTTPXClientInstrumentor.instrument_client(client)

services/api-server/src/simcore_service_api_server/core/application.py

@@ -82,19 +82,22 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
 
     setup_rabbitmq(app)
 
-    if settings.API_SERVER_WEBSERVER:
-        webserver.setup(app, settings.API_SERVER_WEBSERVER)
+    add_tracing = False
     if app.state.settings.API_SERVER_TRACING:
+        add_tracing = True
         setup_tracing(app, app.state.settings.API_SERVER_TRACING, APP_NAME)
 
+    if settings.API_SERVER_WEBSERVER:
+        webserver.setup(app, settings.API_SERVER_WEBSERVER, add_tracing=add_tracing)
+
     if settings.API_SERVER_CATALOG:
-        catalog.setup(app, settings.API_SERVER_CATALOG)
+        catalog.setup(app, settings.API_SERVER_CATALOG, add_tracing=add_tracing)
 
     if settings.API_SERVER_STORAGE:
-        storage.setup(app, settings.API_SERVER_STORAGE)
+        storage.setup(app, settings.API_SERVER_STORAGE, add_tracing=add_tracing)
 
     if settings.API_SERVER_DIRECTOR_V2:
-        director_v2.setup(app, settings.API_SERVER_DIRECTOR_V2)
+        director_v2.setup(app, settings.API_SERVER_DIRECTOR_V2, add_tracing=add_tracing)
 
     # setup app
     app.add_event_handler("startup", create_start_app_handler(app))

services/api-server/src/simcore_service_api_server/services/catalog.py

@@ -209,10 +209,14 @@ async def get_latest_release(
 # MODULES APP SETUP -------------------------------------------------------------
 
 
-def setup(app: FastAPI, settings: CatalogSettings) -> None:
+def setup(app: FastAPI, settings: CatalogSettings, add_tracing: bool = False) -> None:
     if not settings:
         settings = CatalogSettings()
 
     setup_client_instance(
-        app, CatalogApi, api_baseurl=settings.api_base_url, service_name="catalog"
+        app,
+        CatalogApi,
+        api_baseurl=settings.api_base_url,
+        service_name="catalog",
+        add_tracing=add_tracing,
     )

services/api-server/src/simcore_service_api_server/services/director_v2.py

@@ -191,11 +191,14 @@ async def get_computation_logs(
 # MODULES APP SETUP -------------------------------------------------------------
 
 
-def setup(app: FastAPI, settings: DirectorV2Settings) -> None:
+def setup(
+    app: FastAPI, settings: DirectorV2Settings, add_tracing: bool = False
+) -> None:
     setup_client_instance(
         app,
         DirectorV2Api,
         # WARNING: it has /v0 and /v2 prefixes
         api_baseurl=settings.base_url,
         service_name="director_v2",
+        add_tracing=add_tracing,
     )

services/api-server/src/simcore_service_api_server/services/storage.py

@@ -209,12 +209,16 @@ async def create_soft_link(
 # MODULES APP SETUP -------------------------------------------------------------
 
 
-def setup(app: FastAPI, settings: StorageSettings) -> None:
+def setup(app: FastAPI, settings: StorageSettings, add_tracing: bool = False) -> None:
     if not settings:
         settings = StorageSettings()
 
     setup_client_instance(
-        app, StorageApi, api_baseurl=settings.api_base_url, service_name="storage"
+        app,
+        StorageApi,
+        api_baseurl=settings.api_base_url,
+        service_name="storage",
+        add_tracing=add_tracing,
     )
 
 

services/api-server/src/simcore_service_api_server/services/webserver.py

@@ -588,19 +588,21 @@ async def get_service_pricing_plan(
 # MODULES APP SETUP -------------------------------------------------------------
 
 
-def setup(app: FastAPI, settings: WebServerSettings) -> None:
+def setup(app: FastAPI, settings: WebServerSettings, add_tracing: bool = False) -> None:
 
     setup_client_instance(
         app,
         WebserverApi,
         api_baseurl=settings.api_base_url,
         service_name="webserver",
+        add_tracing=add_tracing,
     )
     setup_client_instance(
         app,
         LongRunningTasksClient,
         api_baseurl="",
         service_name="long_running_tasks_client",
+        add_tracing=add_tracing,
     )
 
     def _on_startup() -> None:

services/api-server/src/simcore_service_api_server/utils/client_base.py

@@ -4,6 +4,7 @@
 import httpx
 from fastapi import FastAPI
 from httpx import AsyncClient
+from servicelib.fastapi.tracing import setup_httpx_client_tracing
 
 from .app_data import AppDataMixin
 
@@ -43,6 +44,7 @@ def setup_client_instance(
     api_cls: type[BaseServiceClientApi],
     api_baseurl,
     service_name: str,
+    add_tracing: bool = False,
     **extra_fields,
 ) -> None:
     """Helper to add init/cleanup of ServiceClientApi instances in the app lifespam"""
@@ -51,6 +53,8 @@ def setup_client_instance(
 
     # NOTE: this term is mocked in tests. If you need to modify pay attention to the mock
     client = AsyncClient(base_url=api_baseurl)
+    if add_tracing:
+        setup_httpx_client_tracing(client)
 
     # events
     def _create_instance() -> None: