⚗️ Add tracing spans to simcore services

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -2,7 +2,16 @@
 
 """
 
+import importlib
+import importlib.machinery
+import inspect
 import logging
+import sys
+from functools import wraps
+from importlib.abc import Loader, MetaPathFinder
+from importlib.machinery import ModuleSpec
+from types import ModuleType
+from typing import Callable, Sequence
 
 from fastapi import FastAPI
 from httpx import AsyncClient, Client
@@ -127,3 +136,66 @@ def setup_tracing(
 
 def setup_httpx_client_tracing(client: AsyncClient | Client):
     HTTPXClientInstrumentor.instrument_client(client)
+
+
+def _opentelemetry_function_span(func: Callable):
+    """Decorator that wraps a function call in an OpenTelemetry span."""
+    tracer = trace.get_tracer(__name__)
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        with tracer.start_as_current_span(f"{func.__module__}.{func.__name__}"):
+            return func(*args, **kwargs)
+
+    @wraps(func)
+    async def async_wrapper(*args, **kwargs):
+        with tracer.start_as_current_span(f"{func.__module__}.{func.__name__}"):
+            return await func(*args, **kwargs)
+
+    if inspect.iscoroutinefunction(func):
+        return async_wrapper
+    else:
+        return wrapper
+
+
+def _opentelemetry_method_span(cls):
+    for name, value in cls.__dict__.items():
+        if callable(value) and not name.startswith("_"):
+            setattr(cls, name, _opentelemetry_function_span(value))
+    return cls
+
+
+class _AddTracingSpansLoader(Loader):
+    def __init__(self, loader: Loader):
+        self.loader = loader
+
+    def exec_module(self, module: ModuleType):
+        self.loader.exec_module(module)
+        for name, func in inspect.getmembers(module, inspect.isfunction):
+            if name in module.__dict__:
+                setattr(module, name, _opentelemetry_function_span(func))
+        for name, cls in inspect.getmembers(module, inspect.isclass):
+            if name in module.__dict__ and cls.__module__ == module.__name__:
+                setattr(module, name, _opentelemetry_method_span(cls))
+
+
+class _AddTracingSpansFinder(MetaPathFinder):
+    def find_spec(
+        self,
+        fullname: str,
+        path: Sequence[str] | None,
+        target: ModuleType | None = None,
+    ) -> ModuleSpec | None:
+        if fullname.startswith("simcore_service"):
+            spec = importlib.machinery.PathFinder.find_spec(
+                fullname=fullname, path=path
+            )
+            if spec and spec.loader:
+                spec.loader = _AddTracingSpansLoader(spec.loader)
+                return spec
+
+        return None
+
+
+def setup_tracing_spans_for_simcore_service_functions():
+    sys.meta_path.insert(0, _AddTracingSpansFinder())

services/api-server/src/simcore_service_api_server/core/application.py

@@ -5,10 +5,16 @@
 from models_library.basic_types import BootModeEnum
 from packaging.version import Version
 from servicelib.fastapi.profiler_middleware import ProfilerMiddleware
-from servicelib.fastapi.tracing import setup_tracing
+from servicelib.fastapi.tracing import (
+    setup_tracing,
+    setup_tracing_spans_for_simcore_service_functions,
+)
 from servicelib.logging_utils import config_all_loggers
+from simcore_service_api_server import exceptions
+
+setup_tracing_spans_for_simcore_service_functions()
+# isort: split
 
-from .. import exceptions
 from .._meta import API_VERSION, API_VTAG, APP_NAME
 from ..api.root import create_router
 from ..api.routes.health import router as health_router