🐛✨ Opentelemetry instrument asyncpg for aiohttp servers and introduce decorator to generate opentelemtry profile span

packages/aws-library/requirements/_base.txt

@@ -140,6 +140,7 @@ opentelemetry-api==1.34.1
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-botocore
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
@@ -160,12 +161,15 @@ opentelemetry-exporter-otlp-proto-http==1.34.1
 opentelemetry-instrumentation==0.55b1
     # via
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-botocore
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
 opentelemetry-instrumentation-aio-pika==0.55b1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-instrumentation-asyncpg==0.55b1
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-botocore==0.55b1
     # via -r requirements/_base.in
 opentelemetry-instrumentation-logging==0.55b1
@@ -189,6 +193,7 @@ opentelemetry-sdk==1.34.1
 opentelemetry-semantic-conventions==0.55b1
     # via
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-botocore
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests

packages/celery-library/requirements/_base.txt

@@ -132,6 +132,7 @@ opentelemetry-api==1.34.1
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
@@ -150,11 +151,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1
 opentelemetry-instrumentation==0.55b1
     # via
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
 opentelemetry-instrumentation-aio-pika==0.55b1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-instrumentation-asyncpg==0.55b1
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-logging==0.55b1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-redis==0.55b1
@@ -174,6 +178,7 @@ opentelemetry-sdk==1.34.1
 opentelemetry-semantic-conventions==0.55b1
     # via
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-sdk

packages/notifications-library/requirements/_base.txt

@@ -26,8 +26,6 @@ idna==3.10
     # via
     #   email-validator
     #   yarl
-importlib-metadata==8.5.0
-    # via opentelemetry-api
 jinja2==3.1.5
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -64,19 +62,6 @@ mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via yarl
-opentelemetry-api==1.34.1
-    # via
-    #   opentelemetry-instrumentation
-    #   opentelemetry-instrumentation-asyncpg
-    #   opentelemetry-semantic-conventions
-opentelemetry-instrumentation==0.55b1
-    # via opentelemetry-instrumentation-asyncpg
-opentelemetry-instrumentation-asyncpg==0.55b1
-    # via -r requirements/../../../packages/postgres-database/requirements/_base.in
-opentelemetry-semantic-conventions==0.55b1
-    # via
-    #   opentelemetry-instrumentation
-    #   opentelemetry-instrumentation-asyncpg
 orjson==3.10.15
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -92,8 +77,6 @@ orjson==3.10.15
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
-packaging==24.2
-    # via opentelemetry-instrumentation
 propcache==0.3.0
     # via yarl
 psycopg2-binary==2.9.10
@@ -187,15 +170,9 @@ types-python-dateutil==2.9.0.20241206
 typing-extensions==4.14.1
     # via
     #   alembic
-    #   opentelemetry-api
-    #   opentelemetry-semantic-conventions
     #   pydantic
     #   pydantic-core
     #   pydantic-extra-types
     #   typer
-wrapt==1.17.2
-    # via opentelemetry-instrumentation
 yarl==1.18.3
     # via -r requirements/../../../packages/postgres-database/requirements/_base.in
-zipp==3.21.0
-    # via importlib-metadata

packages/notifications-library/requirements/_test.txt

@@ -54,7 +54,6 @@ mypy-extensions==1.1.0
     # via mypy
 packaging==24.2
     # via
-    #   -c requirements/_base.txt
     #   pytest
     #   pytest-sugar
 pathspec==0.12.1

packages/notifications-library/requirements/_tools.txt

@@ -40,7 +40,6 @@ nodeenv==1.9.1
     # via pre-commit
 packaging==24.2
     # via
-    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   black
     #   build

packages/postgres-database/requirements/_base.in

@@ -6,7 +6,6 @@
 --requirement ../../../packages/common-library/requirements/_base.in
 
 alembic
-opentelemetry-instrumentation-asyncpg
 pydantic
 sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43
 yarl

packages/postgres-database/requirements/_base.txt

@@ -8,8 +8,6 @@ greenlet==3.1.1
     # via sqlalchemy
 idna==3.10
     # via yarl
-importlib-metadata==8.5.0
-    # via opentelemetry-api
 mako==1.3.10
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -19,26 +17,11 @@ markupsafe==3.0.2
     # via mako
 multidict==6.1.0
     # via yarl
-opentelemetry-api==1.34.1
-    # via
-    #   opentelemetry-instrumentation
-    #   opentelemetry-instrumentation-asyncpg
-    #   opentelemetry-semantic-conventions
-opentelemetry-instrumentation==0.55b1
-    # via opentelemetry-instrumentation-asyncpg
-opentelemetry-instrumentation-asyncpg==0.55b1
-    # via -r requirements/_base.in
-opentelemetry-semantic-conventions==0.55b1
-    # via
-    #   opentelemetry-instrumentation
-    #   opentelemetry-instrumentation-asyncpg
 orjson==3.10.15
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/common-library/requirements/_base.in
-packaging==24.2
-    # via opentelemetry-instrumentation
 propcache==0.3.0
     # via yarl
 psycopg2-binary==2.9.10
@@ -63,14 +46,8 @@ sqlalchemy==1.4.54
 typing-extensions==4.14.1
     # via
     #   alembic
-    #   opentelemetry-api
-    #   opentelemetry-semantic-conventions
     #   pydantic
     #   pydantic-core
     #   pydantic-extra-types
-wrapt==1.17.2
-    # via opentelemetry-instrumentation
 yarl==1.18.3
     # via -r requirements/_base.in
-zipp==3.21.0
-    # via importlib-metadata

packages/postgres-database/requirements/_test.txt

@@ -24,9 +24,7 @@ mypy==1.16.1
 mypy-extensions==1.1.0
     # via mypy
 packaging==24.2
-    # via
-    #   -c requirements/_base.txt
-    #   pytest
+    # via pytest
 pathspec==0.12.1
     # via mypy
 pluggy==1.5.0

packages/postgres-database/requirements/_tools.txt

@@ -39,7 +39,6 @@ nodeenv==1.9.1
     # via pre-commit
 packaging==24.2
     # via
-    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   black
     #   build

packages/service-library/requirements/_base.in

@@ -19,6 +19,7 @@ faststream
 opentelemetry-api
 opentelemetry-exporter-otlp
 opentelemetry-instrumentation-aio-pika
+opentelemetry-instrumentation-asyncpg
 opentelemetry-instrumentation-logging
 opentelemetry-instrumentation-redis
 opentelemetry-instrumentation-requests

packages/service-library/requirements/_base.txt

@@ -98,6 +98,7 @@ opentelemetry-api==1.34.1
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
@@ -116,11 +117,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1
 opentelemetry-instrumentation==0.55b1
     # via
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
 opentelemetry-instrumentation-aio-pika==0.55b1
     # via -r requirements/_base.in
+opentelemetry-instrumentation-asyncpg==0.55b1
+    # via -r requirements/_base.in
 opentelemetry-instrumentation-logging==0.55b1
     # via -r requirements/_base.in
 opentelemetry-instrumentation-redis==0.55b1
@@ -140,6 +144,7 @@ opentelemetry-sdk==1.34.1
 opentelemetry-semantic-conventions==0.55b1
     # via
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-sdk

packages/service-library/src/servicelib/aiohttp/tracing.py

@@ -37,6 +37,13 @@
     HAS_AIOPG = True
 except ImportError:
     HAS_AIOPG = False
+try:
+    from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
+
+    HAS_ASYNCPG = True
+except ImportError:
+    HAS_ASYNCPG = False
+
 try:
     from opentelemetry.instrumentation.requests import RequestsInstrumentor
 
@@ -130,6 +137,13 @@ def _startup(
             msg="Attempting to add aio-pg opentelemetry autoinstrumentation...",
         ):
             AiopgInstrumentor().instrument()
+    if HAS_ASYNCPG:
+        with log_context(
+            _logger,
+            logging.INFO,
+            msg="Attempting to add asyncpg opentelemetry autoinstrumentation...",
+        ):
+            AsyncPGInstrumentor().instrument()
     if HAS_BOTOCORE:
         with log_context(
             _logger,
@@ -180,6 +194,11 @@ def _shutdown() -> None:
             AiopgInstrumentor().uninstrument()
         except Exception:  # pylint:disable=broad-exception-caught
             _logger.exception("Failed to uninstrument AiopgInstrumentor")
+    if HAS_ASYNCPG:
+        try:
+            AsyncPGInstrumentor().uninstrument()
+        except Exception:  # pylint:disable=broad-exception-caught
+            _logger.exception("Failed to uninstrument AsyncPGInstrumentor")
     if HAS_BOTOCORE:
         try:
             BotocoreInstrumentor().uninstrument()

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -24,7 +24,7 @@
 _logger = logging.getLogger(__name__)
 
 try:
-    from opentelemetry.instrumentation.asyncpg import (  # type: ignore[import-not-found]
+    from opentelemetry.instrumentation.asyncpg import (
         AsyncPGInstrumentor,
     )
 

packages/service-library/src/servicelib/tracing.py

@@ -1,15 +1,21 @@
+from collections.abc import Callable, Coroutine
 from contextlib import contextmanager
 from contextvars import Token
-from typing import TypeAlias
+from functools import wraps
+from typing import Any, Final, TypeAlias
 
+import pyinstrument
+import pyinstrument.renderers
 from opentelemetry import context as otcontext
 from opentelemetry import trace
 from opentelemetry.instrumentation.logging import LoggingInstrumentor
 from settings_library.tracing import TracingSettings
 
 TracingContext: TypeAlias = otcontext.Context | None
 
-_OSPARC_TRACE_ID_HEADER = "x-osparc-trace-id"
+_TRACER_NAME: Final[str] = "servicelib.tracing"
+_PROFILE_ATTRIBUTE_NAME: Final[str] = "pyinstrument.profile"
+_OSPARC_TRACE_ID_HEADER: Final[str] = "x-osparc-trace-id"
 
 
 def _is_tracing() -> bool:
@@ -49,3 +55,41 @@ def get_trace_id_header() -> dict[str, str] | None:
         )  # Convert trace_id to 32-character hex string
         return {_OSPARC_TRACE_ID_HEADER: trace_id_hex}
     return None
+
+
+def with_profiled_span(
+    func: Callable[..., Coroutine[Any, Any, Any]],
+) -> Callable[..., Coroutine[Any, Any, Any]]:
+    """Decorator that wraps an async function in an OpenTelemetry span with pyinstrument profiling."""
+
+    @wraps(func)
+    async def wrapper(*args: Any, **kwargs: Any) -> Any:
+        if not _is_tracing():
+            return await func(*args, **kwargs)
+
+        tracer = trace.get_tracer(_TRACER_NAME)
+        span_name = f"{func.__module__}.{func.__qualname__}"
+
+        with tracer.start_as_current_span(span_name) as span:
+            profiler = pyinstrument.Profiler(async_mode="enabled")
+            profiler.start()
+
+            try:
+                return await func(*args, **kwargs)
+
+            except Exception as e:
+                span.record_exception(e)
+                span.set_status(trace.Status(trace.StatusCode.ERROR, f"{e}"))
+                raise
+
+            finally:
+                profiler.stop()
+                renderer = pyinstrument.renderers.ConsoleRenderer(
+                    unicode=True, color=False, show_all=True
+                )
+                span.set_attribute(
+                    _PROFILE_ATTRIBUTE_NAME,
+                    profiler.output(renderer=renderer),
+                )
+
+    return wrapper