diff --git a/.env-devel b/.env-devel index 1825a8d7628..167b4e36c59 100644 --- a/.env-devel +++ b/.env-devel @@ -17,12 +17,12 @@ AGENT_VOLUMES_CLEANUP_S3_ENDPOINT=http://172.17.0.1:9001 AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO AGENT_VOLUMES_CLEANUP_S3_REGION=us-east-1 AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678 -AGENT_TRACING=null +AGENT_TRACING={} API_SERVER_DEV_FEATURES_ENABLED=0 API_SERVER_LOGLEVEL=INFO API_SERVER_PROFILING=1 -API_SERVER_TRACING=null +API_SERVER_TRACING={} TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25 AUTOSCALING_DASK=null @@ -35,7 +35,7 @@ AUTOSCALING_LOGLEVEL=INFO AUTOSCALING_NODES_MONITORING=null AUTOSCALING_POLL_INTERVAL="00:00:10" AUTOSCALING_SSM_ACCESS=null -AUTOSCALING_TRACING=null +AUTOSCALING_TRACING={} AWS_S3_CLI_S3=null @@ -47,7 +47,7 @@ CATALOG_PORT=8000 CATALOG_PROFILING=1 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}' CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}' -CATALOG_TRACING=null +CATALOG_TRACING={} CELERY_RESULT_EXPIRES=P7D @@ -63,7 +63,7 @@ CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null CLUSTERS_KEEPER_TASK_INTERVAL=00:00:30 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null -CLUSTERS_KEEPER_TRACING=null +CLUSTERS_KEEPER_TRACING={} DASK_SCHEDULER_HOST=dask-scheduler DASK_SCHEDULER_PORT=8786 @@ -83,7 +83,7 @@ DIRECTOR_PUBLISHED_HOST_NAME="127.0.0.1:9081" DIRECTOR_REGISTRY_CACHING_TTL=00:15:00 DIRECTOR_REGISTRY_CACHING=True DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null -DIRECTOR_TRACING=null +DIRECTOR_TRACING={} DOCKER_API_PROXY_HOST=docker-api-proxy DOCKER_API_PROXY_PASSWORD=admin @@ -98,11 +98,11 @@ EFS_GROUP_NAME=efs-group EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com EFS_MOUNTED_PATH=/tmp/efs EFS_PROJECT_SPECIFIC_DATA_DIRECTORY=project-specific-data -EFS_GUARDIAN_TRACING=null +EFS_GUARDIAN_TRACING={} EFS_DEFAULT_USER_SERVICE_SIZE_BYTES=10000 # DATCORE_ADAPTER -DATCORE_ADAPTER_TRACING=null +DATCORE_ADAPTER_TRACING={} # DIRECTOR_V2 ---- COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' @@ -128,14 +128,14 @@ DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[] DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={} DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00 -DIRECTOR_V2_TRACING=null +DIRECTOR_V2_TRACING={} # DYNAMIC_SCHEDULER ---- DYNAMIC_SCHEDULER_LOGLEVEL=INFO DYNAMIC_SCHEDULER_PROFILING=1 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00 -DYNAMIC_SCHEDULER_TRACING=null +DYNAMIC_SCHEDULER_TRACING={} DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' @@ -158,13 +158,13 @@ INVITATIONS_PORT=8000 INVITATIONS_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key=' INVITATIONS_SWAGGER_API_DOC_ENABLED=1 INVITATIONS_USERNAME=admin -INVITATIONS_TRACING=null +INVITATIONS_TRACING={} LOG_FORMAT_LOCAL_DEV_ENABLED=1 LOG_FILTER_MAPPING='{"gunicorn.access":[" /v0/ ", " /v0/health "], "uvicorn.access":[" / "]}' NOTIFICATIONS_LOGLEVEL=INFO -NOTIFICATIONS_TRACING=null +NOTIFICATIONS_TRACING={} PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676 @@ -186,7 +186,7 @@ PAYMENTS_STRIPE_API_SECRET='REPLACE_ME_with_api_secret' PAYMENTS_STRIPE_URL=https://api.stripe.com PAYMENTS_SWAGGER_API_DOC_ENABLED=1 PAYMENTS_USERNAME=admin -PAYMENTS_TRACING=null +PAYMENTS_TRACING={} POSTGRES_DB=simcoredb POSTGRES_ENDPOINT=postgres:5432 @@ -227,7 +227,7 @@ RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED=1 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL=6 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC=300 RESOURCE_USAGE_TRACKER_S3=null -RESOURCE_USAGE_TRACKER_TRACING=null +RESOURCE_USAGE_TRACKER_TRACING={} # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface. R_CLONE_OPTION_BUFFER_SIZE=16M @@ -259,7 +259,7 @@ STORAGE_HOST=storage STORAGE_LOGLEVEL=INFO STORAGE_PORT=8080 STORAGE_PROFILING=1 -STORAGE_TRACING=null +STORAGE_TRACING={} # STORAGE ---- SWARM_STACK_NAME=master-simcore @@ -300,7 +300,7 @@ WB_GC_SOCKETIO=1 WB_GC_STATICWEB=null WB_GC_STUDIES_DISPATCHER=null WB_GC_TAGS=0 -WB_GC_TRACING=null +WB_GC_TRACING={} WB_GC_USERS={} WB_GC_WALLETS=0 @@ -330,7 +330,7 @@ WB_DB_EL_STATICWEB=null WB_DB_EL_STORAGE=null WB_DB_EL_STUDIES_DISPATCHER=null WB_DB_EL_TAGS=0 -WB_DB_EL_TRACING=null +WB_DB_EL_TRACING={} WB_DB_EL_USERS={} WB_DB_EL_WALLETS=0 @@ -401,5 +401,5 @@ WEBSERVER_SOCKETIO=1 WEBSERVER_STATICWEB={} WEBSERVER_STUDIES_DISPATCHER={} WEBSERVER_TAGS=1 -WEBSERVER_TRACING=null +WEBSERVER_TRACING={} WEBSERVER_USERS={} diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 6cc79c2876e..4e2ae377fbf 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -140,6 +140,7 @@ opentelemetry-api==1.34.1 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis @@ -160,12 +161,15 @@ opentelemetry-exporter-otlp-proto-http==1.34.1 opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-botocore==0.55b1 # via -r requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 @@ -189,6 +193,7 @@ opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests diff --git a/packages/celery-library/requirements/_base.txt b/packages/celery-library/requirements/_base.txt index ed5e9dbcc25..167e64a4283 100644 --- a/packages/celery-library/requirements/_base.txt +++ b/packages/celery-library/requirements/_base.txt @@ -132,6 +132,7 @@ opentelemetry-api==1.34.1 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests @@ -150,11 +151,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1 opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.55b1 @@ -174,6 +178,7 @@ opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt index db3bfe854ae..a037802b4af 100644 --- a/packages/notifications-library/requirements/_base.txt +++ b/packages/notifications-library/requirements/_base.txt @@ -26,8 +26,6 @@ idna==3.10 # via # email-validator # yarl -importlib-metadata==8.5.0 - # via opentelemetry-api jinja2==3.1.5 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -64,19 +62,6 @@ mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 # via yarl -opentelemetry-api==1.34.1 - # via - # opentelemetry-instrumentation - # opentelemetry-instrumentation-asyncpg - # opentelemetry-semantic-conventions -opentelemetry-instrumentation==0.55b1 - # via opentelemetry-instrumentation-asyncpg -opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in -opentelemetry-semantic-conventions==0.55b1 - # via - # opentelemetry-instrumentation - # opentelemetry-instrumentation-asyncpg orjson==3.10.15 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -92,8 +77,6 @@ orjson==3.10.15 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.2 - # via opentelemetry-instrumentation propcache==0.3.0 # via yarl psycopg2-binary==2.9.10 @@ -187,15 +170,9 @@ types-python-dateutil==2.9.0.20241206 typing-extensions==4.14.1 # via # alembic - # opentelemetry-api - # opentelemetry-semantic-conventions # pydantic # pydantic-core # pydantic-extra-types # typer -wrapt==1.17.2 - # via opentelemetry-instrumentation yarl==1.18.3 # via -r requirements/../../../packages/postgres-database/requirements/_base.in -zipp==3.21.0 - # via importlib-metadata diff --git a/packages/notifications-library/requirements/_test.txt b/packages/notifications-library/requirements/_test.txt index 8ab227a3e5d..b85f8f7ebcf 100644 --- a/packages/notifications-library/requirements/_test.txt +++ b/packages/notifications-library/requirements/_test.txt @@ -54,7 +54,6 @@ mypy-extensions==1.1.0 # via mypy packaging==24.2 # via - # -c requirements/_base.txt # pytest # pytest-sugar pathspec==0.12.1 diff --git a/packages/notifications-library/requirements/_tools.txt b/packages/notifications-library/requirements/_tools.txt index 1738c3a0c1d..16f34752f51 100644 --- a/packages/notifications-library/requirements/_tools.txt +++ b/packages/notifications-library/requirements/_tools.txt @@ -40,7 +40,6 @@ nodeenv==1.9.1 # via pre-commit packaging==24.2 # via - # -c requirements/_base.txt # -c requirements/_test.txt # black # build diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index c5aa128b710..0294edf9114 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -6,7 +6,6 @@ --requirement ../../../packages/common-library/requirements/_base.in alembic -opentelemetry-instrumentation-asyncpg pydantic sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43 yarl diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index 4d9a6684694..40b17728545 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -8,8 +8,6 @@ greenlet==3.1.1 # via sqlalchemy idna==3.10 # via yarl -importlib-metadata==8.5.0 - # via opentelemetry-api mako==1.3.10 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -19,26 +17,11 @@ markupsafe==3.0.2 # via mako multidict==6.1.0 # via yarl -opentelemetry-api==1.34.1 - # via - # opentelemetry-instrumentation - # opentelemetry-instrumentation-asyncpg - # opentelemetry-semantic-conventions -opentelemetry-instrumentation==0.55b1 - # via opentelemetry-instrumentation-asyncpg -opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/_base.in -opentelemetry-semantic-conventions==0.55b1 - # via - # opentelemetry-instrumentation - # opentelemetry-instrumentation-asyncpg orjson==3.10.15 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/common-library/requirements/_base.in -packaging==24.2 - # via opentelemetry-instrumentation propcache==0.3.0 # via yarl psycopg2-binary==2.9.10 @@ -63,14 +46,8 @@ sqlalchemy==1.4.54 typing-extensions==4.14.1 # via # alembic - # opentelemetry-api - # opentelemetry-semantic-conventions # pydantic # pydantic-core # pydantic-extra-types -wrapt==1.17.2 - # via opentelemetry-instrumentation yarl==1.18.3 # via -r requirements/_base.in -zipp==3.21.0 - # via importlib-metadata diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt index 3595b7824e9..75384119e83 100644 --- a/packages/postgres-database/requirements/_test.txt +++ b/packages/postgres-database/requirements/_test.txt @@ -24,9 +24,7 @@ mypy==1.16.1 mypy-extensions==1.1.0 # via mypy packaging==24.2 - # via - # -c requirements/_base.txt - # pytest + # via pytest pathspec==0.12.1 # via mypy pluggy==1.5.0 diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt index 174237b764a..3a5b7eecff4 100644 --- a/packages/postgres-database/requirements/_tools.txt +++ b/packages/postgres-database/requirements/_tools.txt @@ -39,7 +39,6 @@ nodeenv==1.9.1 # via pre-commit packaging==24.2 # via - # -c requirements/_base.txt # -c requirements/_test.txt # black # build diff --git a/packages/pytest-simcore/src/pytest_simcore/tracing.py b/packages/pytest-simcore/src/pytest_simcore/tracing.py new file mode 100644 index 00000000000..e9bed749c2c --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/tracing.py @@ -0,0 +1,38 @@ +import pytest +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from pytest_mock import MockerFixture + + +@pytest.fixture +async def setup_tracing_fastapi( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> InMemorySpanExporter: + memory_exporter = InMemorySpanExporter() + span_processor = SimpleSpanProcessor(memory_exporter) + mocker.patch( + "servicelib.fastapi.tracing._create_span_processor", return_value=span_processor + ) + + monkeypatch.setenv( + "TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT", "http://opentelemetry-collector" + ) + monkeypatch.setenv("TRACING_OPENTELEMETRY_COLLECTOR_PORT", "4318") + return memory_exporter + + +@pytest.fixture +async def setup_tracing_aiohttp( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> InMemorySpanExporter: + memory_exporter = InMemorySpanExporter() + span_processor = SimpleSpanProcessor(memory_exporter) + mocker.patch( + "servicelib.aiohttp.tracing._create_span_processor", return_value=span_processor + ) + + monkeypatch.setenv( + "TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT", "http://opentelemetry-collector" + ) + monkeypatch.setenv("TRACING_OPENTELEMETRY_COLLECTOR_PORT", "4318") + return memory_exporter diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in index 24222e414b8..d094798f8a4 100644 --- a/packages/service-library/requirements/_base.in +++ b/packages/service-library/requirements/_base.in @@ -19,6 +19,7 @@ faststream opentelemetry-api opentelemetry-exporter-otlp opentelemetry-instrumentation-aio-pika +opentelemetry-instrumentation-asyncpg opentelemetry-instrumentation-logging opentelemetry-instrumentation-redis opentelemetry-instrumentation-requests diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index 6c3abc92732..8cc93d935c9 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -98,6 +98,7 @@ opentelemetry-api==1.34.1 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests @@ -116,11 +117,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1 opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/_base.in +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 # via -r requirements/_base.in opentelemetry-instrumentation-redis==0.55b1 @@ -140,6 +144,7 @@ opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk diff --git a/packages/service-library/src/servicelib/aiohttp/tracing.py b/packages/service-library/src/servicelib/aiohttp/tracing.py index d4e2d3610cd..9aa8b9e5728 100644 --- a/packages/service-library/src/servicelib/aiohttp/tracing.py +++ b/packages/service-library/src/servicelib/aiohttp/tracing.py @@ -37,6 +37,13 @@ HAS_AIOPG = True except ImportError: HAS_AIOPG = False +try: + from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor + + HAS_ASYNCPG = True +except ImportError: + HAS_ASYNCPG = False + try: from opentelemetry.instrumentation.requests import RequestsInstrumentor @@ -130,6 +137,13 @@ def _startup( msg="Attempting to add aio-pg opentelemetry autoinstrumentation...", ): AiopgInstrumentor().instrument() + if HAS_ASYNCPG: + with log_context( + _logger, + logging.INFO, + msg="Attempting to add asyncpg opentelemetry autoinstrumentation...", + ): + AsyncPGInstrumentor().instrument() if HAS_BOTOCORE: with log_context( _logger, @@ -180,6 +194,11 @@ def _shutdown() -> None: AiopgInstrumentor().uninstrument() except Exception: # pylint:disable=broad-exception-caught _logger.exception("Failed to uninstrument AiopgInstrumentor") + if HAS_ASYNCPG: + try: + AsyncPGInstrumentor().uninstrument() + except Exception: # pylint:disable=broad-exception-caught + _logger.exception("Failed to uninstrument AsyncPGInstrumentor") if HAS_BOTOCORE: try: BotocoreInstrumentor().uninstrument() diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index 06224618a63..9943ed81022 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -24,7 +24,7 @@ _logger = logging.getLogger(__name__) try: - from opentelemetry.instrumentation.asyncpg import ( # type: ignore[import-not-found] + from opentelemetry.instrumentation.asyncpg import ( AsyncPGInstrumentor, ) diff --git a/packages/service-library/src/servicelib/tracing.py b/packages/service-library/src/servicelib/tracing.py index 355a5cf3b64..a95f386495e 100644 --- a/packages/service-library/src/servicelib/tracing.py +++ b/packages/service-library/src/servicelib/tracing.py @@ -1,7 +1,11 @@ +from collections.abc import Callable, Coroutine from contextlib import contextmanager from contextvars import Token -from typing import TypeAlias +from functools import wraps +from typing import Any, Final, TypeAlias +import pyinstrument +import pyinstrument.renderers from opentelemetry import context as otcontext from opentelemetry import trace from opentelemetry.instrumentation.logging import LoggingInstrumentor @@ -9,7 +13,9 @@ TracingContext: TypeAlias = otcontext.Context | None -_OSPARC_TRACE_ID_HEADER = "x-osparc-trace-id" +_TRACER_NAME: Final[str] = "servicelib.tracing" +_PROFILE_ATTRIBUTE_NAME: Final[str] = "pyinstrument.profile" +_OSPARC_TRACE_ID_HEADER: Final[str] = "x-osparc-trace-id" def _is_tracing() -> bool: @@ -49,3 +55,41 @@ def get_trace_id_header() -> dict[str, str] | None: ) # Convert trace_id to 32-character hex string return {_OSPARC_TRACE_ID_HEADER: trace_id_hex} return None + + +def with_profiled_span( + func: Callable[..., Coroutine[Any, Any, Any]], +) -> Callable[..., Coroutine[Any, Any, Any]]: + """Decorator that wraps an async function in an OpenTelemetry span with pyinstrument profiling.""" + + @wraps(func) + async def wrapper(*args: Any, **kwargs: Any) -> Any: + if not _is_tracing(): + return await func(*args, **kwargs) + + tracer = trace.get_tracer(_TRACER_NAME) + span_name = f"{func.__module__}.{func.__qualname__}" + + with tracer.start_as_current_span(span_name) as span: + profiler = pyinstrument.Profiler(async_mode="enabled") + profiler.start() + + try: + return await func(*args, **kwargs) + + except Exception as e: + span.record_exception(e) + span.set_status(trace.Status(trace.StatusCode.ERROR, f"{e}")) + raise + + finally: + profiler.stop() + renderer = pyinstrument.renderers.ConsoleRenderer( + unicode=True, color=False, show_all=True + ) + span.set_attribute( + _PROFILE_ATTRIBUTE_NAME, + profiler.output(renderer=renderer), + ) + + return wrapper diff --git a/packages/service-library/tests/aiohttp/test_tracing.py b/packages/service-library/tests/aiohttp/test_tracing.py index 8e297427923..c9fb30d7de8 100644 --- a/packages/service-library/tests/aiohttp/test_tracing.py +++ b/packages/service-library/tests/aiohttp/test_tracing.py @@ -3,7 +3,7 @@ # pylint: disable=unused-variable import importlib -from collections.abc import Callable, Iterator +from collections.abc import Callable from functools import partial from typing import Any @@ -55,10 +55,10 @@ def set_and_clean_settings_env_vars( indirect=True, ) async def test_valid_tracing_settings( + mock_otel_collector: InMemorySpanExporter, aiohttp_client: Callable, set_and_clean_settings_env_vars: Callable, tracing_settings_in, - uninstrument_opentelemetry: Iterator[None], ) -> TestClient: app = web.Application() service_name = "simcore_service_webserver" @@ -79,10 +79,10 @@ async def test_valid_tracing_settings( indirect=True, ) async def test_invalid_tracing_settings( + mock_otel_collector: InMemorySpanExporter, aiohttp_client: Callable, set_and_clean_settings_env_vars: Callable, tracing_settings_in, - uninstrument_opentelemetry: Iterator[None], ) -> TestClient: with pytest.raises(ValidationError): TracingSettings() @@ -128,11 +128,11 @@ def manage_package(request): indirect=True, ) async def test_tracing_setup_package_detection( + mock_otel_collector: InMemorySpanExporter, aiohttp_client: Callable, set_and_clean_settings_env_vars: Callable[[], None], tracing_settings_in: Callable[[], dict[str, Any]], manage_package, - uninstrument_opentelemetry: Iterator[None], ): package_name = manage_package importlib.import_module(package_name) @@ -169,7 +169,6 @@ async def test_trace_id_in_response_header( aiohttp_client: Callable, set_and_clean_settings_env_vars: Callable, tracing_settings_in, - uninstrument_opentelemetry: Iterator[None], server_response: web.Response | web.HTTPException, ) -> None: app = web.Application() diff --git a/packages/service-library/tests/conftest.py b/packages/service-library/tests/conftest.py index 9c04c4cc902..45a456fed02 100644 --- a/packages/service-library/tests/conftest.py +++ b/packages/service-library/tests/conftest.py @@ -105,60 +105,3 @@ async def _cleanup_redis_data(clients_manager: RedisClientsManager) -> None: await _cleanup_redis_data(clients_manager) yield _ await _cleanup_redis_data(clients_manager) - - -@pytest.fixture() -def uninstrument_opentelemetry(): - yield - try: - from opentelemetry.instrumentation.redis import RedisInstrumentor - - RedisInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.botocore import BotocoreInstrumentor - - BotocoreInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.requests import RequestsInstrumentor - - RequestsInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.aiopg import AiopgInstrumentor - - AiopgInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor - - AsyncPGInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor - - FastAPIInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.aiohttp_client import ( - AioHttpClientInstrumentor, - ) - - AioHttpClientInstrumentor().uninstrument() - except ImportError: - pass - try: - from opentelemetry.instrumentation.aiohttp_server import ( - AioHttpServerInstrumentor, - ) - - AioHttpServerInstrumentor().uninstrument() - except ImportError: - pass diff --git a/packages/service-library/tests/fastapi/conftest.py b/packages/service-library/tests/fastapi/conftest.py index 66db13b8664..9074321fa9b 100644 --- a/packages/service-library/tests/fastapi/conftest.py +++ b/packages/service-library/tests/fastapi/conftest.py @@ -3,7 +3,7 @@ # pylint: disable=unused-variable import socket -from collections.abc import AsyncIterator, Callable, Iterator +from collections.abc import AsyncIterator, Callable from typing import cast import arrow @@ -61,10 +61,10 @@ def go() -> int: @pytest.fixture -def mock_otel_collector(mocker: MockerFixture) -> Iterator[InMemorySpanExporter]: +def mock_otel_collector(mocker: MockerFixture) -> InMemorySpanExporter: memory_exporter = InMemorySpanExporter() span_processor = SimpleSpanProcessor(memory_exporter) mocker.patch( "servicelib.fastapi.tracing._create_span_processor", return_value=span_processor ) - yield memory_exporter + return memory_exporter diff --git a/packages/service-library/tests/fastapi/test_tracing.py b/packages/service-library/tests/fastapi/test_tracing.py index 16becc3a1b6..148f4e43776 100644 --- a/packages/service-library/tests/fastapi/test_tracing.py +++ b/packages/service-library/tests/fastapi/test_tracing.py @@ -4,7 +4,7 @@ import importlib import random import string -from collections.abc import Callable, Iterator +from collections.abc import Callable from functools import partial from typing import Any @@ -21,7 +21,11 @@ get_tracing_instrumentation_lifespan, initialize_fastapi_app_tracing, ) -from servicelib.tracing import _OSPARC_TRACE_ID_HEADER +from servicelib.tracing import ( + _OSPARC_TRACE_ID_HEADER, + _PROFILE_ATTRIBUTE_NAME, + with_profiled_span, +) from settings_library.tracing import TracingSettings @@ -68,9 +72,9 @@ def set_and_clean_settings_env_vars( ) async def test_valid_tracing_settings( mocked_app: FastAPI, + mock_otel_collector: InMemorySpanExporter, set_and_clean_settings_env_vars: Callable[[], None], tracing_settings_in: Callable[[], dict[str, Any]], - uninstrument_opentelemetry: Iterator[None], ): tracing_settings = TracingSettings() async for _ in get_tracing_instrumentation_lifespan( @@ -102,9 +106,9 @@ async def test_valid_tracing_settings( ) async def test_invalid_tracing_settings( mocked_app: FastAPI, + mock_otel_collector: InMemorySpanExporter, set_and_clean_settings_env_vars: Callable[[], None], tracing_settings_in: Callable[[], dict[str, Any]], - uninstrument_opentelemetry: Iterator[None], ): app = mocked_app with pytest.raises((BaseException, ValidationError, TypeError)): # noqa: PT012 @@ -157,9 +161,9 @@ def manage_package(request): ) async def test_tracing_setup_package_detection( mocked_app: FastAPI, + mock_otel_collector: InMemorySpanExporter, set_and_clean_settings_env_vars: Callable[[], None], tracing_settings_in: Callable[[], dict[str, Any]], - uninstrument_opentelemetry: Iterator[None], manage_package, ): package_name = manage_package @@ -196,7 +200,6 @@ async def test_trace_id_in_response_header( mocked_app: FastAPI, set_and_clean_settings_env_vars: Callable, tracing_settings_in: Callable, - uninstrument_opentelemetry: Iterator[None], server_response: PlainTextResponse | HTTPException, ) -> None: tracing_settings = TracingSettings() @@ -225,3 +228,59 @@ async def handler(handler_data: dict): trace_id = response.headers[_OSPARC_TRACE_ID_HEADER] assert len(trace_id) == 32 # Ensure trace ID is a 32-character hex string assert trace_id == handler_data[_OSPARC_TRACE_ID_HEADER] + + +@pytest.mark.parametrize( + "tracing_settings_in", + [ + ("http://opentelemetry-collector", 4318), + ], + indirect=True, +) +@pytest.mark.parametrize( + "server_response", + [ + PlainTextResponse("ok"), + HTTPException(status_code=400, detail="error"), + ], +) +async def test_with_profile_span( + mock_otel_collector: InMemorySpanExporter, + mocked_app: FastAPI, + set_and_clean_settings_env_vars: Callable[[], None], + tracing_settings_in: Callable, + server_response: PlainTextResponse | HTTPException, +): + tracing_settings = TracingSettings() + + handler_data = dict() + + @with_profiled_span + async def handler(handler_data: dict): + current_span = trace.get_current_span() + handler_data[_OSPARC_TRACE_ID_HEADER] = format( + current_span.get_span_context().trace_id, "032x" + ) + if isinstance(server_response, HTTPException): + raise server_response + return server_response + + mocked_app.get("/")(partial(handler, handler_data)) + + async for _ in get_tracing_instrumentation_lifespan( + tracing_settings=tracing_settings, + service_name="Mock-OpenTelemetry-Pytest", + )(app=mocked_app): + initialize_fastapi_app_tracing(mocked_app, add_response_trace_id_header=True) + client = TestClient(mocked_app) + _ = client.get("/") + trace_id = handler_data.get(_OSPARC_TRACE_ID_HEADER) + assert trace_id is not None + + spans = mock_otel_collector.get_finished_spans() + assert any( + span.context.trace_id == int(trace_id, 16) + and _PROFILE_ATTRIBUTE_NAME in span.attributes.keys() + for span in spans + if span.context is not None and span.attributes is not None + ) diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 737ca9cbc25..4b5ae267892 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -202,7 +202,7 @@ opentelemetry-instrumentation==0.55b1 opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.55b1 diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index f5c397a104c..37b6ae2dd8a 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -184,6 +184,7 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -205,6 +206,7 @@ opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -214,6 +216,8 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 @@ -238,6 +242,7 @@ opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index f54e495b207..2557c5cbe34 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -385,8 +385,8 @@ opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index ee0eba1f396..c822689d19e 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -312,6 +312,7 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx @@ -337,6 +338,7 @@ opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx @@ -349,6 +351,10 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.55b1 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-botocore==0.55b1 # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 @@ -384,6 +390,7 @@ opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 9a710aa48c5..8b3b2f50637 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -254,7 +254,7 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 7a21d972268..a9da7581879 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -310,6 +310,7 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx @@ -335,6 +336,7 @@ opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx @@ -347,6 +349,10 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.55b1 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-botocore==0.55b1 # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 @@ -382,6 +388,7 @@ opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 9381e2dede4..e2292cd53d6 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -213,6 +213,7 @@ opentelemetry-api==1.34.1 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests @@ -231,11 +232,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1 opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.55b1 @@ -255,6 +259,7 @@ opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index f26f3216ddc..8a4f8b52b1a 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -196,6 +196,7 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -217,6 +218,7 @@ opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -226,6 +228,8 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 @@ -250,6 +254,7 @@ opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index ce7ec40e7fc..cae7f534f57 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -434,8 +434,8 @@ opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/director/requirements/_base.txt b/services/director/requirements/_base.txt index 3b2bdba69ce..9ead22c1ff0 100644 --- a/services/director/requirements/_base.txt +++ b/services/director/requirements/_base.txt @@ -184,6 +184,7 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -205,6 +206,7 @@ opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -214,6 +216,8 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 @@ -238,6 +242,7 @@ opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis diff --git a/services/docker-api-proxy/requirements/_test.txt b/services/docker-api-proxy/requirements/_test.txt index 79458194086..9c32679316a 100644 --- a/services/docker-api-proxy/requirements/_test.txt +++ b/services/docker-api-proxy/requirements/_test.txt @@ -135,6 +135,7 @@ opentelemetry-api==1.34.1 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests @@ -153,11 +154,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1 opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.55b1 @@ -177,6 +181,7 @@ opentelemetry-sdk==1.34.1 opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index d76446c3921..01fe088016e 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -269,7 +269,7 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 70a01739a17..9de72f9d794 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -359,8 +359,8 @@ opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 690dcb7ef3c..fbbf5d7ab4d 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -358,7 +358,9 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-botocore==0.55b1 # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 4b0de5fb9c8..1a28a3a84e2 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -199,6 +199,7 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -220,6 +221,7 @@ opentelemetry-instrumentation==0.55b1 # via # opentelemetry-instrumentation-aio-pika # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -229,6 +231,8 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.55b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 @@ -253,6 +257,7 @@ opentelemetry-semantic-conventions==0.55b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 7f9853c4e22..cc5d217978e 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -248,7 +248,7 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index 11d4a30ae9a..6239ddf3480 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -275,7 +275,7 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index b124cd1dcc1..8143f9be8e2 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -379,7 +379,9 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-botocore==0.55b1 # via -r requirements/../../../packages/aws-library/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index b889fdb0ece..0dc381669ef 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -464,7 +464,10 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-botocore==0.55b1 # via # -r requirements/../../../packages/aws-library/requirements/_base.in diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index e7f365b66cb..8fbf7c550e9 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -46,6 +46,7 @@ from models_library.projects_nodes_io import LocationID, SimcoreS3FileID, StorageFileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.fastapi import url_from_operation_id @@ -105,6 +106,7 @@ "pytest_simcore.simcore_storage_data_models", "pytest_simcore.simcore_storage_datcore_adapter", "pytest_simcore.simcore_storage_service", + "pytest_simcore.tracing", ] CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -188,6 +190,15 @@ def disabled_rabbitmq(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPa monkeypatch.setenv("STORAGE_RABBITMQ", "null") +@pytest.fixture +def enable_tracing( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + setup_tracing_fastapi: InMemorySpanExporter, +): + monkeypatch.setenv("STORAGE_TRACING", "{}") + + @pytest.fixture def enabled_rabbitmq( app_environment: EnvVarsDict, rabbit_service: RabbitSettings @@ -203,6 +214,7 @@ async def mocked_redis_server(mocker: MockerFixture) -> None: @pytest.fixture def app_settings( + enable_tracing, app_environment: EnvVarsDict, enabled_rabbitmq: RabbitSettings, sqlalchemy_async_engine: AsyncEngine, diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index cc1dca9cf85..a2130885916 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -466,9 +466,8 @@ opentelemetry-instrumentation-aiopg==0.55b1 # -r requirements/_base.in opentelemetry-instrumentation-asyncpg==0.55b1 # via - # -r requirements/../../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-dbapi==0.55b1 # via opentelemetry-instrumentation-aiopg opentelemetry-instrumentation-logging==0.55b1 diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py index 0e01a4b414b..a0b99d9b995 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py @@ -28,6 +28,7 @@ from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.request_keys import RQT_USERID_KEY from servicelib.rest_constants import RESPONSE_MODEL_POLICY +from servicelib.tracing import with_profiled_span from ..._meta import API_VTAG as VTAG from ...constants import RQ_PRODUCT_KEY @@ -202,6 +203,7 @@ async def list_computations_latest_iteration_tasks( name="list_computation_collection_runs", ) @login_required +@with_profiled_span @permission_required("services.pipeline.*") @permission_required("project.read") async def list_computation_collection_runs(request: web.Request) -> web.Response: diff --git a/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py b/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py index 8f4f920168e..ee2479b7377 100644 --- a/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py @@ -12,6 +12,7 @@ parse_request_path_parameters_as, ) from servicelib.redis import handle_redis_returns_union_types +from servicelib.tracing import with_profiled_span from .._meta import API_VTAG from ..login.decorators import login_required @@ -123,6 +124,7 @@ async def mark_notification_as_read(request: web.Request) -> web.Response: @routes.get(f"/{API_VTAG}/me/permissions", name="list_user_permissions") @login_required +@with_profiled_span @permission_required("user.permissions.read") async def list_user_permissions(request: web.Request) -> web.Response: req_ctx = UsersRequestContext.model_validate(request) diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 2ecdddbaf66..2078f6a695d 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -292,7 +292,9 @@ opentelemetry-instrumentation-aio-pika==0.55b1 # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-logging==0.55b1 # via # -r requirements/../../../packages/service-library/requirements/_base.in