🐛 Fix aiohttp server autoinstrumentation

packages/service-library/src/servicelib/aiohttp/tracing.py

@@ -12,8 +12,8 @@
 from opentelemetry.instrumentation.aiohttp_client import (  # pylint:disable=no-name-in-module
     AioHttpClientInstrumentor,
 )
-from opentelemetry.instrumentation.aiohttp_server import (  # pylint:disable=no-name-in-module
-    AioHttpServerInstrumentor,
+from opentelemetry.instrumentation.aiohttp_server import (
+    middleware as aiohttp_server_opentelemetry_middleware,  # pylint:disable=no-name-in-module
 )
 from opentelemetry.instrumentation.aiopg import (  # pylint:disable=no-name-in-module
     AiopgInstrumentor,
@@ -72,8 +72,24 @@ def setup_tracing(
 
     # Add the span processor to the tracer provider
     tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))  # type: ignore[attr-defined] # https://github.com/open-telemetry/opentelemetry-python/issues/3713
-    # Instrument aiohttp server and client
-    AioHttpServerInstrumentor().instrument()
+    # Instrument aiohttp server
+    # Explanation for custom middleware call DK 10/2024:
+    # OpenTelemetry Aiohttp autoinstrumentation is meant to be used by only calling `AioHttpServerInstrumentor().instrument()`
+    # The call `AioHttpServerInstrumentor().instrument()` monkeypatches the __init__() of aiohttp's web.application() to inject the tracing middleware, in it's `__init__()`.
+    # In simcore, we want to switch tracing on or off using the simcore-settings-library.
+    # The simcore-settings library in turn depends on the instance of web.application(), i.e. the aiohttp webserver, to exist. So here we face a hen-and-egg problem.
+    # At the time when the instrumentation should be configured, the instance of web.application already exists and the overwrite to the __init__() is never called
+    #
+    # Since the code that is provided (monkeypatched) in the __init__ that the opentelemetry-autoinstrumentation-library provides is only 4 lines,
+    # just adding a middleware, we are free to simply execute this "missed call" [since we can't call the monkeypatch'ed __init__()] in this following line:
+    app.middlewares.insert(0, aiohttp_server_opentelemetry_middleware)
+    # Code of the aiohttp server instrumentation: github.com/open-telemetry/opentelemetry-python-contrib/blob/eccb05c808a7d797ef5b6ecefed3590664426fbf/instrumentation/opentelemetry-instrumentation-aiohttp-server/src/opentelemetry/instrumentation/aiohttp_server/__init__.py#L246
+    # For reference, the above statement was written for:
+    # - osparc-simcore 1.77.x
+    # - opentelemetry-api==1.27.0
+    # - opentelemetry-instrumentation==0.48b0
+
+    # Instrument aiohttp client
     AioHttpClientInstrumentor().instrument()
     if instrument_aiopg:
         AiopgInstrumentor().instrument()

services/datcore-adapter/Dockerfile

@@ -4,7 +4,7 @@ FROM python:${PYTHON_VERSION}-slim-bookworm as base
 
 #
 #  USAGE:
-#     cd sercices/datcore-adapter
+#     cd services/datcore-adapter
 #     docker build -f Dockerfile -t datcore-adapter:prod --target production ../../
 #     docker run datcore-adapter:prod
 #

services/web/server/tests/unit/isolated/test_application_settings.py

@@ -8,12 +8,16 @@
 import pytest
 from aiohttp import web
 from models_library.utils.json_serialization import json_dumps
+from opentelemetry.instrumentation.aiohttp_server import (
+    middleware as aiohttp_opentelemetry_middleware,
+)
 from pydantic import HttpUrl, parse_obj_as
 from pytest_simcore.helpers.monkeypatch_envs import (
     setenvs_from_dict,
     setenvs_from_envfile,
 )
 from pytest_simcore.helpers.typing_env import EnvVarsDict
+from simcore_service_webserver.application import create_application
 from simcore_service_webserver.application_settings import (
     APP_SETTINGS_KEY,
     ApplicationSettings,
@@ -251,3 +255,38 @@ def test_avoid_sensitive_info_in_public(app_settings: ApplicationSettings):
     assert not any("token" in key for key in app_settings.public_dict())
     assert not any("secret" in key for key in app_settings.public_dict())
     assert not any("private" in key for key in app_settings.public_dict())
+
+
+@pytest.fixture
+def tracing_settings_in(request):
+    return request.param
+
+
+@pytest.mark.parametrize(
+    "tracing_settings_in",
+    [
+        ("http://opentelemetry-collector", 4318),
+    ],
+    indirect=True,
+)
+async def test_middleware_restirctions_opentelemetry_is_second_middleware(
+    mock_webserver_service_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    tracing_settings_in,
+):
+    monkeypatch.setenv(
+        "TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT", f"{tracing_settings_in[0]}"
+    )
+    monkeypatch.setenv(
+        "TRACING_OPENTELEMETRY_COLLECTOR_PORT", f"{tracing_settings_in[1]}"
+    )
+    settings = ApplicationSettings.create_from_envs()
+    assert settings.WEBSERVER_TRACING
+    app = create_application()
+    assert app.middlewares
+    assert (
+        app.middlewares[0].__middleware_name__
+        == "servicelib.aiohttp.monitoring.monitor_simcore_service_webserver"
+    )
+    assert app.middlewares[1] == aiohttp_opentelemetry_middleware
+    assert len(app.middlewares) > 1