🚨✨ Implement tracing sampling strategy (🚧 devops 🚧)

.env-devel

@@ -377,7 +377,7 @@ TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE=2
 TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT=http://opentelemetry-collector
 TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318
 TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318
-TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100
+TRACING_SAMPLING_PROBABILITY=1.0
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 TRASH_RETENTION_DAYS=7
 TWILIO_ACCOUNT_SID=DUMMY

packages/service-library/src/servicelib/aiohttp/tracing.py

@@ -18,6 +18,7 @@
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
 from settings_library.tracing import TracingSettings
 from yarl import URL
 
@@ -100,7 +101,11 @@ def _startup(
         )
         raise RuntimeError(msg)
     resource = Resource(attributes={"service.name": service_name})
-    trace.set_tracer_provider(TracerProvider(resource=resource))
+    sampler = ParentBased(
+        root=TraceIdRatioBased(tracing_settings.TRACING_SAMPLING_PROBABILITY)
+    )
+    tracer_provider = TracerProvider(resource=resource, sampler=sampler)
+    trace.set_tracer_provider(tracer_provider=tracer_provider)
     tracer_provider: trace.TracerProvider = trace.get_tracer_provider()
 
     tracing_destination: str = (

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -15,6 +15,7 @@
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
 from settings_library.tracing import TracingSettings
 from starlette.middleware.base import BaseHTTPMiddleware
 from yarl import URL
@@ -87,7 +88,11 @@ def _startup(tracing_settings: TracingSettings, service_name: str) -> None:
         return
     # Set up the tracer provider
     resource = Resource(attributes={"service.name": service_name})
-    trace.set_tracer_provider(TracerProvider(resource=resource))
+    sampler = ParentBased(
+        root=TraceIdRatioBased(tracing_settings.TRACING_SAMPLING_PROBABILITY)
+    )
+    trace_provider = TracerProvider(resource=resource, sampler=sampler)
+    trace.set_tracer_provider(trace_provider)
     global_tracer_provider = trace.get_tracer_provider()
     assert isinstance(global_tracer_provider, TracerProvider)  # nosec
 

packages/service-library/tests/aiohttp/test_tracing.py

@@ -2,6 +2,7 @@
 # pylint: disable=unused-argument
 # pylint: disable=unused-variable
 
+import asyncio
 import importlib
 from collections.abc import Callable
 from functools import partial
@@ -40,17 +41,23 @@ def set_and_clean_settings_env_vars(
         monkeypatch.setenv(
             "TRACING_OPENTELEMETRY_COLLECTOR_PORT", f"{tracing_settings_in[1]}"
         )
+    sampling_probability_mocked = False
+    if tracing_settings_in[2]:
+        sampling_probability_mocked = True
+        monkeypatch.setenv("TRACING_SAMPLING_PROBABILITY", tracing_settings_in[2])
     yield
     if endpoint_mocked:
         monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT")
     if port_mocked:
         monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_PORT")
+    if sampling_probability_mocked:
+        monkeypatch.delenv("TRACING_SAMPLING_PROBABILITY")
 
 
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 4318),
+        ("http://opentelemetry-collector", 4318, 1.0),
     ],
     indirect=True,
 )
@@ -72,9 +79,9 @@ async def test_valid_tracing_settings(
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 80),
-        ("opentelemetry-collector", 4318),
-        ("httsdasp://ot@##el-collector", 4318),
+        ("http://opentelemetry-collector", 80, 1.0),
+        ("opentelemetry-collector", 4318, 1.0),
+        ("httsdasp://ot@##el-collector", 4318, 1.0),
     ],
     indirect=True,
 )
@@ -111,14 +118,14 @@ def manage_package(request):
     "tracing_settings_in, manage_package",
     [
         (
-            ("http://opentelemetry-collector", 4318),
+            ("http://opentelemetry-collector", 4318, 1.0),
             (
                 "opentelemetry-instrumentation-botocore",
                 "opentelemetry.instrumentation.botocore",
             ),
         ),
         (
-            ("http://opentelemetry-collector", "4318"),
+            ("http://opentelemetry-collector", "4318", 1.0),
             (
                 "opentelemetry-instrumentation-aiopg",
                 "opentelemetry.instrumentation.aiopg",
@@ -157,7 +164,7 @@ async def test_tracing_setup_package_detection(
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 4318),
+        ("http://opentelemetry-collector", 4318, 1.0),
     ],
     indirect=True,
 )
@@ -201,3 +208,61 @@ async def handler(handler_data: dict, request: web.Request) -> web.Response:
         assert (
             trace_id == handler_data[_OSPARC_TRACE_ID_HEADER]
         )  # Ensure trace IDs match
+
+
+@pytest.mark.parametrize(
+    "tracing_settings_in",
+    [
+        ("http://opentelemetry-collector", 4318, 0.05),
+    ],
+    indirect=True,
+)
+async def test_tracing_sampling_probability_effective(
+    mock_otel_collector: InMemorySpanExporter,
+    aiohttp_client: Callable,
+    set_and_clean_settings_env_vars: Callable[[], None],
+    tracing_settings_in,
+):
+    """
+    This test checks that the TRACING_SAMPLING_PROBABILITY setting in TracingSettings
+    is effective by sending 1000 requests and verifying that the number of collected traces
+    is close to 0.05 * 1000 (with some tolerance).
+    """
+    n_requests = 1000
+    tolerance_probability = 0.5
+
+    app = web.Application()
+    service_name = "simcore_service_webserver"
+    tracing_settings = TracingSettings()
+
+    async def handler(request: web.Request) -> web.Response:
+        return web.Response(text="ok")
+
+    app.router.add_get("/", handler)
+
+    async for _ in get_tracing_lifespan(
+        app=app,
+        service_name=service_name,
+        tracing_settings=tracing_settings,
+    )(app):
+        client = await aiohttp_client(app)
+
+        async def make_request():
+            await client.get("/")
+
+        await asyncio.gather(*(make_request() for _ in range(n_requests)))
+        spans = mock_otel_collector.get_finished_spans()
+        trace_ids = set()
+        for span in spans:
+            if span.context is not None:
+                trace_ids.add(span.context.trace_id)
+        num_traces = len(trace_ids)
+        expected_num_traces = int(
+            tracing_settings.TRACING_SAMPLING_PROBABILITY * n_requests
+        )
+        tolerance = int(tolerance_probability * expected_num_traces)
+        assert (
+            expected_num_traces - tolerance
+            <= num_traces
+            <= expected_num_traces + tolerance
+        ), f"Expected roughly {expected_num_traces} distinct trace ids, got {num_traces}"

packages/service-library/tests/fastapi/test_tracing.py

@@ -55,18 +55,24 @@ def set_and_clean_settings_env_vars(
         monkeypatch.setenv(
             "TRACING_OPENTELEMETRY_COLLECTOR_PORT", f"{tracing_settings_in[1]}"
         )
+    sampling_probability_mocked = False
+    if tracing_settings_in[2]:
+        sampling_probability_mocked = True
+        monkeypatch.setenv("TRACING_SAMPLING_PROBABILITY", f"{tracing_settings_in[2]}")
     yield
     if endpoint_mocked:
         monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT")
     if port_mocked:
         monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_PORT")
+    if sampling_probability_mocked:
+        monkeypatch.delenv("TRACING_SAMPLING_PROBABILITY")
 
 
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 4318),
-        ("http://opentelemetry-collector", "4318"),
+        ("http://opentelemetry-collector", 4318, 1.0),
+        ("http://opentelemetry-collector", "4318", 1.0),
     ],
     indirect=True,
 )
@@ -91,15 +97,16 @@ async def test_valid_tracing_settings(
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 80),
-        ("http://opentelemetry-collector", 1238712936),
-        ("opentelemetry-collector", 4318),
-        ("httsdasp://ot@##el-collector", 4318),
-        (" !@#$%^&*()[]{};:,<>?\\|`~+=/'\"", 4318),
+        ("http://opentelemetry-collector", 80, 0.5),
+        ("http://opentelemetry-collector", 1238712936, 0.5),
+        ("opentelemetry-collector", 4318, 0.5),
+        ("httsdasp://ot@##el-collector", 4318, 0.5),
+        (" !@#$%^&*()[]{};:,<>?\\|`~+=/'\"", 4318, 0.5),
         # The following exceeds max DNS name length
         (
             "".join(random.choice(string.ascii_letters) for _ in range(300)),
             "1238712936",
+            0.5,
         ),  # noqa: S311
     ],
     indirect=True,
@@ -143,14 +150,14 @@ def manage_package(request):
     "tracing_settings_in, manage_package",
     [
         (
-            ("http://opentelemetry-collector", 4318),
+            ("http://opentelemetry-collector", 4318, 1.0),
             (
                 "opentelemetry-instrumentation-botocore",
                 "opentelemetry.instrumentation.botocore",
             ),
         ),
         (
-            ("http://opentelemetry-collector", "4318"),
+            ("http://opentelemetry-collector", "4318", 1.0),
             (
                 "opentelemetry-instrumentation-aiopg",
                 "opentelemetry.instrumentation.aiopg",
@@ -184,7 +191,7 @@ async def test_tracing_setup_package_detection(
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 4318),
+        ("http://opentelemetry-collector", 4318, 1.0),
     ],
     indirect=True,
 )
@@ -233,7 +240,7 @@ async def handler(handler_data: dict):
 @pytest.mark.parametrize(
     "tracing_settings_in",
     [
-        ("http://opentelemetry-collector", 4318),
+        ("http://opentelemetry-collector", 4318, 1.0),
     ],
     indirect=True,
 )
@@ -284,3 +291,57 @@ async def handler(handler_data: dict):
             for span in spans
             if span.context is not None and span.attributes is not None
         )
+
+
+@pytest.mark.parametrize(
+    "tracing_settings_in",
+    [
+        ("http://opentelemetry-collector", 4318, 0.05),
+    ],
+    indirect=True,
+)
+async def test_tracing_sampling_probability_effective(
+    mock_otel_collector: InMemorySpanExporter,
+    mocked_app: FastAPI,
+    set_and_clean_settings_env_vars: Callable[[], None],
+    tracing_settings_in: Callable[[], dict[str, Any]],
+):
+    """
+    This test checks that the TRACING_SAMPLING_PROBABILITY setting in TracingSettings
+    is effective by sending 1000 requests and verifying that the number of collected traces
+    is close to 0.05 * 1000 (with some tolerance).
+    """
+    n_requests = 1000
+    tolerance_probability = 0.5
+
+    tracing_settings = TracingSettings()
+
+    async def handler():
+        return PlainTextResponse("ok")
+
+    mocked_app.get("/")(handler)
+
+    async for _ in get_tracing_instrumentation_lifespan(
+        tracing_settings=tracing_settings,
+        service_name="Mock-OpenTelemetry-Pytest",
+    )(app=mocked_app):
+        initialize_fastapi_app_tracing(mocked_app, add_response_trace_id_header=True)
+        client = TestClient(mocked_app)
+        for _ in range(n_requests):
+            client.get("/")
+        spans = mock_otel_collector.get_finished_spans()
+        trace_ids = set()
+        for span in spans:
+            if span.context is not None:
+                trace_ids.add(span.context.trace_id)
+        num_traces = len(trace_ids)
+        expected_num_traces = int(
+            tracing_settings.TRACING_SAMPLING_PROBABILITY * n_requests
+        )
+        # Allow a 50% tolerance due to randomness
+        tolerance = int(tolerance_probability * expected_num_traces)
+        assert (
+            expected_num_traces - tolerance
+            <= num_traces
+            <= expected_num_traces + tolerance
+        ), f"Expected roughly {expected_num_traces} distinct trace ids, got {num_traces}"

packages/settings-library/src/settings_library/tracing.py

@@ -15,3 +15,7 @@ class TracingSettings(BaseCustomSettings):
     TRACING_OPENTELEMETRY_COLLECTOR_PORT: Annotated[
         RegisteredPortInt, Field(description="Opentelemetry compatible collector port")
     ]
+    TRACING_SAMPLING_PROBABILITY: Annotated[
+        float,
+        Field(description="Probability of sampling traces (0.0 - 1.0)", ge=0.0, le=1.0),
+    ]

services/director/tests/unit/test_core_settings.py

@@ -100,7 +100,6 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch):
         SWARM_STACK_NAME=master-simcore
         TERM=xterm
         TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318
-        TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=50
         TRAEFIK_SIMCORE_ZONE=master_internal_simcore_stack
         VIRTUAL_ENV=/home/scu/.venv
         LOG_FORMAT_LOCAL_DEV_ENABLED=1
@@ -150,7 +149,6 @@ def test_docker_compose_environment_sample(
             "STORAGE_ENDPOINT": "master_storage:8080",
             "SWARM_STACK_NAME": "master-simcore",
             "TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT": "http://jaeger:4318",
-            "TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE": "50",
             "TRAEFIK_SIMCORE_ZONE": "master_internal_simcore_stack",
         },
     )

services/docker-compose-ops.yml

@@ -117,7 +117,6 @@ services:
       - interactive_services_subnet
     environment:
       TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE}
-      TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE}
       TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT}
 volumes:
   minio_data:

services/docker-compose.yml

@@ -10,7 +10,7 @@ x-tracing-open-telemetry: &tracing_open_telemetry_environs
   TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}
   TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE}
   TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
-  TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE}
+  TRACING_SAMPLING_PROBABILITY: ${TRACING_SAMPLING_PROBABILITY}
 
 x-webserver-diagnostics: &webserver_diagnostics_environs
   DIAGNOSTICS_HEALTHCHECK_ENABLED: ${DIAGNOSTICS_HEALTHCHECK_ENABLED}
@@ -1503,6 +1503,7 @@ services:
       - "--tracing.addinternals"
       - "--tracing.otlp=true"
       - "--tracing.otlp.http=true"
+      - "--tracing.sampleRate=${TRACING_SAMPLING_PROBABILITY}"
     healthcheck:
       # NOTE: this healthcheck to check if traefik is up and running must be run on the ping entrypoint defined in command!
       test: traefik healthcheck --ping --ping.entryPoint=ping --entryPoints.ping.address=:9082

services/opentelemetry-collector-config.yaml

@@ -19,7 +19,7 @@ processors:
     timeout: 5s
     send_batch_size: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE}
   probabilistic_sampler:
-    sampling_percentage: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE}
+    sampling_percentage: 100
   filter/drop_healthcheck:
     error_mode: ignore
     traces: