✨ instrument (opentelemetry) httpx clients (#6715)

Author: bisgaard-itis

packages/aws-library/requirements/_base.txt

@@ -44,8 +44,6 @@ arrow==1.3.0
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/_base.in
-async-timeout==4.0.3
-    # via redis
 attrs==24.2.0
     # via
     #   aiohttp

packages/notifications-library/requirements/_base.txt

@@ -16,6 +16,10 @@ attrs==24.2.0
     #   referencing
 click==8.1.7
     # via typer
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-semantic-conventions
 dnspython==2.6.1
     # via email-validator
 email-validator==2.2.0
@@ -26,6 +30,8 @@ idna==3.10
     # via
     #   email-validator
     #   yarl
+importlib-metadata==8.5.0
+    # via opentelemetry-api
 jinja2==3.1.4
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -54,13 +60,28 @@ mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via yarl
+opentelemetry-api==1.28.1
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
+    #   opentelemetry-semantic-conventions
+opentelemetry-instrumentation==0.49b1
+    # via opentelemetry-instrumentation-asyncpg
+opentelemetry-instrumentation-asyncpg==0.49b1
+    # via -r requirements/../../../packages/postgres-database/requirements/_base.in
+opentelemetry-semantic-conventions==0.49b1
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
 orjson==3.10.7
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
+packaging==24.2
+    # via opentelemetry-instrumentation
 psycopg2-binary==2.9.9
     # via sqlalchemy
 pydantic==1.10.18
@@ -109,5 +130,11 @@ typing-extensions==4.12.2
     #   alembic
     #   pydantic
     #   typer
+wrapt==1.16.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
 yarl==1.12.1
     # via -r requirements/../../../packages/postgres-database/requirements/_base.in
+zipp==3.21.0
+    # via importlib-metadata

packages/notifications-library/requirements/_test.txt

@@ -28,8 +28,9 @@ mypy==1.12.0
     # via sqlalchemy
 mypy-extensions==1.0.0
     # via mypy
-packaging==24.1
+packaging==24.2
     # via
+    #   -c requirements/_base.txt
     #   pytest
     #   pytest-sugar
 pluggy==1.5.0

packages/notifications-library/requirements/_tools.txt

@@ -38,8 +38,9 @@ mypy-extensions==1.0.0
     #   mypy
 nodeenv==1.9.1
     # via pre-commit
-packaging==24.1
+packaging==24.2
     # via
+    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   black
     #   build

packages/service-library/requirements/_base.txt

@@ -28,8 +28,6 @@ arrow==1.3.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/_base.in
-async-timeout==4.0.3
-    # via redis
 attrs==24.2.0
     # via
     #   aiohttp

packages/service-library/requirements/_fastapi.in

@@ -9,6 +9,7 @@
 fastapi
 httpx
 opentelemetry-instrumentation-fastapi
+opentelemetry-instrumentation-httpx
 prometheus-client
 prometheus-fastapi-instrumentator
 uvicorn

packages/service-library/requirements/_fastapi.txt

@@ -47,23 +47,29 @@ opentelemetry-api==1.27.0
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-instrumentation-httpx
     #   opentelemetry-semantic-conventions
 opentelemetry-instrumentation==0.48b0
     # via
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-instrumentation-httpx
 opentelemetry-instrumentation-asgi==0.48b0
     # via opentelemetry-instrumentation-fastapi
 opentelemetry-instrumentation-fastapi==0.48b0
     # via -r requirements/_fastapi.in
+opentelemetry-instrumentation-httpx==0.48b0
+    # via -r requirements/_fastapi.in
 opentelemetry-semantic-conventions==0.48b0
     # via
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-instrumentation-httpx
 opentelemetry-util-http==0.48b0
     # via
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-instrumentation-httpx
 prometheus-client==0.21.0
     # via
     #   -r requirements/_fastapi.in

packages/service-library/src/servicelib/fastapi/http_client_thin.py

@@ -8,6 +8,8 @@
 from httpx import AsyncClient, ConnectError, HTTPError, PoolTimeout, Response
 from httpx._types import TimeoutTypes, URLTypes
 from pydantic.errors import PydanticErrorMixin
+from servicelib.fastapi.tracing import setup_httpx_client_tracing
+from settings_library.tracing import TracingSettings
 from tenacity import RetryCallState
 from tenacity.asyncio import AsyncRetrying
 from tenacity.before_sleep import before_sleep_log
@@ -201,6 +203,7 @@ def __init__(
         base_url: URLTypes | None = None,
         default_http_client_timeout: TimeoutTypes | None = None,
         extra_allowed_method_names: set[str] | None = None,
+        tracing_settings: TracingSettings | None,
     ) -> None:
         _assert_public_interface(self, extra_allowed_method_names)
 
@@ -220,7 +223,10 @@ def __init__(
         if default_http_client_timeout:
             client_args["timeout"] = default_http_client_timeout
 
-        super().__init__(client=AsyncClient(**client_args))
+        client = AsyncClient(**client_args)
+        if tracing_settings:
+            setup_httpx_client_tracing(client)
+        super().__init__(client=client)
 
     async def __aenter__(self):
         await self.setup_client()

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -5,11 +5,13 @@
 import logging
 
 from fastapi import FastAPI
+from httpx import AsyncClient, Client
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
     OTLPSpanExporter as OTLPSpanExporterHTTP,
 )
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
@@ -121,3 +123,7 @@ def setup_tracing(
             msg="Attempting to add requests opentelemetry autoinstrumentation...",
         ):
             RequestsInstrumentor().instrument()
+
+
+def setup_httpx_client_tracing(client: AsyncClient | Client):
+    HTTPXClientInstrumentor.instrument_client(client)

packages/service-library/tests/fastapi/test_http_client_thin.py

@@ -71,7 +71,9 @@ def request_timeout() -> int:
 
 @pytest.fixture
 async def thick_client(request_timeout: int) -> AsyncIterable[FakeThickClient]:
-    async with FakeThickClient(total_retry_interval=request_timeout) as client:
+    async with FakeThickClient(
+        total_retry_interval=request_timeout, tracing_settings=None
+    ) as client:
         yield client
 
 
@@ -95,7 +97,9 @@ async def test_retry_on_errors(
     test_url: AnyHttpUrl,
     caplog_info_level: pytest.LogCaptureFixture,
 ) -> None:
-    client = FakeThickClient(total_retry_interval=request_timeout)
+    client = FakeThickClient(
+        total_retry_interval=request_timeout, tracing_settings=None
+    )
 
     with pytest.raises(ClientHttpError):
         await client.get_provided_url(test_url)
@@ -119,7 +123,7 @@ async def raises_request_error(self) -> Response:
                 request=Request(method="GET", url=test_url),
             )
 
-    client = ATestClient(total_retry_interval=request_timeout)
+    client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None)
 
     with pytest.raises(ClientHttpError):
         await client.raises_request_error()
@@ -145,7 +149,7 @@ async def raises_http_error(self) -> Response:
             msg = "mock_http_error"
             raise HTTPError(msg)
 
-    client = ATestClient(total_retry_interval=request_timeout)
+    client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None)
 
     with pytest.raises(ClientHttpError):
         await client.raises_http_error()
@@ -159,21 +163,25 @@ async def public_method_ok(self) -> Response:  # type: ignore
             """this method will be ok even if no code is used"""
 
     # OK
-    OKTestClient(total_retry_interval=request_timeout)
+    OKTestClient(total_retry_interval=request_timeout, tracing_settings=None)
 
     class FailWrongAnnotationTestClient(BaseThinClient):
         async def public_method_wrong_annotation(self) -> None:
             """this method will raise an error"""
 
     with pytest.raises(AssertionError, match="should return an instance"):
-        FailWrongAnnotationTestClient(total_retry_interval=request_timeout)
+        FailWrongAnnotationTestClient(
+            total_retry_interval=request_timeout, tracing_settings=None
+        )
 
     class FailNoAnnotationTestClient(BaseThinClient):
         async def public_method_no_annotation(self):
             """this method will raise an error"""
 
     with pytest.raises(AssertionError, match="should return an instance"):
-        FailNoAnnotationTestClient(total_retry_interval=request_timeout)
+        FailNoAnnotationTestClient(
+            total_retry_interval=request_timeout, tracing_settings=None
+        )
 
 
 async def test_expect_state_decorator(
@@ -197,7 +205,9 @@ async def get_wrong_state(self) -> Response:
     respx_mock.get(url_get_200_ok).mock(return_value=Response(codes.OK))
     respx_mock.get(get_wrong_state).mock(return_value=Response(codes.OK))
 
-    test_client = ATestClient(total_retry_interval=request_timeout)
+    test_client = ATestClient(
+        total_retry_interval=request_timeout, tracing_settings=None
+    )
 
     # OK
     response = await test_client.get_200_ok()
@@ -218,7 +228,9 @@ async def test_retry_timeout_overwrite(
     request_timeout: int,
     caplog_info_level: pytest.LogCaptureFixture,
 ) -> None:
-    client = FakeThickClient(total_retry_interval=request_timeout)
+    client = FakeThickClient(
+        total_retry_interval=request_timeout, tracing_settings=None
+    )
 
     caplog_info_level.clear()
     start = arrow.utcnow()