httpx: add support for OTEL_PYTHON_EXCLUDED_URLS / OTEL_PYTHON_HTTPX_EXCLUDED_URLS (open-telemetry#3837)

* httpx: add support for OTEL_PYTHON_EXCLUDED_URLS / OTEL_PYTHON_HTTPX_EXCLUDED_URLS

* Add changelog

* Fix lint

* Revert a change to a teardown, add a comment instead

* Test both environment variables

Author: xrmx

CHANGELOG.md

@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - `opentelemetry-instrumentation-aiohttp-client`: add support for url exclusions via `OTEL_PYTHON_EXCLUDED_URLS` / `OTEL_PYTHON_AIOHTTP_CLIENT_EXCLUDED_URLS`
   ([#3850](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3850))
+- `opentelemetry-instrumentation-httpx`: add support for url exclusions via `OTEL_PYTHON_EXCLUDED_URLS` / `OTEL_PYTHON_HTTPX_EXCLUDED_URLS`
+  ([#3837](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3837))
 
 ### Fixed
 

instrumentation/opentelemetry-instrumentation-httpx/src/opentelemetry/instrumentation/httpx/__init__.py

@@ -199,6 +199,24 @@ async def async_response_hook(span, request, response):
         response_hook=async_response_hook
     )
 
+
+Configuration
+-------------
+
+Exclude lists
+*************
+To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_HTTPX_EXCLUDED_URLS``
+(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the
+URLs.
+
+For example,
+
+::
+
+    export OTEL_PYTHON_HTTPX_EXCLUDED_URLS="client/.*/info,healthcheck"
+
+will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``.
+
 API
 ---
 """
@@ -259,7 +277,12 @@ async def async_response_hook(span, request, response):
 from opentelemetry.trace import SpanKind, Tracer, TracerProvider, get_tracer
 from opentelemetry.trace.span import Span
 from opentelemetry.trace.status import StatusCode
-from opentelemetry.util.http import redact_url, sanitize_method
+from opentelemetry.util.http import (
+    ExcludeList,
+    get_excluded_urls,
+    redact_url,
+    sanitize_method,
+)
 
 _logger = logging.getLogger(__name__)
 
@@ -304,7 +327,7 @@ def _extract_parameters(
     args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]
 ) -> tuple[
     bytes,
-    httpx.URL,
+    httpx.URL | tuple[bytes, bytes, int | None, bytes],
     httpx.Headers | None,
     httpx.SyncByteStream | httpx.AsyncByteStream | None,
     dict[str, typing.Any],
@@ -330,6 +353,22 @@ def _extract_parameters(
     return method, url, headers, stream, extensions
 
 
+def _normalize_url(
+    url: httpx.URL | tuple[bytes, bytes, int | None, bytes],
+) -> str:
+    if isinstance(url, tuple):
+        scheme, host, port, path = [
+            part.decode() if isinstance(part, bytes) else part for part in url
+        ]
+        return (
+            f"{scheme}://{host}:{port}{path}"
+            if port
+            else f"{scheme}://{host}{path}"
+        )
+
+    return str(url)
+
+
 def _inject_propagation_headers(headers, args, kwargs):
     _headers = _prepare_headers(headers)
     inject(_headers)
@@ -533,6 +572,7 @@ def __init__(
             )
         self._request_hook = request_hook
         self._response_hook = response_hook
+        self._excluded_urls = get_excluded_urls("HTTPX")
 
     def __enter__(self) -> SyncOpenTelemetryTransport:
         self._transport.__enter__()
@@ -562,6 +602,12 @@ def handle_request(
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if self._excluded_urls and self._excluded_urls.url_disabled(
+            _normalize_url(url)
+        ):
+            return self._transport.handle_request(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -726,6 +772,7 @@ def __init__(
 
         self._request_hook = request_hook
         self._response_hook = response_hook
+        self._excluded_urls = get_excluded_urls("HTTPX")
 
     async def __aenter__(self) -> "AsyncOpenTelemetryTransport":
         await self._transport.__aenter__()
@@ -753,6 +800,12 @@ async def handle_async_request(
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if self._excluded_urls and self._excluded_urls.url_disabled(
+            _normalize_url(url)
+        ):
+            return await self._transport.handle_async_request(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -900,6 +953,7 @@ def _instrument(self, **kwargs: typing.Any):
             if iscoroutinefunction(async_response_hook)
             else None
         )
+        excluded_urls = get_excluded_urls("HTTPX")
 
         _OpenTelemetrySemanticConventionStability._initialize()
         sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode(
@@ -948,6 +1002,7 @@ def _instrument(self, **kwargs: typing.Any):
                 sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                 request_hook=request_hook,
                 response_hook=response_hook,
+                excluded_urls=excluded_urls,
             ),
         )
         wrap_function_wrapper(
@@ -961,6 +1016,7 @@ def _instrument(self, **kwargs: typing.Any):
                 sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                 async_request_hook=async_request_hook,
                 async_response_hook=async_response_hook,
+                excluded_urls=excluded_urls,
             ),
         )
 
@@ -980,13 +1036,18 @@ def _handle_request_wrapper(  # pylint: disable=too-many-locals
         sem_conv_opt_in_mode: _StabilityMode,
         request_hook: RequestHook,
         response_hook: ResponseHook,
+        excluded_urls: ExcludeList | None,
     ):
         if not is_http_instrumentation_enabled():
             return wrapped(*args, **kwargs)
 
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if excluded_urls and excluded_urls.url_disabled(_normalize_url(url)):
+            return wrapped(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -1096,13 +1157,18 @@ async def _handle_async_request_wrapper(  # pylint: disable=too-many-locals
         sem_conv_opt_in_mode: _StabilityMode,
         async_request_hook: AsyncRequestHook,
         async_response_hook: AsyncResponseHook,
+        excluded_urls: ExcludeList | None,
     ):
         if not is_http_instrumentation_enabled():
             return await wrapped(*args, **kwargs)
 
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if excluded_urls and excluded_urls.url_disabled(_normalize_url(url)):
+            return await wrapped(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -1198,7 +1264,7 @@ async def _handle_async_request_wrapper(  # pylint: disable=too-many-locals
 
         return response
 
-    # pylint: disable=too-many-branches
+    # pylint: disable=too-many-branches,too-many-locals
     @classmethod
     def instrument_client(
         cls,
@@ -1274,6 +1340,8 @@ def instrument_client(
             # response_hook already set
             async_response_hook = None
 
+        excluded_urls = get_excluded_urls("HTTPX")
+
         if hasattr(client._transport, "handle_request"):
             wrap_function_wrapper(
                 client._transport,
@@ -1286,6 +1354,7 @@ def instrument_client(
                     sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                     request_hook=request_hook,
                     response_hook=response_hook,
+                    excluded_urls=excluded_urls,
                 ),
             )
             for transport in client._mounts.values():
@@ -1301,6 +1370,7 @@ def instrument_client(
                             sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                             request_hook=request_hook,
                             response_hook=response_hook,
+                            excluded_urls=excluded_urls,
                         ),
                     )
             client._is_instrumented_by_opentelemetry = True
@@ -1316,6 +1386,7 @@ def instrument_client(
                     sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                     async_request_hook=async_request_hook,
                     async_response_hook=async_response_hook,
+                    excluded_urls=excluded_urls,
                 ),
             )
             for transport in client._mounts.values():
@@ -1331,6 +1402,7 @@ def instrument_client(
                             sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                             async_request_hook=async_request_hook,
                             async_response_hook=async_response_hook,
+                            excluded_urls=excluded_urls,
                         ),
                     )
             client._is_instrumented_by_opentelemetry = True

instrumentation/opentelemetry-instrumentation-httpx/tests/test_httpx_integration.py

@@ -738,6 +738,24 @@ def test_if_headers_equals_none(self):
             self.assertEqual(result.text, "Hello!")
             self.assert_span()
 
+        def test_ignores_excluded_urls(self):
+            for env_var in (
+                "OTEL_PYTHON_HTTPX_EXCLUDED_URLS",
+                "OTEL_PYTHON_EXCLUDED_URLS",
+            ):
+                with self.subTest(env_var=env_var):
+                    with mock.patch.dict(
+                        "os.environ", {env_var: self.URL}, clear=True
+                    ):
+                        client = self.create_client()
+                        HTTPXClientInstrumentor().instrument_client(
+                            client=client
+                        )
+                        self.perform_request(self.URL, client=client)
+                    self.assert_span(num_spans=0)
+                    self.assert_metrics(num_metrics=0)
+                    HTTPXClientInstrumentor().uninstrument_client(client)
+
     class BaseManualTest(BaseTest, metaclass=abc.ABCMeta):
         @abc.abstractmethod
         def create_transport(
@@ -972,6 +990,26 @@ def test_client_mounts_with_instrumented_transport(self):
             self.assertEqual(spans[0].attributes[HTTP_URL], self.URL)
             self.assertEqual(spans[1].attributes[HTTP_URL], https_url)
 
+        def test_ignores_excluded_urls(self):
+            for env_var in (
+                "OTEL_PYTHON_HTTPX_EXCLUDED_URLS",
+                "OTEL_PYTHON_EXCLUDED_URLS",
+            ):
+                with self.subTest(env_var=env_var):
+                    with mock.patch.dict(
+                        "os.environ", {env_var: self.URL}, clear=True
+                    ):
+                        client = self.create_client()
+                        HTTPXClientInstrumentor().instrument_client(
+                            client=client
+                        )
+                        self.perform_request(self.URL, client=client)
+                    self.assert_span(num_spans=0)
+                    self.assert_metrics(num_metrics=0)
+                    HTTPXClientInstrumentor().uninstrument_client(
+                        client=client
+                    )
+
     @mock.patch.dict("os.environ", {"NO_PROXY": ""}, clear=True)
     class BaseInstrumentorTest(BaseTest, metaclass=abc.ABCMeta):
         @abc.abstractmethod
@@ -998,6 +1036,8 @@ def setUp(self):
             HTTPXClientInstrumentor().instrument_client(self.client)
 
         def tearDown(self):
+            # TODO: uninstrument() is required in order to avoid leaks for instrumentations
+            # but we should audit the single tests and fix any missing uninstrumentation
             HTTPXClientInstrumentor().uninstrument()
 
         def create_proxy_mounts(self):
@@ -1329,6 +1369,24 @@ def test_uninstrument_client_with_proxy(self):
             self.assertEqual(result.text, "Hello!")
             self.assert_span()
 
+        def test_ignores_excluded_urls(self):
+            for env_var in (
+                "OTEL_PYTHON_HTTPX_EXCLUDED_URLS",
+                "OTEL_PYTHON_EXCLUDED_URLS",
+            ):
+                with self.subTest(env_var=env_var):
+                    client = self.create_client()
+                    with mock.patch.dict(
+                        "os.environ", {env_var: self.URL}, clear=True
+                    ):
+                        HTTPXClientInstrumentor().instrument_client(
+                            client=client
+                        )
+                        self.perform_request(self.URL, client=client)
+                    self.assert_span(num_spans=0)
+                    self.assert_metrics(num_metrics=0)
+                    HTTPXClientInstrumentor().uninstrument_client(client)
+
 
 class TestSyncIntegration(BaseTestCases.BaseManualTest):
     def setUp(self):