capture_headers (#318)

Author: alexmojaki

docs/integrations/use_cases/web_frameworks.md

@@ -15,7 +15,10 @@ Otherwise, check if your server uses [WSGI](../wsgi.md) or [ASGI](../asgi.md) an
 
 ## Capturing HTTP server request and response headers
 
-There are three environment variables to tell the OpenTelemetry instrumentation libraries to capture request and response headers:
+Some methods (e.g. `logfire.instrument_fastapi()`) allow you to pass `capture_headers=True` to record all request and response headers in the spans,
+and that's all you usually need.
+
+If you want more control, there are three environment variables to tell the OpenTelemetry instrumentation libraries to capture request and response headers:
 
 - `OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`
 - `OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`
@@ -30,6 +33,8 @@ OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST=".*"
 OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE=".*"
 ```
 
+(this is what `capture_headers=True` does)
+
 To specifically capture the `content-type` request header and request headers starting with `X-`:
 
 ```
@@ -42,6 +47,8 @@ To replace the `Authorization` header value with `[REDACTED]` to avoid leaking u
 OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS="Authorization"
 ```
 
+(although usually it's better to rely on **Logfire**'s [scrubbing](../../guides/advanced/scrubbing.md) feature)
+
 ## Query HTTP requests duration per percentile
 
 It's usually interesting to visualize HTTP requests duration per percentile. Instead of having an average, which may be influenced by extreme values, percentiles allow us know the maximum duration for 50%, 90%, 95% or 99% of the requests.

logfire/_internal/integrations/django.py

@@ -1,5 +1,7 @@
 from typing import Any
 
+from logfire._internal.utils import maybe_capture_server_headers
+
 try:
     from opentelemetry.instrumentation.django import DjangoInstrumentor
 except ModuleNotFoundError:
@@ -10,9 +12,10 @@
     )
 
 
-def instrument_django(**kwargs: Any):
+def instrument_django(*, capture_headers: bool = False, **kwargs: Any):
     """Instrument the `django` module so that spans are automatically created for each web request.
 
     See the `Logfire.instrument_django` method for details.
     """
+    maybe_capture_server_headers(capture_headers)
     DjangoInstrumentor().instrument(**kwargs)  # type: ignore[reportUnknownMemberType]

logfire/_internal/integrations/fastapi.py

@@ -16,6 +16,7 @@
 
 from ..main import Logfire
 from ..stack_info import StackInfo, get_code_object_info
+from ..utils import maybe_capture_server_headers
 
 try:
     from opentelemetry.instrumentation.asgi import get_host_port_url_tuple  # type: ignore
@@ -46,6 +47,7 @@ def instrument_fastapi(
     logfire_instance: Logfire,
     app: FastAPI,
     *,
+    capture_headers: bool = False,
     request_attributes_mapper: Callable[
         [
             Request | WebSocket,
@@ -68,6 +70,7 @@ def instrument_fastapi(
         excluded_urls = ','.join(excluded_urls)
 
     if use_opentelemetry_instrumentation:  # pragma: no branch
+        maybe_capture_server_headers(capture_headers)
         FastAPIInstrumentor.instrument_app(app, excluded_urls=excluded_urls, **opentelemetry_kwargs)  # type: ignore
 
     registry = patch_fastapi()

logfire/_internal/integrations/flask.py

@@ -5,6 +5,8 @@
 from flask.app import Flask
 from opentelemetry.instrumentation.flask import FlaskInstrumentor
 
+from logfire._internal.utils import maybe_capture_server_headers
+
 if TYPE_CHECKING:
     from wsgiref.types import WSGIEnvironment
 
@@ -25,9 +27,10 @@ class FlaskInstrumentKwargs(TypedDict, total=False):
         commenter_options: dict[str, str] | None
 
 
-def instrument_flask(app: Flask, **kwargs: Unpack[FlaskInstrumentKwargs]):
+def instrument_flask(app: Flask, capture_headers: bool = False, **kwargs: Unpack[FlaskInstrumentKwargs]):
     """Instrument `app` so that spans are automatically created for each request.
 
     See the `Logfire.instrument_flask` method for details.
     """
+    maybe_capture_server_headers(capture_headers)
     FlaskInstrumentor().instrument_app(app, **kwargs)  # type: ignore[reportUnknownMemberType]

logfire/_internal/integrations/starlette.py

@@ -5,6 +5,8 @@
 from opentelemetry.instrumentation.starlette import StarletteInstrumentor
 from starlette.applications import Starlette
 
+from logfire._internal.utils import maybe_capture_server_headers
+
 if TYPE_CHECKING:
     from opentelemetry.trace import Span
     from typing_extensions import Protocol, TypedDict, Unpack
@@ -24,9 +26,10 @@ class StarletteInstrumentKwargs(TypedDict, total=False):
         client_response_hook: ClientResponseHook | None
 
 
-def instrument_starlette(app: Starlette, **kwargs: Unpack[StarletteInstrumentKwargs]):
+def instrument_starlette(app: Starlette, *, capture_headers: bool = False, **kwargs: Unpack[StarletteInstrumentKwargs]):
     """Instrument `app` so that spans are automatically created for each request.
 
     See the `Logfire.instrument_starlette` method for details.
     """
+    maybe_capture_server_headers(capture_headers)
     StarletteInstrumentor().instrument_app(app, **kwargs)  # type: ignore[reportUnknownMemberType]

logfire/_internal/main.py

@@ -88,7 +88,6 @@
 except ImportError:  # pragma: no cover
     ValidationError = None
 
-
 # This is the type of the exc_info/_exc_info parameter of the log methods.
 # sys.exc_info() returns a tuple of (type, value, traceback) or (None, None, None).
 # We just need the exception, but we allow the user to pass the tuple because:
@@ -822,6 +821,7 @@ def instrument_fastapi(
         self,
         app: FastAPI,
         *,
+        capture_headers: bool = False,
         request_attributes_mapper: Callable[
             [
                 Request | WebSocket,
@@ -838,6 +838,7 @@ def instrument_fastapi(
 
         Args:
             app: The FastAPI app to instrument.
+            capture_headers: Set to `True` to capture all request and response headers.
             request_attributes_mapper: A function that takes a [`Request`][fastapi.Request] or [`WebSocket`][fastapi.WebSocket]
                 and a dictionary of attributes and returns a new dictionary of attributes.
                 The input dictionary will contain:
@@ -877,6 +878,7 @@ def instrument_fastapi(
         return instrument_fastapi(
             self,
             app,
+            capture_headers=capture_headers,
             request_attributes_mapper=request_attributes_mapper,
             excluded_urls=excluded_urls,
             use_opentelemetry_instrumentation=use_opentelemetry_instrumentation,
@@ -1064,6 +1066,7 @@ def instrument_celery(self, **kwargs: Unpack[CeleryInstrumentKwargs]) -> None:
 
     def instrument_django(
         self,
+        capture_headers: bool = False,
         is_sql_commentor_enabled: bool | None = None,
         request_hook: Callable[[Span, HttpRequest], None] | None = None,
         response_hook: Callable[[Span, HttpRequest, HttpResponse], None] | None = None,
@@ -1077,6 +1080,7 @@ def instrument_django(
         library.
 
         Args:
+            capture_headers: Set to `True` to capture all request and response headers.
             is_sql_commentor_enabled: Adds comments to SQL queries performed by Django,
                 so that database logs have additional context.
 
@@ -1103,6 +1107,7 @@ def instrument_django(
 
         self._warn_if_not_initialized_for_instrumentation()
         return instrument_django(
+            capture_headers=capture_headers,
             is_sql_commentor_enabled=is_sql_commentor_enabled,
             request_hook=request_hook,
             response_hook=response_hook,
@@ -1147,29 +1152,37 @@ def instrument_psycopg(self, conn_or_module: Any = None, **kwargs: Unpack[Psycop
         self._warn_if_not_initialized_for_instrumentation()
         return instrument_psycopg(conn_or_module, **kwargs)
 
-    def instrument_flask(self, app: Flask, **kwargs: Unpack[FlaskInstrumentKwargs]) -> None:
+    def instrument_flask(
+        self, app: Flask, *, capture_headers: bool = False, **kwargs: Unpack[FlaskInstrumentKwargs]
+    ) -> None:
         """Instrument `app` so that spans are automatically created for each request.
 
+        Set `capture_headers` to `True` to capture all request and response headers.
+
         Uses the
         [OpenTelemetry Flask Instrumentation](https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/flask/flask.html)
         library, specifically `FlaskInstrumentor().instrument_app()`, to which it passes `**kwargs`.
         """
         from .integrations.flask import instrument_flask
 
         self._warn_if_not_initialized_for_instrumentation()
-        return instrument_flask(app, **kwargs)
+        return instrument_flask(app, capture_headers=capture_headers, **kwargs)
 
-    def instrument_starlette(self, app: Starlette, **kwargs: Unpack[StarletteInstrumentKwargs]) -> None:
+    def instrument_starlette(
+        self, app: Starlette, *, capture_headers: bool = False, **kwargs: Unpack[StarletteInstrumentKwargs]
+    ) -> None:
         """Instrument `app` so that spans are automatically created for each request.
 
+        Set `capture_headers` to `True` to capture all request and response headers.
+
         Uses the
         [OpenTelemetry Starlette Instrumentation](https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/starlette/starlette.html)
         library, specifically `StarletteInstrumentor.instrument_app()`, to which it passes `**kwargs`.
         """
         from .integrations.starlette import instrument_starlette
 
         self._warn_if_not_initialized_for_instrumentation()
-        return instrument_starlette(app, **kwargs)
+        return instrument_starlette(app, capture_headers=capture_headers, **kwargs)
 
     def instrument_aiohttp_client(self, **kwargs: Any):
         """Instrument the `aiohttp` module so that spans are automatically created for each client request.

logfire/_internal/utils.py

@@ -247,3 +247,9 @@ def handle_internal_errors():
         yield
     except Exception:
         log_internal_error()
+
+
+def maybe_capture_server_headers(capture: bool):
+    if capture:
+        os.environ['OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST'] = '.*'
+        os.environ['OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE'] = '.*'

tests/otel_integrations/test_starlette.py

@@ -33,10 +33,12 @@ def app():
 
     app = Starlette(routes=routes)
     try:
-        logfire.instrument_starlette(app)
+        logfire.instrument_starlette(app, capture_headers=True)
         yield app
     finally:
         StarletteInstrumentor.uninstrument_app(app)
+        del os.environ['OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST']
+        del os.environ['OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE']
 
 
 @pytest.fixture()
@@ -138,6 +140,13 @@ def test_websocket(client: TestClient, exporter: TestExporter) -> None:
                     'net.peer.ip': 'testclient',
                     'net.peer.port': 50000,
                     'http.route': '/ws',
+                    'http.request.header.host': ('testserver',),
+                    'http.request.header.accept': ('*/*',),
+                    'http.request.header.accept_encoding': ('gzip, deflate',),
+                    'http.request.header.user_agent': ('testclient',),
+                    'http.request.header.connection': ('upgrade',),
+                    'http.request.header.sec_websocket_key': ('testserver==',),
+                    'http.request.header.sec_websocket_version': ('13',),
                     'http.status_code': 200,
                 },
             },