feat(apm): add referrer hostname as span tag (#12778)

Parses referrer hostname from http header when presents and adds it to
span tags as `http.referrer_hostname`. This will be used to identify
frontend clients without leaking the PII which can be contained in the
entire referrer.

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: cgcote

ddtrace/contrib/internal/trace_utils.py

@@ -172,6 +172,30 @@ def _get_request_header_user_agent(headers, headers_are_case_sensitive=False):
     return ""
 
 
+def _get_request_header_referrer_host(headers, headers_are_case_sensitive=False):
+    # type: (Mapping[str, str], bool) -> str
+    """Get referrer host from request headers
+    :param headers: A dict of http headers to be stored in the span
+    :type headers: dict or list
+    :param headers_are_case_sensitive: Whether the headers are case sensitive
+    :type headers_are_case_sensitive: bool
+    :return: The referrer host if found, empty string otherwise
+    :rtype: str
+    """
+    if headers_are_case_sensitive:
+        referrer = _get_header_value_case_insensitive(headers, http.REFERER_HEADER)
+    else:
+        referrer = headers.get(http.REFERER_HEADER)
+    if referrer:
+        try:
+            parsed_url = parse.urlparse(referrer)
+            if parsed_url.hostname:
+                return parsed_url.hostname
+        except (ValueError, AttributeError):
+            return ""
+    return ""
+
+
 def _get_request_header_client_ip(headers, peer_ip=None, headers_are_case_sensitive=False):
     # type: (Optional[Mapping[str, str]], Optional[str], bool) -> str
 
@@ -501,6 +525,11 @@ def set_http_meta(
         if user_agent:
             span.set_tag_str(http.USER_AGENT, user_agent)
 
+        # Extract referrer host if referer header is present
+        referrer_host = _get_request_header_referrer_host(request_headers, headers_are_case_sensitive)
+        if referrer_host:
+            span.set_tag_str(http.REFERRER_HOSTNAME, referrer_host)
+
         # We always collect the IP if appsec is enabled to report it on potential vulnerabilities.
         # https://datadoghq.atlassian.net/wiki/spaces/APS/pages/2118779066/Client+IP+addresses+resolution
         if asm_config._asm_enabled or config._retrieve_client_ip:

ddtrace/ext/http.py

@@ -17,6 +17,10 @@
 VERSION = "http.version"
 CLIENT_IP = "http.client_ip"
 ROUTE = "http.route"
+REFERRER_HOSTNAME = "http.referrer_hostname"
+
+# HTTP headers
+REFERER_HEADER = "referer"
 
 # template render span type
 TEMPLATE = "template"

releasenotes/notes/referrer-hostname-8aa6dc6edb9d53b1.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - tracer: extracts  the referrer hostname from HTTP requests and stored it as `http.referrer_hostname` tag.

tests/tracer/test_trace_utils.py

@@ -252,6 +252,43 @@ def test_whitelist_case_insensitive(self, span, integration_config):
         )
         assert span.get_tag("http.response.headers.content-type") == "some;value"
 
+    @pytest.mark.parametrize(
+        "headers,expected_hostname,case_sensitive",
+        [
+            (
+                {"referer": "https://example.com/path?query=1"},
+                "example.com",
+                False,
+            ),
+            (
+                {"referer": "https://example.com:8080/path?query=1"},
+                "example.com",
+                False,
+            ),
+            (
+                {"Referer": "https://example.com/path?query=1"},
+                "example.com",
+                True,
+            ),
+            (
+                {"other-header": "value"},
+                None,
+                False,
+            ),
+            (
+                {"referer": "not-a-valid-url"},
+                None,
+                False,
+            ),
+        ],
+    )
+    def test_referrer_hostname_extraction(self, span, integration_config, headers, expected_hostname, case_sensitive):
+        """Test that referrer hostname is correctly extracted from referer header"""
+        trace_utils.set_http_meta(
+            span, integration_config, request_headers=headers, headers_are_case_sensitive=case_sensitive
+        )
+        assert span.get_tag("http.referrer_hostname") == expected_hostname
+
 
 @pytest.mark.parametrize(
     "pin,config_val,default,global_service,expected",