Add Support for Detecting Synthetic Source

CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+- `opentelemetry-instrumentation-requests` Detect synthetic sources on requests.
+  ([#3674](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3674))
+
 ### Fixed
 
 - `opentelemetry-instrumentation-dbapi`: fix crash retrieving libpq version when enabling commenter with psycopg

instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py

@@ -258,6 +258,9 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]):
 from opentelemetry.instrumentation.utils import _start_internal_or_server_span
 from opentelemetry.metrics import get_meter
 from opentelemetry.propagators.textmap import Getter, Setter
+from opentelemetry.semconv._incubating.attributes.user_agent_attributes import (
+    USER_AGENT_SYNTHETIC_TYPE,
+)
 from opentelemetry.semconv._incubating.metrics.http_metrics import (
     create_http_server_active_requests,
     create_http_server_request_body_size,
@@ -276,6 +279,7 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]):
     ExcludeList,
     SanitizeValue,
     _parse_url_query,
+    detect_synthetic_user_agent,
     get_custom_headers,
     normalise_request_header_name,
     normalise_response_header_name,
@@ -397,7 +401,13 @@ def collect_request_attributes(
             )
     http_user_agent = asgi_getter.get(scope, "user-agent")
     if http_user_agent:
-        _set_http_user_agent(result, http_user_agent[0], sem_conv_opt_in_mode)
+        user_agent_value = http_user_agent[0]
+        _set_http_user_agent(result, user_agent_value, sem_conv_opt_in_mode)
+
+        # Check for synthetic user agent type
+        synthetic_type = detect_synthetic_user_agent(user_agent_value)
+        if synthetic_type:
+            result[USER_AGENT_SYNTHETIC_TYPE] = synthetic_type
 
     if "client" in scope and scope["client"] is not None:
         _set_http_peer_ip_server(

instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py

@@ -42,6 +42,9 @@
     HistogramDataPoint,
     NumberDataPoint,
 )
+from opentelemetry.semconv._incubating.attributes.user_agent_attributes import (
+    USER_AGENT_SYNTHETIC_TYPE,
+)
 from opentelemetry.semconv.attributes.client_attributes import (
     CLIENT_ADDRESS,
     CLIENT_PORT,
@@ -883,6 +886,145 @@ def update_expected_user_agent(expected):
             new_sem_conv=True,
         )
 
+    async def test_user_agent_synthetic_bot_detection(self):
+        """Test that bot user agents are detected as synthetic with type 'bot'"""
+        test_cases = [
+            b"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+            b"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+            b"googlebot/1.0",
+            b"bingbot/1.0",
+        ]
+
+        # Test each user agent case separately to avoid span accumulation
+        for user_agent in test_cases:
+            with self.subTest(user_agent=user_agent):
+                # Clear headers first
+                self.scope["headers"] = []
+
+                def update_expected_synthetic_bot(
+                    expected, ua: bytes = user_agent
+                ):
+                    expected[3]["attributes"].update(
+                        {
+                            SpanAttributes.HTTP_USER_AGENT: ua.decode("utf8"),
+                            USER_AGENT_SYNTHETIC_TYPE: "bot",
+                        }
+                    )
+                    return expected
+
+                self.scope["headers"].append([b"user-agent", user_agent])
+                app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+                self.seed_app(app)
+                await self.send_default_request()
+                outputs = await self.get_all_output()
+                self.validate_outputs(
+                    outputs, modifiers=[update_expected_synthetic_bot]
+                )
+
+                # Clear spans after each test case to prevent accumulation
+                self.memory_exporter.clear()
+
+    async def test_user_agent_synthetic_test_detection(self):
+        """Test that test user agents are detected as synthetic with type 'test'"""
+        test_cases = [
+            b"alwayson/1.0",
+            b"AlwaysOn/2.0",
+            b"test-alwayson-client",
+        ]
+
+        # Test each user agent case separately to avoid span accumulation
+        for user_agent in test_cases:
+            with self.subTest(user_agent=user_agent):
+                # Clear headers first
+                self.scope["headers"] = []
+
+                def update_expected_synthetic_test(
+                    expected, ua: bytes = user_agent
+                ):
+                    expected[3]["attributes"].update(
+                        {
+                            SpanAttributes.HTTP_USER_AGENT: ua.decode("utf8"),
+                            USER_AGENT_SYNTHETIC_TYPE: "test",
+                        }
+                    )
+                    return expected
+
+                self.scope["headers"].append([b"user-agent", user_agent])
+                app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+                self.seed_app(app)
+                await self.send_default_request()
+                outputs = await self.get_all_output()
+                self.validate_outputs(
+                    outputs, modifiers=[update_expected_synthetic_test]
+                )
+
+                # Clear spans after each test case to prevent accumulation
+                self.memory_exporter.clear()
+
+    async def test_user_agent_non_synthetic(self):
+        """Test that normal user agents are not marked as synthetic"""
+        test_cases = [
+            b"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            b"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
+            b"PostmanRuntime/7.28.4",
+            b"curl/7.68.0",
+        ]
+
+        # Test each user agent case separately to avoid span accumulation
+        for user_agent in test_cases:
+            with self.subTest(user_agent=user_agent):
+                # Clear headers first
+                self.scope["headers"] = []
+
+                def update_expected_non_synthetic(
+                    expected, ua: bytes = user_agent
+                ):
+                    # Should only have the user agent, not synthetic type
+                    expected[3]["attributes"].update(
+                        {
+                            SpanAttributes.HTTP_USER_AGENT: ua.decode("utf8"),
+                        }
+                    )
+                    return expected
+
+                self.scope["headers"].append([b"user-agent", user_agent])
+                app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+                self.seed_app(app)
+                await self.send_default_request()
+                outputs = await self.get_all_output()
+                self.validate_outputs(
+                    outputs, modifiers=[update_expected_non_synthetic]
+                )
+
+                # Clear spans after each test case to prevent accumulation
+                self.memory_exporter.clear()
+
+    async def test_user_agent_synthetic_new_semconv(self):
+        """Test synthetic user agent detection with new semantic conventions"""
+        user_agent = b"Mozilla/5.0 (compatible; Googlebot/2.1)"
+
+        def update_expected_synthetic_new_semconv(expected):
+            expected[3]["attributes"].update(
+                {
+                    USER_AGENT_ORIGINAL: user_agent.decode("utf8"),
+                    USER_AGENT_SYNTHETIC_TYPE: "bot",
+                }
+            )
+            return expected
+
+        self.scope["headers"] = []
+        self.scope["headers"].append([b"user-agent", user_agent])
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        await self.send_default_request()
+        outputs = await self.get_all_output()
+        self.validate_outputs(
+            outputs,
+            modifiers=[update_expected_synthetic_new_semconv],
+            old_sem_conv=False,
+            new_sem_conv=True,
+        )
+
     async def test_traceresponse_header(self):
         """Test a traceresponse header is sent when a global propagator is set."""
 

instrumentation/opentelemetry-instrumentation-requests/src/opentelemetry/instrumentation/requests/__init__.py

@@ -132,6 +132,9 @@ def response_hook(span, request_obj, response):
 )
 from opentelemetry.metrics import Histogram, get_meter
 from opentelemetry.propagate import inject
+from opentelemetry.semconv._incubating.attributes.user_agent_attributes import (
+    USER_AGENT_SYNTHETIC_TYPE,
+)
 from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE
 from opentelemetry.semconv.attributes.network_attributes import (
     NETWORK_PEER_ADDRESS,
@@ -145,6 +148,7 @@ def response_hook(span, request_obj, response):
 from opentelemetry.trace.span import Span
 from opentelemetry.util.http import (
     ExcludeList,
+    detect_synthetic_user_agent,
     get_excluded_urls,
     parse_excluded_urls,
     redact_url,
@@ -234,6 +238,9 @@ def get_or_create_headers():
 
         url = redact_url(request.url)
 
+        # Get headers early for user agent detection
+        headers = get_or_create_headers()
+
         span_attributes = {}
         _set_http_method(
             span_attributes,
@@ -243,6 +250,12 @@ def get_or_create_headers():
         )
         _set_http_url(span_attributes, url, sem_conv_opt_in_mode)
 
+        # Check for synthetic user agent type
+        user_agent = headers.get("User-Agent")
+        synthetic_type = detect_synthetic_user_agent(user_agent)
+        if synthetic_type:
+            span_attributes[USER_AGENT_SYNTHETIC_TYPE] = synthetic_type
+
         metric_labels = {}
         _set_http_method(
             metric_labels,
@@ -297,7 +310,6 @@ def get_or_create_headers():
             if callable(request_hook):
                 request_hook(span, request)
 
-            headers = get_or_create_headers()
             inject(headers)
 
             with suppress_http_instrumentation():