From a94f4d116e9b19ccbd8aa43f065679f96b94cabc Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Thu, 28 Aug 2025 17:13:44 -0700 Subject: [PATCH 1/5] customize Starlette Instrumentor middleware to exclude noisy receive and send spans --- .../distro/patches/_instrumentation_patch.py | 2 +- .../distro/patches/_starlette_patches.py | 18 ++++++++++ .../distro/patches/test_starlette_patches.py | 35 +++++++++++++++++-- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py index df066bca2..7cc5611f7 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py @@ -67,7 +67,7 @@ def apply_instrumentation_patches() -> None: from amazon.opentelemetry.distro.patches._starlette_patches import _apply_starlette_instrumentation_patches # Starlette auto-instrumentation v0.54b includes a strict dependency version check - # This restriction was removed in v1.34.0/0.55b0. Applying temporary patch for Genesis launch + # This restriction was removed in v1.34.0/0.55b0. Applying temporary patch for Bedrock AgentCore launch # TODO: Remove patch after syncing with upstream v1.34.0 or later _apply_starlette_instrumentation_patches() diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py index b3bcd624b..3c758bc43 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py @@ -4,6 +4,8 @@ from logging import Logger, getLogger from typing import Collection +from amazon.opentelemetry.distro._utils import AGENT_OBSERVABILITY_ENABLED + _logger: Logger = getLogger(__name__) @@ -18,6 +20,7 @@ def _apply_starlette_instrumentation_patches() -> None: """ try: # pylint: disable=import-outside-toplevel + from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware from opentelemetry.instrumentation.starlette import StarletteInstrumentor # Patch starlette dependencies version check @@ -28,6 +31,21 @@ def patched_instrumentation_dependencies(self) -> Collection[str]: # Apply the patch StarletteInstrumentor.instrumentation_dependencies = patched_instrumentation_dependencies + # Patch to exclude http recieve/send ASGI event spans, this Middleware instrumentation is injected + # internally by Starlette Instrumentor, see: + # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/51da0a766e5d3cbc746189e10c9573163198cfcd/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py#L573 + if AGENT_OBSERVABILITY_ENABLED: + original_init = OpenTelemetryMiddleware.__init__ + + def patched_init(self, app, **kwargs): + original_init(self, app, **kwargs) + if hasattr(self, "exclude_receive_span"): + self.exclude_receive_span = True + if hasattr(self, "exclude_send_span"): + self.exclude_send_span = True + + OpenTelemetryMiddleware.__init__ = patched_init + _logger.debug("Successfully patched Starlette instrumentation_dependencies method") except Exception as exc: # pylint: disable=broad-except _logger.warning("Failed to apply Starlette instrumentation patches: %s", exc) diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py index e0bbf4270..d26ab7fd2 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py @@ -9,6 +9,7 @@ class TestStarlettePatch(TestCase): """Test the Starlette instrumentation patches.""" + @patch("amazon.opentelemetry.distro.patches._starlette_patches.AGENT_OBSERVABILITY_ENABLED", True) @patch("amazon.opentelemetry.distro.patches._starlette_patches._logger") def test_starlette_patch_applied_successfully(self, mock_logger): """Test that the Starlette instrumentation patch is applied successfully.""" @@ -16,12 +17,28 @@ def test_starlette_patch_applied_successfully(self, mock_logger): mock_instrumentor_class = MagicMock() mock_instrumentor_class.__name__ = "StarletteInstrumentor" - # Create a mock module + class MockMiddleware: + def __init__(self, app, **kwargs): + pass + + mock_middleware_class = MockMiddleware + original_init = mock_middleware_class.__init__ + + # Create mock modules mock_starlette_module = MagicMock() mock_starlette_module.StarletteInstrumentor = mock_instrumentor_class - # Mock the import - with patch.dict("sys.modules", {"opentelemetry.instrumentation.starlette": mock_starlette_module}): + mock_asgi_module = MagicMock() + mock_asgi_module.OpenTelemetryMiddleware = mock_middleware_class + + # Mock the imports + with patch.dict( + "sys.modules", + { + "opentelemetry.instrumentation.starlette": mock_starlette_module, + "opentelemetry.instrumentation.asgi": mock_asgi_module, + }, + ): # Apply the patch _apply_starlette_instrumentation_patches() @@ -33,6 +50,18 @@ def test_starlette_patch_applied_successfully(self, mock_logger): result = mock_instrumentor_class.instrumentation_dependencies(mock_instance) self.assertEqual(result, ("starlette >= 0.13",)) + self.assertNotEqual(mock_middleware_class.__init__, original_init) + + # Test middleware patching sets exclude flags + mock_middleware_instance = MagicMock() + mock_middleware_instance.exclude_receive_span = False + mock_middleware_instance.exclude_send_span = False + + mock_middleware_class.__init__(mock_middleware_instance, "app") + + self.assertTrue(mock_middleware_instance.exclude_receive_span) + self.assertTrue(mock_middleware_instance.exclude_send_span) + # Verify logging mock_logger.debug.assert_called_once_with( "Successfully patched Starlette instrumentation_dependencies method" From 2c13cb3d3d948bb2c944192e79bfeef58fe596f4 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Thu, 28 Aug 2025 17:26:11 -0700 Subject: [PATCH 2/5] lint fix --- .../amazon/opentelemetry/distro/patches/_starlette_patches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py index 3c758bc43..d85136808 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py @@ -31,7 +31,7 @@ def patched_instrumentation_dependencies(self) -> Collection[str]: # Apply the patch StarletteInstrumentor.instrumentation_dependencies = patched_instrumentation_dependencies - # Patch to exclude http recieve/send ASGI event spans, this Middleware instrumentation is injected + # Patch to exclude http receive/send ASGI event spans, this Middleware instrumentation is injected # internally by Starlette Instrumentor, see: # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/51da0a766e5d3cbc746189e10c9573163198cfcd/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py#L573 if AGENT_OBSERVABILITY_ENABLED: From 1e35cc6e34bcb76eeec6f02e26aa96a5c23cbd05 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Fri, 29 Aug 2025 12:56:15 -0700 Subject: [PATCH 3/5] add patch for excluding asgi event spans from starlette --- .../distro/patches/_starlette_patches.py | 10 +- .../distro/patches/test_starlette_patches.py | 111 +++++++++--------- 2 files changed, 63 insertions(+), 58 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py index d85136808..14cdffab8 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py @@ -4,7 +4,7 @@ from logging import Logger, getLogger from typing import Collection -from amazon.opentelemetry.distro._utils import AGENT_OBSERVABILITY_ENABLED +from amazon.opentelemetry.distro._utils import is_agent_observability_enabled _logger: Logger = getLogger(__name__) @@ -31,10 +31,12 @@ def patched_instrumentation_dependencies(self) -> Collection[str]: # Apply the patch StarletteInstrumentor.instrumentation_dependencies = patched_instrumentation_dependencies - # Patch to exclude http receive/send ASGI event spans, this Middleware instrumentation is injected - # internally by Starlette Instrumentor, see: + # Patch to exclude http receive/send ASGI event spans from Bedrock AgentCore, + # this Middleware instrumentation is injected internally by Starlette Instrumentor, see: # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/51da0a766e5d3cbc746189e10c9573163198cfcd/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py#L573 - if AGENT_OBSERVABILITY_ENABLED: + # + # Issue for tracking a feature to customize this setting within Starlette: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3725 + if is_agent_observability_enabled(): original_init = OpenTelemetryMiddleware.__init__ def patched_init(self, app, **kwargs): diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py index d26ab7fd2..3de5f0bde 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py @@ -9,63 +9,66 @@ class TestStarlettePatch(TestCase): """Test the Starlette instrumentation patches.""" - @patch("amazon.opentelemetry.distro.patches._starlette_patches.AGENT_OBSERVABILITY_ENABLED", True) @patch("amazon.opentelemetry.distro.patches._starlette_patches._logger") def test_starlette_patch_applied_successfully(self, mock_logger): """Test that the Starlette instrumentation patch is applied successfully.""" - # Create a mock StarletteInstrumentor class - mock_instrumentor_class = MagicMock() - mock_instrumentor_class.__name__ = "StarletteInstrumentor" - - class MockMiddleware: - def __init__(self, app, **kwargs): - pass - - mock_middleware_class = MockMiddleware - original_init = mock_middleware_class.__init__ - - # Create mock modules - mock_starlette_module = MagicMock() - mock_starlette_module.StarletteInstrumentor = mock_instrumentor_class - - mock_asgi_module = MagicMock() - mock_asgi_module.OpenTelemetryMiddleware = mock_middleware_class - - # Mock the imports - with patch.dict( - "sys.modules", - { - "opentelemetry.instrumentation.starlette": mock_starlette_module, - "opentelemetry.instrumentation.asgi": mock_asgi_module, - }, - ): - # Apply the patch - _apply_starlette_instrumentation_patches() - - # Verify the instrumentation_dependencies method was replaced - self.assertTrue(hasattr(mock_instrumentor_class, "instrumentation_dependencies")) - - # Test the patched method returns the expected value - mock_instance = MagicMock() - result = mock_instrumentor_class.instrumentation_dependencies(mock_instance) - self.assertEqual(result, ("starlette >= 0.13",)) - - self.assertNotEqual(mock_middleware_class.__init__, original_init) - - # Test middleware patching sets exclude flags - mock_middleware_instance = MagicMock() - mock_middleware_instance.exclude_receive_span = False - mock_middleware_instance.exclude_send_span = False - - mock_middleware_class.__init__(mock_middleware_instance, "app") - - self.assertTrue(mock_middleware_instance.exclude_receive_span) - self.assertTrue(mock_middleware_instance.exclude_send_span) - - # Verify logging - mock_logger.debug.assert_called_once_with( - "Successfully patched Starlette instrumentation_dependencies method" - ) + for agent_enabled in [True, False]: + with self.subTest(agent_enabled=agent_enabled): + with patch.dict("os.environ", {"AGENT_OBSERVABILITY_ENABLED": "true" if agent_enabled else "false"}): + # Create a mock StarletteInstrumentor class + mock_instrumentor_class = MagicMock() + mock_instrumentor_class.__name__ = "StarletteInstrumentor" + + def create_middleware_class(): + class MockMiddleware: + def __init__(self, app, **kwargs): + pass + + return MockMiddleware + + mock_middleware_class = create_middleware_class() + + mock_starlette_module = MagicMock() + mock_starlette_module.StarletteInstrumentor = mock_instrumentor_class + + mock_asgi_module = MagicMock() + mock_asgi_module.OpenTelemetryMiddleware = mock_middleware_class + + with patch.dict( + "sys.modules", + { + "opentelemetry.instrumentation.starlette": mock_starlette_module, + "opentelemetry.instrumentation.asgi": mock_asgi_module, + }, + ): + # Apply the patch + _apply_starlette_instrumentation_patches() + + # Verify the instrumentation_dependencies method was replaced + self.assertTrue(hasattr(mock_instrumentor_class, "instrumentation_dependencies")) + + # Test the patched method returns the expected value + mock_instance = MagicMock() + result = mock_instrumentor_class.instrumentation_dependencies(mock_instance) + self.assertEqual(result, ("starlette >= 0.13",)) + + mock_middleware_instance = MagicMock() + mock_middleware_instance.exclude_receive_span = False + mock_middleware_instance.exclude_send_span = False + mock_middleware_class.__init__(mock_middleware_instance, "app") + + # Test middleware patching sets exclude flags + if agent_enabled: + self.assertTrue(mock_middleware_instance.exclude_receive_span) + self.assertTrue(mock_middleware_instance.exclude_send_span) + else: + self.assertFalse(mock_middleware_instance.exclude_receive_span) + self.assertFalse(mock_middleware_instance.exclude_send_span) + + # Verify logging + mock_logger.debug.assert_called_with( + "Successfully patched Starlette instrumentation_dependencies method" + ) @patch("amazon.opentelemetry.distro.patches._starlette_patches._logger") def test_starlette_patch_handles_import_error(self, mock_logger): From 6036db836edca935e27ad1487be1eaf02f185fec Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Fri, 29 Aug 2025 13:13:26 -0700 Subject: [PATCH 4/5] linting fix --- .../amazon/opentelemetry/distro/patches/_starlette_patches.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py index 14cdffab8..8c580c2ba 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py @@ -31,6 +31,7 @@ def patched_instrumentation_dependencies(self) -> Collection[str]: # Apply the patch StarletteInstrumentor.instrumentation_dependencies = patched_instrumentation_dependencies + # pylint: disable=line-too-long # Patch to exclude http receive/send ASGI event spans from Bedrock AgentCore, # this Middleware instrumentation is injected internally by Starlette Instrumentor, see: # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/51da0a766e5d3cbc746189e10c9573163198cfcd/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py#L573 From 05f3cd8d0f03e75c28aed983aeb43d49baa0b310 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Fri, 29 Aug 2025 13:15:47 -0700 Subject: [PATCH 5/5] lint fix --- .../amazon/opentelemetry/distro/patches/_starlette_patches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py index 8c580c2ba..385fb0b59 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py @@ -36,7 +36,8 @@ def patched_instrumentation_dependencies(self) -> Collection[str]: # this Middleware instrumentation is injected internally by Starlette Instrumentor, see: # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/51da0a766e5d3cbc746189e10c9573163198cfcd/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py#L573 # - # Issue for tracking a feature to customize this setting within Starlette: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3725 + # Issue for tracking a feature to customize this setting within Starlette: + # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3725 if is_agent_observability_enabled(): original_init = OpenTelemetryMiddleware.__init__