Suppress appsignal metrics and attributes from boto3sqs spans (#127)

*Issue*
- Boto3 calls use botocore underneath and OTel offers instrumentation
for both libraries -
[boto3sqs](https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-boto3sqs)
(specifically SQS relates spans) and
[botocore](https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-botocore)
- For boto3 operations on SQS (send, receive, delete), we generate the
following set of spans:
- SendMessage: 1 PRODUCER span (from boto3sqs) + 1 CLIENT span (from
botocore) per operation
- ReceiveMessages: 1 CONSUMER receive span (from boto3sqs) per operation
+ 1 CONSUMER process span (from boto3sqs) per message received + 1
CLIENT span (from botocore) per operation
  - DeleteMessage: 1 CLIENT span (from botocore) per operation

The above behavior causes the current logic to generate multiple
dependency metrics per operation whereas we want to to generate only 1
dependency metrics per operation.

*Note: the below image doesn't include the SendMessage operation so
there are no spans for it*
<img width="1707" alt="before"
src="https://github.com/aws-observability/aws-otel-python-instrumentation/assets/50466688/09ad0e1a-d69d-40c3-8e56-d796dd2f54e7">


*Description of changes:*
- Identify if the span is generated by the `boto3sqs` instrumentation by
checking the instrumentation scope.
- If yes, then we don't generate any service and dependency metrics from
such span. We also do not populate any `AWS_*` attribute on the span.

After making the changes, this is what the dependency metrics looks
like.

<img width="1812" alt="Screenshot 2024-03-27 at 2 48 59 PM"
src="https://github.com/aws-observability/aws-otel-python-instrumentation/assets/50466688/53a95217-3822-49e5-b7be-56208350f4aa">


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: srprash

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py

@@ -20,8 +20,7 @@
 LOCAL_ROOT: str = "LOCAL_ROOT"
 
 # Useful constants
-_SQS_RECEIVE_MESSAGE_SPAN_NAME: str = "Sqs.ReceiveMessage"
-_AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX: str = "io.opentelemetry.aws-sdk-"
+_BOTO3SQS_INSTRUMENTATION_SCOPE: str = "opentelemetry.instrumentation.boto3sqs"
 
 # Max keyword length supported by parsing into remote_operation from DB_STATEMENT
 MAX_KEYWORD_LENGTH = 27
@@ -87,14 +86,14 @@ def is_aws_sdk_span(span: ReadableSpan) -> bool:
 
 
 def should_generate_service_metric_attributes(span: ReadableSpan) -> bool:
-    return (is_local_root(span) and not _is_sqs_receive_message_consumer_span(span)) or SpanKind.SERVER == span.kind
+    return (is_local_root(span) and not _is_boto3sqs_span(span)) or SpanKind.SERVER == span.kind
 
 
 def should_generate_dependency_metric_attributes(span: ReadableSpan) -> bool:
     return (
         SpanKind.CLIENT == span.kind
-        or SpanKind.PRODUCER == span.kind
-        or (_is_dependency_consumer_span(span) and not _is_sqs_receive_message_consumer_span(span))
+        or (SpanKind.PRODUCER == span.kind and not _is_boto3sqs_span(span))
+        or (_is_dependency_consumer_span(span) and not _is_boto3sqs_span(span))
     )
 
 
@@ -118,17 +117,18 @@ def is_local_root(span: ReadableSpan) -> bool:
     return span.parent is None or not span.parent.is_valid or span.parent.is_remote
 
 
-def _is_sqs_receive_message_consumer_span(span: ReadableSpan) -> bool:
-    """To identify the SQS consumer spans produced by AWS SDK instrumentation"""
-    messaging_operation: str = span.attributes.get(SpanAttributes.MESSAGING_OPERATION)
+def _is_boto3sqs_span(span: ReadableSpan) -> bool:
+    """
+    To identify if the span produced is from the boto3sqs instrumentation.
+    We use this to identify the boto3sqs spans and not generate metrics from the since we will generate
+    the same metrics from botocore spans.
+    """
+    # TODO: Evaluate if we can bring the boto3sqs spans back to generate metrics and not have to suppress them.
     instrumentation_scope: InstrumentationScope = span.instrumentation_scope
-
     return (
-        (span.name is not None and _SQS_RECEIVE_MESSAGE_SPAN_NAME.casefold() == span.name.casefold())
-        and SpanKind.CONSUMER == span.kind
-        and instrumentation_scope is not None
-        and instrumentation_scope.name.startswith(_AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX)
-        and (messaging_operation is None or messaging_operation == MessagingOperationValues.PROCESS)
+        instrumentation_scope is not None
+        and instrumentation_scope.name is not None
+        and _BOTO3SQS_INSTRUMENTATION_SCOPE.casefold() == instrumentation_scope.name.casefold()
     )
 
 

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py

@@ -53,12 +53,12 @@ class TestAwsMetricAttributeGenerator(TestCase):
     def setUp(self):
         self.attributes_mock: Attributes = MagicMock()
         self.instrumentation_scope_info_mock: InstrumentationScope = MagicMock()
-        self.instrumentation_scope_info_mock._name = "Scope name"
+        self.instrumentation_scope_info_mock.name = "Scope name"
         self.span_mock: ReadableSpan = MagicMock()
         self.span_mock.name = None
         self.span_mock.attributes = self.attributes_mock
         self.attributes_mock.get.return_value = None
-        self.span_mock._instrumentation_scope = self.instrumentation_scope_info_mock
+        self.span_mock.instrumentation_scope = self.instrumentation_scope_info_mock
         self.span_mock.get_span_context.return_value = MagicMock()
         self.parent_span_context: SpanContext = MagicMock()
         self.parent_span_context.is_valid = True
@@ -772,6 +772,39 @@ def test_both_metric_when_local_root_consumer_process(self):
         self.assertIsNotNone(service_attributes)
         self.assertIsNotNone(dependency_attributes)
 
+    def test_local_root_boto3_span(self):
+        self._update_resource_with_service_name()
+        self.parent_span_context.is_valid = False
+        self.span_mock.kind = SpanKind.PRODUCER
+        self.span_mock.instrumentation_scope.name = "opentelemetry.instrumentation.boto3sqs"
+
+        actual_attributes: Attributes = _GENERATOR.generate_metric_attributes_dict_from_span(
+            self.span_mock, self.resource
+        )
+        service_attributes: Attributes = actual_attributes.get(SERVICE_METRIC)
+        dependency_attributes: Attributes = actual_attributes.get(DEPENDENCY_METRIC)
+
+        # boto3sqs spans shouldn't generate aws service attributes even local root
+        self.assertIsNone(service_attributes)
+        # boto3sqs spans shouldn't generate aws dependency attributes
+        self.assertIsNone(dependency_attributes)
+
+    def test_non_local_root_boto3_span(self):
+        self._update_resource_with_service_name()
+        self.span_mock.kind = SpanKind.CONSUMER
+        self.span_mock.instrumentation_scope.name = "opentelemetry.instrumentation.boto3sqs"
+
+        actual_attributes: Attributes = _GENERATOR.generate_metric_attributes_dict_from_span(
+            self.span_mock, self.resource
+        )
+        service_attributes: Attributes = actual_attributes.get(SERVICE_METRIC)
+        dependency_attributes: Attributes = actual_attributes.get(DEPENDENCY_METRIC)
+
+        # boto3sqs spans shouldn't generate aws service attributes
+        self.assertIsNone(service_attributes)
+        # boto3sqs spans shouldn't generate aws dependency attributes
+        self.assertIsNone(dependency_attributes)
+
     def test_normalize_service_name_non_aws_sdk_span(self):
         service_name: str = "non aws service"
         self._mock_attribute([SpanAttributes.RPC_SERVICE], [service_name])

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_span_processing_util.py

@@ -307,45 +307,32 @@ def attributes_get_side_effect(key):
 
         self.assertTrue(is_consumer_process_span(self.span_data_mock))
 
-    # check that AWS SDK v1 SQS ReceiveMessage consumer spans metrics are suppressed
-    def test_no_metric_attributes_for_sqs_consumer_span_aws_sdk_v1(self):
+    # check that boto3 SQS spans metrics are suppressed
+    def test_no_metric_attributes_for_boto3sqs_producer_span(self):
         instrumentation_scope_mock: InstrumentationScope = MagicMock()
-        instrumentation_scope_mock.name = "io.opentelemetry.aws-sdk-1.11"
+        instrumentation_scope_mock.name = "opentelemetry.instrumentation.boto3sqs"
         self.span_data_mock.instrumentation_scope = instrumentation_scope_mock
-        self.span_data_mock.kind = SpanKind.CONSUMER
-        self.span_data_mock.name = "SQS.ReceiveMessage"
+        self.span_data_mock.kind = SpanKind.PRODUCER
+        self.span_data_mock.name = "testQueue send"
         self.assertFalse(should_generate_service_metric_attributes(self.span_data_mock))
         self.assertFalse(should_generate_dependency_metric_attributes(self.span_data_mock))
 
-    # check that AWS SDK v1 SQS ReceiveMessage consumer spans metrics are suppressed
-    def test_no_metric_attributes_for_sqs_consumer_span_aws_sdk_v2(self):
+    def test_no_metric_attributes_for_boto3sqs_consumer_span(self):
         instrumentation_scope_mock: InstrumentationScope = MagicMock()
-        instrumentation_scope_mock.name = "io.opentelemetry.aws-sdk-2.2"
+        instrumentation_scope_mock.name = "opentelemetry.instrumentation.boto3sqs"
         self.span_data_mock.instrumentation_scope = instrumentation_scope_mock
         self.span_data_mock.kind = SpanKind.CONSUMER
-        self.span_data_mock.name = "SQS.ReceiveMessage"
+        self.span_data_mock.name = "testQueue receive"
+
         self.assertFalse(should_generate_service_metric_attributes(self.span_data_mock))
         self.assertFalse(should_generate_dependency_metric_attributes(self.span_data_mock))
 
-    # check that SQS ReceiveMessage consumer spans metrics are still generated for other instrumentation
-    def test_metric_attributes_generated_for_other_instrumentation_sqs_consumer_span(self):
+    def test_no_metric_attributes_for_boto3sqs_process_span(self):
         instrumentation_scope_info_mock = MagicMock()
-        instrumentation_scope_info_mock.name = "my-instrumentation"
+        instrumentation_scope_info_mock.name = "opentelemetry.instrumentation.boto3sqs"
         self.span_data_mock.instrumentation_scope = instrumentation_scope_info_mock
         self.span_data_mock.kind = SpanKind.CONSUMER
-        self.span_data_mock.name = "Sqs.ReceiveMessage"
-
-        self.assertTrue(should_generate_service_metric_attributes(self.span_data_mock))
-        self.assertTrue(should_generate_dependency_metric_attributes(self.span_data_mock))
-
-    # check that SQS ReceiveMessage consumer span metrics are suppressed if messaging operation
-    # is process and not receive
-    def test_no_metric_attributes_for_aws_sdk_sqs_consumer_process_span(self):
-        instrumentation_scope_info_mock = MagicMock()
-        instrumentation_scope_info_mock.name = "io.opentelemetry.aws-sdk-2.2"
-        self.span_data_mock.instrumentation_scope = instrumentation_scope_info_mock
-        self.span_data_mock.kind = SpanKind.CONSUMER
-        self.span_data_mock.name = "Sqs.ReceiveMessage"
+        self.span_data_mock.name = "testQueue process"
 
         def attributes_get_side_effect_process(key):
             if key == SpanAttributes.MESSAGING_OPERATION:
@@ -358,12 +345,14 @@ def attributes_get_side_effect_process(key):
         self.assertFalse(should_generate_service_metric_attributes(self.span_data_mock))
         self.assertFalse(should_generate_dependency_metric_attributes(self.span_data_mock))
 
-        def attributes_get_side_effect_receive(key):
-            if key == SpanAttributes.MESSAGING_OPERATION:
-                return MessagingOperationValues.RECEIVE
-            return None
+    # check that consumer spans metrics are still generated for other instrumentation
+    def test_metric_attributes_generated_for_instrumentation_other_than_boto3sqs(self):
+        instrumentation_scope_info_mock = MagicMock()
+        instrumentation_scope_info_mock.name = "my-instrumentation"
+        self.span_data_mock.instrumentation_scope = instrumentation_scope_info_mock
+        self.span_data_mock.kind = SpanKind.CONSUMER
+        self.span_data_mock.name = "testQueue receive"
 
-        self.attributes_mock.get.side_effect = attributes_get_side_effect_receive
         self.assertTrue(should_generate_service_metric_attributes(self.span_data_mock))
         self.assertTrue(should_generate_dependency_metric_attributes(self.span_data_mock))