Remove spans with ec2 metadata ip address from metrics (#323)

*Issue #, if available:*
ADOT SDK resource detectors by default have enabled a few AWS resource
detector which will call EC2 metadata API endpoints. These activities
have been captured by auto-instrumentation and generated AppSignals
metrics.

*Description of changes:*
Suppress AwsSpanMetricsProcessor from generating metrics when the
RemoteService points to 169.254.169.254

*Testing*:
Tested using Java Adot:
aws-observability/aws-otel-java-instrumentation#1015

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: harrryr

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_span_metrics_processor.py

@@ -4,6 +4,7 @@
 
 from typing_extensions import override
 
+from amazon.opentelemetry.distro._aws_attribute_keys import AWS_REMOTE_SERVICE
 from amazon.opentelemetry.distro.metric_attribute_generator import MetricAttributeGenerator
 from opentelemetry.context import Context
 from opentelemetry.metrics import Histogram
@@ -21,6 +22,11 @@
 _FAULT_CODE_UPPER_BOUND: int = 599
 
 
+# EC2 Metadata API IP Address
+# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instancedata-inside-access
+_EC2_METADATA_API_IP: str = "169.254.169.254"
+
+
 class AwsSpanMetricsProcessor(SpanProcessor):
     """AwsSpanMetricsProcessor is SpanProcessor that generates metrics from spans
 
@@ -79,6 +85,7 @@ def on_end(self, span: ReadableSpan) -> None:
         attribute_dict: Dict[str, BoundedAttributes] = self._generator.generate_metric_attributes_dict_from_span(
             span, self._resource
         )
+
         for attributes in attribute_dict.values():
             self._record_metrics(span, attributes)
 
@@ -93,7 +100,7 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool:
 
     def _record_metrics(self, span: ReadableSpan, attributes: BoundedAttributes) -> None:
         # Only record metrics if non-empty attributes are returned.
-        if len(attributes) > 0:
+        if len(attributes) > 0 and not _is_ec2_metadata_api_span(attributes):
             self._record_error_or_fault(span, attributes)
             self._record_latency(span, attributes)
 
@@ -130,3 +137,7 @@ def _is_not_error_or_fault(http_status_code: int) -> bool:
         or http_status_code < _ERROR_CODE_LOWER_BOUND
         or http_status_code > _FAULT_CODE_UPPER_BOUND
     )
+
+
+def _is_ec2_metadata_api_span(attributes: BoundedAttributes) -> bool:
+    return attributes.get(AWS_REMOTE_SERVICE) == _EC2_METADATA_API_IP

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_span_metrics_processor.py

@@ -5,6 +5,7 @@
 from unittest import TestCase
 from unittest.mock import MagicMock, call
 
+from amazon.opentelemetry.distro._aws_attribute_keys import AWS_REMOTE_SERVICE
 from amazon.opentelemetry.distro._aws_span_processing_util import (
     should_generate_dependency_metric_attributes,
     should_generate_service_metric_attributes,
@@ -238,6 +239,15 @@ def test_on_end_metrics_generation_with_status_data_ok(self):
         self._validate_metrics_generated_for_status_data_ok(599, self.ExpectedStatusMetric.FAULT)
         self._validate_metrics_generated_for_status_data_ok(600, self.ExpectedStatusMetric.NEITHER)
 
+    def test_on_end_metrics_generation_from_ec2_metadata_api(self):
+        span_attributes: Attributes = {AWS_REMOTE_SERVICE: "169.254.169.254"}
+        span: ReadableSpan = _build_readable_span_mock(span_attributes)
+        metric_attributes_dict = _build_ec2_metadata_api_metric_attributes()
+        self._configure_mock_for_on_end(span, metric_attributes_dict)
+
+        self.aws_span_metrics_processor.on_end(span)
+        self._verify_histogram_record(metric_attributes_dict, 0, 0)
+
     def _configure_mock_for_on_end(self, span: Span, attribute_map: {str: Attributes}):
         def generate_m_a_from_span_side_effect(input_span, resource):
             if input_span == span and resource == self.test_resource:
@@ -373,3 +383,10 @@ def _build_metric_attributes(contain_attributes: bool, span: Span) -> Attributes
             attributes = {"new dependency key": "new dependency value"}
             attribute_map[DEPENDENCY_METRIC] = attributes
     return attribute_map
+
+
+def _build_ec2_metadata_api_metric_attributes() -> Attributes:
+    attribute_map: Attributes = {}
+    attributes = {AWS_REMOTE_SERVICE: "169.254.169.254"}
+    attribute_map[DEPENDENCY_METRIC] = attributes
+    return attribute_map