From 9efb2d5160642591ffb6101868de41623883333d Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Thu, 23 Jan 2025 14:52:42 -0800 Subject: [PATCH 1/2] lambda topology issue fix --- .../distro/_aws_attribute_keys.py | 1 + .../distro/_aws_metric_attribute_generator.py | 18 +++++ .../test_aws_metric_attribute_generator.py | 77 +++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py index 323ccbe56..879f86653 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py @@ -7,6 +7,7 @@ AWS_REMOTE_DB_USER: str = "aws.remote.db.user" AWS_REMOTE_SERVICE: str = "aws.remote.service" AWS_REMOTE_OPERATION: str = "aws.remote.operation" +AWS_REMOTE_ENVIRONMENT: str = "aws.remote.environment" AWS_REMOTE_RESOURCE_TYPE: str = "aws.remote.resource.type" AWS_REMOTE_RESOURCE_IDENTIFIER: str = "aws.remote.resource.identifier" AWS_SDK_DESCENDANT: str = "aws.sdk.descendant" diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py index 93424379c..fe3944ebc 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +import os import re from logging import DEBUG, Logger, getLogger from typing import Match, Optional @@ -13,10 +14,13 @@ AWS_BEDROCK_KNOWLEDGE_BASE_ID, AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, AWS_KINESIS_STREAM_NAME, + AWS_LAMBDA_FUNCTION_ARN, + AWS_LAMBDA_FUNCTION_NAME, AWS_LAMBDA_RESOURCEMAPPING_ID, AWS_LOCAL_OPERATION, AWS_LOCAL_SERVICE, AWS_REMOTE_DB_USER, + AWS_REMOTE_ENVIRONMENT, AWS_REMOTE_OPERATION, AWS_REMOTE_RESOURCE_IDENTIFIER, AWS_REMOTE_RESOURCE_TYPE, @@ -445,6 +449,20 @@ def _set_remote_type_and_identifier(span: ReadableSpan, attributes: BoundedAttri ":" )[-1] cloudformation_primary_identifier = _escape_delimiters(span.attributes.get(AWS_STEPFUNCTIONS_ACTIVITY_ARN)) + elif is_key_present(span, AWS_LAMBDA_FUNCTION_NAME): + # To fix lambda topology issue, we handle the downstream lambda as a service ONLY IF the method + # call is "Invoke". Otherwise, we treat the downstream lambda as an AWS resource. + if span.attributes.get(_RPC_METHOD) == "Invoke": + attributes[AWS_REMOTE_SERVICE] = os.environ.get( + "AWS_LAMBDA_REMOTE_SERVICE", span.attributes.get(AWS_LAMBDA_FUNCTION_NAME) + ) + attributes[AWS_REMOTE_ENVIRONMENT] = ( + f"lambda:{os.environ.get('AWS_LAMBDA_REMOTE_ENVIRONMENT', 'default')}" + ) + else: + remote_resource_type = _NORMALIZED_LAMBDA_SERVICE_NAME + "::Function" + remote_resource_identifier = _escape_delimiters(span.attributes.get(AWS_LAMBDA_FUNCTION_NAME)) + cloudformation_primary_identifier = _escape_delimiters(span.attributes.get(AWS_LAMBDA_FUNCTION_ARN)) elif is_key_present(span, AWS_LAMBDA_RESOURCEMAPPING_ID): remote_resource_type = _NORMALIZED_LAMBDA_SERVICE_NAME + "::EventSourceMapping" remote_resource_identifier = _escape_delimiters(span.attributes.get(AWS_LAMBDA_RESOURCEMAPPING_ID)) diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py index 9cc625d46..10dbb9501 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py @@ -3,6 +3,7 @@ # pylint: disable=too-many-lines +import os from typing import Dict, List, Optional from unittest import TestCase from unittest.mock import MagicMock @@ -12,12 +13,16 @@ AWS_BEDROCK_DATA_SOURCE_ID, AWS_BEDROCK_GUARDRAIL_ID, AWS_BEDROCK_KNOWLEDGE_BASE_ID, + AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, AWS_CONSUMER_PARENT_SPAN_KIND, AWS_KINESIS_STREAM_NAME, + AWS_LAMBDA_FUNCTION_ARN, + AWS_LAMBDA_FUNCTION_NAME, AWS_LAMBDA_RESOURCEMAPPING_ID, AWS_LOCAL_OPERATION, AWS_LOCAL_SERVICE, AWS_REMOTE_DB_USER, + AWS_REMOTE_ENVIRONMENT, AWS_REMOTE_OPERATION, AWS_REMOTE_RESOURCE_IDENTIFIER, AWS_REMOTE_RESOURCE_TYPE, @@ -1152,6 +1157,78 @@ def test_sdk_client_span_with_remote_resource_attributes(self): self._validate_remote_resource_attributes("AWS::Lambda::EventSourceMapping", "aws_event_source_mapping_id") self._mock_attribute([AWS_LAMBDA_RESOURCEMAPPING_ID], [None]) + # Test AWS Lambda Invoke scenario with default lambda remote environment + self.span_mock.kind = SpanKind.CLIENT + self._mock_attribute( + [AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], + ["test_downstream_lambda1", "Invoke"], + keys, + values, + ) + dependency_attributes = _GENERATOR.generate_metric_attributes_dict_from_span(self.span_mock, self.resource).get( + DEPENDENCY_METRIC + ) + self.assertEqual(dependency_attributes.get(AWS_REMOTE_SERVICE), "test_downstream_lambda1") + self.assertEqual(dependency_attributes.get(AWS_REMOTE_ENVIRONMENT), "lambda:default") + self.assertNotIn(AWS_REMOTE_RESOURCE_TYPE, dependency_attributes) + self.assertNotIn(AWS_REMOTE_RESOURCE_IDENTIFIER, dependency_attributes) + self.assertNotIn(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, dependency_attributes) + self._mock_attribute([AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], [None, None]) + + # Test AWS Lambda Invoke scenario with user-configured lambda remote environment + os.environ["AWS_LAMBDA_REMOTE_ENVIRONMENT"] = "test" + self.span_mock.kind = SpanKind.CLIENT + self._mock_attribute( + [AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], + ["testLambdaFunction", "Invoke"], + keys, + values, + ) + dependency_attributes = _GENERATOR.generate_metric_attributes_dict_from_span(self.span_mock, self.resource).get( + DEPENDENCY_METRIC + ) + self.assertEqual(dependency_attributes.get(AWS_REMOTE_SERVICE), "testLambdaFunction") + self.assertEqual(dependency_attributes.get(AWS_REMOTE_ENVIRONMENT), "lambda:test") + self.assertNotIn(AWS_REMOTE_RESOURCE_TYPE, dependency_attributes) + self.assertNotIn(AWS_REMOTE_RESOURCE_IDENTIFIER, dependency_attributes) + self.assertNotIn(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, dependency_attributes) + self._mock_attribute([AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], [None, None]) + os.environ.pop("AWS_LAMBDA_REMOTE_ENVIRONMENT", None) + + # Test AWS Lambda Invoke scenario with user-configured lambda remote service + os.environ["AWS_LAMBDA_REMOTE_SERVICE"] = "test_downstream_lambda2" + self.span_mock.kind = SpanKind.CLIENT + self._mock_attribute( + [AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], + ["testLambdaFunction", "Invoke"], + keys, + values, + ) + dependency_attributes = _GENERATOR.generate_metric_attributes_dict_from_span(self.span_mock, self.resource).get( + DEPENDENCY_METRIC + ) + self.assertEqual(dependency_attributes.get(AWS_REMOTE_SERVICE), "test_downstream_lambda2") + self.assertEqual(dependency_attributes.get(AWS_REMOTE_ENVIRONMENT), "lambda:default") + self.assertNotIn(AWS_REMOTE_RESOURCE_TYPE, dependency_attributes) + self.assertNotIn(AWS_REMOTE_RESOURCE_IDENTIFIER, dependency_attributes) + self.assertNotIn(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, dependency_attributes) + self._mock_attribute([AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], [None, None]) + os.environ.pop("AWS_LAMBDA_REMOTE_SERVICE", None) + + # Test AWS Lambda non-Invoke scenario + self.span_mock.kind = SpanKind.CLIENT + lambda_arn = "arn:aws:lambda:us-east-1:123456789012:function:testLambda" + self._mock_attribute( + [AWS_LAMBDA_FUNCTION_NAME, AWS_LAMBDA_FUNCTION_ARN, SpanAttributes.RPC_METHOD], + ["testLambdaFunction", lambda_arn, "GetFunction"], + keys, + values, + ) + self._validate_remote_resource_attributes("AWS::Lambda::Function", "testLambdaFunction") + self._mock_attribute( + [AWS_LAMBDA_FUNCTION_NAME, AWS_LAMBDA_FUNCTION_ARN, SpanAttributes.RPC_METHOD], [None, None, None] + ) + self._mock_attribute([SpanAttributes.RPC_SYSTEM], [None]) def test_client_db_span_with_remote_resource_attributes(self): From 7cb74386ac1b8059242d561459f4b9a82b8fee72 Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Tue, 4 Feb 2025 09:16:53 -0800 Subject: [PATCH 2/2] update remote service and remote environment env var names --- .../distro/_aws_metric_attribute_generator.py | 16 ++++++++++++---- .../test_aws_metric_attribute_generator.py | 8 ++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py index fe3944ebc..d196a3a43 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py @@ -450,14 +450,22 @@ def _set_remote_type_and_identifier(span: ReadableSpan, attributes: BoundedAttri )[-1] cloudformation_primary_identifier = _escape_delimiters(span.attributes.get(AWS_STEPFUNCTIONS_ACTIVITY_ARN)) elif is_key_present(span, AWS_LAMBDA_FUNCTION_NAME): - # To fix lambda topology issue, we handle the downstream lambda as a service ONLY IF the method - # call is "Invoke". Otherwise, we treat the downstream lambda as an AWS resource. + # Handling downstream Lambda as a service vs. an AWS resource: + # - If the method call is "Invoke", we treat downstream Lambda as a service. + # - Otherwise, we treat it as an AWS resource. + # + # This addresses a Lambda topology issue in Application Signals. + # More context in PR: https://github.com/aws-observability/aws-otel-python-instrumentation/pull/319 + # + # NOTE: The env vars LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE and + # LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT were introduced as part of this fix. + # They are optional and allow users to override the default values if needed. if span.attributes.get(_RPC_METHOD) == "Invoke": attributes[AWS_REMOTE_SERVICE] = os.environ.get( - "AWS_LAMBDA_REMOTE_SERVICE", span.attributes.get(AWS_LAMBDA_FUNCTION_NAME) + "LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE", span.attributes.get(AWS_LAMBDA_FUNCTION_NAME) ) attributes[AWS_REMOTE_ENVIRONMENT] = ( - f"lambda:{os.environ.get('AWS_LAMBDA_REMOTE_ENVIRONMENT', 'default')}" + f'lambda:{os.environ.get("LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT", "default")}' ) else: remote_resource_type = _NORMALIZED_LAMBDA_SERVICE_NAME + "::Function" diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py index 10dbb9501..0bb738fa4 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_metric_attribute_generator.py @@ -1176,7 +1176,7 @@ def test_sdk_client_span_with_remote_resource_attributes(self): self._mock_attribute([AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], [None, None]) # Test AWS Lambda Invoke scenario with user-configured lambda remote environment - os.environ["AWS_LAMBDA_REMOTE_ENVIRONMENT"] = "test" + os.environ["LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT"] = "test" self.span_mock.kind = SpanKind.CLIENT self._mock_attribute( [AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], @@ -1193,10 +1193,10 @@ def test_sdk_client_span_with_remote_resource_attributes(self): self.assertNotIn(AWS_REMOTE_RESOURCE_IDENTIFIER, dependency_attributes) self.assertNotIn(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, dependency_attributes) self._mock_attribute([AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], [None, None]) - os.environ.pop("AWS_LAMBDA_REMOTE_ENVIRONMENT", None) + os.environ.pop("LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT", None) # Test AWS Lambda Invoke scenario with user-configured lambda remote service - os.environ["AWS_LAMBDA_REMOTE_SERVICE"] = "test_downstream_lambda2" + os.environ["LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE"] = "test_downstream_lambda2" self.span_mock.kind = SpanKind.CLIENT self._mock_attribute( [AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], @@ -1213,7 +1213,7 @@ def test_sdk_client_span_with_remote_resource_attributes(self): self.assertNotIn(AWS_REMOTE_RESOURCE_IDENTIFIER, dependency_attributes) self.assertNotIn(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, dependency_attributes) self._mock_attribute([AWS_LAMBDA_FUNCTION_NAME, SpanAttributes.RPC_METHOD], [None, None]) - os.environ.pop("AWS_LAMBDA_REMOTE_SERVICE", None) + os.environ.pop("LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE", None) # Test AWS Lambda non-Invoke scenario self.span_mock.kind = SpanKind.CLIENT