Skip to content

Fix: Lambda Topology Issue (#1016) #1085

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiyuan-he merged 14 commits into from
Jun 3, 2025
Merged

Fix: Lambda Topology Issue (#1016) #1085

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
0db4b31
Fix: Lambda Topology Issue (#1016)
yiyuan-he Feb 11, 2025
9df4092
Merge branch 'main' into cherry-pick-lambda
yiyuan-he May 28, 2025
1c78149
Merge branch 'main' into cherry-pick-lambda
yiyuan-he May 29, 2025
d4ac453
address pr comments
yiyuan-he May 30, 2025
13026de
further implementation refactoring and enhance unit tests
yiyuan-he May 30, 2025
12e4e54
address follow-up pr comments
yiyuan-he Jun 3, 2025
87df166
move lambda environment test logic
yiyuan-he Jun 3, 2025
3135477
merge guardrail tests and add test case for fallback cfn primary id
yiyuan-he Jun 3, 2025
4d730e0
fix lambda invoke with arn case comment
yiyuan-he Jun 3, 2025
789d392
add coverage for fallback unknown remote service
yiyuan-he Jun 3, 2025
3f4fa46
update to use isLambdaInvokeOperation
yiyuan-he Jun 3, 2025
1f4f161
refactor fallback cfn id test case and add lambda cases
yiyuan-he Jun 3, 2025
af4f9e4
remove unecessary mock attribute clean up
yiyuan-he Jun 3, 2025
e86898e
add docstring to setRemoteEnvironment
yiyuan-he Jun 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ...are/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@
public final class AwsApplicationSignalsCustomizerProvider
implements AutoConfigurationCustomizerProvider {
static final String AWS_LAMBDA_FUNCTION_NAME_CONFIG = "AWS_LAMBDA_FUNCTION_NAME";
static final String LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT =
"LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT";

private static final Duration DEFAULT_METRIC_EXPORT_INTERVAL = Duration.ofMinutes(1);
private static final Logger logger =
Expand Down
17 changes: 9 additions & 8 deletions ...n/java/software/amazon/opentelemetry/javaagent/providers/AwsMetricAttributeGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import static io.opentelemetry.semconv.SemanticAttributes.SERVER_SOCKET_ADDRESS;
import static io.opentelemetry.semconv.SemanticAttributes.SERVER_SOCKET_PORT;
import static io.opentelemetry.semconv.SemanticAttributes.URL_FULL;
import static software.amazon.opentelemetry.javaagent.providers.AwsApplicationSignalsCustomizerProvider.LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_AGENT_ID;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_BUCKET_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER;
Expand Down Expand Up @@ -135,11 +136,6 @@ final class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
// Constants for Lambda operations
private static final String LAMBDA_INVOKE_OPERATION = "Invoke";

// Environment variable to specify remote environment for Lambda services in Application Signals
// topology
private static final String LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT =
"LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT";

// Special DEPENDENCY attribute value if GRAPHQL_OPERATION_TYPE attribute key is present.
private static final String GRAPHQL = "graphql";

Expand Down Expand Up @@ -403,6 +399,9 @@ private static String generateRemoteService(SpanData span) {
}

private static boolean isLambdaInvokeOperation(SpanData span) {
if (!isAwsSDKSpan(span)) {
return false;
}
String rpcService = getRemoteService(span, RPC_SERVICE);
return ("AWSLambda".equals(rpcService) || "Lambda".equals(rpcService))
&& LAMBDA_INVOKE_OPERATION.equals(span.getAttributes().get(RPC_METHOD));
Expand Down Expand Up @@ -460,9 +459,11 @@ private static String normalizeRemoteServiceName(SpanData span, String serviceNa
// SDK calls accept both formats
Optional<String> lambdaFunctionName =
getLambdaFunctionNameFromArn(
Optional.ofNullable(
escapeDelimiters(span.getAttributes().get(AWS_LAMBDA_NAME))));
return lambdaFunctionName.get();
Optional.ofNullable(span.getAttributes().get(AWS_LAMBDA_NAME)));
// If Lambda name is not present, use UnknownRemoteService
// This is intentional - we want to clearly indicate when the Lambda function name
// is missing rather than falling back to a generic service name
return lambdaFunctionName.orElse(UNKNOWN_REMOTE_SERVICE);
} else {
return NORMALIZED_LAMBDA_SERVICE_NAME;
}
Expand Down
210 changes: 177 additions & 33 deletions ...va/software/amazon/opentelemetry/javaagent/providers/AwsMetricAttributeGeneratorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -488,31 +488,6 @@ public void testRemoteAttributesCombinations() {
// Validate behaviour of various combinations of RPC attributes, then remove them.
validateAndRemoveRemoteAttributes(RPC_SERVICE, "RPC service", RPC_METHOD, "RPC method");

// Validate behaviour of Lambda non-Invoke operations (treated as normalized service)
mockAttribute(RPC_SYSTEM, "aws-api");
mockAttribute(RPC_SERVICE, "Lambda");
mockAttribute(RPC_METHOD, "GetFunction");
validateExpectedRemoteAttributes("AWS::Lambda", "GetFunction");
mockAttribute(RPC_SYSTEM, null);
mockAttribute(RPC_SERVICE, null);
mockAttribute(RPC_METHOD, null);

// Validate behaviour of Lambda Invoke operations (treated as service with function name)
mockAttribute(RPC_SYSTEM, "aws-api");
mockAttribute(RPC_SERVICE, "Lambda");
mockAttribute(RPC_METHOD, "Invoke");
mockAttribute(AWS_LAMBDA_NAME, "testFunction");
validateExpectedRemoteAttributes("testFunction", "Invoke");
// Also validate AWS_REMOTE_ENVIRONMENT is set for Lambda Invoke
when(spanDataMock.getKind()).thenReturn(SpanKind.CLIENT);
Attributes actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isEqualTo("lambda:default");
mockAttribute(RPC_SYSTEM, null);
mockAttribute(RPC_SERVICE, null);
mockAttribute(RPC_METHOD, null);
mockAttribute(AWS_LAMBDA_NAME, null);

// Validate behaviour of various combinations of DB attributes, then remove them.
validateAndRemoveRemoteAttributes(DB_SYSTEM, "DB system", DB_OPERATION, "DB operation");

Expand Down Expand Up @@ -862,21 +837,27 @@ public void testSdkClientSpanWithRemoteResourceAttributes() {
// Validate behaviour of AWS_GUARDRAIL_ID attribute, then remove it.
mockAttribute(AWS_GUARDRAIL_ID, "test_guardrail_id");
validateRemoteResourceAttributes("AWS::Bedrock::Guardrail", "test_guardrail_id");
// Also test with ARN to verify cloudformationPrimaryIdentifier uses ARN
mockAttribute(
AWS_GUARDRAIL_ARN, "arn:aws:bedrock:us-east-1:123456789012:guardrail/test_guardrail_id");
validateRemoteResourceAttributes(
"AWS::Bedrock::Guardrail",
"test_guardrail_id",
"arn:aws:bedrock:us-east-1:123456789012:guardrail/test_guardrail_id");
mockAttribute(AWS_GUARDRAIL_ID, null);
mockAttribute(AWS_GUARDRAIL_ARN, null);

// Validate behaviour of AWS_GUARDRAIL_ID attribute with special chars(^), then remove it.
mockAttribute(AWS_GUARDRAIL_ID, "test_guardrail_^id");
validateRemoteResourceAttributes("AWS::Bedrock::Guardrail", "test_guardrail_^^id");
mockAttribute(AWS_GUARDRAIL_ID, null);

// Validate behaviour of AWS_GUARDRAIL_ARN attribute with AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER
mockAttribute(AWS_GUARDRAIL_ID, "test_guardrail_id");
// Also test with ARN containing special chars to verify delimiter escaping in
// cloudformationPrimaryIdentifier
mockAttribute(
AWS_GUARDRAIL_ARN, "arn:aws:bedrock:us-east-1:123456789012:guardrail/test_guardrail_id");
AWS_GUARDRAIL_ARN, "arn:aws:bedrock:us-east-1:123456789012:guardrail/test_guardrail_^id");
validateRemoteResourceAttributes(
"AWS::Bedrock::Guardrail",
"test_guardrail_id",
"arn:aws:bedrock:us-east-1:123456789012:guardrail/test_guardrail_id");
"test_guardrail_^^id",
"arn:aws:bedrock:us-east-1:123456789012:guardrail/test_guardrail_^^id");
mockAttribute(AWS_GUARDRAIL_ID, null);
mockAttribute(AWS_GUARDRAIL_ARN, null);

Expand Down Expand Up @@ -963,7 +944,7 @@ public void testSdkClientSpanWithRemoteResourceAttributes() {
mockAttribute(RPC_METHOD, null);
mockAttribute(AWS_LAMBDA_NAME, null);

// Validate that Lambda Invoke with ARN in AWS_LAMBDA_NAME extracts function name correctly
// Validate behaviour of AWS_LAMBDA_NAME containing ARN for Invoke operations
mockAttribute(RPC_SERVICE, "Lambda");
mockAttribute(RPC_METHOD, "Invoke");
mockAttribute(AWS_LAMBDA_NAME, "arn:aws:lambda:us-east-1:123456789012:function:testLambdaName");
Expand All @@ -980,6 +961,95 @@ public void testSdkClientSpanWithRemoteResourceAttributes() {
mockAttribute(RPC_SYSTEM, "null");
}

@Test
public void testCloudFormationPrimaryIdentifierFallbackToRemoteResourceIdentifier() {
// Test that when cloudformationPrimaryIdentifier is not explicitly set,
// it falls back to use the same value as remoteResourceIdentifier

mockAttribute(RPC_SYSTEM, "aws-api");
when(spanDataMock.getKind()).thenReturn(SpanKind.CLIENT);

// Test case 1: S3 Bucket (no ARN available, should use bucket name for both)
mockAttribute(AWS_BUCKET_NAME, "my-test-bucket");
Attributes actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::S3::Bucket");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER)).isEqualTo("my-test-bucket");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my-test-bucket");

// Test S3 bucket with special characters
mockAttribute(AWS_BUCKET_NAME, "my-test|bucket^name");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::S3::Bucket");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER))
.isEqualTo("my-test^|bucket^^name");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my-test^|bucket^^name");
mockAttribute(AWS_BUCKET_NAME, null);

// Test case 2: SQS Queue by name (no ARN, should use queue name for both)
mockAttribute(AWS_QUEUE_NAME, "my-test-queue");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::SQS::Queue");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER)).isEqualTo("my-test-queue");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my-test-queue");

// Test SQS queue with special characters
mockAttribute(AWS_QUEUE_NAME, "my^queue|name");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::SQS::Queue");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER)).isEqualTo("my^^queue^|name");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my^^queue^|name");
mockAttribute(AWS_QUEUE_NAME, null);

// Test case 3: DynamoDB Table (no ARN, should use table name for both)
mockAttribute(AWS_TABLE_NAME, "my-test-table");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::DynamoDB::Table");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER)).isEqualTo("my-test-table");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my-test-table");

// Test DynamoDB table with special characters
mockAttribute(AWS_TABLE_NAME, "my|test^table");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::DynamoDB::Table");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER)).isEqualTo("my^|test^^table");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my^|test^^table");
mockAttribute(AWS_TABLE_NAME, null);

// Test case 4: Kinesis Stream
mockAttribute(AWS_STREAM_NAME, "my-test-stream");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::Kinesis::Stream");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER)).isEqualTo("my-test-stream");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my-test-stream");

// Test Kinesis stream with special characters
mockAttribute(AWS_STREAM_NAME, "my-stream^with|chars");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_TYPE)).isEqualTo("AWS::Kinesis::Stream");
assertThat(actualAttributes.get(AWS_REMOTE_RESOURCE_IDENTIFIER))
.isEqualTo("my-stream^^with^|chars");
assertThat(actualAttributes.get(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER))
.isEqualTo("my-stream^^with^|chars");
mockAttribute(AWS_STREAM_NAME, null);

mockAttribute(RPC_SYSTEM, null);
}

@Test
public void testDBClientSpanWithRemoteResourceAttributes() {
mockAttribute(DB_SYSTEM, "mysql");
Expand Down Expand Up @@ -1396,6 +1466,25 @@ public void testNormalizeRemoteServiceName_AwsSdk() {
testAwsSdkServiceNormalization("AmazonBedrockRuntime", "AWS::BedrockRuntime");
testAwsSdkServiceNormalization("AWSStepFunctions", "AWS::StepFunctions");

// AWS SDK V1 Lambda tests
testAwsSdkServiceNormalization("Lambda", "AWS::Lambda");
mockAttribute(RPC_METHOD, "Invoke");
mockAttribute(AWS_LAMBDA_NAME, "testFunction");
testAwsSdkServiceNormalization("Lambda", "testFunction");
// Test Lambda Invoke without AWS_LAMBDA_NAME - should fall back to UnknownRemoteService
mockAttribute(AWS_LAMBDA_NAME, null);
testAwsSdkServiceNormalization("Lambda", "UnknownRemoteService");
mockAttribute(RPC_METHOD, null);

testAwsSdkServiceNormalization("AWSLambda", "AWS::Lambda");
mockAttribute(RPC_METHOD, "Invoke");
mockAttribute(AWS_LAMBDA_NAME, "testFunction");
testAwsSdkServiceNormalization("AWSLambda", "testFunction");
// Test Lambda Invoke without AWS_LAMBDA_NAME - should fall back to UnknownRemoteService
mockAttribute(AWS_LAMBDA_NAME, null);
testAwsSdkServiceNormalization("AWSLambda", "UnknownRemoteService");
mockAttribute(RPC_METHOD, null);

// AWS SDK V2
testAwsSdkServiceNormalization("DynamoDb", "AWS::DynamoDB");
testAwsSdkServiceNormalization("Kinesis", "AWS::Kinesis");
Expand All @@ -1417,6 +1506,61 @@ private void testAwsSdkServiceNormalization(String serviceName, String expectedR
assertThat(actualAttributes.get(AWS_REMOTE_SERVICE)).isEqualTo(expectedRemoteService);
}

@Test
public void testSetRemoteEnvironment() {
// Test 1: Setting remote environment when all relevant attributes are present
mockAttribute(RPC_SYSTEM, "aws-api");
mockAttribute(RPC_SERVICE, "Lambda");
mockAttribute(RPC_METHOD, "Invoke");
mockAttribute(AWS_LAMBDA_NAME, "testFunction");
when(spanDataMock.getKind()).thenReturn(SpanKind.CLIENT);

Attributes actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isEqualTo("lambda:default");

// Test 2: NOT setting it when RPC_SYSTEM is missing
mockAttribute(RPC_SYSTEM, null);
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isNull();
mockAttribute(RPC_SYSTEM, "aws-api");

// Test 3: NOT setting it when RPC_METHOD is missing
mockAttribute(RPC_METHOD, null);
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isNull();
mockAttribute(RPC_METHOD, "Invoke");

// Test 4: Still setting it to lambda:default when AWS_LAMBDA_NAME is missing
mockAttribute(AWS_LAMBDA_NAME, null);
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isEqualTo("lambda:default");
mockAttribute(AWS_LAMBDA_NAME, "testFunction");

// Test 5: NOT setting it for non-Lambda services
mockAttribute(RPC_SERVICE, "S3");
mockAttribute(RPC_METHOD, "GetObject");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isNull();

// Test 6: NOT setting it for Lambda non-Invoke operations
mockAttribute(RPC_SERVICE, "Lambda");
mockAttribute(RPC_METHOD, "GetFunction");
actualAttributes =
GENERATOR.generateMetricAttributeMapFromSpan(spanDataMock, resource).get(DEPENDENCY_METRIC);
assertThat(actualAttributes.get(AWS_REMOTE_ENVIRONMENT)).isNull();

// Clean up
mockAttribute(RPC_SYSTEM, null);
mockAttribute(RPC_SERVICE, null);
mockAttribute(RPC_METHOD, null);
mockAttribute(AWS_LAMBDA_NAME, null);
}

@Test
public void testNoMetricWhenConsumerProcessWithConsumerParent() {
when(attributesMock.get(AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND))
Expand Down
Loading