Skip to content

feat: Add auto-instrumentation support for Step Functions, SNS, Secre… #899

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 9, 2024
Merged

feat: Add auto-instrumentation support for Step Functions, SNS, Secre… #899

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cd43a9c
feat: Add auto-instrumentation support for Step Functions, SNS, Secre…
yiyuan-he Oct 1, 2024
7207885
feat: Add remaining unit tests and clean up logic for CFN Primary Ids…
yiyuan-he Oct 2, 2024
a270511
Merge branch 'main' into new-resources
yiyuan-he Oct 7, 2024
e001aea
chore: apply lint to test file
yiyuan-he Oct 7, 2024
9522a68
chore: adding dependency management build changes for PR build
yiyuan-he Oct 7, 2024
0c6b871
fix: local patching environment for contract tests
yiyuan-he Oct 8, 2024
2d84b64
fix: give MavenCentral higher priority
yiyuan-he Oct 8, 2024
3470706
chore: remove unnecessary lambda library import
yiyuan-he Oct 8, 2024
10ecf28
chore: explicitly define aws-java-sdk-core version
yiyuan-he Oct 8, 2024
2f3002f
chore: update aws sdk core library version
yiyuan-he Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions awsagentprovider/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,12 @@ dependencies {
implementation("io.opentelemetry.contrib:opentelemetry-aws-resources")
// Json file reader
implementation("com.fasterxml.jackson.core:jackson-databind:2.16.1")
// Import AWS SDK v1 core for ARN parsing utilities
implementation("com.amazonaws:aws-java-sdk-core")
// Export configuration
compileOnly("io.opentelemetry:opentelemetry-exporter-otlp")
// SDK Lambda for Lambda ARN retrieval
implementation("com.amazonaws:aws-java-sdk-lambda")

testImplementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure")
testImplementation("io.opentelemetry:opentelemetry-sdk-testing")
Expand Down
20 changes: 20 additions & 0 deletions ...der/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsAttributeKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ private AwsAttributeKeys() {}
static final AttributeKey<String> AWS_REMOTE_RESOURCE_IDENTIFIER =
AttributeKey.stringKey("aws.remote.resource.identifier");

static final AttributeKey<String> AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER =
AttributeKey.stringKey("aws.remote.resource.cfn.primary.identifier");

static final AttributeKey<String> AWS_REMOTE_RESOURCE_TYPE =
AttributeKey.stringKey("aws.remote.resource.type");

Expand All @@ -50,6 +53,23 @@ private AwsAttributeKeys() {}
static final AttributeKey<String> AWS_CONSUMER_PARENT_SPAN_KIND =
AttributeKey.stringKey("aws.consumer.parent.span.kind");

static final AttributeKey<String> AWS_STATE_MACHINE_ARN =
AttributeKey.stringKey("aws.stepfunctions.state_machine.arn");

static final AttributeKey<String> AWS_STEP_FUNCTIONS_ACTIVITY_ARN =
AttributeKey.stringKey("aws.stepfunctions.activity.arn");

static final AttributeKey<String> AWS_SNS_TOPIC_ARN = AttributeKey.stringKey("aws.sns.topic.arn");

static final AttributeKey<String> AWS_SECRET_ARN =
AttributeKey.stringKey("aws.secretsmanager.secret.arn");

static final AttributeKey<String> AWS_LAMBDA_NAME =
AttributeKey.stringKey("aws.lambda.function.name");

static final AttributeKey<String> AWS_LAMBDA_RESOURCE_ID =
AttributeKey.stringKey("aws.lambda.resource_mapping.id");

// use the same AWS Resource attribute name defined by OTel java auto-instr for aws_sdk_v_1_1
// TODO: all AWS specific attributes should be defined in semconv package and reused cross all
// otel packages. Related sim -
Expand Down
108 changes: 108 additions & 0 deletions ...n/java/software/amazon/opentelemetry/javaagent/providers/AwsMetricAttributeGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,12 @@
import static io.opentelemetry.semconv.SemanticAttributes.SERVER_SOCKET_PORT;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_AGENT_ID;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_BUCKET_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_DATA_SOURCE_ID;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_GUARDRAIL_ID;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_KNOWLEDGE_BASE_ID;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_LAMBDA_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_LAMBDA_RESOURCE_ID;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_LOCAL_OPERATION;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_LOCAL_SERVICE;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_QUEUE_NAME;
Expand All @@ -55,7 +58,11 @@
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_REMOTE_RESOURCE_IDENTIFIER;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_REMOTE_RESOURCE_TYPE;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_REMOTE_SERVICE;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_SECRET_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_SNS_TOPIC_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_SPAN_KIND;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_STATE_MACHINE_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_STEP_FUNCTIONS_ACTIVITY_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_STREAM_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_TABLE_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsSpanProcessingUtil.GEN_AI_REQUEST_MODEL;
Expand All @@ -69,6 +76,11 @@
import static software.amazon.opentelemetry.javaagent.providers.AwsSpanProcessingUtil.isDBSpan;
import static software.amazon.opentelemetry.javaagent.providers.AwsSpanProcessingUtil.isKeyPresent;

import com.amazonaws.arn.Arn;
import com.amazonaws.services.lambda.AWSLambda;
import com.amazonaws.services.lambda.AWSLambdaClientBuilder;
import com.amazonaws.services.lambda.model.GetFunctionRequest;
import com.amazonaws.services.lambda.model.GetFunctionResult;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.common.AttributesBuilder;
Expand Down Expand Up @@ -113,6 +125,10 @@ final class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
private static final String NORMALIZED_SQS_SERVICE_NAME = "AWS::SQS";
private static final String NORMALIZED_BEDROCK_SERVICE_NAME = "AWS::Bedrock";
private static final String NORMALIZED_BEDROCK_RUNTIME_SERVICE_NAME = "AWS::BedrockRuntime";
private static final String NORMALIZED_STEPFUNCTIONS_SERVICE_NAME = "AWS::StepFunctions";
private static final String NORMALIZED_SNS_SERVICE_NAME = "AWS::SNS";
private static final String NORMALIZED_SECRETSMANAGER_SERVICE_NAME = "AWS::SecretsManager";
private static final String NORMALIZED_LAMBDA_SERVICE_NAME = "AWS::Lambda";

// Special DEPENDENCY attribute value if GRAPHQL_OPERATION_TYPE attribute key is present.
private static final String GRAPHQL = "graphql";
Expand Down Expand Up @@ -384,6 +400,18 @@ private static String normalizeRemoteServiceName(SpanData span, String serviceNa
case "AmazonBedrockRuntime": // AWS SDK v1
case "BedrockRuntime": // AWS SDK v2
return NORMALIZED_BEDROCK_RUNTIME_SERVICE_NAME;
case "AWSStepFunctions": // AWS SDK v1
case "Sfn": // AWS SDK v2
return NORMALIZED_STEPFUNCTIONS_SERVICE_NAME;
case "AmazonSNS":
case "Sns":
return NORMALIZED_SNS_SERVICE_NAME;
case "AWSSecretsManager": // AWS SDK v1
case "SecretsManager": // AWS SDK v2
return NORMALIZED_SECRETSMANAGER_SERVICE_NAME;
case "AWSLambda": // AWS SDK v1
case "Lambda": // AWS SDK v2
return NORMALIZED_LAMBDA_SERVICE_NAME;
default:
return "AWS::" + serviceName;
}
Expand All @@ -405,6 +433,7 @@ private static String normalizeRemoteServiceName(SpanData span, String serviceNa
private static void setRemoteResourceTypeAndIdentifier(SpanData span, AttributesBuilder builder) {
Optional<String> remoteResourceType = Optional.empty();
Optional<String> remoteResourceIdentifier = Optional.empty();
Optional<String> cloudformationPrimaryIdentifier = Optional.empty();

if (isAwsSDKSpan(span)) {
if (isKeyPresent(span, AWS_TABLE_NAME)) {
Expand Down Expand Up @@ -447,6 +476,50 @@ private static void setRemoteResourceTypeAndIdentifier(SpanData span, Attributes
remoteResourceType = Optional.of(NORMALIZED_BEDROCK_SERVICE_NAME + "::Model");
remoteResourceIdentifier =
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(GEN_AI_REQUEST_MODEL)));
} else if (isKeyPresent(span, AWS_STATE_MACHINE_ARN)) {
remoteResourceType = Optional.of(NORMALIZED_STEPFUNCTIONS_SERVICE_NAME + "::StateMachine");
remoteResourceIdentifier =
getSfnResourceNameFromArn(
Optional.ofNullable(
escapeDelimiters(span.getAttributes().get(AWS_STATE_MACHINE_ARN))));
cloudformationPrimaryIdentifier =
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_STATE_MACHINE_ARN)));
} else if (isKeyPresent(span, AWS_STEP_FUNCTIONS_ACTIVITY_ARN)) {
remoteResourceType = Optional.of(NORMALIZED_STEPFUNCTIONS_SERVICE_NAME + "::Activity");
remoteResourceIdentifier =
getSfnResourceNameFromArn(
Optional.ofNullable(
escapeDelimiters(span.getAttributes().get(AWS_STEP_FUNCTIONS_ACTIVITY_ARN))));
cloudformationPrimaryIdentifier =
Optional.ofNullable(
escapeDelimiters(span.getAttributes().get(AWS_STEP_FUNCTIONS_ACTIVITY_ARN)));
} else if (isKeyPresent(span, AWS_SNS_TOPIC_ARN)) {
remoteResourceType = Optional.of(NORMALIZED_SNS_SERVICE_NAME + "::Topic");
remoteResourceIdentifier =
getSnsResourceNameFromArn(
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_SNS_TOPIC_ARN))));
cloudformationPrimaryIdentifier =
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_SNS_TOPIC_ARN)));
} else if (isKeyPresent(span, AWS_SECRET_ARN)) {
remoteResourceType = Optional.of(NORMALIZED_SECRETSMANAGER_SERVICE_NAME + "::Secret");
remoteResourceIdentifier =
getSecretsManagerResourceNameFromArn(
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_SECRET_ARN))));
cloudformationPrimaryIdentifier =
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_SECRET_ARN)));
} else if (isKeyPresent(span, AWS_LAMBDA_NAME)) {
remoteResourceType = Optional.of(NORMALIZED_LAMBDA_SERVICE_NAME + "::Function");
remoteResourceIdentifier =
getLambdaNameFromArbitraryName(
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_LAMBDA_NAME))));
cloudformationPrimaryIdentifier =
getLambdaArnFromArbitraryName(
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_LAMBDA_NAME))));
} else if (isKeyPresent(span, AWS_LAMBDA_RESOURCE_ID)) {
remoteResourceType = Optional.of(NORMALIZED_LAMBDA_SERVICE_NAME + "::EventSourceMapping");
remoteResourceIdentifier =
Optional.ofNullable(escapeDelimiters(span.getAttributes().get(AWS_LAMBDA_RESOURCE_ID)));
cloudformationPrimaryIdentifier = remoteResourceIdentifier;
}
} else if (isDBSpan(span)) {
remoteResourceType = Optional.of(DB_CONNECTION_RESOURCE_TYPE);
Expand All @@ -457,6 +530,41 @@ private static void setRemoteResourceTypeAndIdentifier(SpanData span, Attributes
builder.put(AWS_REMOTE_RESOURCE_TYPE, remoteResourceType.get());
builder.put(AWS_REMOTE_RESOURCE_IDENTIFIER, remoteResourceIdentifier.get());
}
if (cloudformationPrimaryIdentifier.isPresent()) {
builder.put(AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, cloudformationPrimaryIdentifier.get());
}
}

private static Optional<String> getLambdaNameFromArbitraryName(Optional<String> arbitraryName) {
AWSLambda lambdaClient = AWSLambdaClientBuilder.defaultClient();
GetFunctionRequest getFunctionRequest =
new GetFunctionRequest().withFunctionName(arbitraryName.get());
GetFunctionResult getFunctionResult = lambdaClient.getFunction(getFunctionRequest);
return Optional.of(getFunctionResult.getConfiguration().getFunctionName());
}

// NOTE: "name" in this case can be either the lambda name or lambda arn
private static Optional<String> getLambdaArnFromArbitraryName(Optional<String> arbitraryName) {
AWSLambda lambdaClient = AWSLambdaClientBuilder.defaultClient();
GetFunctionRequest getFunctionRequest =
new GetFunctionRequest().withFunctionName(arbitraryName.get());
GetFunctionResult getFunctionResult = lambdaClient.getFunction(getFunctionRequest);
return Optional.of(getFunctionResult.getConfiguration().getFunctionArn());
}

private static Optional<String> getSecretsManagerResourceNameFromArn(Optional<String> stringArn) {
Arn resourceArn = Arn.fromString(stringArn.get());
return Optional.of(resourceArn.getResource().toString().split(":")[1]);
}

private static Optional<String> getSfnResourceNameFromArn(Optional<String> stringArn) {
Arn resourceArn = Arn.fromString(stringArn.get());
return Optional.of(resourceArn.getResource().toString().split(":")[1]);
}

private static Optional<String> getSnsResourceNameFromArn(Optional<String> stringArn) {
Arn resourceArn = Arn.fromString(stringArn.get());
return Optional.of(resourceArn.getResource().toString());
}

/**
Expand Down
31 changes: 31 additions & 0 deletions ...va/software/amazon/opentelemetry/javaagent/providers/AwsMetricAttributeGeneratorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_REMOTE_RESOURCE_IDENTIFIER;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_REMOTE_RESOURCE_TYPE;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_REMOTE_SERVICE;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_SECRET_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_SNS_TOPIC_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_SPAN_KIND;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_STATE_MACHINE_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_STEP_FUNCTIONS_ACTIVITY_ARN;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_STREAM_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsAttributeKeys.AWS_TABLE_NAME;
import static software.amazon.opentelemetry.javaagent.providers.AwsSpanProcessingUtil.GEN_AI_REQUEST_MODEL;
Expand Down Expand Up @@ -757,6 +761,32 @@ public void testSdkClientSpanWithRemoteResourceAttributes() {
mockAttribute(GEN_AI_REQUEST_MODEL, "test.service_^id");
validateRemoteResourceAttributes("AWS::Bedrock::Model", "test.service_^^id");
mockAttribute(GEN_AI_REQUEST_MODEL, null);

// Validate behaviour of AWS_STATE_MACHINE_ARN attribute, then remove it.
mockAttribute(
AWS_STATE_MACHINE_ARN,
"arn:aws:states:us-east-1:123456789012:stateMachine:test_state_machine");
validateRemoteResourceAttributes("AWS::StepFunctions::StateMachine", "test_state_machine");
mockAttribute(AWS_STATE_MACHINE_ARN, null);

// Validate behaviour of AWS_STEPFUNCTIONS_ACTIVITY_ARN, then remove it.
mockAttribute(
AWS_STEP_FUNCTIONS_ACTIVITY_ARN,
"arn:aws:states:us-east-1:007003123456789012:activity:testActivity");
validateRemoteResourceAttributes("AWS::StepFunctions::Activity", "testActivity");
mockAttribute(AWS_STEP_FUNCTIONS_ACTIVITY_ARN, null);

// Validate behaviour of AWS_SNS_TOPIC_ARN, then remove it.
mockAttribute(AWS_SNS_TOPIC_ARN, "arn:aws:sns:us-west-2:012345678901:testTopic");
validateRemoteResourceAttributes("AWS::SNS::Topic", "testTopic");
mockAttribute(AWS_SNS_TOPIC_ARN, null);

// Validate behaviour of AWS_SECRET_ARN, then remove it.
mockAttribute(
AWS_SECRET_ARN, "arn:aws:secretsmanager:us-east-1:123456789012:secret:secretName");
validateRemoteResourceAttributes("AWS::SecretsManager::Secret", "secretName");
mockAttribute(AWS_SECRET_ARN, null);

mockAttribute(RPC_SYSTEM, "null");
}

Expand Down Expand Up @@ -1162,6 +1192,7 @@ public void testNormalizeRemoteServiceName_AwsSdk() {
testAwsSdkServiceNormalization("AWSBedrockAgentRuntime", "AWS::Bedrock");
testAwsSdkServiceNormalization("AWSBedrockAgent", "AWS::Bedrock");
testAwsSdkServiceNormalization("AmazonBedrockRuntime", "AWS::BedrockRuntime");
testAwsSdkServiceNormalization("AWSStepFunctions", "AWS::StepFunctions");

// AWS SDK V2
testAwsSdkServiceNormalization("DynamoDb", "AWS::DynamoDB");
Expand Down
Loading