diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts index 46150b7f..54bd517a 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts @@ -9,6 +9,7 @@ export const AWS_ATTRIBUTE_KEYS: { [key: string]: string } = { AWS_LOCAL_SERVICE: 'aws.local.service', AWS_LOCAL_OPERATION: 'aws.local.operation', AWS_REMOTE_SERVICE: 'aws.remote.service', + AWS_REMOTE_ENVIRONMENT: 'aws.remote.environment', AWS_REMOTE_OPERATION: 'aws.remote.operation', AWS_REMOTE_RESOURCE_TYPE: 'aws.remote.resource.type', AWS_REMOTE_RESOURCE_IDENTIFIER: 'aws.remote.resource.identifier', diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts index ae318a37..0404d5ce 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts @@ -409,6 +409,33 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator { this.extractResourceNameFromArn(activityArn) ); cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(activityArn); + } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME)) { + // Handling downstream Lambda as a service vs. an AWS resource: + // - If the method call is "Invoke", we treat downstream Lambda as a service. + // - Otherwise, we treat it as an AWS resource. + // + // This addresses a Lambda topology issue in Application Signals. + // More context in PR: https://github.com/aws-observability/aws-otel-python-instrumentation/pull/319 + // + // NOTE: The env vars LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE and + // LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT were introduced as part of this fix. + // They are optional and allow users to override the default values if needed. + if (AwsMetricAttributeGenerator.getRemoteOperation(span, SEMATTRS_RPC_METHOD) === 'Invoke') { + attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_SERVICE] = + process.env.LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE || + span.attributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME]; + attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_ENVIRONMENT] = `lambda:${ + process.env.LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT || 'default' + }`; + } else { + remoteResourceType = NORMALIZED_LAMBDA_SERVICE_NAME + '::Function'; + remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters( + span.attributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME] + ); + cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters( + span.attributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN] + ); + } } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID)) { remoteResourceType = NORMALIZED_LAMBDA_SERVICE_NAME + '::EventSourceMapping'; remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters( diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/instrumentation-patch.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/instrumentation-patch.ts index 74879b97..7bf7321e 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/instrumentation-patch.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/instrumentation-patch.ts @@ -10,9 +10,15 @@ import { ROOT_CONTEXT, TextMapGetter, trace, + Span, + Tracer, } from '@opentelemetry/api'; import { Instrumentation } from '@opentelemetry/instrumentation'; -import { AwsSdkInstrumentationConfig, NormalizedRequest } from '@opentelemetry/instrumentation-aws-sdk'; +import { + AwsSdkInstrumentationConfig, + NormalizedRequest, + NormalizedResponse, +} from '@opentelemetry/instrumentation-aws-sdk'; import { AWSXRAY_TRACE_ID_HEADER, AWSXRayPropagator } from '@opentelemetry/propagator-aws-xray'; import { APIGatewayProxyEventHeaders, Context } from 'aws-lambda'; import { AWS_ATTRIBUTE_KEYS } from '../aws-attribute-keys'; @@ -215,10 +221,41 @@ function patchLambdaServiceExtension(lambdaServiceExtension: any): void { if (resourceMappingId) { requestMetadata.spanAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID] = resourceMappingId; } + + const requestFunctionNameFormat = request.commandInput?.FunctionName; + let functionName = requestFunctionNameFormat; + + if (requestFunctionNameFormat) { + if (requestFunctionNameFormat.startsWith('arn:aws:lambda')) { + const split = requestFunctionNameFormat.split(':'); + functionName = split[split.length - 1]; + } + requestMetadata.spanAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME] = functionName; + } } return requestMetadata; }; lambdaServiceExtension.requestPreSpanHook = patchedRequestPreSpanHook; + + if (typeof lambdaServiceExtension.responseHook === 'function') { + const originalResponseHook = lambdaServiceExtension.responseHook; + + lambdaServiceExtension.responseHook = ( + response: NormalizedResponse, + span: Span, + tracer: Tracer, + config: AwsSdkInstrumentationConfig + ): void => { + originalResponseHook.call(lambdaServiceExtension, response, span, tracer, config); + + if (response.data && response.data.Configuration) { + const functionArn = response.data.Configuration.FunctionArn; + if (functionArn) { + span.setAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN, functionArn); + } + } + }; + } } } diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/aws-metric-attribute-generator.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/aws-metric-attribute-generator.test.ts index 0e0a7f28..c51c03ec 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/aws-metric-attribute-generator.test.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/aws-metric-attribute-generator.test.ts @@ -774,6 +774,16 @@ describe('AwsMetricAttributeGeneratorTest', () => { validateRemoteResourceAttributes('AWS::SecretsManager::Secret', 'testSecret'); mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN, undefined); + // Validate behaviour of AWS_LAMBDA_FUNCTION_NAME and AWS_LAMBDA_FUNCTION_ARN + mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME, 'aws_lambda_function_name'); + mockAttribute( + AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN, + 'arn:aws:lambda:us-east-1:123456789012:function:aws_lambda_function_name' + ); + validateRemoteResourceAttributes('AWS::Lambda::Function', 'aws_lambda_function_name'); + mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME, undefined); + mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN, undefined); + // Validate behaviour of AWS_LAMBDA_RESOURCE_MAPPING_ID attribute then remove it. mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID, 'aws_lambda_resource_mapping_id'); validateRemoteResourceAttributes('AWS::Lambda::EventSourceMapping', 'aws_lambda_resource_mapping_id'); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts index 6a731f25..1b725d37 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts @@ -38,6 +38,8 @@ const _SECRETS_ARN: string = 'arn:aws:secretsmanager:us-east-1:123456789123:secr const _UUID: string = 'random-uuid'; const _TOPIC_ARN: string = 'arn:aws:sns:us-east-1:123456789012:mystack-mytopic-NZJ5JSMVGFIE'; const _QUEUE_URL: string = 'https://sqs.us-east-1.amazonaws.com/123412341234/queueName'; +const _FUNCTION_NAME: string = 'testFunction'; +const _FUNCTION_ARN: string = `arn:aws:lambda:us-east-1:123456789012:function:${_FUNCTION_NAME}`; const _BEDROCK_AGENT_ID: string = 'agentId'; const _BEDROCK_DATASOURCE_ID: string = 'DataSourceId'; const _BEDROCK_GUARDRAIL_ID: string = 'GuardrailId'; @@ -165,6 +167,8 @@ describe('InstrumentationPatchTest', () => { const lambdaAttributes: Attributes = doExtractLambdaAttributes(services); expect(lambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID]).toBeUndefined(); + expect(lambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME]).toBeUndefined(); + expect(lambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN]).toBeUndefined(); }); it('SFN without patching', () => { @@ -228,6 +232,9 @@ describe('InstrumentationPatchTest', () => { const services: Map = extractServicesFromAwsSdkInstrumentation(patchedAwsSdkInstrumentation); const requestLambdaAttributes: Attributes = doExtractLambdaAttributes(services); expect(requestLambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID]).toEqual(_UUID); + expect(requestLambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME]).toEqual(_FUNCTION_NAME); + const responseLambdaAttributes: Attributes = doResponseHookLambda(services); + expect(responseLambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN]).toEqual(_FUNCTION_ARN); }); it('SFN with patching', () => { @@ -429,6 +436,7 @@ describe('InstrumentationPatchTest', () => { commandName: 'mockCommandName', commandInput: { UUID: _UUID, + FunctionName: _FUNCTION_NAME, }, }; return doExtractAttributes(services, serviceName, params); @@ -507,6 +515,23 @@ describe('InstrumentationPatchTest', () => { return doResponseHook(services, 'SecretsManager', results as NormalizedResponse); } + function doResponseHookLambda(services: Map): Attributes { + const results: Partial = { + data: { + Configuration: { + FunctionArn: _FUNCTION_ARN, + }, + }, + request: { + commandInput: {}, + commandName: 'dummy_operation', + serviceName: 'Lambda', + }, + }; + + return doResponseHook(services, 'Lambda', results as NormalizedResponse); + } + function doResponseHookBedrock( services: Map, serviceName: string,