Fix: Lambda Topology Issue

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts

@@ -9,6 +9,7 @@ export const AWS_ATTRIBUTE_KEYS: { [key: string]: string } = {
   AWS_LOCAL_SERVICE: 'aws.local.service',
   AWS_LOCAL_OPERATION: 'aws.local.operation',
   AWS_REMOTE_SERVICE: 'aws.remote.service',
+  AWS_REMOTE_ENVIRONMENT: 'aws.remote.environment',
   AWS_REMOTE_OPERATION: 'aws.remote.operation',
   AWS_REMOTE_RESOURCE_TYPE: 'aws.remote.resource.type',
   AWS_REMOTE_RESOURCE_IDENTIFIER: 'aws.remote.resource.identifier',

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts

@@ -409,6 +409,33 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
           this.extractResourceNameFromArn(activityArn)
         );
         cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(activityArn);
+      } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME)) {
+        // Handling downstream Lambda as a service vs. an AWS resource:
+        // - If the method call is "Invoke", we treat downstream Lambda as a service.
+        // - Otherwise, we treat it as an AWS resource.
+        //
+        // This addresses a Lambda topology issue in Application Signals.
+        // More context in PR: https://github.com/aws-observability/aws-otel-python-instrumentation/pull/319
+        //
+        // NOTE: The env vars LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE and
+        // LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT were introduced as part of this fix.
+        // They are optional and allow users to override the default values if needed.
+        if (AwsMetricAttributeGenerator.getRemoteOperation(span, SEMATTRS_RPC_METHOD) === 'Invoke') {
+          attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_SERVICE] =
+            process.env.LAMBDA_APPLICATION_SIGNALS_REMOTE_SERVICE ||
+            span.attributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME];
+          attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_ENVIRONMENT] = `lambda:${
+            process.env.LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT || 'default'
+          }`;
+        } else {
+          remoteResourceType = NORMALIZED_LAMBDA_SERVICE_NAME + '::Function';
+          remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
+            span.attributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME]
+          );
+          cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
+            span.attributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN]
+          );
+        }
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID)) {
         remoteResourceType = NORMALIZED_LAMBDA_SERVICE_NAME + '::EventSourceMapping';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(

aws-distro-opentelemetry-node-autoinstrumentation/src/patches/instrumentation-patch.ts

@@ -10,9 +10,15 @@ import {
   ROOT_CONTEXT,
   TextMapGetter,
   trace,
+  Span,
+  Tracer,
 } from '@opentelemetry/api';
 import { Instrumentation } from '@opentelemetry/instrumentation';
-import { AwsSdkInstrumentationConfig, NormalizedRequest } from '@opentelemetry/instrumentation-aws-sdk';
+import {
+  AwsSdkInstrumentationConfig,
+  NormalizedRequest,
+  NormalizedResponse,
+} from '@opentelemetry/instrumentation-aws-sdk';
 import { AWSXRAY_TRACE_ID_HEADER, AWSXRayPropagator } from '@opentelemetry/propagator-aws-xray';
 import { APIGatewayProxyEventHeaders, Context } from 'aws-lambda';
 import { AWS_ATTRIBUTE_KEYS } from '../aws-attribute-keys';
@@ -215,10 +221,41 @@ function patchLambdaServiceExtension(lambdaServiceExtension: any): void {
         if (resourceMappingId) {
           requestMetadata.spanAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID] = resourceMappingId;
         }
+
+        const requestFunctionNameFormat = request.commandInput?.FunctionName;
+        let functionName = requestFunctionNameFormat;
+
+        if (requestFunctionNameFormat) {
+          if (requestFunctionNameFormat.startsWith('arn:aws:lambda')) {
+            const split = requestFunctionNameFormat.split(':');
+            functionName = split[split.length - 1];
+          }
+          requestMetadata.spanAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME] = functionName;
+        }
       }
       return requestMetadata;
     };
 
     lambdaServiceExtension.requestPreSpanHook = patchedRequestPreSpanHook;
+
+    if (typeof lambdaServiceExtension.responseHook === 'function') {
+      const originalResponseHook = lambdaServiceExtension.responseHook;
+
+      lambdaServiceExtension.responseHook = (
+        response: NormalizedResponse,
+        span: Span,
+        tracer: Tracer,
+        config: AwsSdkInstrumentationConfig
+      ): void => {
+        originalResponseHook.call(lambdaServiceExtension, response, span, tracer, config);
+
+        if (response.data && response.data.Configuration) {
+          const functionArn = response.data.Configuration.FunctionArn;
+          if (functionArn) {
+            span.setAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN, functionArn);
+          }
+        }
+      };
+    }
   }
 }

aws-distro-opentelemetry-node-autoinstrumentation/test/aws-metric-attribute-generator.test.ts

@@ -774,6 +774,16 @@ describe('AwsMetricAttributeGeneratorTest', () => {
     validateRemoteResourceAttributes('AWS::SecretsManager::Secret', 'testSecret');
     mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN, undefined);
 
+    // Validate behaviour of AWS_LAMBDA_FUNCTION_NAME and AWS_LAMBDA_FUNCTION_ARN
+    mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME, 'aws_lambda_function_name');
+    mockAttribute(
+      AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN,
+      'arn:aws:lambda:us-east-1:123456789012:function:aws_lambda_function_name'
+    );
+    validateRemoteResourceAttributes('AWS::Lambda::Function', 'aws_lambda_function_name');
+    mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME, undefined);
+    mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN, undefined);
+
     // Validate behaviour of AWS_LAMBDA_RESOURCE_MAPPING_ID attribute then remove it.
     mockAttribute(AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID, 'aws_lambda_resource_mapping_id');
     validateRemoteResourceAttributes('AWS::Lambda::EventSourceMapping', 'aws_lambda_resource_mapping_id');

aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts

@@ -38,6 +38,8 @@ const _SECRETS_ARN: string = 'arn:aws:secretsmanager:us-east-1:123456789123:secr
 const _UUID: string = 'random-uuid';
 const _TOPIC_ARN: string = 'arn:aws:sns:us-east-1:123456789012:mystack-mytopic-NZJ5JSMVGFIE';
 const _QUEUE_URL: string = 'https://sqs.us-east-1.amazonaws.com/123412341234/queueName';
+const _FUNCTION_NAME: string = 'testFunction';
+const _FUNCTION_ARN: string = `arn:aws:lambda:us-east-1:123456789012:function:${_FUNCTION_NAME}`;
 const _BEDROCK_AGENT_ID: string = 'agentId';
 const _BEDROCK_DATASOURCE_ID: string = 'DataSourceId';
 const _BEDROCK_GUARDRAIL_ID: string = 'GuardrailId';
@@ -165,6 +167,8 @@ describe('InstrumentationPatchTest', () => {
 
     const lambdaAttributes: Attributes = doExtractLambdaAttributes(services);
     expect(lambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID]).toBeUndefined();
+    expect(lambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME]).toBeUndefined();
+    expect(lambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN]).toBeUndefined();
   });
 
   it('SFN without patching', () => {
@@ -228,6 +232,9 @@ describe('InstrumentationPatchTest', () => {
     const services: Map<string, any> = extractServicesFromAwsSdkInstrumentation(patchedAwsSdkInstrumentation);
     const requestLambdaAttributes: Attributes = doExtractLambdaAttributes(services);
     expect(requestLambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_RESOURCE_MAPPING_ID]).toEqual(_UUID);
+    expect(requestLambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME]).toEqual(_FUNCTION_NAME);
+    const responseLambdaAttributes: Attributes = doResponseHookLambda(services);
+    expect(responseLambdaAttributes[AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN]).toEqual(_FUNCTION_ARN);
   });
 
   it('SFN with patching', () => {
@@ -429,6 +436,7 @@ describe('InstrumentationPatchTest', () => {
       commandName: 'mockCommandName',
       commandInput: {
         UUID: _UUID,
+        FunctionName: _FUNCTION_NAME,
       },
     };
     return doExtractAttributes(services, serviceName, params);
@@ -507,6 +515,23 @@ describe('InstrumentationPatchTest', () => {
     return doResponseHook(services, 'SecretsManager', results as NormalizedResponse);
   }
 
+  function doResponseHookLambda(services: Map<string, ServiceExtension>): Attributes {
+    const results: Partial<NormalizedResponse> = {
+      data: {
+        Configuration: {
+          FunctionArn: _FUNCTION_ARN,
+        },
+      },
+      request: {
+        commandInput: {},
+        commandName: 'dummy_operation',
+        serviceName: 'Lambda',
+      },
+    };
+
+    return doResponseHook(services, 'Lambda', results as NormalizedResponse);
+  }
+
   function doResponseHookBedrock(
     services: Map<string, ServiceExtension>,
     serviceName: string,