feat: Extract resource arn and remote resource access key for cross-account support (#229)

*Description of changes:*
1. Add auto-instrumentation support for the following AWS resources.
- Populate `aws.stream.arn` in Span by extracting StreamArn from the
request body.
- Populate `aws.table.arn` in Span by extracting TableArn from the
response body.

2. Add auto-instrumentation support for remote resource access key.
- Populate `aws.auth.account.access_key` and `aws.auth.region` in Span
from STS credentials in client config
 
3. Generate cross-account metrics attributes when remote resource
identifier is present
- If remote resource arn is available, extract account id and region
from arn.
- Otherwise, pass account access key id and region from span to metric
if available.

4. Add unit tests, contract tests. Done E2E tests with CW agent. 

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Jonathan Lee <[email protected]>

Author: blairhyy-amazon

aws-distro-opentelemetry-node-autoinstrumentation/package.json

@@ -78,6 +78,7 @@
     "@aws-sdk/client-s3": "3.632.0",
     "@aws-sdk/client-secrets-manager": "3.632.0",
     "@aws-sdk/client-sfn": "3.632.0",
+    "@aws-sdk/client-sts": "3.632.0",
     "@aws-sdk/client-sns": "3.632.0",
     "@opentelemetry/contrib-test-utils": "^0.45.0",
     "@smithy/protocol-http": "^5.0.1",

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts

@@ -5,12 +5,17 @@ import { SEMATTRS_AWS_DYNAMODB_TABLE_NAMES } from '@opentelemetry/semantic-conve
 
 // Utility class holding attribute keys with special meaning to AWS components
 export const AWS_ATTRIBUTE_KEYS = {
+  AWS_AUTH_ACCOUNT_ACCESS_KEY: 'aws.auth.account.access_key',
+  AWS_AUTH_REGION: 'aws.auth.region',
   AWS_SPAN_KIND: 'aws.span.kind',
   AWS_LOCAL_SERVICE: 'aws.local.service',
   AWS_LOCAL_OPERATION: 'aws.local.operation',
   AWS_REMOTE_SERVICE: 'aws.remote.service',
   AWS_REMOTE_ENVIRONMENT: 'aws.remote.environment',
   AWS_REMOTE_OPERATION: 'aws.remote.operation',
+  AWS_REMOTE_RESOURCE_ACCOUNT_ACCESS_KEY: 'aws.remote.resource.account.access_key',
+  AWS_REMOTE_RESOURCE_ACCOUNT_ID: 'aws.remote.resource.account.id',
+  AWS_REMOTE_RESOURCE_REGION: 'aws.remote.resource.region',
   AWS_REMOTE_RESOURCE_TYPE: 'aws.remote.resource.type',
   AWS_REMOTE_RESOURCE_IDENTIFIER: 'aws.remote.resource.identifier',
   AWS_SDK_DESCENDANT: 'aws.sdk.descendant',
@@ -31,7 +36,9 @@ export const AWS_ATTRIBUTE_KEYS = {
   AWS_S3_BUCKET: 'aws.s3.bucket',
   AWS_SQS_QUEUE_URL: 'aws.sqs.queue.url',
   AWS_SQS_QUEUE_NAME: 'aws.sqs.queue.name',
+  AWS_KINESIS_STREAM_ARN: 'aws.kinesis.stream.arn',
   AWS_KINESIS_STREAM_NAME: 'aws.kinesis.stream.name',
+  AWS_DYNAMODB_TABLE_ARN: 'aws.dynamodb.table.arn',
   AWS_DYNAMODB_TABLE_NAMES: SEMATTRS_AWS_DYNAMODB_TABLE_NAMES,
   AWS_BEDROCK_DATA_SOURCE_ID: 'aws.bedrock.data_source.id',
   AWS_BEDROCK_KNOWLEDGE_BASE_ID: 'aws.bedrock.knowledge_base.id',

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts

@@ -32,6 +32,7 @@ import {
   MetricAttributeGenerator,
   SERVICE_METRIC,
 } from './metric-attribute-generator';
+import { RegionalResourceArnParser } from './regional-resource-arn-parser';
 import { SqsUrlParser } from './sqs-url-parser';
 import { LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT } from './aws-opentelemetry-configurator';
 
@@ -112,8 +113,20 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
     AwsMetricAttributeGenerator.setService(resource, span, attributes);
     AwsMetricAttributeGenerator.setEgressOperation(span, attributes);
     AwsMetricAttributeGenerator.setRemoteServiceAndOperation(span, attributes);
-    AwsMetricAttributeGenerator.setRemoteResourceTypeAndIdentifier(span, attributes);
+    const isRemoteResourceIdentifierPresent = AwsMetricAttributeGenerator.setRemoteResourceTypeAndIdentifier(
+      span,
+      attributes
+    );
     AwsMetricAttributeGenerator.setRemoteEnvironment(span, attributes);
+    if (isRemoteResourceIdentifierPresent) {
+      const isAccountIdAndRegionPresent = AwsMetricAttributeGenerator.setRemoteResourceAccountIdAndRegion(
+        span,
+        attributes
+      );
+      if (!isAccountIdAndRegionPresent) {
+        AwsMetricAttributeGenerator.setRemoteResourceAccessKeyAndRegion(span, attributes);
+      }
+    }
     AwsMetricAttributeGenerator.setSpanKindForDependency(span, attributes);
     AwsMetricAttributeGenerator.setRemoteDbUser(span, attributes);
 
@@ -369,7 +382,7 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
    * href="https://docs.aws.amazon.com/cloudcontrolapi/latest/userguide/supported-resources.html">AWS
    * Cloud Control resource format</a>.
    */
-  private static setRemoteResourceTypeAndIdentifier(span: ReadableSpan, attributes: Attributes): void {
+  private static setRemoteResourceTypeAndIdentifier(span: ReadableSpan, attributes: Attributes): boolean {
     let remoteResourceType: AttributeValue | undefined;
     let remoteResourceIdentifier: AttributeValue | undefined;
     let cloudFormationIdentifier: AttributeValue | undefined;
@@ -383,11 +396,25 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
       ) {
         remoteResourceType = NORMALIZED_DYNAMO_DB_SERVICE_NAME + '::Table';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(awsTableNames[0]);
+      } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_DYNAMODB_TABLE_ARN)) {
+        remoteResourceType = NORMALIZED_DYNAMO_DB_SERVICE_NAME + '::Table';
+        remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
+          RegionalResourceArnParser.extractDynamoDbTableNameFromArn(
+            span.attributes[AWS_ATTRIBUTE_KEYS.AWS_DYNAMODB_TABLE_ARN]
+          )
+        );
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_NAME)) {
         remoteResourceType = NORMALIZED_KINESIS_SERVICE_NAME + '::Stream';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
           span.attributes[AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_NAME]
         );
+      } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_ARN)) {
+        remoteResourceType = NORMALIZED_KINESIS_SERVICE_NAME + '::Stream';
+        remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
+          RegionalResourceArnParser.extractKinesisStreamNameFromArn(
+            span.attributes[AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_ARN]
+          )
+        );
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_S3_BUCKET)) {
         remoteResourceType = NORMALIZED_S3_SERVICE_NAME + '::Bucket';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
@@ -398,31 +425,31 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
 
         remoteResourceType = NORMALIZED_SNS_SERVICE_NAME + '::Topic';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
-          this.extractResourceNameFromArn(snsArn)
+          RegionalResourceArnParser.extractResourceNameFromArn(snsArn)
         );
         cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(snsArn);
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN)) {
         const secretsArn = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN];
 
         remoteResourceType = NORMALIZED_SECRETSMANAGER_SERVICE_NAME + '::Secret';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
-          this.extractResourceNameFromArn(secretsArn)
+          RegionalResourceArnParser.extractResourceNameFromArn(secretsArn)
         );
         cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(secretsArn);
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_STATEMACHINE_ARN)) {
         const stateMachineArn = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_STATEMACHINE_ARN];
 
         remoteResourceType = NORMALIZED_STEPFUNCTIONS_SERVICE_NAME + '::StateMachine';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
-          this.extractResourceNameFromArn(stateMachineArn)
+          RegionalResourceArnParser.extractResourceNameFromArn(stateMachineArn)
         );
         cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(stateMachineArn);
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_ACTIVITY_ARN)) {
         const activityArn = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_ACTIVITY_ARN];
 
         remoteResourceType = NORMALIZED_STEPFUNCTIONS_SERVICE_NAME + '::Activity';
         remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
-          this.extractResourceNameFromArn(activityArn)
+          RegionalResourceArnParser.extractResourceNameFromArn(activityArn)
         );
         cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(activityArn);
       } else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME)) {
@@ -500,7 +527,10 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
       attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_TYPE] = remoteResourceType;
       attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_IDENTIFIER] = remoteResourceIdentifier;
       attributes[AWS_ATTRIBUTE_KEYS.AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER] = cloudFormationIdentifier;
+      return true;
     }
+
+    return false;
   }
 
   /**
@@ -522,6 +552,56 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
     }
   }
 
+  private static setRemoteResourceAccountIdAndRegion(span: ReadableSpan, attributes: Attributes): boolean {
+    const ARN_ATTRIBUTES: string[] = [
+      AWS_ATTRIBUTE_KEYS.AWS_DYNAMODB_TABLE_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_SNS_TOPIC_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_STATEMACHINE_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_ACTIVITY_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN,
+      AWS_ATTRIBUTE_KEYS.AWS_BEDROCK_GUARDRAIL_ARN,
+    ];
+    let remoteResourceAccountId: string | undefined = undefined;
+    let remoteResourceRegion: string | undefined = undefined;
+
+    if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_SQS_QUEUE_URL)) {
+      const sqsQueueUrl = AwsMetricAttributeGenerator.escapeDelimiters(
+        span.attributes[AWS_ATTRIBUTE_KEYS.AWS_SQS_QUEUE_URL]
+      );
+      remoteResourceAccountId = SqsUrlParser.getAccountId(sqsQueueUrl);
+      remoteResourceRegion = SqsUrlParser.getRegion(sqsQueueUrl);
+    } else {
+      for (const attributeKey of ARN_ATTRIBUTES) {
+        if (AwsSpanProcessingUtil.isKeyPresent(span, attributeKey)) {
+          const arn = span.attributes[attributeKey];
+          remoteResourceAccountId = RegionalResourceArnParser.getAccountId(arn);
+          remoteResourceRegion = RegionalResourceArnParser.getRegion(arn);
+          break;
+        }
+      }
+    }
+
+    if (remoteResourceAccountId !== undefined && remoteResourceRegion !== undefined) {
+      attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_ACCOUNT_ID] = remoteResourceAccountId;
+      attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_REGION] = remoteResourceRegion;
+      return true;
+    }
+
+    return false;
+  }
+
+  private static setRemoteResourceAccessKeyAndRegion(span: ReadableSpan, attributes: Attributes): void {
+    if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_AUTH_ACCOUNT_ACCESS_KEY)) {
+      attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_ACCOUNT_ACCESS_KEY] =
+        span.attributes[AWS_ATTRIBUTE_KEYS.AWS_AUTH_ACCOUNT_ACCESS_KEY];
+    }
+    if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_AUTH_REGION)) {
+      attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_REGION] = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_AUTH_REGION];
+    }
+  }
+
   /**
    * RemoteResourceIdentifier is populated with rule <code>
    *     ^[{db.name}|]?{address}[|{port}]?
@@ -649,16 +729,6 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
     return rpcService === 'Lambda' && span.attributes[SEMATTRS_RPC_METHOD] === LAMBDA_INVOKE_OPERATION;
   }
 
-  // Extracts the name of the resource from an arn
-  private static extractResourceNameFromArn(attribute: AttributeValue | undefined): string | undefined {
-    if (typeof attribute === 'string' && attribute.startsWith('arn:aws:')) {
-      const split = attribute.split(':');
-      return split[split.length - 1];
-    }
-
-    return undefined;
-  }
-
   /** Span kind is needed for differentiating metrics in the EMF exporter */
   private static setSpanKindForService(span: ReadableSpan, attributes: Attributes): void {
     let spanKind: string = SpanKind[span.kind];