Merge branch 'main' into upload-layer-zip

Author: yiyuan-he

.github/workflows/daily-scan.yml

@@ -56,9 +56,9 @@ jobs:
         if: always()
         run: |
           gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 259A55407DD6C00299E6607EFFDE55BE73A2D1ED
-          VERSION=$(curl -s https://jeremylong.github.io/DependencyCheck/current.txt)
-          curl -Ls "https://github.com/jeremylong/DependencyCheck/releases/download/v$VERSION/dependency-check-$VERSION-release.zip" --output dependency-check.zip
-          curl -Ls "https://github.com/jeremylong/DependencyCheck/releases/download/v$VERSION/dependency-check-$VERSION-release.zip.asc" --output dependency-check.zip.asc
+          VERSION=$(curl -s https://jeremylong.github.io/DependencyCheck/current.txt | head -n1 | cut -d' ' -f1)
+          curl -Ls "https://github.com/dependency-check/DependencyCheck/releases/download/v$VERSION/dependency-check-$VERSION-release.zip" --output dependency-check.zip
+          curl -Ls "https://github.com/dependency-check/DependencyCheck/releases/download/v$VERSION/dependency-check-$VERSION-release.zip.asc" --output dependency-check.zip.asc
           gpg --verify dependency-check.zip.asc
           unzip dependency-check.zip
           ./dependency-check/bin/dependency-check.sh  --nvdApiKey ${{ env.NVD_API_KEY_NVD_API_KEY }} -s "aws-distro-opentelemetry-node-autoinstrumentation/"

.github/workflows/release-build.yml

@@ -113,10 +113,11 @@ jobs:
              layer_artifact/layer.zip \
              layer_artifact/layer.zip.sha256
 
-      # Publish to npm
-      - name: Publish to npm
+      # Publish '@aws/aws-distro-opentelemetry-node-autoinstrumentation' to npm
+      - name: Publish autoinstrumentation to npm
+        working-directory: aws-distro-opentelemetry-node-autoinstrumentation
         env:
           NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
           NPM_CONFIG_PROVENANCE: true
-        run: npx lerna publish from-package --no-push --no-private --no-git-tag-version --no-verify-access --yes
+        run: npm publish
 

.github/workflows/release-lambda.yml

@@ -174,24 +174,22 @@ jobs:
       - name: generate tf layer
         working-directory: ${{ env.LAYER_NAME }}
         run: |
-          echo "locals {" >> ../layer.tf
-          echo "  sdk_layer_arns = {" >> ../layer.tf
+          echo "locals {" >> ../layer_arns.tf
+          echo "  sdk_layer_arns = {" >> ../layer_arns.tf
           for file in *
           do
           read arn < $file
-          echo "    \""$file"\" = \""$arn"\"" >> ../layer.tf
+          echo "    \""$file"\" = \""$arn"\"" >> ../layer_arns.tf
           done
           cd ..
-          echo "  }" >> layer.tf
-          echo "}" >> layer.tf
-          terraform fmt layer.tf
-          cat layer.tf
-      - name: upload layer tf file
-        uses: actions/upload-artifact@v4
+          echo "  }" >> layer_arns.tf
+          echo "}" >> layer_arns.tf
+          terraform fmt layer_arns.tf
+          cat layer_arns.tf
+      - name: download layer.zip
+        uses: actions/download-artifact@v4
         with:
-          name: layer.tf
-          path: layer.tf
-      
+          name: layer.zip
       - name: Get commit hash
         id: commit
         run: |
@@ -216,7 +214,7 @@ jobs:
             --notes-file release_notes.md \
             --draft \
             "lambda-v${{ github.event.inputs.version }}-${{ steps.commit.outputs.sha_short }}" \
-            lambda-layer/terraform/lambda/layer.tf
+            layer_arns.tf layer.zip
           echo Removing release_notes.md ...
           rm -f release_notes.md
       - name: Upload layer.zip and SHA-256 checksum to SDK Release Notes (tagged with latest)

.github/workflows/release-udp-exporter.yml

@@ -0,0 +1,83 @@
+name: Release ADOT OTLP UDP Exporter
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: The version to tag the release with, e.g., 1.2.0
+        required: true
+
+jobs:
+  build:
+    environment: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Contrib Repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@v4
+
+      - name: Set up Node and run Unit Tests
+        uses: ./.github/actions/set_up
+        with:
+          node_version: "20"
+          package_name: "@aws/aws-otel-otlp-udp-exporter"
+          os: ubuntu-latest
+          run_unit_tests: true
+
+      # Project dependencies and compilation are already done in the previous step
+      - name: Install Dependencies, Compile, and Build Tarball
+        id: staging_tarball_build
+        shell: bash
+        run: |
+          cd exporters/aws-otel-otlp-udp-exporter
+          npm pack
+
+      - name: Download and run X-Ray Daemon
+        run: |
+          mkdir xray-daemon
+          cd xray-daemon
+          wget https://s3.us-west-2.amazonaws.com/aws-xray-assets.us-west-2/xray-daemon/aws-xray-daemon-linux-3.x.zip
+          unzip aws-xray-daemon-linux-3.x.zip
+          ./xray -o -n us-west-2 -f ./daemon-logs.log --log-level debug &
+
+      - name: Setup Sample App
+        run: |
+          cd sample-applications/integ-test-http-server
+          npm install
+          npm install ../../exporters/aws-otel-otlp-udp-exporter/aws-aws-otel-otlp-udp-exporter-*.tgz
+
+      - name: Run Sample App in Background
+        run: |
+          cd sample-applications/integ-test-http-server
+          node udp-exporter-test-server.js &
+          # Wait for test server to initialize
+          sleep 5
+
+      - name: Call Sample App Endpoint
+        id: call-endpoint
+        run: |
+          echo "traceId=$(curl localhost:8080/test)" >> $GITHUB_OUTPUT
+
+      - name: Verify X-Ray daemon received traces
+        run: |
+          sleep 10
+          echo "X-Ray daemon logs:"
+          cat xray-daemon/daemon-logs.log
+          # Check if the daemon received and processed some data
+          if grep -q "sending.*batch" xray-daemon/daemon-logs.log; then
+            echo "✅ X-Ray daemon processed trace data (AWS upload errors are expected)"
+            exit 0
+          elif grep -q "processor:.*segment" xray-daemon/daemon-logs.log; then
+            echo "✅ X-Ray daemon processed segment data (AWS upload errors are expected)"
+            exit 0
+          else
+            echo "❌ No evidence of traces being received by X-Ray daemon"
+            exit 1
+          fi
+
+      # TODO: Uncomment when we make the first release
+      # # Publish OTLP UDP Exporter to npm
+      # - name: Publish to npm
+      #   working-directory: exporters/aws-otel-otlp-udp-exporter
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
+      #     NPM_CONFIG_PROVENANCE: true
+      #   run: npx publish

aws-distro-opentelemetry-node-autoinstrumentation/package.json

@@ -40,7 +40,8 @@
       "src/**/*.ts"
     ],
     "exclude": [
-      "src/third-party/**/*.ts"
+      "src/third-party/**/*.ts",
+      "src/otlp-aws-span-exporter.ts"
     ]
   },
   "bugs": {
@@ -79,13 +80,17 @@
     "@aws-sdk/client-sfn": "^3.632.0",
     "@aws-sdk/client-sns": "^3.632.0",
     "@opentelemetry/contrib-test-utils": "0.41.0",
+    "@smithy/protocol-http": "^5.0.1",
+    "@smithy/signature-v4": "^5.0.1",
     "@types/mocha": "7.0.2",
     "@types/node": "18.6.5",
+    "@types/proxyquire": "^1.3.31",
     "@types/sinon": "10.0.18",
     "expect": "29.2.0",
     "mocha": "7.2.0",
     "nock": "13.2.1",
     "nyc": "15.1.0",
+    "proxyquire": "^2.1.3",
     "rimraf": "5.0.5",
     "sinon": "15.2.0",
     "ts-mocha": "10.0.0",

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts

@@ -56,11 +56,14 @@ import { AttributePropagatingSpanProcessorBuilder } from './attribute-propagatin
 import { AwsBatchUnsampledSpanProcessor } from './aws-batch-unsampled-span-processor';
 import { AwsMetricAttributesSpanExporterBuilder } from './aws-metric-attributes-span-exporter-builder';
 import { AwsSpanMetricsProcessorBuilder } from './aws-span-metrics-processor-builder';
+import { OTLPAwsSpanExporter } from './otlp-aws-span-exporter';
 import { OTLPUdpSpanExporter } from './otlp-udp-exporter';
 import { AwsXRayRemoteSampler } from './sampler/aws-xray-remote-sampler';
 // This file is generated via `npm run compile`
 import { LIB_VERSION } from './version';
 
+const XRAY_OTLP_ENDPOINT_PATTERN = '^https://xray\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/traces$';
+
 const APPLICATION_SIGNALS_ENABLED_CONFIG: string = 'OTEL_AWS_APPLICATION_SIGNALS_ENABLED';
 const APPLICATION_SIGNALS_EXPORTER_ENDPOINT_CONFIG: string = 'OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT';
 const METRIC_EXPORT_INTERVAL_CONFIG: string = 'OTEL_METRIC_EXPORT_INTERVAL';
@@ -235,6 +238,11 @@ export class AwsOpentelemetryConfigurator {
     }
 
     spanProcessors.push(AttributePropagatingSpanProcessorBuilder.create().build());
+
+    if (isXrayOtlpEndpoint(process.env['OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'])) {
+      return;
+    }
+
     const applicationSignalsMetricExporter: PushMetricExporter =
       ApplicationSignalsExporterProvider.Instance.createExporter();
     const periodicExportingMetricReader: PeriodicExportingMetricReader = new PeriodicExportingMetricReader({
@@ -422,6 +430,7 @@ export class AwsSpanProcessorProvider {
   private resource: Resource;
 
   static configureOtlp(): SpanExporter {
+    const otlp_exporter_traces_endpoint = process.env['OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'];
     // eslint-disable-next-line @typescript-eslint/typedef
     let protocol = this.getOtlpProtocol();
 
@@ -438,12 +447,20 @@ export class AwsSpanProcessorProvider {
       case 'http/json':
         return new OTLPHttpTraceExporter();
       case 'http/protobuf':
+        if (otlp_exporter_traces_endpoint && isXrayOtlpEndpoint(otlp_exporter_traces_endpoint)) {
+          diag.debug('Detected XRay OTLP Traces endpoint. Switching exporter to OtlpAwsSpanExporter');
+          return new OTLPAwsSpanExporter(otlp_exporter_traces_endpoint);
+        }
         return new OTLPProtoTraceExporter();
       case 'udp':
         diag.debug('Detected AWS Lambda environment and enabling UDPSpanExporter');
         return new OTLPUdpSpanExporter(getXrayDaemonEndpoint(), FORMAT_OTEL_SAMPLED_TRACES_BINARY_PREFIX);
       default:
         diag.warn(`Unsupported OTLP traces protocol: ${protocol}. Using http/protobuf.`);
+        if (otlp_exporter_traces_endpoint && isXrayOtlpEndpoint(otlp_exporter_traces_endpoint)) {
+          diag.debug('Detected XRay OTLP Traces endpoint. Switching exporter to OtlpAwsSpanExporter');
+          return new OTLPAwsSpanExporter(otlp_exporter_traces_endpoint);
+        }
         return new OTLPProtoTraceExporter();
     }
   }
@@ -652,4 +669,8 @@ function getXrayDaemonEndpoint() {
   return process.env[AWS_XRAY_DAEMON_ADDRESS_CONFIG];
 }
 
+function isXrayOtlpEndpoint(otlpEndpoint: string | undefined) {
+  return otlpEndpoint && new RegExp(XRAY_OTLP_ENDPOINT_PATTERN).test(otlpEndpoint.toLowerCase());
+}
+
 // END The OpenTelemetry Authors code

aws-distro-opentelemetry-node-autoinstrumentation/src/otlp-aws-span-exporter.ts

@@ -0,0 +1,142 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import { OTLPTraceExporter as OTLPProtoTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
+import { diag } from '@opentelemetry/api';
+import { OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base';
+import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer';
+import { ReadableSpan } from '@opentelemetry/sdk-trace-base';
+import { ExportResult } from '@opentelemetry/core';
+import { getNodeVersion } from './utils';
+
+/**
+ * This exporter extends the functionality of the OTLPProtoTraceExporter to allow spans to be exported
+ * to the XRay OTLP endpoint https://xray.[AWSRegion].amazonaws.com/v1/traces. Utilizes the aws-sdk
+ * library to sign and directly inject SigV4 Authentication to the exported request's headers. <a
+ * href="https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-OTLPEndpoint.html">...</a>
+ *
+ * This only works with version >=16 Node.js environments.
+ */
+export class OTLPAwsSpanExporter extends OTLPProtoTraceExporter {
+  private static readonly SERVICE_NAME: string = 'xray';
+  private endpoint: string;
+  private region: string;
+
+  // Holds the dependencies needed to sign the SigV4 headers
+  private defaultProvider: any;
+  private sha256: any;
+  private signatureV4: any;
+  private httpRequest: any;
+
+  // If the required dependencies are installed then we enable SigV4 signing. Otherwise skip it
+  private hasRequiredDependencies: boolean = false;
+
+  constructor(endpoint: string, config?: OTLPExporterNodeConfigBase) {
+    super(OTLPAwsSpanExporter.changeUrlConfig(endpoint, config));
+    this.initDependencies();
+    this.region = endpoint.split('.')[1];
+    this.endpoint = endpoint;
+  }
+
+  /**
+   * Overrides the upstream implementation of export. All behaviors are the same except if the
+   * endpoint is an XRay OTLP endpoint, we will sign the request with SigV4 in headers before
+   * sending it to the endpoint. Otherwise, we will skip signing.
+   */
+  public override async export(items: ReadableSpan[], resultCallback: (result: ExportResult) => void): Promise<void> {
+    // Only do SigV4 Signing if the required dependencies are installed. Otherwise default to the regular http/protobuf exporter.
+    if (this.hasRequiredDependencies) {
+      const url = new URL(this.endpoint);
+      const serializedSpans: Uint8Array | undefined = ProtobufTraceSerializer.serializeRequest(items);
+
+      if (serializedSpans === undefined) {
+        return;
+      }
+
+      /*
+        This is bad practice but there is no other way to access and inject SigV4 headers
+        into the request headers before the traces get exported.
+      */
+      const oldHeaders = (this as any)._transport?._transport?._parameters?.headers;
+
+      if (oldHeaders) {
+        const request = new this.httpRequest({
+          method: 'POST',
+          protocol: 'https',
+          hostname: url.hostname,
+          path: url.pathname,
+          body: serializedSpans,
+          headers: {
+            ...this.removeSigV4Headers(oldHeaders),
+            host: url.hostname,
+          },
+        });
+
+        try {
+          const signer = new this.signatureV4({
+            credentials: this.defaultProvider(),
+            region: this.region,
+            service: OTLPAwsSpanExporter.SERVICE_NAME,
+            sha256: this.sha256,
+          });
+
+          const signedRequest = await signer.sign(request);
+
+          (this as any)._transport._transport._parameters.headers = signedRequest.headers;
+        } catch (exception) {
+          diag.debug(
+            `Failed to sign/authenticate the given exported Span request to OTLP XRay endpoint with error: ${exception}`
+          );
+        }
+      }
+    }
+
+    await super.export(items, resultCallback);
+  }
+
+  // Removes Sigv4 headers from old headers to avoid accidentally copying them to the new headers
+  private removeSigV4Headers(headers: Record<string, string>) {
+    const newHeaders: Record<string, string> = {};
+    const sigV4Headers = ['x-amz-date', 'authorization', 'x-amz-content-sha256', 'x-amz-security-token'];
+
+    for (const key in headers) {
+      if (!sigV4Headers.includes(key.toLowerCase())) {
+        newHeaders[key] = headers[key];
+      }
+    }
+    return newHeaders;
+  }
+
+  private initDependencies(): any {
+    if (getNodeVersion() < 16) {
+      diag.error('SigV4 signing requires atleast Node major version 16');
+      return;
+    }
+
+    try {
+      const awsSdkModule = require('@aws-sdk/credential-provider-node');
+      const awsCryptoModule = require('@aws-crypto/sha256-js');
+      const signatureModule = require('@smithy/signature-v4');
+      const httpModule = require('@smithy/protocol-http');
+
+      (this.defaultProvider = awsSdkModule.defaultProvider),
+        (this.sha256 = awsCryptoModule.Sha256),
+        (this.signatureV4 = signatureModule.SignatureV4),
+        (this.httpRequest = httpModule.HttpRequest);
+      this.hasRequiredDependencies = true;
+    } catch (error) {
+      diag.error(`Failed to load required AWS dependency for SigV4 Signing: ${error}`);
+    }
+  }
+
+  private static changeUrlConfig(endpoint: string, config?: OTLPExporterNodeConfigBase) {
+    const newConfig =
+      config === undefined
+        ? { url: endpoint }
+        : {
+            ...config,
+            url: endpoint,
+          };
+
+    return newConfig;
+  }
+}