[SVLS-8054] add integration testing

.dockerignore

@@ -1 +1,2 @@
-/bottlecap/target/
+bottlecap/target/
+integration-tests/

.gitlab/Dockerfile

@@ -4,6 +4,10 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
   curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime libclang-dev \
   clang llvm-dev
 
+# Install Node.js 20 from NodeSource
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+  apt-get install -y nodejs
+
 # Install AWS CLI
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
 RUN unzip awscliv2.zip && ./aws/install

.gitlab/scripts/get_secrets.sh

@@ -37,6 +37,24 @@ export DD_API_KEY=$(aws ssm get-parameter \
     --query "Parameter.Value" \
     --out text)
 
+printf "Getting DD API KEY Secret ARN...\n"
+
+export DATADOG_API_SECRET_ARN=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name ci.datadog-lambda-extension.dd-api-key-secret-arn \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Getting DD APP KEY...\n"
+
+export DD_APP_KEY=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name ci.datadog-lambda-extension.dd-app-key \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
 printf "Assuming role...\n"
 
 export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \

.gitlab/templates/pipeline.yaml.tpl

@@ -2,6 +2,7 @@ stages:
   - test
   - compile
   - build
+  - integration-tests
   - self-monitoring
   - sign
   - publish
@@ -322,3 +323,152 @@ signed layer bundle:
     - rm -rf datadog_extension-signed-bundle-${CI_JOB_ID}
     - mkdir -p datadog_extension-signed-bundle-${CI_JOB_ID}
     - cp .layers/datadog_extension-*.zip datadog_extension-signed-bundle-${CI_JOB_ID}
+
+# Integration Tests - Publish arm64 layer with integration test prefix
+publish integration layer (arm64):
+  stage: integration-tests
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  rules:
+    - when: on_success
+  needs:
+    - layer (arm64)
+  dependencies:
+    - layer (arm64)
+  variables:
+    LAYER_NAME_BASE_SUFFIX: ""
+    ARCHITECTURE: arm64
+    LAYER_FILE: datadog_extension-arm64.zip
+    REGION: us-east-1
+    ADD_LAYER_VERSION_PERMISSIONS: "0"
+    AUTOMATICALLY_BUMP_VERSION: "1"
+  {{ with $environment := (ds "environments").environments.sandbox }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+    - export PIPELINE_LAYER_SUFFIX="ARM-${CI_COMMIT_SHORT_SHA}"
+    - echo "Publishing layer with suffix - ${PIPELINE_LAYER_SUFFIX}"
+  script:
+    - .gitlab/scripts/publish_layers.sh
+    - LAYER_ARN=$(aws lambda list-layer-versions --layer-name "Datadog-Extension-ARM-${CI_COMMIT_SHORT_SHA}" --query 'LayerVersions[0].LayerVersionArn' --output text --region us-east-1)
+    - echo "Published layer ARN - ${LAYER_ARN}"
+  {{ end }}
+
+# Integration Tests - Deploy CDK stacks with commit hash prefix
+integration-deploy:
+  stage: integration-tests
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  rules:
+    - when: on_success
+  needs:
+    - publish integration layer (arm64)
+  variables:
+    IDENTIFIER: ${CI_COMMIT_SHORT_SHA}
+    AWS_DEFAULT_REGION: us-east-1
+  {{ with $environment := (ds "environments").environments.sandbox }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+    - curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+    - apt-get install -y nodejs
+    - cd integration-tests
+    - npm ci
+  {{ end }}
+  script:
+    - echo "Deploying CDK stacks with identifier ${IDENTIFIER}..."
+    - export EXTENSION_LAYER_ARN=$(aws lambda list-layer-versions --layer-name "Datadog-Extension-ARM-${CI_COMMIT_SHORT_SHA}" --query 'LayerVersions[0].LayerVersionArn' --output text --region us-east-1)
+    - echo "Using integration test layer - ${EXTENSION_LAYER_ARN}"
+    - export CDK_DEFAULT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
+    - export CDK_DEFAULT_REGION=us-east-1
+    - npm run build
+    - npx cdk deploy "integ-$IDENTIFIER-*" --require-approval never
+
+# Integration Tests - Run Jest test suite
+integration-test:
+  stage: integration-tests
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  rules:
+    - when: on_success
+  needs:
+    - integration-deploy
+  variables:
+    IDENTIFIER: ${CI_COMMIT_SHORT_SHA}
+    DD_SITE: datadoghq.com
+  {{ with $environment := (ds "environments").environments.sandbox }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+    - curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+    - apt-get install -y nodejs
+    - cd integration-tests
+    - npm ci
+  script:
+    - echo "Running integration tests with identifier ${IDENTIFIER}..."
+    - npm run test:ci
+  {{ end }}
+  artifacts:
+    when: always
+    paths:
+      - integration-tests/test-results/
+    reports:
+      junit: integration-tests/test-results/junit.xml
+    expire_in: 30 days
+
+# Integration Tests - Cleanup stacks
+integration-cleanup-stacks:
+  stage: integration-tests
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  when: always
+  rules:
+    - when: always
+  needs:
+    - integration-test
+  variables:
+    IDENTIFIER: ${CI_COMMIT_SHORT_SHA}
+  {{ with $environment := (ds "environments").environments.sandbox }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+    - curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+    - apt-get install -y nodejs
+    - cd integration-tests
+  {{ end }}
+  script:
+    - echo "Destroying CDK stacks with identifier ${IDENTIFIER}..."
+    - npx cdk destroy "integ-$IDENTIFIER-*" --force || echo "Failed to destroy some stacks, but continuing..."
+
+# Integration Tests - Cleanup layer
+integration-cleanup-layer:
+  stage: integration-tests
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  when: always
+  rules:
+    - when: always
+  needs:
+    - integration-cleanup-stacks
+  variables:
+    IDENTIFIER: ${CI_COMMIT_SHORT_SHA}
+  {{ with $environment := (ds "environments").environments.sandbox }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+  {{ end }}
+  script:
+    - echo "Deleting integration test layer with identifier ${IDENTIFIER}..."
+    - |
+      LAYER_NAME="Datadog-Extension-${IDENTIFIER}"
+      echo "Looking for layer: ${LAYER_NAME}"
+
+      # Get all versions of the layer
+      VERSIONS=$(aws lambda list-layer-versions --layer-name "${LAYER_NAME}" --query 'LayerVersions[*].Version' --output text --region us-east-1 2>/dev/null || echo "")
+
+      if [ -z "$VERSIONS" ]; then
+        echo "No versions found for layer ${LAYER_NAME}"
+      else
+        echo "Found versions: ${VERSIONS}"
+        for VERSION in $VERSIONS; do
+          echo "Deleting ${LAYER_NAME} version ${VERSION}..."
+          aws lambda delete-layer-version --layer-name "${LAYER_NAME}" --version-number "${VERSION}" --region us-east-1 || echo "Failed to delete version ${VERSION}, continuing..."
+        done
+        echo "Successfully deleted all versions of ${LAYER_NAME}"
+      fi
+

integration-tests/.gitignore

@@ -0,0 +1,40 @@
+# CDK
+*.js
+!jest.config.js
+!lambda/**/*.js
+*.d.ts
+node_modules
+cdk.out
+.cdk.staging
+
+# Compiled output
+dist/
+*.tsbuildinfo
+
+# Logs
+*.log
+npm-debug.log*
+
+# Testing
+coverage/
+.nyc_output/
+test-results/
+
+# Environment
+.env
+.env.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Lambda artifacts
+response.json
+lambda-bundle.zip

integration-tests/README.md

@@ -0,0 +1,70 @@
+# Datadog Lambda Extension Integration Tests
+
+This directory contains integration tests for the Datadog Lambda Extension. 
+
+The general flow is:
+1. Deploy test setup using CDK.
+2. Invoke lambda functions.
+3. Wait for data to propagate to Datadog.
+4. Call Datadog to get telemetry data and check the data based on test requirements.
+
+For simplicity, integraiton tests are setup to only test against ARM runtimes.
+
+## Guidelines
+
+### Naming
+* An `identifier` is used to differentiate the different stacks. For local development, the identifier is automatically set using the command `whoami` and parses the user's first name. For gitlab pipelines, the identifier is the git commit short sha.
+* Stacks should be named `integ-<identifier>-<stack name>`
+* Lambda functions should be named `<stack-id>-<function name>`
+
+### Stacks
+* Stacks automatically get destroyed at the end of the gitlab integration tests step. Stack should be setup to not retain resources. A helper function `createLogGroup` exists with `removalPolicy: cdk.RemovalPolicy.DESTROY`. 
+
+
+## Local Development
+
+### Prerequisites Set env variables
+**Datadog API Keys**: Set environment variables
+   ```bash
+   export DD_API_KEY="your-datadog-api-key"
+   export DD_APP_KEY="your-datadog-app-key"
+   export DATADOG_API_SECRET_ARN="arn:aws:secretsmanager:us-east-1:ACCOUNT_ID:secret:YOUR_SECRET"
+   ```
+
+### 1. Build and Deploy Extension Layer
+
+First, publish your extension layer. 
+
+```bash
+./scripts/local_publish.sh
+```
+
+This will create and publish `Datadog-Extension-ARM-<your name>`.
+
+### 2. Deploy Test Stacks
+
+Deploy the CDK stacks that create Lambda functions for testing.
+
+```bash
+./scripts/local_deploy.sh <stack name> 
+```
+
+This will create `integ-<your name>-<stack name>`. The stacks will use the lambda extension created in the previous step.
+
+### 3. Run Integration Tests
+
+Run Jest tests that invoke Lambda functions and verify Datadog telemetry:
+
+```bash
+# All tests
+npm test
+
+# Single test
+npm test -- <my test file>
+```
+
+
+
+**Note**: Tests wait for a few minutes after Lambda invocation to allow telemetry to appear in Datadog. 
+
+

integration-tests/bin/app.ts

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from 'aws-cdk-lib';
+import { BaseNodeStack } from '../lib/stacks/base-node-stack';
+import { BasePythonStack } from '../lib/stacks/base-python-stack';
+import { getIdentifier } from '../tests/utils/config';
+
+const app = new cdk.App();
+
+const env = {
+  account: process.env.CDK_DEFAULT_ACCOUNT || process.env.AWS_ACCOUNT_ID,
+  region: process.env.CDK_DEFAULT_REGION || process.env.AWS_REGION || 'us-east-1',
+};
+
+const identifier = getIdentifier();
+
+new BaseNodeStack(app, `integ-${identifier}-base-node`, {
+  env,
+});
+
+new BasePythonStack(app, `integ-${identifier}-base-python`, {
+  env,
+});
+
+app.synth();