update versions

Author: jchrostek-dd

.gitlab/scripts/publish_layers.sh

@@ -43,7 +43,7 @@ publish_layer() {
     compatible_architectures=$4
 
     version_nbr=$(aws lambda publish-layer-version --layer-name $layer \
-        --description "Datadog Lambda Extension" \
+        --description "${LAYER_DESCRIPTION:-Datadog Lambda Extension}" \
         --compatible-architectures $compatible_architectures \
         --zip-file "fileb://${file}" \
         --region $region \

.gitlab/templates/pipeline.yaml.tpl

@@ -424,12 +424,48 @@ publish integration layer (arm64):
   {{ with $environment := (ds "environments").environments.sandbox }}
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
-    - export PIPELINE_LAYER_SUFFIX="ARM-${CI_COMMIT_SHORT_SHA}"
-    - echo "Publishing layer with suffix - ${PIPELINE_LAYER_SUFFIX}"
+    # Use fixed layer suffix for integration tests
+    - export PIPELINE_LAYER_SUFFIX="IntegTests-ARM"
+    # Set description to just the commit SHA
+    - export LAYER_DESCRIPTION="${CI_COMMIT_SHORT_SHA}"
+    - echo "Publishing to layer Datadog-Extension-IntegTests-ARM with commit SHA ${LAYER_DESCRIPTION}"
   script:
     - .gitlab/scripts/publish_layers.sh
-    - LAYER_ARN=$(aws lambda list-layer-versions --layer-name "Datadog-Extension-ARM-${CI_COMMIT_SHORT_SHA}" --query 'LayerVersions[0].LayerVersionArn' --output text --region us-east-1)
-    - echo "Published layer ARN - ${LAYER_ARN}"
+    # Find the version we just published by matching commit SHA in description
+    - |
+      LAYER_NAME="Datadog-Extension-IntegTests-ARM"
+      COMMIT_SHA="${CI_COMMIT_SHORT_SHA}"
+
+      echo "Searching for layer version with commit SHA: ${COMMIT_SHA}"
+
+      # List all versions and find the one with our commit SHA as the description
+      LAYER_INFO=$(aws lambda list-layer-versions \
+        --layer-name "${LAYER_NAME}" \
+        --region us-east-1 \
+        --query "LayerVersions[?Description=='${COMMIT_SHA}'].[Version, LayerVersionArn, Description] | [0]" \
+        --output json)
+
+      if [ "$LAYER_INFO" = "null" ] || [ -z "$LAYER_INFO" ]; then
+        echo "ERROR: Could not find layer version with commit SHA ${COMMIT_SHA}"
+        echo "Available versions:"
+        aws lambda list-layer-versions --layer-name "${LAYER_NAME}" --region us-east-1 --query 'LayerVersions[*].[Version, Description]' --output table
+        exit 1
+      fi
+
+      LAYER_VERSION=$(echo "$LAYER_INFO" | jq -r '.[0]')
+      LAYER_ARN=$(echo "$LAYER_INFO" | jq -r '.[1]')
+      LAYER_DESC=$(echo "$LAYER_INFO" | jq -r '.[2]')
+
+      echo "Found layer version ${LAYER_VERSION} with ARN ${LAYER_ARN}"
+      echo "Description: ${LAYER_DESC}"
+
+      # Export for use by integration tests via dotenv artifact
+      echo "EXTENSION_LAYER_ARN=${LAYER_ARN}" >> layer.env
+      echo "LAYER_VERSION=${LAYER_VERSION}" >> layer.env
+      echo "COMMIT_SHA=${COMMIT_SHA}" >> layer.env
+  artifacts:
+    reports:
+      dotenv: layer.env
   {{ end }}
 
 # Integration Tests - Deploy, test, and cleanup each suite independently (parallel by test suite)
@@ -443,7 +479,8 @@ integration-suite:
   rules:
     - when: on_success
   needs:
-    - publish integration layer (arm64)
+    - job: publish integration layer (arm64)
+      artifacts: true
     - build java lambdas
     - build dotnet lambdas
     - build python lambdas
@@ -468,8 +505,16 @@ integration-suite:
   script:
     # Deploy
     - echo "Deploying ${TEST_SUITE} CDK stack with identifier ${IDENTIFIER}..."
-    - export EXTENSION_LAYER_ARN=$(aws lambda list-layer-versions --layer-name "Datadog-Extension-ARM-${CI_COMMIT_SHORT_SHA}" --query 'LayerVersions[0].LayerVersionArn' --output text --region us-east-1)
-    - echo "Using integration test layer - ${EXTENSION_LAYER_ARN}"
+    # EXTENSION_LAYER_ARN is available from the dotenv artifact (passed from publish job)
+    - echo "Using integration test layer - ${EXTENSION_LAYER_ARN} (version ${LAYER_VERSION}, commit ${COMMIT_SHA})"
+    # Validate the layer ARN is set
+    - |
+      if [ -z "$EXTENSION_LAYER_ARN" ]; then
+        echo "ERROR: EXTENSION_LAYER_ARN not set from publish job dotenv artifact"
+        echo "This indicates the publish job failed to export the layer ARN"
+        exit 1
+      fi
+      echo "✓ Layer ARN inherited from publish job"
     - export CDK_DEFAULT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
     - export CDK_DEFAULT_REGION=us-east-1
     - npm run build
@@ -514,7 +559,7 @@ integration-suite:
       junit: integration-tests/test-results/junit-*.xml
     expire_in: 30 days
 
-# Integration Tests - Cleanup layer
+# Integration Tests - Cleanup old layer versions
 integration-cleanup-layer:
   stage: integration-tests
   tags: ["arch:amd64"]
@@ -524,28 +569,48 @@ integration-cleanup-layer:
   needs:
     - job: integration-suite
   variables:
-    IDENTIFIER: ${CI_COMMIT_SHORT_SHA}
+    LAYER_NAME: "Datadog-Extension-IntegTests-ARM"
+    KEEP_VERSIONS: 500
   {{ with $environment := (ds "environments").environments.sandbox }}
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   {{ end }}
   script:
-    - echo "Deleting integration test layer with identifier ${IDENTIFIER}..."
+    - echo "Cleaning up old versions of ${LAYER_NAME}..."
     - |
-      LAYER_NAME="Datadog-Extension-ARM-${IDENTIFIER}"
-      echo "Looking for layer: ${LAYER_NAME}"
+      # Get all layer versions sorted by version number (newest first)
+      ALL_VERSIONS=$(aws lambda list-layer-versions \
+        --layer-name "${LAYER_NAME}" \
+        --query 'LayerVersions[*].Version' \
+        --output text \
+        --region us-east-1 2>/dev/null || echo "")
+
+      if [ -z "$ALL_VERSIONS" ]; then
+        echo "No versions found for layer ${LAYER_NAME}"
+        exit 0
+      fi
 
-      # Get all versions of the layer
-      VERSIONS=$(aws lambda list-layer-versions --layer-name "${LAYER_NAME}" --query 'LayerVersions[*].Version' --output text --region us-east-1 2>/dev/null || echo "")
+      # Convert to array
+      VERSION_ARRAY=($ALL_VERSIONS)
+      TOTAL_VERSIONS=${#VERSION_ARRAY[@]}
 
-      if [ -z "$VERSIONS" ]; then
-        echo "No versions found for layer ${LAYER_NAME}"
-      else
-        echo "Found versions: ${VERSIONS}"
-        for VERSION in $VERSIONS; do
+      echo "Found ${TOTAL_VERSIONS} versions of ${LAYER_NAME}"
+      echo "Keeping the ${KEEP_VERSIONS} most recent versions"
+
+      # Delete old versions (keep only the most recent KEEP_VERSIONS)
+      if [ $TOTAL_VERSIONS -gt $KEEP_VERSIONS ]; then
+        VERSIONS_TO_DELETE=${VERSION_ARRAY[@]:$KEEP_VERSIONS}
+
+        for VERSION in $VERSIONS_TO_DELETE; do
           echo "Deleting ${LAYER_NAME} version ${VERSION}..."
-          aws lambda delete-layer-version --layer-name "${LAYER_NAME}" --version-number "${VERSION}" --region us-east-1 || echo "Failed to delete version ${VERSION}, continuing..."
+          aws lambda delete-layer-version \
+            --layer-name "${LAYER_NAME}" \
+            --version-number "${VERSION}" \
+            --region us-east-1 || echo "Failed to delete version ${VERSION}, continuing..."
         done
-        echo "Successfully deleted all versions of ${LAYER_NAME}"
+
+        echo "Successfully cleaned up old versions"
+      else
+        echo "Total versions (${TOTAL_VERSIONS}) <= keep versions (${KEEP_VERSIONS}), no cleanup needed"
       fi