[Python] Add Performance Testing Workflow for Lambda Layer

.github/workflows/python-lambda-layer-perf-test.yml

@@ -0,0 +1,103 @@
+## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+## SPDX-License-Identifier: Apache-2.0
+name: Python Lambda Layer Performance Test
+on:
+  workflow_dispatch:
+    inputs:
+      test_runs:
+        description: 'Number of test runs to perform'
+        required: true
+        default: 20
+        type: number
+jobs:
+  python-lambda-layer-performance-test:
+    runs-on: ubuntu-latest 
+    permissions:
+        id-token: write
+        contents: read   
+
+    env:
+      NUM_TEST_RUNS: ${{ github.event.inputs.test_runs }}
+
+    steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: arn:aws:iam::${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
+        aws-region: us-east-1     
+
+    - name: Checkout aws-otel-python-instrumentation
+      uses: actions/checkout@v4
+      with:
+        repository: aws-observability/aws-otel-python-instrumentation
+        path: aws-otel-python-instrumentation
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v2
+
+    - name: Build Lambda Layer
+      run: |
+        cd aws-otel-python-instrumentation/lambda-layer
+        chmod +x build.sh
+        ./build.sh
+
+    - name: Checkout current repository
+      uses: actions/checkout@v4
+      with:
+        path: current-repo
+
+    - name: Run Cold Start Iterations for Base Lambda + Lambda Layer
+      run: |
+        cd current-repo
+        chmod +x lambda-layer-perf-test/lambda-layer-run.sh
+        ./lambda-layer-perf-test/lambda-layer-run.sh false aws-opentelemetry-distro-python
+
+    - name: Remove Application Signals Lambda Layer
+      run: |
+          echo "Removing Lambda layer..."
+
+          OUTPUT=$(aws lambda update-function-configuration \
+              --function-name aws-opentelemetry-distro-python \
+              --layers [])
+
+          echo "Lambda configuration:"
+          echo "$OUTPUT"
+
+          LAYERS=$(echo "$OUTPUT" | jq -r '.Layers | length')
+
+          if [ "$LAYERS" -ne 0 ]; then
+              echo "::error::Found $LAYERS layer(s) still attached to the function"
+              echo "::error::Layer details:"
+              echo "$OUTPUT" | jq -r '.Layers'
+              exit 1
+          else
+              echo "✅ Layers successfully removed"
+          fi
+
+    - name: Run Cold Start Iterations for Base Lambda
+      run: |
+        cd current-repo
+        chmod +x lambda-layer-perf-test/lambda-layer-run.sh
+        ./lambda-layer-perf-test/lambda-layer-run.sh true aws-opentelemetry-distro-python
+
+    - name: Upload test results
+      uses: actions/upload-artifact@v4
+      with:
+        name: performance-test-results
+        path: |
+          no_layer_results.txt
+          layer_results.txt
+        retention-days: 90
+
+    - name: Cleanup Terraform Resources
+      if: success() || failure() || cancelled()
+      run: |
+        cd aws-otel-python-instrumentation/lambda-layer/terraform/lambda
+        echo "Starting Terraform cleanup..."
+        terraform init
+        terraform destroy -auto-approve

lambda-layer-perf-test/lambda-layer-run.sh

@@ -0,0 +1,110 @@
+## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+## SPDX-License-Identifier: Apache-2.0
+
+#!/bin/bash
+
+IS_BASE_RUN=${1:-false}
+FUNCTION_NAME=${2:-none}
+SLEEP_TIME_SECONDS=300
+TEST_RUNS=${NUM_TEST_RUNS:-20}
+
+echo "Running $TEST_RUNS cold start test iterations"
+
+CW_LOGS_QUERY_START_TIME=$(date +%s)
+
+echo "Sleeping for $SLEEP_TIME_SECONDS seconds to ensure logs are within CloudWatch Insights query timeframe"
+sleep "$SLEEP_TIME_SECONDS"
+
+ACTUAL_START_TIME=$(date +%s)
+echo "Start time for test run: $ACTUAL_START_TIME"
+
+for i in $(seq 1 "$TEST_RUNS"); do
+  if $IS_BASE_RUN; then
+      ENV_JSON="{\"Variables\":{\"FOO\":\"BAR_$i\"}}"
+  else
+      ENV_JSON="{\"Variables\":{\"AWS_LAMBDA_EXEC_WRAPPER\":\"/opt/otel-instrument\",\"FOO\":\"BAR_$i\"}}"
+  fi
+
+  echo "Iteration $i: Updating environment variables to simulate cold start"
+
+  # Update environment variables
+  aws lambda update-function-configuration \
+    --function-name $FUNCTION_NAME \
+    --environment "$ENV_JSON" > /dev/null
+
+  # Wait a short time for changes to take effect
+  sleep 5
+
+  # Invoke the Lambda function
+  echo "Iteration $i: Invoking Lambda function"
+  aws lambda invoke --function-name $FUNCTION_NAME --payload '{}' response.json > /dev/null
+
+  # Wait to simulate cold start reset
+  echo "Iteration $i: Waiting for cold start reset"
+  sleep 5
+done
+
+ACTUAL_END_TIME=$(date +%s)
+echo "End time for test run: $ACTUAL_END_TIME"
+echo "Cold start test completed"
+
+echo "Sleeping for $SLEEP_TIME_SECONDS seconds to ensure logs are within CloudWatch Insights query timeframe"
+sleep $SLEEP_TIME_SECONDS
+
+CW_LOGS_QUERY_END_TIME=$(date +%s)
+
+echo "Running CloudWatch Logs Insights query..."
+
+QUERY='fields @timestamp, @message, @initDuration
+| filter @message like "REPORT RequestId"
+| stats
+    count(@initDuration) as Sample_Count,
+    avg(@initDuration) as Average_Init_Duration,
+    min(@initDuration) as Min_Init_Duration,
+    max(@initDuration) as Max_Init_Duration,
+    pct(@initDuration, 50) as P50_Init_Duration,
+    pct(@initDuration, 90) as P90_Init_Duration,
+    pct(@initDuration, 99) as P99_Init_Duration'
+
+# Start the query
+QUERY_ID=$(aws logs start-query \
+  --log-group-name "/aws/lambda/$FUNCTION_NAME" \
+  --start-time "$CW_LOGS_QUERY_START_TIME" \
+  --end-time "$CW_LOGS_QUERY_END_TIME" \
+  --query-string "$QUERY" \
+  --output text)
+
+echo "Query ID: $QUERY_ID"
+
+STATUS="Running"
+while [ "$STATUS" = "Running" ]; do
+  echo "Waiting for query results..."
+  sleep 3
+  RESULT=$(aws logs get-query-results --query-id "$QUERY_ID")
+  STATUS=$(echo "$RESULT" | jq -r .status)
+done
+
+if [ "$STATUS" = "Complete" ]; then
+  echo "Query completed. Results:"
+  echo "$RESULT"
+else
+  echo "Query failed with status: $STATUS"
+fi
+
+FLATTENED=$(echo "$RESULT" | jq -r '
+    .results[0] | 
+    map({(.field): .value}) | 
+    add | 
+    del(.Sample_Count) |
+    to_entries | 
+    map("\"\(.key)\": \"\(.value)\"") | 
+    join(",\n")
+')
+
+if $IS_BASE_RUN; then
+    echo "$FLATTENED" > ../no_layer_results.txt
+    echo "Results saved to no_layer_results.txt"
+else
+    echo "$FLATTENED" > ../layer_results.txt
+    echo "Results saved to layer_results.txt"
+fi