diff --git a/.github/workflows/python-lambda-layer-perf-test.yml b/.github/workflows/python-lambda-layer-perf-test.yml new file mode 100644 index 000000000..9288ce610 --- /dev/null +++ b/.github/workflows/python-lambda-layer-perf-test.yml @@ -0,0 +1,103 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 +name: Python Lambda Layer Performance Test +on: + workflow_dispatch: + inputs: + test_runs: + description: 'Number of test runs to perform' + required: true + default: 20 + type: number +jobs: + python-lambda-layer-performance-test: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + + env: + NUM_TEST_RUNS: ${{ github.event.inputs.test_runs }} + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }} + aws-region: us-east-1 + + - name: Checkout aws-otel-python-instrumentation + uses: actions/checkout@v4 + with: + repository: aws-observability/aws-otel-python-instrumentation + path: aws-otel-python-instrumentation + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + + - name: Build Lambda Layer + run: | + cd aws-otel-python-instrumentation/lambda-layer + chmod +x build.sh + ./build.sh + + - name: Checkout current repository + uses: actions/checkout@v4 + with: + path: current-repo + + - name: Run Cold Start Iterations for Base Lambda + Lambda Layer + run: | + cd current-repo + chmod +x lambda-layer-perf-test/lambda-layer-run.sh + ./lambda-layer-perf-test/lambda-layer-run.sh false aws-opentelemetry-distro-python + + - name: Remove Application Signals Lambda Layer + run: | + echo "Removing Lambda layer..." + + OUTPUT=$(aws lambda update-function-configuration \ + --function-name aws-opentelemetry-distro-python \ + --layers []) + + echo "Lambda configuration:" + echo "$OUTPUT" + + LAYERS=$(echo "$OUTPUT" | jq -r '.Layers | length') + + if [ "$LAYERS" -ne 0 ]; then + echo "::error::Found $LAYERS layer(s) still attached to the function" + echo "::error::Layer details:" + echo "$OUTPUT" | jq -r '.Layers' + exit 1 + else + echo "✅ Layers successfully removed" + fi + + - name: Run Cold Start Iterations for Base Lambda + run: | + cd current-repo + chmod +x lambda-layer-perf-test/lambda-layer-run.sh + ./lambda-layer-perf-test/lambda-layer-run.sh true aws-opentelemetry-distro-python + + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + name: performance-test-results + path: | + no_layer_results.txt + layer_results.txt + retention-days: 90 + + - name: Cleanup Terraform Resources + if: success() || failure() || cancelled() + run: | + cd aws-otel-python-instrumentation/lambda-layer/terraform/lambda + echo "Starting Terraform cleanup..." + terraform init + terraform destroy -auto-approve \ No newline at end of file diff --git a/lambda-layer-perf-test/lambda-layer-run.sh b/lambda-layer-perf-test/lambda-layer-run.sh new file mode 100755 index 000000000..f2176ab9e --- /dev/null +++ b/lambda-layer-perf-test/lambda-layer-run.sh @@ -0,0 +1,110 @@ +## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +## SPDX-License-Identifier: Apache-2.0 + +#!/bin/bash + +IS_BASE_RUN=${1:-false} +FUNCTION_NAME=${2:-none} +SLEEP_TIME_SECONDS=300 +TEST_RUNS=${NUM_TEST_RUNS:-20} + +echo "Running $TEST_RUNS cold start test iterations" + +CW_LOGS_QUERY_START_TIME=$(date +%s) + +echo "Sleeping for $SLEEP_TIME_SECONDS seconds to ensure logs are within CloudWatch Insights query timeframe" +sleep "$SLEEP_TIME_SECONDS" + +ACTUAL_START_TIME=$(date +%s) +echo "Start time for test run: $ACTUAL_START_TIME" + +for i in $(seq 1 "$TEST_RUNS"); do + if $IS_BASE_RUN; then + ENV_JSON="{\"Variables\":{\"FOO\":\"BAR_$i\"}}" + else + ENV_JSON="{\"Variables\":{\"AWS_LAMBDA_EXEC_WRAPPER\":\"/opt/otel-instrument\",\"FOO\":\"BAR_$i\"}}" + fi + + echo "Iteration $i: Updating environment variables to simulate cold start" + + # Update environment variables + aws lambda update-function-configuration \ + --function-name $FUNCTION_NAME \ + --environment "$ENV_JSON" > /dev/null + + # Wait a short time for changes to take effect + sleep 5 + + # Invoke the Lambda function + echo "Iteration $i: Invoking Lambda function" + aws lambda invoke --function-name $FUNCTION_NAME --payload '{}' response.json > /dev/null + + # Wait to simulate cold start reset + echo "Iteration $i: Waiting for cold start reset" + sleep 5 +done + +ACTUAL_END_TIME=$(date +%s) +echo "End time for test run: $ACTUAL_END_TIME" +echo "Cold start test completed" + +echo "Sleeping for $SLEEP_TIME_SECONDS seconds to ensure logs are within CloudWatch Insights query timeframe" +sleep $SLEEP_TIME_SECONDS + +CW_LOGS_QUERY_END_TIME=$(date +%s) + +echo "Running CloudWatch Logs Insights query..." + +QUERY='fields @timestamp, @message, @initDuration +| filter @message like "REPORT RequestId" +| stats + count(@initDuration) as Sample_Count, + avg(@initDuration) as Average_Init_Duration, + min(@initDuration) as Min_Init_Duration, + max(@initDuration) as Max_Init_Duration, + pct(@initDuration, 50) as P50_Init_Duration, + pct(@initDuration, 90) as P90_Init_Duration, + pct(@initDuration, 99) as P99_Init_Duration' + +# Start the query +QUERY_ID=$(aws logs start-query \ + --log-group-name "/aws/lambda/$FUNCTION_NAME" \ + --start-time "$CW_LOGS_QUERY_START_TIME" \ + --end-time "$CW_LOGS_QUERY_END_TIME" \ + --query-string "$QUERY" \ + --output text) + +echo "Query ID: $QUERY_ID" + +STATUS="Running" +while [ "$STATUS" = "Running" ]; do + echo "Waiting for query results..." + sleep 3 + RESULT=$(aws logs get-query-results --query-id "$QUERY_ID") + STATUS=$(echo "$RESULT" | jq -r .status) +done + +if [ "$STATUS" = "Complete" ]; then + echo "Query completed. Results:" + echo "$RESULT" +else + echo "Query failed with status: $STATUS" +fi + +FLATTENED=$(echo "$RESULT" | jq -r ' + .results[0] | + map({(.field): .value}) | + add | + del(.Sample_Count) | + to_entries | + map("\"\(.key)\": \"\(.value)\"") | + join(",\n") +') + +if $IS_BASE_RUN; then + echo "$FLATTENED" > ../no_layer_results.txt + echo "Results saved to no_layer_results.txt" +else + echo "$FLATTENED" > ../layer_results.txt + echo "Results saved to layer_results.txt" +fi \ No newline at end of file