diff --git a/.github/actions/artifacts_build/action.yml b/.github/actions/artifacts_build/action.yml index 7543b04e3..c0795757d 100644 --- a/.github/actions/artifacts_build/action.yml +++ b/.github/actions/artifacts_build/action.yml @@ -53,7 +53,7 @@ runs: - name: Configure AWS Credentials if: ${{ inputs.push_image == true || inputs.push_image == 'true' }} - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 with: role-to-assume: ${{ inputs.snapshot-ecr-role }} aws-region: ${{ inputs.aws-region }} @@ -68,14 +68,14 @@ runs: python -m build --outdir ../dist - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #3.6.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #3.11.1 - name: Login to private AWS ECR if: ${{ inputs.push_image == true || inputs.push_image == 'true' }} - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 with: registry: ${{ inputs.image_registry }} env: @@ -91,7 +91,7 @@ runs: run: docker logout public.ecr.aws - name: Build and push image according to input - uses: docker/build-push-action@v5 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #6.18.0 with: push: ${{ inputs.push_image }} context: . diff --git a/.github/actions/image_scan/action.yml b/.github/actions/image_scan/action.yml index 31d5a78fe..519f6a708 100644 --- a/.github/actions/image_scan/action.yml +++ b/.github/actions/image_scan/action.yml @@ -32,7 +32,7 @@ runs: run: docker logout public.ecr.aws - name: Run Trivy vulnerability scanner on image - uses: aquasecurity/trivy-action@master + uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 with: image-ref: ${{ inputs.image-ref }} severity: ${{ inputs.severity }} diff --git a/.github/actions/set_up/action.yml b/.github/actions/set_up/action.yml index dd4948518..433367f95 100644 --- a/.github/actions/set_up/action.yml +++ b/.github/actions/set_up/action.yml @@ -21,7 +21,7 @@ runs: using: "composite" steps: - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0 with: python-version: ${{ inputs.python_version }} @@ -31,7 +31,7 @@ runs: - name: Cache tox environment # Preserves .tox directory between runs for faster installs - uses: actions/cache@v3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 #4.2.4 with: path: | .tox diff --git a/.github/workflows/application-signals-e2e-test.yml b/.github/workflows/application-signals-e2e-test.yml index 25b2d6f72..e42b6586a 100644 --- a/.github/workflows/application-signals-e2e-test.yml +++ b/.github/workflows/application-signals-e2e-test.yml @@ -29,12 +29,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 with: role-to-assume: arn:aws:iam::${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}:role/${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }} aws-region: us-east-1 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 #5.0.0 with: name: ${{ inputs.staging-wheel-name }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index dacf9c831..a1e6ce0af 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -60,11 +60,11 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@16df4fbc19aea13d921737861d6c622bf3cefe23 #v2.23.0 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -92,6 +92,41 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@16df4fbc19aea13d921737861d6c622bf3cefe23 #v2.23.0 with: category: "/language:${{matrix.language}}" + + all-codeql-checks-pass: + runs-on: ubuntu-latest + needs: [analyze] + if: always() + steps: + - name: Checkout to get workflow file + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + + - name: Check all jobs succeeded and none missing + run: | + # Check if all needed jobs succeeded + results='${{ toJSON(needs) }}' + if echo "$results" | jq -r '.[] | .result' | grep -v success; then + echo "Some jobs failed" + exit 1 + fi + + # Extract all job names from workflow (excluding this gate job) + all_jobs=$(yq eval '.jobs | keys | .[]' .github/workflows/codeql.yml | grep -v "all-codeql-checks-pass" | sort) + + # Extract job names from needs array + needed_jobs='${{ toJSON(needs) }}' + needs_list=$(echo "$needed_jobs" | jq -r 'keys[]' | sort) + + # Check if any jobs are missing from needs + missing_jobs=$(comm -23 <(echo "$all_jobs") <(echo "$needs_list")) + if [ -n "$missing_jobs" ]; then + echo "ERROR: Jobs missing from needs array in all-codeql-checks-pass:" + echo "$missing_jobs" + echo "Please add these jobs to the needs array of all-codeql-checks-pass" + exit 1 + fi + + echo "All CodeQL checks passed and no jobs missing from gate!" diff --git a/.github/workflows/daily-scan.yml b/.github/workflows/daily-scan.yml index 79d826202..a84ed7569 100644 --- a/.github/workflows/daily-scan.yml +++ b/.github/workflows/daily-scan.yml @@ -26,12 +26,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo for dependency scan - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: fetch-depth: 0 - name: Set up Python for dependency scan - uses: actions/setup-python@v4 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0 with: python-version: "3.10" @@ -44,19 +44,19 @@ jobs: less aws-opentelemetry-distro/requirements.txt - name: Install java for dependency scan - uses: actions/setup-java@v4 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 #v5.0.0 with: java-version: 17 distribution: 'temurin' - name: Configure AWS credentials for dependency scan - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 with: role-to-assume: ${{ secrets.SECRET_MANAGER_ROLE_ARN }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Get NVD API key for dependency scan - uses: aws-actions/aws-secretsmanager-get-secrets@v1 + uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 #v2.0.10 id: nvd_api_key with: secret-ids: ${{ secrets.NVD_API_KEY_SECRET_ARN }} @@ -80,13 +80,13 @@ jobs: run: less dependency-check-report.html - name: Configure AWS credentials for image scan - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 with: role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Login to Public ECR - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 with: registry: public.ecr.aws @@ -110,7 +110,7 @@ jobs: - name: Configure AWS Credentials for emitting metrics if: always() - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 with: role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }} aws-region: ${{ env.AWS_DEFAULT_REGION }} diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index f7c01e440..7e1df8fef 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -33,7 +33,7 @@ jobs: staging_wheel_file: ${{ steps.staging_wheel_output.outputs.STAGING_WHEEL}} steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 - name: Get Python Distro Output id: python_output @@ -87,7 +87,7 @@ jobs: aws s3 cp dist/${{ steps.staging_wheel_output.outputs.STAGING_WHEEL}} s3://${{ env.STAGING_S3_BUCKET }} - name: Upload Wheel to GitHub Actions - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #v4.6.2 with: name: ${{ steps.staging_wheel_output.outputs.STAGING_WHEEL}} path: dist/${{ steps.staging_wheel_output.outputs.STAGING_WHEEL}} @@ -117,7 +117,7 @@ jobs: if: always() steps: - name: Configure AWS Credentials for emitting metrics - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 with: role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }} aws-region: ${{ env.AWS_DEFAULT_REGION }} diff --git a/.github/workflows/post-release-version-bump.yml b/.github/workflows/post-release-version-bump.yml index d1f4b180b..96264116e 100644 --- a/.github/workflows/post-release-version-bump.yml +++ b/.github/workflows/post-release-version-bump.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout main - uses: actions/checkout@v2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: ref: main fetch-depth: 0 @@ -59,13 +59,13 @@ jobs: needs: check-version steps: - name: Configure AWS credentials for BOT secrets - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #v5.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRETS_MANAGER }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Get Bot secrets - uses: aws-actions/aws-secretsmanager-get-secrets@v1 + uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 #v2.0.10 id: bot_secrets with: secret-ids: | @@ -73,7 +73,7 @@ jobs: parse-json-secrets: true - name: Setup Git - uses: actions/checkout@v2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: fetch-depth: 0 token: ${{ env.BOT_TOKEN_GITHUB_RW_PATOKEN }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index fbd265f50..420e9cd30 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -10,6 +10,61 @@ permissions: contents: read jobs: + static-code-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + fetch-depth: 0 + + - name: Check CHANGELOG + if: always() + run: | + # Check if PR is from workflows bot or dependabot + if [[ "${{ github.event.pull_request.user.login }}" == "aws-application-signals-bot" ]]; then + echo "Skipping check: PR from aws-application-signals-bot" + exit 0 + fi + + if [[ "${{ github.event.pull_request.user.login }}" == "dependabot[bot]" ]]; then + echo "Skipping check: PR from dependabot" + exit 0 + fi + + # Check for skip changelog label + if echo '${{ toJSON(github.event.pull_request.labels.*.name) }}' | jq -r '.[]' | grep -q "skip changelog"; then + echo "Skipping check: skip changelog label found" + exit 0 + fi + + # Fetch base branch and check for CHANGELOG modifications + git fetch origin ${{ github.base_ref }} + if git diff --name-only origin/${{ github.base_ref }}..HEAD | grep -q "CHANGELOG.md"; then + echo "CHANGELOG.md entry found - check passed" + exit 0 + fi + + echo "It looks like you didn't add an entry to CHANGELOG.md. If this change affects the SDK behavior, please update CHANGELOG.md and link this PR in your entry. If this PR does not need a CHANGELOG entry, you can add the 'Skip Changelog' label to this PR." + exit 1 + + - name: Check for versioned GitHub actions + if: always() + run: | + # Get changed GitHub workflow/action files + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}..HEAD | grep -E "^\.github/(workflows|actions)/.*\.ya?ml$" || true) + + if [ -n "$CHANGED_FILES" ]; then + # Check for any versioned actions, excluding comments and this validation script + VIOLATIONS=$(grep -Hn "uses:.*@v" $CHANGED_FILES | grep -v "grep.*uses:.*@v" | grep -v "#.*@v" || true) + if [ -n "$VIOLATIONS" ]; then + echo "Found versioned GitHub actions. Use commit SHAs instead:" + echo "$VIOLATIONS" + exit 1 + fi + fi + + echo "No versioned actions found in changed files" + build: runs-on: ubuntu-latest strategy: @@ -18,7 +73,7 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 - name: Build Wheel and Image Files uses: ./.github/actions/artifacts_build @@ -40,8 +95,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0 if: ${{ matrix.language == 'python' }} with: python-version: '3.x' @@ -63,7 +118,7 @@ jobs: tox-environment: ["spellcheck", "lint"] steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 - name: Install libsnappy-dev if: ${{ matrix.tox-environment == 'lint' }} @@ -84,19 +139,54 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 - name: Gradle validation - uses: gradle/wrapper-validation-action@v1 + uses: gradle/actions/wrapper-validation@ed408507eac070d1f99cc633dbcf757c94c7933a #4.4.3 - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 #v5.0.0 with: java-version: 17 distribution: temurin - name: Setup Gradle - uses: gradle/gradle-build-action@v3 + uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a #4.4.3 - name: Build with Gradle run: cd performance-tests; ./gradlew spotlessCheck + + all-pr-checks-pass: + runs-on: ubuntu-latest + needs: [static-code-checks, lint, spotless, build, build-lambda] + if: always() + steps: + - name: Checkout to get workflow file + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + + - name: Check all jobs succeeded and none missing + run: | + # Check if all needed jobs succeeded + results='${{ toJSON(needs) }}' + if echo "$results" | jq -r '.[] | .result' | grep -v success; then + echo "Some jobs failed" + exit 1 + fi + + # Extract all job names from workflow (excluding this gate job) + all_jobs=$(yq eval '.jobs | keys | .[]' .github/workflows/pr-build.yml | grep -v "all-pr-checks-pass" | sort) + + # Extract job names from needs array + needed_jobs='${{ toJSON(needs) }}' + needs_list=$(echo "$needed_jobs" | jq -r 'keys[]' | sort) + + # Check if any jobs are missing from needs + missing_jobs=$(comm -23 <(echo "$all_jobs") <(echo "$needs_list")) + if [ -n "$missing_jobs" ]; then + echo "ERROR: Jobs missing from needs array in all-pr-checks-pass:" + echo "$missing_jobs" + echo "Please add these jobs to the needs array of all-pr-checks-pass" + exit 1 + fi + + echo "All checks passed and no jobs missing from gate!" diff --git a/.github/workflows/pre-release-prepare.yml b/.github/workflows/pre-release-prepare.yml index a6f83cc73..a2527f052 100644 --- a/.github/workflows/pre-release-prepare.yml +++ b/.github/workflows/pre-release-prepare.yml @@ -25,13 +25,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Configure AWS credentials for BOT secrets - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #v5.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRETS_MANAGER }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Get Bot secrets - uses: aws-actions/aws-secretsmanager-get-secrets@v1 + uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 #v2.0.10 id: bot_secrets with: secret-ids: | @@ -39,7 +39,7 @@ jobs: parse-json-secrets: true - name: Checkout main branch - uses: actions/checkout@v3 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: ref: 'main' token: ${{ env.BOT_TOKEN_GITHUB_RW_PATOKEN }} diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 6d818c0e7..8cc87d77b 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 - name: Check main build status env: @@ -61,13 +61,13 @@ jobs: # https://github.com/aws-observability/aws-otel-java-instrumentation/tree/93870a550ac30988fbdd5d3bf1e8f9f1b37916f5/smoke-tests - name: Configure AWS credentials for PyPI secrets - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #v5.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRETS_MANAGER }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Get PyPI secrets - uses: aws-actions/aws-secretsmanager-get-secrets@v1 + uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 #v2.0.10 id: pypi_secrets with: secret-ids: | @@ -76,24 +76,24 @@ jobs: parse-json-secrets: true - name: Configure AWS credentials for private ECR - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #v5.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN_ECR_RELEASE }} aws-region: ${{ env.AWS_PRIVATE_ECR_REGION }} - name: Log in to AWS private ECR - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 with: registry: ${{ env.RELEASE_PRIVATE_REGISTRY }} - name: Configure AWS credentials for public ECR - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #v5.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN_ECR_RELEASE }} aws-region: ${{ env.AWS_PUBLIC_ECR_REGION }} - name: Log in to AWS public ECR - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 with: registry: public.ecr.aws @@ -119,7 +119,7 @@ jobs: # Publish to public ECR - name: Build and push public ECR image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0 with: push: true context: . @@ -130,7 +130,7 @@ jobs: # Publish to private ECR - name: Build and push private ECR image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0 with: push: true context: . diff --git a/.github/workflows/release-lambda.yml b/.github/workflows/release-lambda.yml index e887d1d1e..800a509c2 100644 --- a/.github/workflows/release-lambda.yml +++ b/.github/workflows/release-lambda.yml @@ -40,8 +40,8 @@ jobs: echo ${MATRIX} echo "aws_regions_json=${MATRIX}" >> $GITHUB_OUTPUT - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0 with: python-version: '3.x' - name: Build layers @@ -51,7 +51,7 @@ jobs: pip install tox tox - name: upload layer - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #v4.6.2 with: name: layer.zip path: lambda-layer/src/build/aws-opentelemetry-python-layer.zip @@ -83,7 +83,7 @@ jobs: fi SECRET_KEY=${SECRET_KEY//-/_} echo "SECRET_KEY=${SECRET_KEY}" >> $GITHUB_ENV - - uses: aws-actions/configure-aws-credentials@v4.0.2 + - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #v5.0.0 with: role-to-assume: ${{ secrets[env.SECRET_KEY] }} role-duration-seconds: 1200 @@ -92,7 +92,7 @@ jobs: run: | echo BUCKET_NAME=python-lambda-layer-${{ github.run_id }}-${{ matrix.aws_region }} | tee --append $GITHUB_ENV - name: download layer.zip - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 #v5.0.0 with: name: layer.zip - name: publish @@ -130,7 +130,7 @@ jobs: --action lambda:GetLayerVersion - name: upload layer arn artifact if: ${{ success() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #v4.6.2 with: name: ${{ env.LAYER_NAME }}-${{ matrix.aws_region }} path: ${{ env.LAYER_NAME }}/${{ matrix.aws_region }} @@ -143,10 +143,10 @@ jobs: needs: publish-prod steps: - name: Checkout Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd #v3.1.2 - name: download layerARNs - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 #v5.0.0 with: pattern: ${{ env.LAYER_NAME }}-* path: ${{ env.LAYER_NAME }} @@ -195,7 +195,7 @@ jobs: echo "}" >> ../layer_cdk cat ../layer_cdk - name: download layer.zip - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 #v5.0.0 with: name: layer.zip - name: Rename layer file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..bc21e089c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +> **Note:** This CHANGELOG was created starting from version 0.12.0. Earlier changes are not documented here. + +For any change that affects end users of this package, please add an entry under the **Unreleased** section. Briefly summarize the change and provide the link to the PR. Example: +- add GenAI attribute support for Amazon Bedrock models + ([#300](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/300)) + +If your change does not need a CHANGELOG entry, add the "skip changelog" label to your PR. + +## Unreleased +- [PATCH] Only decode JSON input buffer in Anthropic Claude streaming + ([#497](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/497)) diff --git a/Dockerfile b/Dockerfile index 784cb908d..7f20849df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN sed -i "/opentelemetry-exporter-otlp-proto-grpc/d" ./aws-opentelemetry-distr RUN mkdir workspace && pip install --target workspace ./aws-opentelemetry-distro # Stage 2: Build the cp-utility binary -FROM public.ecr.aws/docker/library/rust:1.87 as builder +FROM public.ecr.aws/docker/library/rust:1.87 AS builder WORKDIR /usr/src/cp-utility COPY ./tools/cp-utility . diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py index 10fc77182..7001e18af 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py @@ -327,8 +327,89 @@ def patched_extract_tool_calls( tool_calls.append(tool_call) return tool_calls + # TODO: The following code is to patch a bedrock bug that was fixed in + # opentelemetry-instrumentation-botocore==0.60b0 in: + # https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3875 + # Remove this code once we've bumped opentelemetry-instrumentation-botocore dependency to 0.60b0 + def patched_process_anthropic_claude_chunk(self, chunk): + # pylint: disable=too-many-return-statements,too-many-branches + if not (message_type := chunk.get("type")): + return + + if message_type == "message_start": + # {'type': 'message_start', 'message': {'id': 'id', 'type': 'message', 'role': 'assistant', + # 'model': 'claude-2.0', 'content': [], 'stop_reason': None, 'stop_sequence': None, + # 'usage': {'input_tokens': 18, 'output_tokens': 1}}} + if chunk.get("message", {}).get("role") == "assistant": + self._record_message = True + message = chunk["message"] + self._message = { + "role": message["role"], + "content": message.get("content", []), + } + return + + if message_type == "content_block_start": + # {'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text', 'text': ''}} + # {'type': 'content_block_start', 'index': 1, 'content_block': + # {'type': 'tool_use', 'id': 'id', 'name': 'func_name', 'input': {}}} + if self._record_message: + block = chunk.get("content_block", {}) + if block.get("type") == "text": + self._content_block = block + elif block.get("type") == "tool_use": + self._content_block = block + return + + if message_type == "content_block_delta": + # {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': 'Here'}} + # {'type': 'content_block_delta', 'index': 1, 'delta': {'type': 'input_json_delta', 'partial_json': ''}} + if self._record_message: + delta = chunk.get("delta", {}) + if delta.get("type") == "text_delta": + self._content_block["text"] += delta.get("text", "") + elif delta.get("type") == "input_json_delta": + self._tool_json_input_buf += delta.get("partial_json", "") + return + + if message_type == "content_block_stop": + # {'type': 'content_block_stop', 'index': 0} + if self._tool_json_input_buf: + try: + self._content_block["input"] = json.loads(self._tool_json_input_buf) + except json.JSONDecodeError: + self._content_block["input"] = self._tool_json_input_buf + self._message["content"].append(self._content_block) + self._content_block = {} + self._tool_json_input_buf = "" + return + + if message_type == "message_delta": + # {'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, + # 'usage': {'output_tokens': 123}} + if (stop_reason := chunk.get("delta", {}).get("stop_reason")) is not None: + self._response["stopReason"] = stop_reason + return + + if message_type == "message_stop": + # {'type': 'message_stop', 'amazon-bedrock-invocationMetrics': + # {'inputTokenCount': 18, 'outputTokenCount': 123, 'invocationLatency': 5250, 'firstByteLatency': 290}} + if invocation_metrics := chunk.get("amazon-bedrock-invocationMetrics"): + self._process_invocation_metrics(invocation_metrics) + + if self._record_message: + self._response["output"] = {"message": self._message} + self._record_message = False + self._message = None + + self._stream_done_callback(self._response) + return + bedrock_utils.ConverseStreamWrapper.__init__ = patched_init bedrock_utils.ConverseStreamWrapper._process_event = patched_process_event + bedrock_utils.InvokeModelWithResponseStreamWrapper._process_anthropic_claude_chunk = ( + patched_process_anthropic_claude_chunk + ) bedrock_utils.extract_tool_calls = patched_extract_tool_calls # END The OpenTelemetry Authors code diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py index 256ee3673..c82217fca 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +# pylint: disable=too-many-lines import os from importlib.metadata import PackageNotFoundError from typing import Any, Dict @@ -84,10 +85,6 @@ def _run_patch_behaviour_tests(self): self._test_unpatched_botocore_propagator() self._test_unpatched_gevent_instrumentation() self._test_unpatched_starlette_instrumentation() - # TODO: remove these tests once we bump botocore instrumentation version to 0.56b0 - # Bedrock Runtime tests - self._test_unpatched_converse_stream_wrapper() - self._test_unpatched_extract_tool_calls() # Apply patches apply_instrumentation_patches() @@ -178,6 +175,16 @@ def _test_unpatched_botocore_instrumentation(self): # DynamoDB self.assertTrue("dynamodb" in _KNOWN_EXTENSIONS, "Upstream has removed a DynamoDB extension") + # Bedrock Runtime tests + # TODO: remove these tests once we bump botocore instrumentation version to 0.56b0 + self._test_unpatched_converse_stream_wrapper() + self._test_unpatched_extract_tool_calls() + + # TODO: remove these tests once we bump botocore instrumentation version to 0.60b0 + self._test_unpatched_process_anthropic_claude_chunk({"location": "Seattle"}, {"location": "Seattle"}) + self._test_unpatched_process_anthropic_claude_chunk(None, None) + self._test_unpatched_process_anthropic_claude_chunk({}, {}) + def _test_unpatched_gevent_instrumentation(self): self.assertFalse(gevent.monkey.is_module_patched("os"), "gevent os module has been patched") self.assertFalse(gevent.monkey.is_module_patched("thread"), "gevent thread module has been patched") @@ -223,10 +230,14 @@ def _test_patched_botocore_instrumentation(self): # Bedrock Agent Operation self._test_patched_bedrock_agent_instrumentation() - # TODO: remove these tests once we bump botocore instrumentation version to 0.56b0 # Bedrock Runtime + # TODO: remove these tests once we bump botocore instrumentation version to 0.56b0 self._test_patched_converse_stream_wrapper() self._test_patched_extract_tool_calls() + # TODO: remove these tests once we bump botocore instrumentation version to 0.60b0 + self._test_patched_process_anthropic_claude_chunk({"location": "Seattle"}, {"location": "Seattle"}) + self._test_patched_process_anthropic_claude_chunk(None, None) + self._test_patched_process_anthropic_claude_chunk({}, {}) # Bedrock Agent Runtime self.assertTrue("bedrock-agent-runtime" in _KNOWN_EXTENSIONS) @@ -600,6 +611,94 @@ def _test_patched_extract_tool_calls(self): result = bedrock_utils.extract_tool_calls(message_with_string_content, True) self.assertIsNone(result) + # Test with toolUse format to exercise the for loop + message_with_tool_use = {"role": "assistant", "content": [{"toolUse": {"toolUseId": "id1", "name": "func1"}}]} + result = bedrock_utils.extract_tool_calls(message_with_tool_use, True) + self.assertEqual(len(result), 1) + + # Test with tool_use format to exercise the for loop + message_with_type_tool_use = { + "role": "assistant", + "content": [{"type": "tool_use", "id": "id2", "name": "func2"}], + } + result = bedrock_utils.extract_tool_calls(message_with_type_tool_use, True) + self.assertEqual(len(result), 1) + + def _test_patched_process_anthropic_claude_chunk( + self, input_value: Dict[str, str], expected_output: Dict[str, str] + ): + self._test_process_anthropic_claude_chunk(input_value, expected_output, False) + + def _test_unpatched_process_anthropic_claude_chunk( + self, input_value: Dict[str, str], expected_output: Dict[str, str] + ): + self._test_process_anthropic_claude_chunk(input_value, expected_output, True) + + def _test_process_anthropic_claude_chunk( + self, input_value: Dict[str, str], expected_output: Dict[str, str], expect_exception: bool + ): + """Test that _process_anthropic_claude_chunk handles various tool_use input formats.""" + wrapper = bedrock_utils.InvokeModelWithResponseStreamWrapper( + stream=MagicMock(), + stream_done_callback=MagicMock, + stream_error_callback=MagicMock, + model_id="anthropic.claude-3-5-sonnet-20240620-v1:0", + ) + + # Simulate message_start + wrapper._process_anthropic_claude_chunk( + { + "type": "message_start", + "message": { + "role": "assistant", + "content": [], + }, + } + ) + + # Simulate content_block_start with specified input + content_block = { + "type": "tool_use", + "id": "test_id", + "name": "test_tool", + } + if input_value is not None: + content_block["input"] = input_value + + wrapper._process_anthropic_claude_chunk( + { + "type": "content_block_start", + "index": 0, + "content_block": content_block, + } + ) + + # Simulate content_block_stop + try: + wrapper._process_anthropic_claude_chunk({"type": "content_block_stop", "index": 0}) + except TypeError: + if expect_exception: + return + raise + + # Verify the message content + self.assertEqual(len(wrapper._message["content"]), 1) + tool_block = wrapper._message["content"][0] + self.assertEqual(tool_block["type"], "tool_use") + self.assertEqual(tool_block["id"], "test_id") + self.assertEqual(tool_block["name"], "test_tool") + + if expected_output is not None: + self.assertEqual(tool_block["input"], expected_output) + self.assertIsInstance(tool_block["input"], dict) + else: + self.assertNotIn("input", tool_block) + + # Just adding this to do basic sanity checks and increase code coverage + wrapper._process_anthropic_claude_chunk({"type": "content_block_delta", "index": 0}) + wrapper._process_anthropic_claude_chunk({"type": "message_delta"}) + wrapper._process_anthropic_claude_chunk({"type": "message_stop"}) + def _test_patched_bedrock_agent_instrumentation(self): """For bedrock-agent service, both extract_attributes and on_success provides attributes, the attributes depend on the API being invoked."""