From cf59208241a265e8e67d94b1641ae63ca0e43646 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 19 Feb 2025 13:43:30 +0100 Subject: [PATCH 1/6] Removed AWS auth related code --- scripts/aws-cleanup.sh | 18 ----------- scripts/populate_tox/tox.jinja | 2 -- .../split_tox_gh_actions.py | 12 -------- .../split_tox_gh_actions/templates/base.jinja | 22 -------------- .../templates/check_permissions.jinja | 30 ------------------- .../templates/test_group.jinja | 10 ------- tests/integrations/aws_lambda/client.py | 8 ----- tox.ini | 2 -- 8 files changed, 104 deletions(-) delete mode 100755 scripts/aws-cleanup.sh delete mode 100644 scripts/split_tox_gh_actions/templates/check_permissions.jinja diff --git a/scripts/aws-cleanup.sh b/scripts/aws-cleanup.sh deleted file mode 100755 index 982835c283..0000000000 --- a/scripts/aws-cleanup.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# -# Helper script to clean up AWS Lambda functions created -# by the test suite (tests/integrations/aws_lambda/test_aws.py). -# -# This will delete all Lambda functions named `test_function_*`. -# - -export AWS_DEFAULT_REGION="us-east-1" -export AWS_ACCESS_KEY_ID="$SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID" -export AWS_SECRET_ACCESS_KEY="$SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY" - -for func in $(aws lambda list-functions --output text --query 'Functions[?starts_with(FunctionName, `test_`) == `true`].FunctionName'); do - echo "Deleting $func" - aws lambda delete-function --function-name "$func" -done - -echo "All done! Have a nice day!" diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index 328d37ee9a..813eb6c7df 100644 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -533,8 +533,6 @@ setenv = socket: TESTPATH=tests/integrations/socket passenv = - SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID - SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY SENTRY_PYTHON_TEST_POSTGRES_HOST SENTRY_PYTHON_TEST_POSTGRES_USER SENTRY_PYTHON_TEST_POSTGRES_PASSWORD diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index 5218b0675f..da02594f7b 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -43,14 +43,6 @@ "clickhouse_driver", } -FRAMEWORKS_NEEDING_AWS = { - "aws_lambda", -} - -FRAMEWORKS_NEEDING_GITHUB_SECRETS = { - "aws_lambda", -} - # Frameworks grouped here will be tested together to not hog all GitHub runners. # If you add or remove a group, make sure to git rm the generated YAML file as # well. @@ -292,13 +284,9 @@ def render_template(group, frameworks, py_versions_pinned, py_versions_latest): "group": group, "frameworks": frameworks, "categories": sorted(categories), - "needs_aws_credentials": bool(set(frameworks) & FRAMEWORKS_NEEDING_AWS), "needs_clickhouse": bool(set(frameworks) & FRAMEWORKS_NEEDING_CLICKHOUSE), "needs_postgres": bool(set(frameworks) & FRAMEWORKS_NEEDING_POSTGRES), "needs_redis": bool(set(frameworks) & FRAMEWORKS_NEEDING_REDIS), - "needs_github_secrets": bool( - set(frameworks) & FRAMEWORKS_NEEDING_GITHUB_SECRETS - ), "py_versions": { category: [f'"{version}"' for version in _normalize_py_versions(versions)] for category, versions in py_versions.items() diff --git a/scripts/split_tox_gh_actions/templates/base.jinja b/scripts/split_tox_gh_actions/templates/base.jinja index e69b6f9134..75c988e32a 100644 --- a/scripts/split_tox_gh_actions/templates/base.jinja +++ b/scripts/split_tox_gh_actions/templates/base.jinja @@ -13,15 +13,7 @@ on: - release/** - potel-base - {% if needs_github_secrets %} - # XXX: We are using `pull_request_target` instead of `pull_request` because we want - # this to run on forks with access to the secrets necessary to run the test suite. - # Prefer to use `pull_request` when possible. - pull_request_target: - types: [labeled, opened, reopened, synchronize] - {% else %} pull_request: - {% endif %} # Cancel in progress workflows on pull_requests. # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value @@ -31,27 +23,13 @@ concurrency: permissions: contents: read - {% if needs_github_secrets %} - # `write` is needed to remove the `Trigger: tests using secrets` label - pull-requests: write - {% endif %} env: -{% if needs_aws_credentials %} -{% raw %} - SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID }} - SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY }} -{% endraw %} -{% endif %} BUILD_CACHE_KEY: {% raw %}${{ github.sha }}{% endraw %} CACHED_BUILD_PATHS: | {% raw %}${{ github.workspace }}/dist-serverless{% endraw %} jobs: -{% if needs_github_secrets %} -{% include "check_permissions.jinja" %} -{% endif %} - {% for category in categories %} {% include "test_group.jinja" %} {% endfor %} diff --git a/scripts/split_tox_gh_actions/templates/check_permissions.jinja b/scripts/split_tox_gh_actions/templates/check_permissions.jinja deleted file mode 100644 index 390f447856..0000000000 --- a/scripts/split_tox_gh_actions/templates/check_permissions.jinja +++ /dev/null @@ -1,30 +0,0 @@ - check-permissions: - name: permissions check - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4.2.2 - with: - persist-credentials: false - - - name: Check permissions on PR - if: github.event_name == 'pull_request_target' - run: | - {% raw %} - python3 -uS .github/workflows/scripts/trigger_tests_on_label.py \ - --repo-id ${{ github.event.repository.id }} \ - --pr ${{ github.event.number }} \ - --event ${{ github.event.action }} \ - --username "$ARG_USERNAME" \ - --label-names "$ARG_LABEL_NAMES" - {% endraw %} - env: - {% raw %} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # these can contain special characters - ARG_USERNAME: ${{ github.event.pull_request.user.login }} - ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }} - {% endraw %} - - - name: Check permissions on repo branch - if: github.event_name == 'push' - run: true diff --git a/scripts/split_tox_gh_actions/templates/test_group.jinja b/scripts/split_tox_gh_actions/templates/test_group.jinja index 66e346511d..55f791c637 100644 --- a/scripts/split_tox_gh_actions/templates/test_group.jinja +++ b/scripts/split_tox_gh_actions/templates/test_group.jinja @@ -12,10 +12,6 @@ # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 os: [ubuntu-20.04] - {% if needs_github_secrets %} - needs: check-permissions - {% endif %} - {% if needs_postgres %} services: postgres: @@ -40,12 +36,6 @@ steps: - uses: actions/checkout@v4.2.2 - {% if needs_github_secrets %} - {% raw %} - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - {% endraw %} - {% endif %} - uses: actions/setup-python@v5 with: python-version: {% raw %}${{ matrix.python-version }}{% endraw %} diff --git a/tests/integrations/aws_lambda/client.py b/tests/integrations/aws_lambda/client.py index 20d2102779..f918bd8c8b 100644 --- a/tests/integrations/aws_lambda/client.py +++ b/tests/integrations/aws_lambda/client.py @@ -1,11 +1,3 @@ -import os - -AWS_CREDENTIALS = { - "aws_access_key_id": os.environ["SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID"], - "aws_secret_access_key": os.environ["SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY"], -} - - # This is for inspecting new Python runtime environments in AWS Lambda # If you need to debug a new runtime, use this REPL to run arbitrary Python or bash commands # in that runtime in a Lambda function: diff --git a/tox.ini b/tox.ini index 58d7d44eed..a3f04d3af9 100644 --- a/tox.ini +++ b/tox.ini @@ -798,8 +798,6 @@ setenv = socket: TESTPATH=tests/integrations/socket passenv = - SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID - SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY SENTRY_PYTHON_TEST_POSTGRES_HOST SENTRY_PYTHON_TEST_POSTGRES_USER SENTRY_PYTHON_TEST_POSTGRES_PASSWORD From 6d94dc287df213270f87a4f1d1912e14c24ddbf2 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 19 Feb 2025 13:47:41 +0100 Subject: [PATCH 2/6] Deleted more aws related config --- .../scripts/trigger_tests_on_label.py | 72 ------------------- .github/workflows/test-integrations-aws.yml | 38 ---------- tox.ini | 6 +- 3 files changed, 3 insertions(+), 113 deletions(-) delete mode 100644 .github/workflows/scripts/trigger_tests_on_label.py diff --git a/.github/workflows/scripts/trigger_tests_on_label.py b/.github/workflows/scripts/trigger_tests_on_label.py deleted file mode 100644 index f6039fd16a..0000000000 --- a/.github/workflows/scripts/trigger_tests_on_label.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -from urllib.parse import quote -from urllib.request import Request, urlopen - -LABEL = "Trigger: tests using secrets" - - -def _has_write(repo_id: int, username: str, *, token: str) -> bool: - req = Request( - f"https://api.github.com/repositories/{repo_id}/collaborators/{username}/permission", - headers={"Authorization": f"token {token}"}, - ) - contents = json.load(urlopen(req, timeout=10)) - - return contents["permission"] in {"admin", "write"} - - -def _remove_label(repo_id: int, pr: int, label: str, *, token: str) -> None: - quoted_label = quote(label) - req = Request( - f"https://api.github.com/repositories/{repo_id}/issues/{pr}/labels/{quoted_label}", - method="DELETE", - headers={"Authorization": f"token {token}"}, - ) - urlopen(req) - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--repo-id", type=int, required=True) - parser.add_argument("--pr", type=int, required=True) - parser.add_argument("--event", required=True) - parser.add_argument("--username", required=True) - parser.add_argument("--label-names", type=json.loads, required=True) - args = parser.parse_args() - - token = os.environ["GITHUB_TOKEN"] - - write_permission = _has_write(args.repo_id, args.username, token=token) - - if ( - not write_permission - # `reopened` is included here due to close => push => reopen - and args.event in {"synchronize", "reopened"} - and LABEL in args.label_names - ): - print(f"Invalidating label [{LABEL}] due to code change...") - _remove_label(args.repo_id, args.pr, LABEL, token=token) - args.label_names.remove(LABEL) - - if write_permission or LABEL in args.label_names: - print("Permissions passed!") - print(f"- has write permission: {write_permission}") - print(f"- has [{LABEL}] label: {LABEL in args.label_names}") - return 0 - else: - print("Permissions failed!") - print(f"- has write permission: {write_permission}") - print(f"- has [{LABEL}] label: {LABEL in args.label_names}") - print(f"- args.label_names: {args.label_names}") - print( - f"Please have a collaborator add the [{LABEL}] label once they " - f"have reviewed the code to trigger tests." - ) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/workflows/test-integrations-aws.yml b/.github/workflows/test-integrations-aws.yml index f20125fc23..5c138348e7 100644 --- a/.github/workflows/test-integrations-aws.yml +++ b/.github/workflows/test-integrations-aws.yml @@ -9,9 +9,6 @@ on: - master - release/** - potel-base - # XXX: We are using `pull_request_target` instead of `pull_request` because we want - # this to run on forks with access to the secrets necessary to run the test suite. - # Prefer to use `pull_request` when possible. pull_request: # Cancel in progress workflows on pull_requests. # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value @@ -20,39 +17,11 @@ concurrency: cancel-in-progress: true permissions: contents: read - # `write` is needed to remove the `Trigger: tests using secrets` label - pull-requests: write env: - SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID }} - SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY }} BUILD_CACHE_KEY: ${{ github.sha }} CACHED_BUILD_PATHS: | ${{ github.workspace }}/dist-serverless jobs: - check-permissions: - name: permissions check - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4.2.2 - with: - persist-credentials: false - - name: Check permissions on PR - if: github.event_name == 'pull_request_target' - run: | - python3 -uS .github/workflows/scripts/trigger_tests_on_label.py \ - --repo-id ${{ github.event.repository.id }} \ - --pr ${{ github.event.number }} \ - --event ${{ github.event.action }} \ - --username "$ARG_USERNAME" \ - --label-names "$ARG_LABEL_NAMES" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # these can contain special characters - ARG_USERNAME: ${{ github.event.pull_request.user.login }} - ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }} - - name: Check permissions on repo branch - if: github.event_name == 'push' - run: true test-aws-pinned: name: AWS (pinned) timeout-minutes: 30 @@ -66,15 +35,8 @@ jobs: # ubuntu-20.04 is the last version that supported python3.6 # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 os: [ubuntu-20.04] - needs: check-permissions - services: - docker: - image: docker:dind # Docker-in-Docker - options: --privileged steps: - uses: actions/checkout@v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/tox.ini b/tox.ini index a3f04d3af9..52e9c97445 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-02-18T12:57:32.874168+00:00 +# Last generated: 2025-02-19T12:44:40.907388+00:00 [tox] requires = @@ -211,7 +211,7 @@ envlist = {py3.8,py3.10,py3.11}-ariadne-v0.20.1 {py3.8,py3.11,py3.12}-ariadne-v0.22 {py3.8,py3.11,py3.12}-ariadne-v0.24.0 - {py3.8,py3.11,py3.12}-ariadne-v0.25.2 + {py3.9,py3.12,py3.13}-ariadne-v0.26.0 {py3.6,py3.9,py3.10}-gql-v3.4.1 {py3.7,py3.11,py3.12}-gql-v3.5.0 @@ -597,7 +597,7 @@ deps = ariadne-v0.20.1: ariadne==0.20.1 ariadne-v0.22: ariadne==0.22 ariadne-v0.24.0: ariadne==0.24.0 - ariadne-v0.25.2: ariadne==0.25.2 + ariadne-v0.26.0: ariadne==0.26.0 ariadne: fastapi ariadne: flask ariadne: httpx From 4f1e51bcda9e573a32df702cf41caf289fae4586 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 19 Feb 2025 14:34:18 +0100 Subject: [PATCH 3/6] Moved AWS into cloud group again, because it is now a normal test suite --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/test-integrations-aws.yml | 91 ------------------- .github/workflows/test-integrations-cloud.yml | 10 +- scripts/populate_tox/tox.jinja | 11 +-- .../split_tox_gh_actions.py | 6 +- tox.ini | 11 +-- 6 files changed, 19 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/test-integrations-aws.yml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f0002fe486..12db62315a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,4 +4,4 @@ Thank you for contributing to `sentry-python`! Please add tests to validate your changes, and lint your code using `tox -e linters`. -Running the test suite on your PR might require maintainer approval. The AWS Lambda tests additionally require a maintainer to add a special label, and they will fail until this label is added. +Running the test suite on your PR might require maintainer approval. \ No newline at end of file diff --git a/.github/workflows/test-integrations-aws.yml b/.github/workflows/test-integrations-aws.yml deleted file mode 100644 index 5c138348e7..0000000000 --- a/.github/workflows/test-integrations-aws.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Do not edit this YAML file. This file is generated automatically by executing -# python scripts/split_tox_gh_actions/split_tox_gh_actions.py -# The template responsible for it is in -# scripts/split_tox_gh_actions/templates/base.jinja -name: Test AWS -on: - push: - branches: - - master - - release/** - - potel-base - pull_request: -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true -permissions: - contents: read -env: - BUILD_CACHE_KEY: ${{ github.sha }} - CACHED_BUILD_PATHS: | - ${{ github.workspace }}/dist-serverless -jobs: - test-aws-pinned: - name: AWS (pinned) - timeout-minutes: 30 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.8","3.10","3.12","3.13"] - # python3.6 reached EOL and is no longer being supported on - # new versions of hosted runners on Github Actions - # ubuntu-20.04 is the last version that supported python3.6 - # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 - os: [ubuntu-20.04] - steps: - - uses: actions/checkout@v4.2.2 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Setup Test Env - run: | - pip install "coverage[toml]" tox - - name: Erase coverage - run: | - coverage erase - - name: Test aws_lambda pinned - run: | - set -x # print commands that are executed - ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-aws_lambda" - - name: Generate coverage XML (Python 3.6) - if: ${{ !cancelled() && matrix.python-version == '3.6' }} - run: | - export COVERAGE_RCFILE=.coveragerc36 - coverage combine .coverage-sentry-* - coverage xml --ignore-errors - - name: Generate coverage XML - if: ${{ !cancelled() && matrix.python-version != '3.6' }} - run: | - coverage combine .coverage-sentry-* - coverage xml - - name: Upload coverage to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5.3.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - # make sure no plugins alter our coverage reports - plugin: noop - verbose: true - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: .junitxml - verbose: true - check_required_tests: - name: All pinned AWS tests passed - needs: test-aws-pinned - # Always run this, even if a dependent job failed - if: always() - runs-on: ubuntu-20.04 - steps: - - name: Check for failures - if: contains(needs.test-aws-pinned.result, 'failure') || contains(needs.test-aws-pinned.result, 'skipped') - run: | - echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/test-integrations-cloud.yml b/.github/workflows/test-integrations-cloud.yml index b929b8d899..26779537e1 100644 --- a/.github/workflows/test-integrations-cloud.yml +++ b/.github/workflows/test-integrations-cloud.yml @@ -47,6 +47,10 @@ jobs: - name: Erase coverage run: | coverage erase + - name: Test aws_lambda latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda-latest" - name: Test boto3 latest run: | set -x # print commands that are executed @@ -97,7 +101,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6","3.7","3.9","3.11","3.12","3.13"] + python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12","3.13"] # python3.6 reached EOL and is no longer being supported on # new versions of hosted runners on Github Actions # ubuntu-20.04 is the last version that supported python3.6 @@ -115,6 +119,10 @@ jobs: - name: Erase coverage run: | coverage erase + - name: Test aws_lambda pinned + run: | + set -x # print commands that are executed + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-aws_lambda" - name: Test boto3 pinned run: | set -x # print commands that are executed diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index 813eb6c7df..9da986a35a 100644 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -57,10 +57,7 @@ envlist = {py3.8,py3.11,py3.12}-asyncpg-latest # AWS Lambda - # The aws_lambda tests deploy to the real AWS and have their own - # matrix of Python versions to run the test lambda function in. - # see `lambda_runtime` fixture in tests/integrations/aws_lambda.py - {py3.8,py3.10,py3.12,py3.13}-aws_lambda + {py3.8,py3.9,py3.11,py3.13}-aws_lambda # Beam {py3.7}-beam-v{2.12} @@ -250,12 +247,12 @@ deps = asyncpg: pytest-asyncio # AWS Lambda + aws_lambda: aws-cdk-lib + aws_lambda: aws-sam-cli aws_lambda: boto3 aws_lambda: fastapi - aws_lambda: uvicorn - aws_lambda: aws-sam-cli - aws_lambda: aws-cdk-lib aws_lambda: requests + aws_lambda: uvicorn # Beam beam-v2.12: apache-beam~=2.12.0 diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index da02594f7b..bc5994c90a 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -57,12 +57,8 @@ "openai", "huggingface_hub", ], - "AWS": [ - # this is separate from Cloud Computing because only this one test suite - # needs to run with access to GitHub secrets - "aws_lambda", - ], "Cloud": [ + "aws_lambda", "boto3", "chalice", "cloud_resource_context", diff --git a/tox.ini b/tox.ini index 52e9c97445..3f78f7bf87 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-02-19T12:44:40.907388+00:00 +# Last generated: 2025-02-19T13:16:21.819887+00:00 [tox] requires = @@ -57,9 +57,6 @@ envlist = {py3.8,py3.11,py3.12}-asyncpg-latest # AWS Lambda - # The aws_lambda tests deploy to the real AWS and have their own - # matrix of Python versions to run the test lambda function in. - # see `lambda_runtime` fixture in tests/integrations/aws_lambda.py {py3.8,py3.10,py3.12,py3.13}-aws_lambda # Beam @@ -362,12 +359,12 @@ deps = asyncpg: pytest-asyncio # AWS Lambda + aws_lambda: aws-cdk-lib + aws_lambda: aws-sam-cli aws_lambda: boto3 aws_lambda: fastapi - aws_lambda: uvicorn - aws_lambda: aws-sam-cli - aws_lambda: aws-cdk-lib aws_lambda: requests + aws_lambda: uvicorn # Beam beam-v2.12: apache-beam~=2.12.0 From 002139a81a5d7e2d6c8cc62f97469dae73bd02a4 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 19 Feb 2025 14:41:12 +0100 Subject: [PATCH 4/6] Added docker config to yml files again --- .github/workflows/test-integrations-cloud.yml | 8 ++++++++ scripts/split_tox_gh_actions/split_tox_gh_actions.py | 5 +++++ scripts/split_tox_gh_actions/templates/test_group.jinja | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/.github/workflows/test-integrations-cloud.yml b/.github/workflows/test-integrations-cloud.yml index 26779537e1..5dc4acb8ca 100644 --- a/.github/workflows/test-integrations-cloud.yml +++ b/.github/workflows/test-integrations-cloud.yml @@ -35,6 +35,10 @@ jobs: # ubuntu-20.04 is the last version that supported python3.6 # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 os: [ubuntu-20.04] + services: + docker: + image: docker:dind # Required for Docker network management + options: --privileged # Required for Docker-in-Docker operations steps: - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 @@ -107,6 +111,10 @@ jobs: # ubuntu-20.04 is the last version that supported python3.6 # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 os: [ubuntu-20.04] + services: + docker: + image: docker:dind # Required for Docker network management + options: --privileged # Required for Docker-in-Docker operations steps: - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index bc5994c90a..293af897c9 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -43,6 +43,10 @@ "clickhouse_driver", } +FRAMEWORKS_NEEDING_DOCKER = { + "aws_lambda", +} + # Frameworks grouped here will be tested together to not hog all GitHub runners. # If you add or remove a group, make sure to git rm the generated YAML file as # well. @@ -281,6 +285,7 @@ def render_template(group, frameworks, py_versions_pinned, py_versions_latest): "frameworks": frameworks, "categories": sorted(categories), "needs_clickhouse": bool(set(frameworks) & FRAMEWORKS_NEEDING_CLICKHOUSE), + "needs_docker": bool(set(frameworks) & FRAMEWORKS_NEEDING_DOCKER), "needs_postgres": bool(set(frameworks) & FRAMEWORKS_NEEDING_POSTGRES), "needs_redis": bool(set(frameworks) & FRAMEWORKS_NEEDING_REDIS), "py_versions": { diff --git a/scripts/split_tox_gh_actions/templates/test_group.jinja b/scripts/split_tox_gh_actions/templates/test_group.jinja index 55f791c637..ac768db9ea 100644 --- a/scripts/split_tox_gh_actions/templates/test_group.jinja +++ b/scripts/split_tox_gh_actions/templates/test_group.jinja @@ -12,6 +12,12 @@ # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 os: [ubuntu-20.04] + {% if needs_docker %} + services: + docker: + image: docker:dind # Required for Docker network management + options: --privileged # Required for Docker-in-Docker operations + {% endif %} {% if needs_postgres %} services: postgres: From 2cde31a2e71e674516e46bfd9dd34e2e63aa1db8 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 19 Feb 2025 14:45:14 +0100 Subject: [PATCH 5/6] nicer diffs for other pr --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 58d7d44eed..c6f313bdc4 100644 --- a/tox.ini +++ b/tox.ini @@ -362,12 +362,12 @@ deps = asyncpg: pytest-asyncio # AWS Lambda + aws_lambda: aws-cdk-lib + aws_lambda: aws-sam-cli aws_lambda: boto3 aws_lambda: fastapi - aws_lambda: uvicorn - aws_lambda: aws-sam-cli - aws_lambda: aws-cdk-lib aws_lambda: requests + aws_lambda: uvicorn # Beam beam-v2.12: apache-beam~=2.12.0 From bad499413e83c8c55d60d17fcc5659ca9dc1687d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 24 Feb 2025 12:29:40 +0100 Subject: [PATCH 6/6] Cleaning up some AWS related scripts --- .../aws-attach-layer-to-lambda-function.sh | 0 .../aws-delete-lambda-layer-versions.sh | 1 + scripts/{ => aws}/aws-deploy-local-layer.sh | 3 +- scripts/aws_lambda_functions/README.md | 4 -- .../sentryPythonDeleteTestFunctions/README.md | 13 ----- .../lambda_function.py | 55 ------------------- 6 files changed, 2 insertions(+), 74 deletions(-) rename scripts/{ => aws}/aws-attach-layer-to-lambda-function.sh (100%) rename scripts/{ => aws}/aws-delete-lambda-layer-versions.sh (95%) rename scripts/{ => aws}/aws-deploy-local-layer.sh (81%) delete mode 100644 scripts/aws_lambda_functions/README.md delete mode 100644 scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/README.md delete mode 100644 scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/lambda_function.py diff --git a/scripts/aws-attach-layer-to-lambda-function.sh b/scripts/aws/aws-attach-layer-to-lambda-function.sh similarity index 100% rename from scripts/aws-attach-layer-to-lambda-function.sh rename to scripts/aws/aws-attach-layer-to-lambda-function.sh diff --git a/scripts/aws-delete-lambda-layer-versions.sh b/scripts/aws/aws-delete-lambda-layer-versions.sh similarity index 95% rename from scripts/aws-delete-lambda-layer-versions.sh rename to scripts/aws/aws-delete-lambda-layer-versions.sh index f467f9398b..dcbd2f9c65 100755 --- a/scripts/aws-delete-lambda-layer-versions.sh +++ b/scripts/aws/aws-delete-lambda-layer-versions.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # Deletes all versions of the layer specified in LAYER_NAME in one region. +# Use with caution! # set -euo pipefail diff --git a/scripts/aws-deploy-local-layer.sh b/scripts/aws/aws-deploy-local-layer.sh similarity index 81% rename from scripts/aws-deploy-local-layer.sh rename to scripts/aws/aws-deploy-local-layer.sh index 56f2087596..ee7b3e45c0 100755 --- a/scripts/aws-deploy-local-layer.sh +++ b/scripts/aws/aws-deploy-local-layer.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash # -# Builds and deploys the Sentry AWS Lambda layer (including the Sentry SDK and the Sentry Lambda Extension) +# Builds and deploys the `SentryPythonServerlessSDK-local-dev` AWS Lambda layer (containing the Sentry SDK) # # The currently checked out version of the SDK in your local directory is used. -# The latest version of the Lambda Extension is fetched from the Sentry Release Registry. # set -euo pipefail diff --git a/scripts/aws_lambda_functions/README.md b/scripts/aws_lambda_functions/README.md deleted file mode 100644 index e07b445d5b..0000000000 --- a/scripts/aws_lambda_functions/README.md +++ /dev/null @@ -1,4 +0,0 @@ -aws_lambda_functions -==================== - -In this directory you can place AWS Lambda functions that are used for administrative tasks (or whatever) \ No newline at end of file diff --git a/scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/README.md b/scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/README.md deleted file mode 100644 index de1120a026..0000000000 --- a/scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/README.md +++ /dev/null @@ -1,13 +0,0 @@ -sentryPythonDeleteTestFunctions -=============================== - -This AWS Lambda function deletes all AWS Lambda functions in the current AWS account that are prefixed with `test_`. -The functions that are deleted are created by the Google Actions CI checks running on every PR of the `sentry-python` repository. - -The Lambda function has been deployed here: -- AWS Account ID: `943013980633` -- Region: `us-east-1` -- Function ARN: `arn:aws:lambda:us-east-1:943013980633:function:sentryPythonDeleteTestFunctions` - -This function also emits Sentry Metrics and Sentry Crons checkins to the `sentry-python` project in the `Sentry SDKs` organisation on Sentry.io: -https://sentry-sdks.sentry.io/projects/sentry-python/?project=5461230 \ No newline at end of file diff --git a/scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/lambda_function.py b/scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/lambda_function.py deleted file mode 100644 index ce7afb6aa4..0000000000 --- a/scripts/aws_lambda_functions/sentryPythonDeleteTestFunctions/lambda_function.py +++ /dev/null @@ -1,55 +0,0 @@ -import boto3 -import sentry_sdk - - -monitor_slug = "python-sdk-aws-lambda-tests-cleanup" -monitor_config = { - "schedule": { - "type": "crontab", - "value": "0 12 * * 0", # 12 o'clock on Sunday - }, - "timezone": "UTC", - "checkin_margin": 2, - "max_runtime": 20, - "failure_issue_threshold": 1, - "recovery_threshold": 1, -} - - -@sentry_sdk.crons.monitor(monitor_slug=monitor_slug) -def delete_lambda_functions(prefix="test_"): - """ - Delete all AWS Lambda functions in the current account - where the function name matches the prefix - """ - client = boto3.client("lambda", region_name="us-east-1") - functions_deleted = 0 - - functions_paginator = client.get_paginator("list_functions") - for functions_page in functions_paginator.paginate(): - for func in functions_page["Functions"]: - function_name = func["FunctionName"] - if function_name.startswith(prefix): - try: - response = client.delete_function( - FunctionName=func["FunctionArn"], - ) - functions_deleted += 1 - except Exception as ex: - print(f"Got exception: {ex}") - - return functions_deleted - - -def lambda_handler(event, context): - functions_deleted = delete_lambda_functions() - - sentry_sdk.metrics.gauge( - key="num_aws_functions_deleted", - value=functions_deleted, - ) - - return { - "statusCode": 200, - "body": f"{functions_deleted} AWS Lambda functions deleted successfully.", - }