From 2174c51f918e5f0528aa73dfe50135b56070773a Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 20 Nov 2024 15:02:06 -0500 Subject: [PATCH 1/5] PYTHON-4965 - Consolidate startup and teardown tasks --- .evergreen/config.yml | 80 ++++++++++++------------------------------- 1 file changed, 21 insertions(+), 59 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 59b8a543fd..931265e017 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -37,6 +37,8 @@ functions: # Applies the subitted patch, if any # Deprecated. Should be removed. But still needed for certain agents (ZAP) - command: git.apply_patch + + "setup system": # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: @@ -49,13 +51,19 @@ functions: - command: expansions.update params: file: src/expansion.yml - - "prepare resources": - command: subprocess.exec params: + include_expansions_in_env: ["PROJECT_DIRECTORY", "DRIVERS_TOOLS"] binary: bash args: - src/.evergreen/scripts/prepare-resources.sh + # Run drivers-evergreen-tools system setup + - command: subprocess.exec + params: + include_expansions_in_env: ["PROJECT_DIRECTORY", "DRIVERS_TOOLS"] + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/setup.sh "upload coverage" : - command: ec2.assume_role @@ -519,34 +527,6 @@ functions: args: - ${DRIVERS_TOOLS}/.evergreen/teardown.sh - "fix absolute paths": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/fix-absolute-paths.sh - - "windows fix": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/windows-fix.sh - - "make files executable": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/make-files-executable.sh - - "init test-results": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/init-test-results.sh - "install dependencies": - command: subprocess.exec params: @@ -665,11 +645,7 @@ functions: pre: - func: "fetch source" - - func: "prepare resources" - - func: "windows fix" - - func: "fix absolute paths" - - func: "init test-results" - - func: "make files executable" + - func: "setup system" - func: "install dependencies" - func: "assume ec2 role" @@ -691,7 +667,7 @@ task_groups: setup_group_timeout_secs: 1800 # 30 minutes setup_group: - func: "fetch source" - - func: "prepare resources" + - func: "setup system" - command: subprocess.exec params: binary: bash @@ -714,9 +690,7 @@ task_groups: setup_group_timeout_secs: 1800 # 30 minutes setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - command: subprocess.exec params: binary: bash @@ -735,9 +709,7 @@ task_groups: - name: testazurekms_task_group setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - command: subprocess.exec params: binary: bash @@ -761,9 +733,7 @@ task_groups: - name: testazureoidc_task_group setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - command: subprocess.exec params: binary: bash @@ -785,9 +755,7 @@ task_groups: - name: testgcpoidc_task_group setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - command: subprocess.exec params: binary: bash @@ -809,9 +777,7 @@ task_groups: - name: testk8soidc_task_group setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - command: ec2.assume_role params: role_arn: ${aws_test_secrets_role} @@ -835,9 +801,7 @@ task_groups: - name: testoidc_task_group setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - func: "assume ec2 role" - command: subprocess.exec params: @@ -859,7 +823,7 @@ task_groups: - name: test_aws_lambda_task_group setup_group: - func: fetch source - - func: prepare resources + - func: setup system - func: setup atlas teardown_task: - func: teardown atlas @@ -871,9 +835,7 @@ task_groups: - name: test_atlas_task_group_search_indexes setup_group: - func: fetch source - - func: prepare resources - - func: fix absolute paths - - func: make files executable + - func: setup system - func: setup atlas teardown_task: - func: teardown atlas @@ -1584,7 +1546,7 @@ tasks: - name: testazurekms-fail-task commands: - func: fetch source - - func: make files executable + - func: setup system - func: "bootstrap mongo-orchestration" vars: VERSION: "latest" From 313f95466da26d77cdf9ff8c6d4110f301511cc3 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 21 Nov 2024 13:44:35 -0500 Subject: [PATCH 2/5] Add executable-shell pre-commit hook --- .evergreen/combine-coverage.sh | 0 .evergreen/hatch.sh | 0 .evergreen/install-dependencies.sh | 0 .evergreen/run-azurekms-fail-test.sh | 0 .evergreen/run-azurekms-test.sh | 0 .evergreen/run-deployed-lambda-aws-tests.sh | 0 .evergreen/run-gcpkms-test.sh | 0 .evergreen/run-perf-tests.sh | 0 .evergreen/scripts/archive-mongodb-logs.sh | 0 .evergreen/scripts/bootstrap-mongo-orchestration.sh | 0 .evergreen/scripts/check-import-time.sh | 0 .evergreen/scripts/cleanup.sh | 0 .evergreen/scripts/configure-env.sh | 0 .evergreen/scripts/download-and-merge-coverage.sh | 0 .evergreen/scripts/fix-absolute-paths.sh | 0 .evergreen/scripts/init-test-results.sh | 0 .evergreen/scripts/install-dependencies.sh | 0 .evergreen/scripts/make-files-executable.sh | 0 .evergreen/scripts/prepare-resources.sh | 0 .evergreen/scripts/run-atlas-tests.sh | 0 .evergreen/scripts/run-aws-ecs-auth-test.sh | 0 .evergreen/scripts/run-doctests.sh | 0 .evergreen/scripts/run-enterprise-auth-tests.sh | 0 .evergreen/scripts/run-gcpkms-fail-test.sh | 0 .evergreen/scripts/run-getdata.sh | 0 .evergreen/scripts/run-load-balancer.sh | 0 .evergreen/scripts/run-mockupdb-tests.sh | 0 .evergreen/scripts/run-mod-wsgi-tests.sh | 0 .evergreen/scripts/run-ocsp-test.sh | 0 .evergreen/scripts/run-perf-tests.sh | 0 .evergreen/scripts/run-tests.sh | 0 .evergreen/scripts/run-with-env.sh | 0 .evergreen/scripts/setup-encryption.sh | 0 .evergreen/scripts/setup-tests.sh | 0 .evergreen/scripts/stop-load-balancer.sh | 0 .evergreen/scripts/teardown-aws.sh | 0 .evergreen/scripts/teardown-docker.sh | 0 .evergreen/scripts/upload-coverage-report.sh | 0 .evergreen/scripts/windows-fix.sh | 0 .evergreen/setup-encryption.sh | 0 .evergreen/teardown-encryption.sh | 0 .pre-commit-config.yaml | 12 ++++++++++++ tools/synchro.sh | 0 43 files changed, 12 insertions(+) mode change 100644 => 100755 .evergreen/combine-coverage.sh mode change 100644 => 100755 .evergreen/hatch.sh mode change 100644 => 100755 .evergreen/install-dependencies.sh mode change 100644 => 100755 .evergreen/run-azurekms-fail-test.sh mode change 100644 => 100755 .evergreen/run-azurekms-test.sh mode change 100644 => 100755 .evergreen/run-deployed-lambda-aws-tests.sh mode change 100644 => 100755 .evergreen/run-gcpkms-test.sh mode change 100644 => 100755 .evergreen/run-perf-tests.sh mode change 100644 => 100755 .evergreen/scripts/archive-mongodb-logs.sh mode change 100644 => 100755 .evergreen/scripts/bootstrap-mongo-orchestration.sh mode change 100644 => 100755 .evergreen/scripts/check-import-time.sh mode change 100644 => 100755 .evergreen/scripts/cleanup.sh mode change 100644 => 100755 .evergreen/scripts/configure-env.sh mode change 100644 => 100755 .evergreen/scripts/download-and-merge-coverage.sh mode change 100644 => 100755 .evergreen/scripts/fix-absolute-paths.sh mode change 100644 => 100755 .evergreen/scripts/init-test-results.sh mode change 100644 => 100755 .evergreen/scripts/install-dependencies.sh mode change 100644 => 100755 .evergreen/scripts/make-files-executable.sh mode change 100644 => 100755 .evergreen/scripts/prepare-resources.sh mode change 100644 => 100755 .evergreen/scripts/run-atlas-tests.sh mode change 100644 => 100755 .evergreen/scripts/run-aws-ecs-auth-test.sh mode change 100644 => 100755 .evergreen/scripts/run-doctests.sh mode change 100644 => 100755 .evergreen/scripts/run-enterprise-auth-tests.sh mode change 100644 => 100755 .evergreen/scripts/run-gcpkms-fail-test.sh mode change 100644 => 100755 .evergreen/scripts/run-getdata.sh mode change 100644 => 100755 .evergreen/scripts/run-load-balancer.sh mode change 100644 => 100755 .evergreen/scripts/run-mockupdb-tests.sh mode change 100644 => 100755 .evergreen/scripts/run-mod-wsgi-tests.sh mode change 100644 => 100755 .evergreen/scripts/run-ocsp-test.sh mode change 100644 => 100755 .evergreen/scripts/run-perf-tests.sh mode change 100644 => 100755 .evergreen/scripts/run-tests.sh mode change 100644 => 100755 .evergreen/scripts/run-with-env.sh mode change 100644 => 100755 .evergreen/scripts/setup-encryption.sh mode change 100644 => 100755 .evergreen/scripts/setup-tests.sh mode change 100644 => 100755 .evergreen/scripts/stop-load-balancer.sh mode change 100644 => 100755 .evergreen/scripts/teardown-aws.sh mode change 100644 => 100755 .evergreen/scripts/teardown-docker.sh mode change 100644 => 100755 .evergreen/scripts/upload-coverage-report.sh mode change 100644 => 100755 .evergreen/scripts/windows-fix.sh mode change 100644 => 100755 .evergreen/setup-encryption.sh mode change 100644 => 100755 .evergreen/teardown-encryption.sh mode change 100644 => 100755 tools/synchro.sh diff --git a/.evergreen/combine-coverage.sh b/.evergreen/combine-coverage.sh old mode 100644 new mode 100755 diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh old mode 100644 new mode 100755 diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/check-import-time.sh b/.evergreen/scripts/check-import-time.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/download-and-merge-coverage.sh b/.evergreen/scripts/download-and-merge-coverage.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/fix-absolute-paths.sh b/.evergreen/scripts/fix-absolute-paths.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/init-test-results.sh b/.evergreen/scripts/init-test-results.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/make-files-executable.sh b/.evergreen/scripts/make-files-executable.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-aws-ecs-auth-test.sh b/.evergreen/scripts/run-aws-ecs-auth-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-doctests.sh b/.evergreen/scripts/run-doctests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-load-balancer.sh b/.evergreen/scripts/run-load-balancer.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-mockupdb-tests.sh b/.evergreen/scripts/run-mockupdb-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-mod-wsgi-tests.sh b/.evergreen/scripts/run-mod-wsgi-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-perf-tests.sh b/.evergreen/scripts/run-perf-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/stop-load-balancer.sh b/.evergreen/scripts/stop-load-balancer.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/teardown-aws.sh b/.evergreen/scripts/teardown-aws.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/teardown-docker.sh b/.evergreen/scripts/teardown-docker.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/upload-coverage-report.sh b/.evergreen/scripts/upload-coverage-report.sh old mode 100644 new mode 100755 diff --git a/.evergreen/scripts/windows-fix.sh b/.evergreen/scripts/windows-fix.sh old mode 100644 new mode 100755 diff --git a/.evergreen/setup-encryption.sh b/.evergreen/setup-encryption.sh old mode 100644 new mode 100755 diff --git a/.evergreen/teardown-encryption.sh b/.evergreen/teardown-encryption.sh old mode 100644 new mode 100755 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6e2b497e59..4f6759bc5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -102,3 +102,15 @@ repos: # - test/versioned-api/crud-api-version-1-strict.json:514: nin ==> inn, min, bin, nine # - test/test_client.py:188: te ==> the, be, we, to args: ["-L", "fle,fo,infinit,isnt,nin,te,aks"] + +- repo: local + hooks: + - id: executable-shell + name: executable-shell + entry: chmod +x + language: system + types: [shell] + exclude: | + (?x)( + .evergreen/retry-with-backoff.sh + ) diff --git a/tools/synchro.sh b/tools/synchro.sh old mode 100644 new mode 100755 From 9cd3a6175a7ccb3f67a55fd8e3569bb6581dddba Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Mon, 25 Nov 2024 11:06:10 -0500 Subject: [PATCH 3/5] Consolidate teardowns --- .evergreen/config.yml | 17 +++++------------ .evergreen/scripts/teardown.sh | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) create mode 100755 .evergreen/scripts/teardown.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 931265e017..dca9337ea3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -519,7 +519,7 @@ functions: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/cleanup.sh - "teardown": + "teardown system": - command: subprocess.exec params: binary: bash @@ -601,20 +601,13 @@ functions: args: - src/.evergreen/scripts/stop-load-balancer.sh - "teardown_docker": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/teardown-docker.sh - - "teardown_aws": + "teardown": - command: subprocess.exec params: binary: bash args: - src/.evergreen/scripts/run-with-env.sh - - src/.evergreen/scripts/teardown-aws.sh + - src/.evergreen/scripts/teardown.sh "teardown atlas": - command: subprocess.exec @@ -652,14 +645,13 @@ pre: post: # Disabled, causing timeouts # - func: "upload working dir" + - func: "teardown system" - func: "teardown" - func: "upload coverage" - func: "upload mo artifacts" - func: "upload test results" - func: "stop mongo-orchestration" - - func: "teardown_aws" - func: "cleanup" - - func: "teardown_docker" task_groups: - name: serverless_task_group @@ -1602,6 +1594,7 @@ tasks: params: binary: bash working_dir: src + include_expansions_in_env: ["PYTHON_BINARY"] args: - .evergreen/scripts/check-import-time.sh - ${revision} diff --git a/.evergreen/scripts/teardown.sh b/.evergreen/scripts/teardown.sh new file mode 100755 index 0000000000..998e174a26 --- /dev/null +++ b/.evergreen/scripts/teardown.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Teardown AWS +pushd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || true +if [ -f "./aws_e2e_setup.json" ]; then + . ./activate-authawsvenv.sh + python ./lib/aws_assign_instance_profile.py +fi + +popd || true + +# Remove all Docker images +DOCKER=$(command -v docker) || true +if [ -n "$DOCKER" ]; then + docker rmi -f "$(docker images -a -q)" &> /dev/null || true +fi From c136fbce6a8b8a9f969923901027eae64ac14292 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Mon, 2 Dec 2024 11:31:02 -0500 Subject: [PATCH 4/5] Use drivers-evergreen-tools teardown --- .evergreen/config.yml | 28 +++++++++++++++++++--------- .evergreen/scripts/teardown.sh | 16 ---------------- 2 files changed, 19 insertions(+), 25 deletions(-) delete mode 100755 .evergreen/scripts/teardown.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index dca9337ea3..8fde98af08 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -520,6 +520,25 @@ functions: - .evergreen/scripts/cleanup.sh "teardown system": + - command: subprocess.exec + params: + binary: bash + working_dir: "src" + args: + # Ensure the instance profile is reassigned for aws tests. + - ${DRIVERS_TOOLS}/.evergreen/auth_aws/teardown.sh + - command: subprocess.exec + params: + binary: bash + working_dir: "src" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh + - command: subprocess.exec + params: + binary: bash + working_dir: "src" + args: + - ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh - command: subprocess.exec params: binary: bash @@ -601,14 +620,6 @@ functions: args: - src/.evergreen/scripts/stop-load-balancer.sh - "teardown": - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/run-with-env.sh - - src/.evergreen/scripts/teardown.sh - "teardown atlas": - command: subprocess.exec params: @@ -646,7 +657,6 @@ post: # Disabled, causing timeouts # - func: "upload working dir" - func: "teardown system" - - func: "teardown" - func: "upload coverage" - func: "upload mo artifacts" - func: "upload test results" diff --git a/.evergreen/scripts/teardown.sh b/.evergreen/scripts/teardown.sh deleted file mode 100755 index 998e174a26..0000000000 --- a/.evergreen/scripts/teardown.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Teardown AWS -pushd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || true -if [ -f "./aws_e2e_setup.json" ]; then - . ./activate-authawsvenv.sh - python ./lib/aws_assign_instance_profile.py -fi - -popd || true - -# Remove all Docker images -DOCKER=$(command -v docker) || true -if [ -n "$DOCKER" ]; then - docker rmi -f "$(docker images -a -q)" &> /dev/null || true -fi From 2f10c0fc6bdfb3503af3418e810aca0cf390ed19 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Mon, 2 Dec 2024 11:45:16 -0500 Subject: [PATCH 5/5] Removed unused teardown-aws and teardown-docker scripts --- .evergreen/scripts/teardown-aws.sh | 7 ------- .evergreen/scripts/teardown-docker.sh | 7 ------- 2 files changed, 14 deletions(-) delete mode 100755 .evergreen/scripts/teardown-aws.sh delete mode 100755 .evergreen/scripts/teardown-docker.sh diff --git a/.evergreen/scripts/teardown-aws.sh b/.evergreen/scripts/teardown-aws.sh deleted file mode 100755 index 634d1e5724..0000000000 --- a/.evergreen/scripts/teardown-aws.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || exit -if [ -f "./aws_e2e_setup.json" ]; then - . ./activate-authawsvenv.sh - python ./lib/aws_assign_instance_profile.py -fi diff --git a/.evergreen/scripts/teardown-docker.sh b/.evergreen/scripts/teardown-docker.sh deleted file mode 100755 index 733779d058..0000000000 --- a/.evergreen/scripts/teardown-docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Remove all Docker images -DOCKER=$(command -v docker) || true -if [ -n "$DOCKER" ]; then - docker rmi -f "$(docker images -a -q)" &> /dev/null || true -fi