From 14d26bc660afa81f440f585919954c6fd42f885b Mon Sep 17 00:00:00 2001 From: Jib Date: Fri, 6 Sep 2024 09:49:00 -0400 Subject: [PATCH 01/47] [1/x] Migrate all pre and most post scripts to subprocess in /scripts --- .evergreen/config.yml | 248 ++++-------------- .evergreen/scripts/archive-mongodb-logs.sh | 8 + .../scripts/boostrap-mongo-orchestration.sh | 47 ++++ .evergreen/scripts/fix-absolute-paths.sh | 7 + .evergreen/scripts/init-test-results.sh | 5 + .evergreen/scripts/install-dependencies.sh | 7 + .evergreen/scripts/make-files-executable.sh | 7 + .evergreen/scripts/prepare-resources.sh | 12 + .evergreen/scripts/run-aws-ecs-auth-test.sh | 15 ++ .evergreen/scripts/run-tests.sh | 61 +++++ .evergreen/scripts/windows-fix.sh | 10 + 11 files changed, 227 insertions(+), 200 deletions(-) create mode 100644 .evergreen/scripts/archive-mongodb-logs.sh create mode 100644 .evergreen/scripts/boostrap-mongo-orchestration.sh create mode 100644 .evergreen/scripts/fix-absolute-paths.sh create mode 100644 .evergreen/scripts/init-test-results.sh create mode 100644 .evergreen/scripts/install-dependencies.sh create mode 100644 .evergreen/scripts/make-files-executable.sh create mode 100644 .evergreen/scripts/prepare-resources.sh create mode 100644 .evergreen/scripts/run-aws-ecs-auth-test.sh create mode 100644 .evergreen/scripts/run-tests.sh create mode 100644 .evergreen/scripts/windows-fix.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c8e314f6c8..f22d50a093 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -48,19 +48,11 @@ functions: file: src/expansion.yml "prepare resources": - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - rm -rf $DRIVERS_TOOLS - if [ "$PROJECT" = "drivers-tools" ]; then - # If this was a patch build, doing a fresh clone would not actually test the patch - cp -R ${PROJECT_DIRECTORY}/ ${DRIVERS_TOOLS} - else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} - fi - echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config + binary: bash + args: + - src/.evergreen/scripts/prepare-resources.sh "upload coverage" : - command: ec2.assume_role @@ -124,15 +116,12 @@ functions: - command: ec2.assume_role params: role_arn: ${assume_role_arn} - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - mkdir out_dir - find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; - tar zcvf mongodb-logs.tar.gz -C out_dir/ . - rm -rf out_dir + binary: bash + args: + - src/.evergreen/scripts/archive-mongodb-logs.sh + - command: archive.targz_pack params: target: "mongo-coredumps.tgz" @@ -226,54 +215,11 @@ functions: file: "src/xunit-results/TEST-*.xml" "bootstrap mongo-orchestration": - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - - # Enable core dumps if enabled on the machine - # Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml - if [ -f /proc/self/coredump_filter ]; then - # Set the shell process (and its children processes) to dump ELF headers (bit 4), - # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). - echo 0x13 > /proc/self/coredump_filter - - if [ -f /sbin/sysctl ]; then - # Check that the core pattern is set explicitly on our distro image instead - # of being the OS's default value. This ensures that coredump names are consistent - # across distros and can be picked up by Evergreen. - core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") - if [ "$core_pattern" = "dump_%e.%p.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited - fi - fi - fi - - if [ $(uname -s) = "Darwin" ]; then - core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") - if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited - fi - fi - - if [ -n "${skip_crypt_shared}" ]; then - export SKIP_CRYPT_SHARED=1 - fi - - MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - STORAGE_ENGINE=${STORAGE_ENGINE} \ - DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - LOAD_BALANCER=${LOAD_BALANCER} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - # run-orchestration generates expansion file with the MONGODB_URI for the cluster + binary: bash + args: + - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh - command: expansions.update params: file: mo-expansion.yml @@ -284,28 +230,25 @@ functions: value: "1" "bootstrap data lake": - - command: shell.exec + - command: subprocess.exec type: setup params: - script: | - . src/.evergreen/scripts/env.sh - bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - - command: shell.exec + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh + - command: subprocess.exec type: setup params: - script: | - . src/.evergreen/scripts/env.sh - bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh - sleep 1 - docker ps + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh "stop mongo-orchestration": - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh "run mod_wsgi tests": - command: shell.exec @@ -341,76 +284,14 @@ functions: PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test "run tests": - - command: shell.exec - params: - working_dir: "src" - shell: bash - background: true - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - . .evergreen/scripts/env.sh - if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup - fi - - command: shell.exec + - command: subprocess.exec type: test params: + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" - shell: bash - script: | - # Disable xtrace - set +x - . .evergreen/scripts/env.sh - if [ -n "${MONGODB_STARTED}" ]; then - export PYMONGO_MUST_CONNECT=true - fi - if [ -n "${DISABLE_TEST_COMMANDS}" ]; then - export PYMONGO_DISABLE_TEST_COMMANDS=1 - fi - if [ -n "${test_encryption}" ]; then - # Disable xtrace (just in case it was accidentally set). - set +x - bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh - export TEST_ENCRYPTION=1 - if [ -n "${test_encryption_pyopenssl}" ]; then - export TEST_ENCRYPTION_PYOPENSSL=1 - fi - fi - if [ -n "${test_crypt_shared}" ]; then - export TEST_CRYPT_SHARED=1 - export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} - fi - if [ -n "${test_pyopenssl}" ]; then - export TEST_PYOPENSSL=1 - fi - if [ -n "${SETDEFAULTENCODING}" ]; then - export SETDEFAULTENCODING="${SETDEFAULTENCODING}" - fi - if [ -n "${test_loadbalancer}" ]; then - export TEST_LOADBALANCER=1 - export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" - export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" - fi - if [ -n "${test_serverless}" ]; then - export TEST_SERVERLESS=1 - fi - if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then - export TEST_INDEX_MANAGEMENT=1 - fi - if [ -n "${SKIP_CSOT_TESTS}" ]; then - export SKIP_CSOT_TESTS=1 - fi - - GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ - PYTHON_BINARY=${PYTHON_BINARY} \ - NO_EXT=${NO_EXT} \ - COVERAGE=${COVERAGE} \ - COMPRESSORS=${COMPRESSORS} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - TEST_DATA_LAKE=${TEST_DATA_LAKE} \ - MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg + binary: bash + args: + - .evergreen/scripts/run-tests.sh "run enterprise auth tests": - command: shell.exec @@ -535,25 +416,13 @@ functions: .evergreen/run-mongodb-aws-test.sh session-creds "run aws ECS auth test": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" + binary: "bash" working_dir: "src" - script: | - if [ "${skip_ECS_auth_test}" = "true" ]; then - echo "This platform does not support the ECS auth test, skipping..." - exit 0 - fi - . .evergreen/scripts/env.sh - set -ex - cd ${DRIVERS_TOOLS}/.evergreen/auth_aws - . ./activate-authawsvenv.sh - . aws_setup.sh ecs - export MONGODB_BINARIES="$MONGODB_BINARIES"; - export PROJECT_DIRECTORY="${PROJECT_DIRECTORY}"; - python aws_tester.py ecs - cd - + args: + - .evergreen/scripts/run-aws-ecs-auth-test.sh "cleanup": - command: shell.exec @@ -568,54 +437,33 @@ functions: "fix absolute paths": - command: shell.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename - done + args: + - src/.evergreen/scripts/fix-absolute-paths.sh "windows fix": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - cat $i | tr -d '\r' > $i.new - mv $i.new $i - done - # Copy client certificate because symlinks do not work on Windows. - cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem + args: + - src/.evergreen/scripts/windows-fix.sh "make files executable": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - chmod +x $i - done + args: + - src/.evergreen/scripts/make-files-executable.sh "init test-results": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json + args: + - src/.evergreen/scripts/init-test-results.sh "install dependencies": - - command: shell.exec + - command: subprocess.exec params: working_dir: "src" - script: | - . .evergreen/scripts/env.sh - set -o xtrace - file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" - # Don't use ${file} syntax here because evergreen treats it as an empty expansion. - [ -f "$file" ] && bash $file || echo "$file not available, skipping" + args: + - .evergreen/scripts/install-dependencies.sh "assume ec2 role": - command: ec2.assume_role diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh new file mode 100644 index 0000000000..c25e9b6c0b --- /dev/null +++ b/.evergreen/scripts/archive-mongodb-logs.sh @@ -0,0 +1,8 @@ +#!bin/bash + +. src/.evergreen/scripts/env.sh +set -o xtrace +mkdir out_dir +find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; +tar zcvf mongodb-logs.tar.gz -C out_dir/ . +rm -rf out_dir diff --git a/.evergreen/scripts/boostrap-mongo-orchestration.sh b/.evergreen/scripts/boostrap-mongo-orchestration.sh new file mode 100644 index 0000000000..02b635a6d1 --- /dev/null +++ b/.evergreen/scripts/boostrap-mongo-orchestration.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +. src/.evergreen/scripts/env.sh +set -o xtrace + +# Enable core dumps if enabled on the machine +# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml +if [ -f /proc/self/coredump_filter ]; then + # Set the shell process (and its children processes) to dump ELF headers (bit 4), + # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). + echo 0x13 >/proc/self/coredump_filter + + if [ -f /sbin/sysctl ]; then + # Check that the core pattern is set explicitly on our distro image instead + # of being the OS's default value. This ensures that coredump names are consistent + # across distros and can be picked up by Evergreen. + core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") + if [ "$core_pattern" = "dump_%e.%p.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi + fi +fi + +if [ $(uname -s) = "Darwin" ]; then + core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") + if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi +fi + +if [ -n "${skip_crypt_shared}" ]; then + export SKIP_CRYPT_SHARED=1 +fi + +MONGODB_VERSION=${VERSION} \ + TOPOLOGY=${TOPOLOGY} \ + AUTH=${AUTH} \ + SSL=${SSL} \ + STORAGE_ENGINE=${STORAGE_ENGINE} \ + DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ + ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ + REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ + LOAD_BALANCER=${LOAD_BALANCER} \ + bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh +# run-orchestration generates expansion file with the MONGODB_URI for the cluster diff --git a/.evergreen/scripts/fix-absolute-paths.sh b/.evergreen/scripts/fix-absolute-paths.sh new file mode 100644 index 0000000000..c00c275b60 --- /dev/null +++ b/.evergreen/scripts/fix-absolute-paths.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +for filename in $(find $DRIVERS_TOOLS -name \*.json); do + perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename +done diff --git a/.evergreen/scripts/init-test-results.sh b/.evergreen/scripts/init-test-results.sh new file mode 100644 index 0000000000..666ac60620 --- /dev/null +++ b/.evergreen/scripts/init-test-results.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh new file mode 100644 index 0000000000..c7cdbe67cf --- /dev/null +++ b/.evergreen/scripts/install-dependencies.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +set -o xtrace +file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh" +# Don't use ${file} syntax here because evergreen treats it as an empty expansion. +[ -f "$file" ] && bash $file || echo "$file not available, skipping" diff --git a/.evergreen/scripts/make-files-executable.sh b/.evergreen/scripts/make-files-executable.sh new file mode 100644 index 0000000000..72957e32ef --- /dev/null +++ b/.evergreen/scripts/make-files-executable.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +for i in $(find $DRIVERS_TOOLS/.evergreen $PROJECT_DIRECTORY/.evergreen -name \*.sh); do + chmod +x $i +done diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh new file mode 100644 index 0000000000..33394b55ff --- /dev/null +++ b/.evergreen/scripts/prepare-resources.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +. src/.evergreen/scripts/env.sh +set -o xtrace +rm -rf $DRIVERS_TOOLS +if [ "$PROJECT" = "drivers-tools" ]; then + # If this was a patch build, doing a fresh clone would not actually test the patch + cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS +else + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS +fi +echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config diff --git a/.evergreen/scripts/run-aws-ecs-auth-test.sh b/.evergreen/scripts/run-aws-ecs-auth-test.sh new file mode 100644 index 0000000000..cd692862f6 --- /dev/null +++ b/.evergreen/scripts/run-aws-ecs-auth-test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ "${skip_ECS_auth_test}" = "true" ]; then + echo "This platform does not support the ECS auth test, skipping..." + exit 0 +fi +. .evergreen/scripts/env.sh +set -ex +cd $DRIVERS_TOOLS/.evergreen/auth_aws +. ./activate-authawsvenv.sh +. aws_setup.sh ecs +export MONGODB_BINARIES="$MONGODB_BINARIES" +export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" +python aws_tester.py ecs +cd - diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh new file mode 100644 index 0000000000..6c80b8c211 --- /dev/null +++ b/.evergreen/scripts/run-tests.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh + + +if [ -n "${test_encryption}" ]; then + ./.evergreen/hatch.sh encryption:setup +fi + +# Disable xtrace +set +x +if [ -n "${MONGODB_STARTED}" ]; then + export PYMONGO_MUST_CONNECT=true +fi +if [ -n "${DISABLE_TEST_COMMANDS}" ]; then + export PYMONGO_DISABLE_TEST_COMMANDS=1 +fi +if [ -n "${test_encryption}" ]; then + # Disable xtrace (just in case it was accidentally set). + set +x + bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh + export TEST_ENCRYPTION=1 + if [ -n "${test_encryption_pyopenssl}" ]; then + export TEST_ENCRYPTION_PYOPENSSL=1 + fi +fi +if [ -n "${test_crypt_shared}" ]; then + export TEST_CRYPT_SHARED=1 + export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} +fi +if [ -n "${test_pyopenssl}" ]; then + export TEST_PYOPENSSL=1 +fi +if [ -n "${SETDEFAULTENCODING}" ]; then + export SETDEFAULTENCODING="${SETDEFAULTENCODING}" +fi +if [ -n "${test_loadbalancer}" ]; then + export TEST_LOADBALANCER=1 + export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" + export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" +fi +if [ -n "${test_serverless}" ]; then + export TEST_SERVERLESS=1 +fi +if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then + export TEST_INDEX_MANAGEMENT=1 +fi +if [ -n "${SKIP_CSOT_TESTS}" ]; then + export SKIP_CSOT_TESTS=1 +fi + +GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ + PYTHON_BINARY=${PYTHON_BINARY} \ + NO_EXT=${NO_EXT} \ + COVERAGE=${COVERAGE} \ + COMPRESSORS=${COMPRESSORS} \ + AUTH=${AUTH} \ + SSL=${SSL} \ + TEST_DATA_LAKE=${TEST_DATA_LAKE} \ + MONGODB_API_VERSION=${MONGODB_API_VERSION} \ + bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/windows-fix.sh b/.evergreen/scripts/windows-fix.sh new file mode 100644 index 0000000000..5e489e9c9c --- /dev/null +++ b/.evergreen/scripts/windows-fix.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +for i in $(find $DRIVERS_TOOLS/.evergreen $PROJECT_DIRECTORY/.evergreen -name \*.sh); do + cat $i | tr -d '\r' >$i.new + mv $i.new $i +done +# Copy client certificate because symlinks do not work on Windows. +cp $DRIVERS_TOOLS/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem From 56fafb817f62cae9a03d9aa0d2fbef42147c429e Mon Sep 17 00:00:00 2001 From: Jib Date: Fri, 6 Sep 2024 10:05:56 -0400 Subject: [PATCH 02/47] command-type fix --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f22d50a093..608f5db7a9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -435,7 +435,7 @@ functions: rm -f ./secrets-export.sh || true "fix absolute paths": - - command: shell.exec + - command: subprocess.exec params: args: - src/.evergreen/scripts/fix-absolute-paths.sh From f957a69ccdb3e3602cce075f0796ece0b5c8e349 Mon Sep 17 00:00:00 2001 From: Jib Date: Fri, 6 Sep 2024 11:59:59 -0400 Subject: [PATCH 03/47] remove .evergreen/scripts/run-tests from this PR --- .evergreen/config.yml | 11 ++++++ .evergreen/scripts/run-tests.sh | 61 --------------------------------- 2 files changed, 11 insertions(+), 61 deletions(-) delete mode 100644 .evergreen/scripts/run-tests.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c13bda219a..7fa6710c6f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -284,6 +284,17 @@ functions: PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test "run tests": + - command: shell.exec + params: + working_dir: "src" + shell: bash + background: true + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + script: | + . .evergreen/scripts/env.sh + if [ -n "${test_encryption}" ]; then + ./.evergreen/hatch.sh encryption:setup + fi - command: shell.exec type: test params: diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh deleted file mode 100644 index 6c80b8c211..0000000000 --- a/.evergreen/scripts/run-tests.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -. .evergreen/scripts/env.sh - - -if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup -fi - -# Disable xtrace -set +x -if [ -n "${MONGODB_STARTED}" ]; then - export PYMONGO_MUST_CONNECT=true -fi -if [ -n "${DISABLE_TEST_COMMANDS}" ]; then - export PYMONGO_DISABLE_TEST_COMMANDS=1 -fi -if [ -n "${test_encryption}" ]; then - # Disable xtrace (just in case it was accidentally set). - set +x - bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh - export TEST_ENCRYPTION=1 - if [ -n "${test_encryption_pyopenssl}" ]; then - export TEST_ENCRYPTION_PYOPENSSL=1 - fi -fi -if [ -n "${test_crypt_shared}" ]; then - export TEST_CRYPT_SHARED=1 - export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} -fi -if [ -n "${test_pyopenssl}" ]; then - export TEST_PYOPENSSL=1 -fi -if [ -n "${SETDEFAULTENCODING}" ]; then - export SETDEFAULTENCODING="${SETDEFAULTENCODING}" -fi -if [ -n "${test_loadbalancer}" ]; then - export TEST_LOADBALANCER=1 - export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" - export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" -fi -if [ -n "${test_serverless}" ]; then - export TEST_SERVERLESS=1 -fi -if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then - export TEST_INDEX_MANAGEMENT=1 -fi -if [ -n "${SKIP_CSOT_TESTS}" ]; then - export SKIP_CSOT_TESTS=1 -fi - -GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ - PYTHON_BINARY=${PYTHON_BINARY} \ - NO_EXT=${NO_EXT} \ - COVERAGE=${COVERAGE} \ - COMPRESSORS=${COMPRESSORS} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - TEST_DATA_LAKE=${TEST_DATA_LAKE} \ - MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg From 051e733ff378004b2086876c53e2908bdbf192ff Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 Nov 2024 11:44:34 -0500 Subject: [PATCH 04/47] download_and_merge_coverage --- .evergreen/config.yml | 8 ++++---- .evergreen/scripts/download_and_merge_coverage.sh | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 .evergreen/scripts/download_and_merge_coverage.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d83e5caef8..4ffe6f7967 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -80,14 +80,14 @@ functions: - command: ec2.assume_role params: role_arn: ${assume_role_arn} - - command: shell.exec + - command: subprocess.exec params: + binary: bash silent: true working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - # Download all the task coverage files. - aws s3 cp --recursive s3://${bucket_name}/coverage/${revision}/${version_id}/coverage/ coverage/ + args: + - src/.evergreen/scripts/download_and_merge_coverage.sh ${bucket_name} ${revision} ${version_id} - command: subprocess.exec params: working_dir: "src" diff --git a/.evergreen/scripts/download_and_merge_coverage.sh b/.evergreen/scripts/download_and_merge_coverage.sh new file mode 100644 index 0000000000..0083f2899a --- /dev/null +++ b/.evergreen/scripts/download_and_merge_coverage.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +# Download all the task coverage files. +aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/ From ea5dff86557a238d73671c09363aaf6c5200fb1c Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 Nov 2024 11:57:21 -0500 Subject: [PATCH 05/47] mockupdb, doctests, atlas --- .evergreen/config.yml | 37 +++++++------------ .evergreen/scripts/archive-mongodb-logs.sh | 6 +-- ...on.sh => bootstrap-mongo-orchestration.sh} | 0 ...rage.sh => download-and-merge-coverage.sh} | 0 .evergreen/scripts/run-atlas-tests.sh | 7 ++++ .evergreen/scripts/run-doctests.sh | 5 +++ .evergreen/scripts/run-mockupdb-tests.sh | 6 +++ 7 files changed, 35 insertions(+), 26 deletions(-) rename .evergreen/scripts/{boostrap-mongo-orchestration.sh => bootstrap-mongo-orchestration.sh} (100%) rename .evergreen/scripts/{download_and_merge_coverage.sh => download-and-merge-coverage.sh} (100%) create mode 100644 .evergreen/scripts/run-atlas-tests.sh create mode 100644 .evergreen/scripts/run-doctests.sh create mode 100644 .evergreen/scripts/run-mockupdb-tests.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4ffe6f7967..28b162b2c0 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -87,7 +87,7 @@ functions: working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - - src/.evergreen/scripts/download_and_merge_coverage.sh ${bucket_name} ${revision} ${version_id} + - src/.evergreen/scripts/download-and-merge-coverage.sh ${bucket_name} ${revision} ${version_id} - command: subprocess.exec params: working_dir: "src" @@ -267,25 +267,22 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh "run mockupdb tests": - - command: shell.exec + - command: subprocess.exec type: test params: working_dir: "src" - script: | - . .evergreen/scripts/env.sh - set -o xtrace - export PYTHON_BINARY=${PYTHON_BINARY} - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb + binary: bash + args: + - .evergreen/scripts/run-mockupdb-tests.sh "run doctests": - - command: shell.exec + - command: subprocess.exec type: test params: working_dir: "src" - script: | - . .evergreen/scripts/env.sh - set -o xtrace - PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test + binary: bash + args: + - .evergreen/scripts/run-doctests.sh "run tests": - command: shell.exec @@ -379,20 +376,14 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg "run atlas tests": - - command: shell.exec + - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: bash + include_expansions_in_env: ["DRIVERS_TOOLS", "PROJECT_DIRECTORY", "PYTHON_BINARY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" - script: | - # Disable xtrace for security reasons (just in case it was accidentally set). - set +x - set -o errexit - bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect - PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ - PYTHON_BINARY="${PYTHON_BINARY}" \ - TEST_ATLAS=1 \ - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg + args: + - .evergreen/scripts/run-atlas-tests.sh "get aws auth secrets": - command: subprocess.exec diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh index c25e9b6c0b..8033354ed4 100644 --- a/.evergreen/scripts/archive-mongodb-logs.sh +++ b/.evergreen/scripts/archive-mongodb-logs.sh @@ -1,8 +1,8 @@ -#!bin/bash +#!/bin/bash -. src/.evergreen/scripts/env.sh +. .evergreen/scripts/env.sh set -o xtrace mkdir out_dir -find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; +find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; tar zcvf mongodb-logs.tar.gz -C out_dir/ . rm -rf out_dir diff --git a/.evergreen/scripts/boostrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh similarity index 100% rename from .evergreen/scripts/boostrap-mongo-orchestration.sh rename to .evergreen/scripts/bootstrap-mongo-orchestration.sh diff --git a/.evergreen/scripts/download_and_merge_coverage.sh b/.evergreen/scripts/download-and-merge-coverage.sh similarity index 100% rename from .evergreen/scripts/download_and_merge_coverage.sh rename to .evergreen/scripts/download-and-merge-coverage.sh diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh new file mode 100644 index 0000000000..98a19f047f --- /dev/null +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Disable xtrace for security reasons (just in case it was accidentally set). +set +x +set -o errexit +bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect +TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/run-doctests.sh b/.evergreen/scripts/run-doctests.sh new file mode 100644 index 0000000000..2e2b6b10b3 --- /dev/null +++ b/.evergreen/scripts/run-doctests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +set -o xtrace +PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test diff --git a/.evergreen/scripts/run-mockupdb-tests.sh b/.evergreen/scripts/run-mockupdb-tests.sh new file mode 100644 index 0000000000..1148e96fc4 --- /dev/null +++ b/.evergreen/scripts/run-mockupdb-tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +set -o xtrace +export PYTHON_BINARY=${PYTHON_BINARY} +bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb From b10bd37e9195b4287f293f2058f64f7b6b0cd9d5 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 Nov 2024 13:41:26 -0500 Subject: [PATCH 06/47] modwsgi, enterprise auth --- .evergreen/config.yml | 28 +++++++------------ .../scripts/run-enterprise-auth-tests.sh | 6 ++++ .evergreen/scripts/run-modwsgi-tests.sh | 9 ++++++ 3 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 .evergreen/scripts/run-enterprise-auth-tests.sh create mode 100644 .evergreen/scripts/run-modwsgi-tests.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 28b162b2c0..178182186b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -255,16 +255,14 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh "run mod_wsgi tests": - - command: shell.exec + - command: subprocess.exec type: test params: + include_expansions_in_env: ["PROJECT_DIRECTORY"] working_dir: "src" - script: | - . .evergreen/scripts/env.sh - set -o xtrace - PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} \ - MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh + binary: bash + args: + - .evergreen/scripts/run-modwsgi-tests.sh "run mockupdb tests": - command: subprocess.exec @@ -360,20 +358,14 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg "run enterprise auth tests": - - command: shell.exec + - command: subprocess.exec type: test params: + binary: bash working_dir: "src" - include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - # Disable xtrace for security reasons (just in case it was accidentally set). - set +x - bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth - PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ - PYTHON_BINARY="${PYTHON_BINARY}" \ - TEST_ENTERPRISE_AUTH=1 \ - AUTH=auth \ - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg + include_expansions_in_env: ["DRIVERS_TOOLS", "PROJECT_DIRECTORY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + args: + - .evergreen/scripts/run-enterprise-auth-tests.sh "run atlas tests": - command: subprocess.exec diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh new file mode 100644 index 0000000000..37298a2bdb --- /dev/null +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Disable xtrace for security reasons (just in case it was accidentally set). +set +x +bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth +TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/run-modwsgi-tests.sh b/.evergreen/scripts/run-modwsgi-tests.sh new file mode 100644 index 0000000000..3e0f27c3e5 --- /dev/null +++ b/.evergreen/scripts/run-modwsgi-tests.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +set -o xtrace +PYTHON_BINARY=${PYTHON_BINARY} +MOD_WSGI_VERSION=${MOD_WSGI_VERSION} +MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} +PROJECT_DIRECTORY=${PROJECT_DIRECTORY} +bash "${PROJECT_DIRECTORY}"/.evergreen/run-mod-wsgi-tests.sh From a9b1c07bea5ffa2848098dc66df6b76d2b10b8a1 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 Nov 2024 14:04:25 -0500 Subject: [PATCH 07/47] Fix modwsgi, aws auth --- .evergreen/config.yml | 95 +++++++++---------- .../{ => scripts}/run-mod-wsgi-tests.sh | 2 + .evergreen/scripts/run-modwsgi-tests.sh | 9 -- .../{ => scripts}/run-mongodb-aws-test.sh | 7 ++ test/mod_wsgi_test/README.rst | 2 +- 5 files changed, 53 insertions(+), 62 deletions(-) rename .evergreen/{ => scripts}/run-mod-wsgi-tests.sh (98%) delete mode 100644 .evergreen/scripts/run-modwsgi-tests.sh rename .evergreen/{ => scripts}/run-mongodb-aws-test.sh (76%) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 178182186b..f927b6165d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -258,11 +258,11 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["PROJECT_DIRECTORY"] + include_expansions_in_env: ["PROJECT_DIRECTORY", "PYTHON_BINARY", "MOD_WSGI_VERSION", "MOD_WSGI_EMBEDDED"] working_dir: "src" binary: bash args: - - .evergreen/scripts/run-modwsgi-tests.sh + - .evergreen/scripts/run-mod-wsgi-tests.sh "run mockupdb tests": - command: subprocess.exec @@ -387,55 +387,66 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh "run aws auth test with regular aws credentials": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" + include_expansions_in_env: ["DRIVERS_TOOLS"] + binary: bash working_dir: "src" - script: | - . .evergreen/scripts/env.sh - .evergreen/run-mongodb-aws-test.sh regular + args: + - .evergreen/scripts/run-mongodb-aws-test.sh regular "run aws auth test with assume role credentials": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" + include_expansions_in_env: ["DRIVERS_TOOLS"] + binary: bash working_dir: "src" - script: | - . .evergreen/scripts/env.sh - .evergreen/run-mongodb-aws-test.sh assume-role + args: + - .evergreen/scripts/run-mongodb-aws-test.sh assume-role "run aws auth test with aws EC2 credentials": - - command: shell.exec + - command: subprocess.exec type: test params: + include_expansions_in_env: ["DRIVERS_TOOLS"] + binary: bash working_dir: "src" - shell: "bash" - script: | - if [ "${skip_EC2_auth_test}" = "true" ]; then - echo "This platform does not support the EC2 auth test, skipping..." - exit 0 - fi - . .evergreen/scripts/env.sh - .evergreen/run-mongodb-aws-test.sh ec2 + args: + - .evergreen/scripts/run-mongodb-aws-test.sh ec2 "run aws auth test with aws web identity credentials": - - command: shell.exec + - command: subprocess.exec type: test params: + include_expansions_in_env: ["DRIVERS_TOOLS"] + binary: bash working_dir: "src" - shell: "bash" - script: | - if [ "${skip_EC2_auth_test}" = "true" ]; then - echo "This platform does not support the web identity auth test, skipping..." - exit 0 - fi - . .evergreen/scripts/env.sh + args: # Test with and without AWS_ROLE_SESSION_NAME set. - .evergreen/run-mongodb-aws-test.sh web-identity - AWS_ROLE_SESSION_NAME="test" \ - .evergreen/run-mongodb-aws-test.sh web-identity + - .evergreen/scripts/run-mongodb-aws-test.sh web-identity + - AWS_ROLE_SESSION_NAME="test" .evergreen/scripts/run-mongodb-aws-test.sh web-identity + + "run aws auth test with aws credentials as environment variables": + - command: subprocess.exec + type: test + params: + include_expansions_in_env: ["DRIVERS_TOOLS"] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/run-mongodb-aws-test.sh env-creds + + "run aws auth test with aws credentials and session token as environment variables": + - command: subprocess.exec + type: test + params: + include_expansions_in_env: ["DRIVERS_TOOLS"] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/run-mongodb-aws-test.sh session-creds "run oidc auth test with test credentials": - command: subprocess.exec @@ -459,26 +470,6 @@ functions: args: - ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-oidc-remote-test.sh - "run aws auth test with aws credentials as environment variables": - - command: shell.exec - type: test - params: - working_dir: "src" - shell: bash - script: | - . .evergreen/scripts/env.sh - .evergreen/run-mongodb-aws-test.sh env-creds - - "run aws auth test with aws credentials and session token as environment variables": - - command: shell.exec - type: test - params: - working_dir: "src" - shell: bash - script: | - . .evergreen/scripts/env.sh - .evergreen/run-mongodb-aws-test.sh session-creds - "run aws ECS auth test": - command: subprocess.exec type: test diff --git a/.evergreen/run-mod-wsgi-tests.sh b/.evergreen/scripts/run-mod-wsgi-tests.sh similarity index 98% rename from .evergreen/run-mod-wsgi-tests.sh rename to .evergreen/scripts/run-mod-wsgi-tests.sh index e1f5238110..3fcae568e5 100644 --- a/.evergreen/run-mod-wsgi-tests.sh +++ b/.evergreen/scripts/run-mod-wsgi-tests.sh @@ -2,6 +2,8 @@ set -o xtrace set -o errexit +. .evergreen/scripts/env.sh + APACHE=$(command -v apache2 || command -v /usr/lib/apache2/mpm-prefork/apache2) || true if [ -n "$APACHE" ]; then APACHE_CONFIG=apache24ubuntu161404.conf diff --git a/.evergreen/scripts/run-modwsgi-tests.sh b/.evergreen/scripts/run-modwsgi-tests.sh deleted file mode 100644 index 3e0f27c3e5..0000000000 --- a/.evergreen/scripts/run-modwsgi-tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -. .evergreen/scripts/env.sh -set -o xtrace -PYTHON_BINARY=${PYTHON_BINARY} -MOD_WSGI_VERSION=${MOD_WSGI_VERSION} -MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} -PROJECT_DIRECTORY=${PROJECT_DIRECTORY} -bash "${PROJECT_DIRECTORY}"/.evergreen/run-mod-wsgi-tests.sh diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh similarity index 76% rename from .evergreen/run-mongodb-aws-test.sh rename to .evergreen/scripts/run-mongodb-aws-test.sh index c4051bb34a..ca6c19a378 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -13,6 +13,13 @@ set -o errexit # Exit the script with error if any of the commands fail # mechanism. # PYTHON_BINARY The Python version to use. +# shellcheck disable=SC2154 +if [ "${skip_EC2_auth_test}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then + echo "This platform does not support the EC2 auth test, skipping..." + exit 0 +fi + +. .evergreen/scripts/env.sh echo "Running MONGODB-AWS authentication tests" # Handle credentials and environment setup. diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index 2c204f7ac5..e96db9406c 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -107,4 +107,4 @@ Automation At MongoDB, Inc. we use a continuous integration job that tests each combination in the matrix. The job starts up Apache, starts a single server or replica set, and runs ``test_client.py`` with the proper arguments. -See `run-mod-wsgi-tests.sh `_ +See `run-mod-wsgi-tests.sh `_ From c0175818713b49b24ac4e6e7ee61aa679133eb17 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 6 Nov 2024 14:23:56 -0500 Subject: [PATCH 08/47] Fix aws auth, ocsp --- .evergreen/config.yml | 22 +++++++++++----------- .evergreen/scripts/run-mongodb-aws-test.sh | 4 ++-- .evergreen/scripts/run-ocsp-test.sh | 9 +++++++++ 3 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 .evergreen/scripts/run-ocsp-test.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f927b6165d..9b20dd3e39 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -87,7 +87,10 @@ functions: working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - - src/.evergreen/scripts/download-and-merge-coverage.sh ${bucket_name} ${revision} ${version_id} + - src/.evergreen/scripts/download-and-merge-coverage.sh + - ${bucket_name} + - ${revision} + - ${version_id} - command: subprocess.exec params: working_dir: "src" @@ -394,7 +397,8 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-mongodb-aws-test.sh regular + - .evergreen/scripts/run-mongodb-aws-test.sh + - regular "run aws auth test with assume role credentials": - command: subprocess.exec @@ -542,18 +546,14 @@ functions: file: atlas-expansion.yml "run-ocsp-test": - - command: shell.exec + - command: subprocess.exec type: test params: + include_expansions_in_env: ["DRIVERS_TOOLS", "PROJECT_DIRECTORY", "OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED"] + binary: bash working_dir: "src" - script: | - . .evergreen/scripts/env.sh - TEST_OCSP=1 \ - PYTHON_BINARY=${PYTHON_BINARY} \ - CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ - OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg - bash ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh + args: + - .evergreen/scripts/run-ocsp-test.sh "run-ocsp-server": - command: subprocess.exec diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index ca6c19a378..375ed854f1 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -20,10 +20,10 @@ if [ "${skip_EC2_auth_test}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-i fi . .evergreen/scripts/env.sh -echo "Running MONGODB-AWS authentication tests" +echo "Running MONGODB-AWS authentication tests for $1" # Handle credentials and environment setup. -. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1 +. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1" # show test output set -x diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh new file mode 100644 index 0000000000..c7a5239cbd --- /dev/null +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh + +TEST_OCSP=1 \ +CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ +OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ +bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh From 6ce1f86c095f6eb123ce224c7a34018a9eaf66ff Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 Nov 2024 10:12:09 -0500 Subject: [PATCH 09/47] Standardize binary to bash --- .evergreen/config.yml | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9b20dd3e39..6fcee5d8ea 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -83,7 +83,6 @@ functions: - command: subprocess.exec params: binary: bash - silent: true working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: @@ -261,7 +260,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["PROJECT_DIRECTORY", "PYTHON_BINARY", "MOD_WSGI_VERSION", "MOD_WSGI_EMBEDDED"] + include_expansions_in_env: ["PYTHON_BINARY", "MOD_WSGI_VERSION", "MOD_WSGI_EMBEDDED"] working_dir: "src" binary: bash args: @@ -366,7 +365,7 @@ functions: params: binary: bash working_dir: "src" - include_expansions_in_env: ["DRIVERS_TOOLS", "PROJECT_DIRECTORY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - .evergreen/scripts/run-enterprise-auth-tests.sh @@ -375,7 +374,7 @@ functions: type: test params: binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS", "PROJECT_DIRECTORY", "PYTHON_BINARY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["PYTHON_BINARY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" args: - .evergreen/scripts/run-atlas-tests.sh @@ -393,7 +392,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "PYTHON_BINARY", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: @@ -404,7 +403,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: @@ -414,7 +413,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: @@ -424,7 +423,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: @@ -436,7 +435,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: @@ -446,7 +445,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS"] + include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: @@ -478,7 +477,7 @@ functions: - command: subprocess.exec type: test params: - binary: "bash" + binary: bash working_dir: "src" args: - .evergreen/scripts/run-aws-ecs-auth-test.sh @@ -498,30 +497,35 @@ functions: "fix absolute paths": - command: subprocess.exec params: + binary: bash args: - src/.evergreen/scripts/fix-absolute-paths.sh "windows fix": - command: subprocess.exec params: + binary: bash args: - src/.evergreen/scripts/windows-fix.sh "make files executable": - command: subprocess.exec params: + binary: bash args: - src/.evergreen/scripts/make-files-executable.sh "init test-results": - command: subprocess.exec params: + binary: bash args: - src/.evergreen/scripts/init-test-results.sh "install dependencies": - command: subprocess.exec params: + binary: bash working_dir: "src" args: - .evergreen/scripts/install-dependencies.sh @@ -549,7 +553,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS", "PROJECT_DIRECTORY", "OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED"] + include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED"] binary: bash working_dir: "src" args: @@ -560,7 +564,7 @@ functions: params: background: true binary: bash - include_expansions_in_env: [SERVER_TYPE, OCSP_ALGORITHM] + include_expansions_in_env: ["SERVER_TYPE", "OCSP_ALGORITHM"] args: - ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh @@ -683,13 +687,13 @@ task_groups: - func: make files executable - command: subprocess.exec params: - binary: "bash" + binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/create-and-setup-instance.sh teardown_task: - command: subprocess.exec params: - binary: "bash" + binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh - func: "upload test results" @@ -1535,7 +1539,7 @@ tasks: type: setup params: working_dir: "src" - binary: "bash" + binary: bash include_expansions_in_env: ["DRIVERS_TOOLS"] args: - .evergreen/run-gcpkms-test.sh From ec3c2c456b9b7e403903511b4fa1044c3888a111 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 Nov 2024 11:41:32 -0500 Subject: [PATCH 10/47] All but run_tests --- .evergreen/config.yml | 126 +++++++----------- .evergreen/scripts/archive-mongodb-logs.sh | 4 +- .../scripts/bootstrap-mongo-orchestration.sh | 2 +- .evergreen/scripts/check-import-time.sh | 7 + .evergreen/scripts/cleanup.sh | 8 ++ .evergreen/scripts/fix-absolute-paths.sh | 1 + .evergreen/scripts/make-files-executable.sh | 5 +- .evergreen/scripts/run-atlas-tests.sh | 1 + .evergreen/scripts/run-aws-ecs-auth-test.sh | 1 + .evergreen/scripts/run-gcpkms-fail-test.sh | 7 + .evergreen/scripts/run-getdata.sh | 22 +++ .evergreen/scripts/run-load-balancer.sh | 4 + .evergreen/scripts/run-ocsp-test.sh | 1 + .evergreen/scripts/run-perf-tests.sh | 5 + .evergreen/scripts/stop-load-balancer.sh | 5 + .evergreen/scripts/teardown-aws.sh | 9 ++ .evergreen/scripts/teardown-docker.sh | 7 + .evergreen/scripts/upload-coverage-report.sh | 3 + .evergreen/scripts/windows-fix.sh | 9 +- 19 files changed, 140 insertions(+), 87 deletions(-) create mode 100644 .evergreen/scripts/check-import-time.sh create mode 100644 .evergreen/scripts/cleanup.sh create mode 100644 .evergreen/scripts/run-gcpkms-fail-test.sh create mode 100644 .evergreen/scripts/run-getdata.sh create mode 100644 .evergreen/scripts/run-load-balancer.sh create mode 100644 .evergreen/scripts/run-perf-tests.sh create mode 100644 .evergreen/scripts/stop-load-balancer.sh create mode 100644 .evergreen/scripts/teardown-aws.sh create mode 100644 .evergreen/scripts/teardown-docker.sh create mode 100644 .evergreen/scripts/upload-coverage-report.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6fcee5d8ea..37000af7ea 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -20,10 +20,9 @@ exec_timeout_secs: 3600 # 60 minutes is the longest we'll ever run (primarily # What to do when evergreen hits the timeout (`post:` tasks are run automatically) timeout: - - command: shell.exec + - command: subprocess.exec params: - script: | - ls -la + binary: ls -la include: - filename: .evergreen/generated_configs/tasks.yml @@ -97,13 +96,17 @@ functions: args: - .evergreen/combine-coverage.sh # Upload the resulting html coverage report. - - command: shell.exec + - command: subprocess.exec params: silent: true + binary: bash working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - aws s3 cp htmlcov/ s3://${bucket_name}/coverage/${revision}/${version_id}/htmlcov/ --recursive --acl public-read --region us-east-1 + args: + - src/.evergreen/scripts/upload-coverage-report.sh + - ${bucket_name} + - ${revision} + - ${version_id} # Attach the index.html with s3.put so it shows up in the Evergreen UI. - command: s3.put params: @@ -483,16 +486,12 @@ functions: - .evergreen/scripts/run-aws-ecs-auth-test.sh "cleanup": - - command: shell.exec + - command: subprocess.exec params: + binary: bash working_dir: "src" - script: | - . .evergreen/scripts/env.sh - if [ -f $DRIVERS_TOOLS/.evergreen/csfle/secrets-export.sh ]; then - . .evergreen/hatch.sh encryption:teardown - fi - rm -rf ${DRIVERS_TOOLS} || true - rm -f ./secrets-export.sh || true + args: + - .evergreen/scripts/cleanup.sh "fix absolute paths": - command: subprocess.exec @@ -553,7 +552,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED"] + include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED", "PYTHON_BINARY"] binary: bash working_dir: "src" args: @@ -569,42 +568,36 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh "run load-balancer": - - command: shell.exec + - command: subprocess.exec params: - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start + binary: bash + include_expansions_in_env: ["MONGODB_URI"] + args: + - src/.evergreen/scripts/run-load-balancer.sh - command: expansions.update params: file: lb-expansion.yml "stop load-balancer": - - command: shell.exec + - command: subprocess.exec params: - script: | - cd ${DRIVERS_TOOLS}/.evergreen - DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop + binary: bash + args: + - src/.evergreen/scripts/stop-load-balancer.sh "teardown_docker": - - command: shell.exec + - command: subprocess.exec params: - script: | - # Remove all Docker images - DOCKER=$(command -v docker) || true - if [ -n "$DOCKER" ]; then - docker rmi -f $(docker images -a -q) &> /dev/null || true - fi + binary: bash + args: + - src/.evergreen/scripts/teardown-docker.sh "teardown_aws": - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - . src/.evergreen/scripts/env.sh - cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" - if [ -f "./aws_e2e_setup.json" ]; then - . ./activate-authawsvenv.sh - python ./lib/aws_assign_instance_profile.py - fi + binary: bash + args: + - src/.evergreen/scripts/teardown-aws.sh "teardown atlas": - command: subprocess.exec @@ -614,10 +607,13 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh "run perf tests": - - command: shell.exec + - command: subprocess.exec type: test params: working_dir: "src" + binary: bash + args: + - .evergreen/scripts/run-perf-tests.sh script: | . .evergreen/scripts/env.sh PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh @@ -858,31 +854,12 @@ tasks: # Throw it here, and execute this task on all buildvariants - name: getdata commands: - - command: shell.exec + - command: subprocess.exec + binary: bash type: test params: - script: | - set -o xtrace - . ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true - get_distro || true - echo $DISTRO - echo $MARCH - echo $OS - uname -a || true - ls /etc/*release* || true - cc --version || true - gcc --version || true - clang --version || true - gcov --version || true - lcov --version || true - llvm-cov --version || true - echo $PATH - ls -la /usr/local/Cellar/llvm/*/bin/ || true - ls -la /usr/local/Cellar/ || true - scan-build --version || true - genhtml --version || true - valgrind --version || true - + args: + - src/.evergreen/scripts/run-getdata.sh # Standard test tasks {{{ - name: "mockupdb" @@ -1552,17 +1529,13 @@ tasks: vars: VERSION: "latest" TOPOLOGY: "server" - - command: shell.exec + - command: subprocess.exec type: test params: working_dir: "src" - shell: "bash" - script: | - . .evergreen/scripts/env.sh - export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 - export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz - SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh - SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg + binary: "bash" + args: + - .evergreen/scripts/run-gcpkms-fail-test.sh - name: testazurekms-task commands: @@ -1628,18 +1601,15 @@ tasks: - name: "check-import-time" tags: ["pr"] commands: - - command: shell.exec + - command: subprocess.exec type: test params: shell: "bash" working_dir: src - script: | - . .evergreen/scripts/env.sh - set -x - export BASE_SHA=${revision} - export HEAD_SHA=${github_commit} - bash .evergreen/run-import-time-test.sh - + args: + - .evergreen/scripts/check-import-time.sh + - ${revision} + - ${github_commit} - name: "backport-pr" allowed_requesters: ["commit"] commands: diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh index 8033354ed4..29d7ab62f4 100644 --- a/.evergreen/scripts/archive-mongodb-logs.sh +++ b/.evergreen/scripts/archive-mongodb-logs.sh @@ -1,8 +1,8 @@ #!/bin/bash -. .evergreen/scripts/env.sh +. src/.evergreen/scripts/env.sh set -o xtrace mkdir out_dir -find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; +find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x=$1; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' shell {} \; tar zcvf mongodb-logs.tar.gz -C out_dir/ . rm -rf out_dir diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index 02b635a6d1..a7d8cf5da0 100644 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -22,7 +22,7 @@ if [ -f /proc/self/coredump_filter ]; then fi fi -if [ $(uname -s) = "Darwin" ]; then +if [ "$(uname -s)" = "Darwin" ]; then core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then echo "Enabling coredumps" diff --git a/.evergreen/scripts/check-import-time.sh b/.evergreen/scripts/check-import-time.sh new file mode 100644 index 0000000000..cdd2025d59 --- /dev/null +++ b/.evergreen/scripts/check-import-time.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +set -x +export BASE_SHA="$1" +export HEAD_SHA="$2" +bash .evergreen/run-import-time-test.sh diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh new file mode 100644 index 0000000000..3c66a13e36 --- /dev/null +++ b/.evergreen/scripts/cleanup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then + . .evergreen/hatch.sh encryption:teardown +fi +rm -rf "${DRIVERS_TOOLS}" || true +rm -f ./secrets-export.sh || true diff --git a/.evergreen/scripts/fix-absolute-paths.sh b/.evergreen/scripts/fix-absolute-paths.sh index c00c275b60..eb9433c673 100644 --- a/.evergreen/scripts/fix-absolute-paths.sh +++ b/.evergreen/scripts/fix-absolute-paths.sh @@ -2,6 +2,7 @@ set +x . src/.evergreen/scripts/env.sh +# shellcheck disable=SC2044 for filename in $(find $DRIVERS_TOOLS -name \*.json); do perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename done diff --git a/.evergreen/scripts/make-files-executable.sh b/.evergreen/scripts/make-files-executable.sh index 72957e32ef..806be7c599 100644 --- a/.evergreen/scripts/make-files-executable.sh +++ b/.evergreen/scripts/make-files-executable.sh @@ -2,6 +2,7 @@ set +x . src/.evergreen/scripts/env.sh -for i in $(find $DRIVERS_TOOLS/.evergreen $PROJECT_DIRECTORY/.evergreen -name \*.sh); do - chmod +x $i +# shellcheck disable=SC2044 +for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do + chmod +x "$i" done diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index 98a19f047f..816566ab6a 100644 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -1,5 +1,6 @@ #!/bin/bash +. .evergreen/scripts/env.sh # Disable xtrace for security reasons (just in case it was accidentally set). set +x set -o errexit diff --git a/.evergreen/scripts/run-aws-ecs-auth-test.sh b/.evergreen/scripts/run-aws-ecs-auth-test.sh index cd692862f6..295ba8ca54 100644 --- a/.evergreen/scripts/run-aws-ecs-auth-test.sh +++ b/.evergreen/scripts/run-aws-ecs-auth-test.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck disable=SC2154 if [ "${skip_ECS_auth_test}" = "true" ]; then echo "This platform does not support the ECS auth test, skipping..." exit 0 diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh new file mode 100644 index 0000000000..dd9d522c8a --- /dev/null +++ b/.evergreen/scripts/run-gcpkms-fail-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 +export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz +SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh +SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh new file mode 100644 index 0000000000..b2d6ecb476 --- /dev/null +++ b/.evergreen/scripts/run-getdata.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o xtrace +. ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true +get_distro || true +echo $DISTRO +echo $MARCH +echo $OS +uname -a || true +ls /etc/*release* || true +cc --version || true +gcc --version || true +clang --version || true +gcov --version || true +lcov --version || true +llvm-cov --version || true +echo $PATH +ls -la /usr/local/Cellar/llvm/*/bin/ || true +ls -la /usr/local/Cellar/ || true +scan-build --version || true +genhtml --version || true +valgrind --version || true diff --git a/.evergreen/scripts/run-load-balancer.sh b/.evergreen/scripts/run-load-balancer.sh new file mode 100644 index 0000000000..9cbc277697 --- /dev/null +++ b/.evergreen/scripts/run-load-balancer.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +. src/.evergreen/scripts/env.sh +MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh index c7a5239cbd..945ea488f8 100644 --- a/.evergreen/scripts/run-ocsp-test.sh +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -3,6 +3,7 @@ . .evergreen/scripts/env.sh TEST_OCSP=1 \ +PYTHON_BINARY="${PYTHON_BINARY}" \ CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/run-perf-tests.sh b/.evergreen/scripts/run-perf-tests.sh new file mode 100644 index 0000000000..ab0cf9fbc1 --- /dev/null +++ b/.evergreen/scripts/run-perf-tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +PROJECT_DIRECTORY=${PROJECT_DIRECTORY} +bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh diff --git a/.evergreen/scripts/stop-load-balancer.sh b/.evergreen/scripts/stop-load-balancer.sh new file mode 100644 index 0000000000..2d3c5366ec --- /dev/null +++ b/.evergreen/scripts/stop-load-balancer.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd "${DRIVERS_TOOLS}"/.evergreen || exit +DRIVERS_TOOLS=${DRIVERS_TOOLS} +bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop diff --git a/.evergreen/scripts/teardown-aws.sh b/.evergreen/scripts/teardown-aws.sh new file mode 100644 index 0000000000..2386ee2df2 --- /dev/null +++ b/.evergreen/scripts/teardown-aws.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +. src/.evergreen/scripts/env.sh + +cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || exit +if [ -f "./aws_e2e_setup.json" ]; then + . ./activate-authawsvenv.sh + python ./lib/aws_assign_instance_profile.py +fi diff --git a/.evergreen/scripts/teardown-docker.sh b/.evergreen/scripts/teardown-docker.sh new file mode 100644 index 0000000000..733779d058 --- /dev/null +++ b/.evergreen/scripts/teardown-docker.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Remove all Docker images +DOCKER=$(command -v docker) || true +if [ -n "$DOCKER" ]; then + docker rmi -f "$(docker images -a -q)" &> /dev/null || true +fi diff --git a/.evergreen/scripts/upload-coverage-report.sh b/.evergreen/scripts/upload-coverage-report.sh new file mode 100644 index 0000000000..71a2a80bb8 --- /dev/null +++ b/.evergreen/scripts/upload-coverage-report.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aws s3 cp htmlcov/ s3://"$1"/coverage/"$2"/"$3"/htmlcov/ --recursive --acl public-read --region us-east-1 diff --git a/.evergreen/scripts/windows-fix.sh b/.evergreen/scripts/windows-fix.sh index 5e489e9c9c..cb4fa44130 100644 --- a/.evergreen/scripts/windows-fix.sh +++ b/.evergreen/scripts/windows-fix.sh @@ -2,9 +2,10 @@ set +x . src/.evergreen/scripts/env.sh -for i in $(find $DRIVERS_TOOLS/.evergreen $PROJECT_DIRECTORY/.evergreen -name \*.sh); do - cat $i | tr -d '\r' >$i.new - mv $i.new $i +# shellcheck disable=SC2044 +for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do + < "$i" tr -d '\r' >"$i".new + mv "$i".new "$i" done # Copy client certificate because symlinks do not work on Windows. -cp $DRIVERS_TOOLS/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem +cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem From 312266f8f25898cbaf214fb85450fa14f2eda953 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 Nov 2024 13:37:23 -0500 Subject: [PATCH 11/47] Run tests --- .evergreen/config.yml | 76 +++------------------------------ .evergreen/scripts/run-tests.sh | 60 ++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 71 deletions(-) create mode 100644 .evergreen/scripts/run-tests.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 37000af7ea..c20a106013 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -288,79 +288,13 @@ functions: - .evergreen/scripts/run-doctests.sh "run tests": - - command: shell.exec - params: - working_dir: "src" - shell: bash - background: true - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - . .evergreen/scripts/env.sh - if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup - fi - - command: shell.exec - type: test + - command: subprocess.exec params: - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" - shell: bash - script: | - # Disable xtrace - set +x - . .evergreen/scripts/env.sh - if [ -n "${MONGODB_STARTED}" ]; then - export PYMONGO_MUST_CONNECT=true - fi - if [ -n "${DISABLE_TEST_COMMANDS}" ]; then - export PYMONGO_DISABLE_TEST_COMMANDS=1 - fi - if [ -n "${test_encryption}" ]; then - # Disable xtrace (just in case it was accidentally set). - set +x - bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh - export TEST_ENCRYPTION=1 - if [ -n "${test_encryption_pyopenssl}" ]; then - export TEST_ENCRYPTION_PYOPENSSL=1 - fi - fi - if [ -n "${test_crypt_shared}" ]; then - export TEST_CRYPT_SHARED=1 - export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} - fi - if [ -n "${test_pyopenssl}" ]; then - export TEST_PYOPENSSL=1 - fi - if [ -n "${SETDEFAULTENCODING}" ]; then - export SETDEFAULTENCODING="${SETDEFAULTENCODING}" - fi - if [ -n "${test_loadbalancer}" ]; then - export TEST_LOADBALANCER=1 - export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" - export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" - fi - if [ -n "${test_serverless}" ]; then - export TEST_SERVERLESS=1 - fi - if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then - export TEST_INDEX_MANAGEMENT=1 - fi - if [ -n "${SKIP_CSOT_TESTS}" ]; then - export SKIP_CSOT_TESTS=1 - fi - - GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ - PYTHON_BINARY=${PYTHON_BINARY} \ - NO_EXT=${NO_EXT} \ - COVERAGE=${COVERAGE} \ - COMPRESSORS=${COMPRESSORS} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - TEST_DATA_LAKE=${TEST_DATA_LAKE} \ - TEST_SUITES=${TEST_SUITES} \ - MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - SKIP_HATCH=${SKIP_HATCH} \ - bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg + binary: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + args: + - .evergreen/scripts/run-tests.sh "run enterprise auth tests": - command: subprocess.exec diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh new file mode 100644 index 0000000000..312a55edee --- /dev/null +++ b/.evergreen/scripts/run-tests.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +if [ -n "${test_encryption}" ]; then + ./.evergreen/hatch.sh encryption:setup & +fi + +set +x +if [ -n "${MONGODB_STARTED}" ]; then + export PYMONGO_MUST_CONNECT=true +fi +if [ -n "${DISABLE_TEST_COMMANDS}" ]; then + export PYMONGO_DISABLE_TEST_COMMANDS=1 +fi +if [ -n "${test_encryption}" ]; then + # Disable xtrace (just in case it was accidentally set). + set +x + bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh + export TEST_ENCRYPTION=1 + if [ -n "${test_encryption_pyopenssl}" ]; then + export TEST_ENCRYPTION_PYOPENSSL=1 + fi +fi +if [ -n "${test_crypt_shared}" ]; then + export TEST_CRYPT_SHARED=1 + export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} +fi +if [ -n "${test_pyopenssl}" ]; then + export TEST_PYOPENSSL=1 +fi +if [ -n "${SETDEFAULTENCODING}" ]; then + export SETDEFAULTENCODING="${SETDEFAULTENCODING}" +fi +if [ -n "${test_loadbalancer}" ]; then + export TEST_LOADBALANCER=1 + export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" + export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" +fi +if [ -n "${test_serverless}" ]; then + export TEST_SERVERLESS=1 +fi +if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then + export TEST_INDEX_MANAGEMENT=1 +fi +if [ -n "${SKIP_CSOT_TESTS}" ]; then + export SKIP_CSOT_TESTS=1 +fi + +GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ + PYTHON_BINARY=${PYTHON_BINARY} \ + NO_EXT=${NO_EXT} \ + COVERAGE=${COVERAGE} \ + COMPRESSORS=${COMPRESSORS} \ + AUTH=${AUTH} \ + SSL=${SSL} \ + TEST_DATA_LAKE=${TEST_DATA_LAKE} \ + TEST_SUITES=${TEST_SUITES} \ + MONGODB_API_VERSION=${MONGODB_API_VERSION} \ + SKIP_HATCH=${SKIP_HATCH} \ + bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg From 1b1abc8e9fb5f6ad40bd4aa161c83af4b44b4dc9 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 Nov 2024 13:54:10 -0500 Subject: [PATCH 12/47] Cleanup --- .evergreen/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c20a106013..ba1a322760 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -548,9 +548,6 @@ functions: binary: bash args: - .evergreen/scripts/run-perf-tests.sh - script: | - . .evergreen/scripts/env.sh - PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-perf-tests.sh "attach benchmark test results": - command: attach.results @@ -1538,7 +1535,7 @@ tasks: - command: subprocess.exec type: test params: - shell: "bash" + binary: bash working_dir: src args: - .evergreen/scripts/check-import-time.sh From b395f01c3632aaa32ecea743c50991d3d9a8c763 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 7 Nov 2024 14:20:13 -0500 Subject: [PATCH 13/47] Debugging --- .evergreen/config.yml | 4 ++-- .evergreen/scripts/configure-env.sh | 5 ++++- .evergreen/scripts/run-mongodb-aws-test.sh | 6 +++++- .evergreen/scripts/run-ocsp-test.sh | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ba1a322760..8cad5dcace 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -40,7 +40,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "PYTHON_BINARY"] binary: bash working_dir: "src" args: @@ -486,7 +486,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED", "PYTHON_BINARY"] + include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED"] binary: bash working_dir: "src" args: diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 0c9c8bb03a..ea125a72b6 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -eux # Get the current unique version of this checkout # shellcheck disable=SC2154 @@ -38,6 +38,9 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" +export AUTH="${AUTH:-noauth}" +export SSL="${SSL:-nossl}" +export PYTHON_BINARY="${PYTHON_BINARY:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index 375ed854f1..8e15e198a7 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -3,6 +3,8 @@ set -o xtrace set -o errexit # Exit the script with error if any of the commands fail +set -eu + ############################################ # Main Program # ############################################ @@ -14,7 +16,7 @@ set -o errexit # Exit the script with error if any of the commands fail # PYTHON_BINARY The Python version to use. # shellcheck disable=SC2154 -if [ "${skip_EC2_auth_test}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then +if [ "${skip_EC2_auth_test:-}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then echo "This platform does not support the EC2 auth test, skipping..." exit 0 fi @@ -22,6 +24,8 @@ fi . .evergreen/scripts/env.sh echo "Running MONGODB-AWS authentication tests for $1" +python -c "import os; print([key for key in os.environ.keys()])" + # Handle credentials and environment setup. . "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1" diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh index 945ea488f8..ab0ea8e8ef 100644 --- a/.evergreen/scripts/run-ocsp-test.sh +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -1,5 +1,6 @@ #!/bin/bash +set -eu . .evergreen/scripts/env.sh TEST_OCSP=1 \ From 7bda2943596640cd3029c77598de66590eb8d223 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 10:41:37 -0500 Subject: [PATCH 14/47] Fixes --- .evergreen/config.yml | 38 ++++++++++++++++++++------ .evergreen/scripts/configure-env.sh | 19 +++++++++++++ .evergreen/scripts/run-atlas-tests.sh | 1 - .evergreen/scripts/run-with-env.sh | 13 +++++++++ .evergreen/scripts/setup-encryption.sh | 8 ++++++ 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 .evergreen/scripts/run-with-env.sh create mode 100644 .evergreen/scripts/setup-encryption.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8cad5dcace..d75837dfe1 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -288,6 +288,14 @@ functions: - .evergreen/scripts/run-doctests.sh "run tests": + - command: subprocess.exec + params: + working_dir: "src" + binary: bash + background: true + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + args: + - .evergreen/scripts/setup-encryption.sh - command: subprocess.exec params: working_dir: "src" @@ -314,6 +322,7 @@ functions: include_expansions_in_env: ["PYTHON_BINARY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-atlas-tests.sh "get aws auth secrets": @@ -344,7 +353,8 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-mongodb-aws-test.sh assume-role + - .evergreen/scripts/run-mongodb-aws-test.sh + - assume-role "run aws auth test with aws EC2 credentials": - command: subprocess.exec @@ -354,9 +364,11 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-mongodb-aws-test.sh ec2 + - .evergreen/scripts/run-mongodb-aws-test.sh + - ec2 "run aws auth test with aws web identity credentials": + - # Test with and without AWS_ROLE_SESSION_NAME set. - command: subprocess.exec type: test params: @@ -364,9 +376,17 @@ functions: binary: bash working_dir: "src" args: - # Test with and without AWS_ROLE_SESSION_NAME set. - - .evergreen/scripts/run-mongodb-aws-test.sh web-identity - - AWS_ROLE_SESSION_NAME="test" .evergreen/scripts/run-mongodb-aws-test.sh web-identity + - .evergreen/scripts/run-mongodb-aws-test.sh + - web-identity + - command: subprocess.exec + type: test + params: + include_expansions_in_env: [ "DRIVERS_TOOLS", "skip_EC2_auth_test" ] + binary: bash + working_dir: "src" + args: + - AWS_ROLE_SESSION_NAME="test" .evergreen/scripts/run-mongodb-aws-test.sh + - web-identity "run aws auth test with aws credentials as environment variables": - command: subprocess.exec @@ -376,7 +396,8 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-mongodb-aws-test.sh env-creds + - .evergreen/scripts/run-mongodb-aws-test.sh + - env-creds "run aws auth test with aws credentials and session token as environment variables": - command: subprocess.exec @@ -386,7 +407,8 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-mongodb-aws-test.sh session-creds + - .evergreen/scripts/run-mongodb-aws-test.sh + - session-creds "run oidc auth test with test credentials": - command: subprocess.exec @@ -497,7 +519,7 @@ functions: params: background: true binary: bash - include_expansions_in_env: ["SERVER_TYPE", "OCSP_ALGORITHM"] + include_expansions_in_env: [SERVER_TYPE, OCSP_ALGORITHM] args: - ${DRIVERS_TOOLS}/.evergreen/ocsp/setup.sh diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index ea125a72b6..f6b8b65f2c 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -41,6 +41,25 @@ export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export AUTH="${AUTH:-noauth}" export SSL="${SSL:-nossl}" export PYTHON_BINARY="${PYTHON_BINARY:-}" +export test_encryption="${test_encryption:-}" +export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" +export test_crypt_shared="${test_crypt_shared:-}" +export test_pyopenssl="${test_pyopenssl:-}" +export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" +export test_loadbalancer="${test_loadbalancer:-}" +export test_serverless="${test_serverless:-}" +export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" +export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" +export MONGODB_STARTED="${MONGODB_STARTED:-}" +export DISABLE_TEST_COMMANDS="${DISABLE_TEST_COMMANDS:-}" +export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}" +export NO_EXT="${NO_EXT:-}" +export COVERAGE="${COVERAGE:-}" +export COMPRESSORS="${COMPRESSORS:-}" +export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" +export TEST_SUITES="${TEST_SUITES:-}" +export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}" +export SKIP_HATCH="${SKIP_HATCH:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index 816566ab6a..98a19f047f 100644 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -1,6 +1,5 @@ #!/bin/bash -. .evergreen/scripts/env.sh # Disable xtrace for security reasons (just in case it was accidentally set). set +x set -o errexit diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh new file mode 100644 index 0000000000..360941caf4 --- /dev/null +++ b/.evergreen/scripts/run-with-env.sh @@ -0,0 +1,13 @@ +#!/bin/bash -eu + +# Example use: bash run-with-env.sh run-tests.sh {args...} + +# Parameter expansion to get just the current directory's name +if [ "${PWD##*/}" == "src" ]; then + . .evergreen/scripts/env.sh +else + . src/.evergreen/scripts/env.sh +fi + +# shellcheck source=/dev/null +. "$1" "${@:2}" diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh new file mode 100644 index 0000000000..4dc3b44f73 --- /dev/null +++ b/.evergreen/scripts/setup-encryption.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eu + +. .evergreen/scripts/env.sh +if [ -n "${test_encryption}" ]; then + ./.evergreen/hatch.sh encryption:setup & +fi From 3ef0084f862b718226de14b0cf231287c2769ecc Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 10:57:58 -0500 Subject: [PATCH 15/47] Add run-with-env.sh --- .evergreen/config.yml | 25 +++++++++++++++++-- .../scripts/bootstrap-mongo-orchestration.sh | 1 - .evergreen/scripts/cleanup.sh | 1 - .evergreen/scripts/configure-env.sh | 7 ++++++ .evergreen/scripts/install-dependencies.sh | 3 +-- .evergreen/scripts/run-aws-ecs-auth-test.sh | 3 +-- .evergreen/scripts/run-doctests.sh | 3 +-- .../scripts/run-enterprise-auth-tests.sh | 2 +- .evergreen/scripts/run-load-balancer.sh | 1 - .evergreen/scripts/run-mockupdb-tests.sh | 3 +-- .evergreen/scripts/run-mod-wsgi-tests.sh | 2 -- .evergreen/scripts/run-mongodb-aws-test.sh | 5 ---- .evergreen/scripts/run-ocsp-test.sh | 3 --- .evergreen/scripts/run-perf-tests.sh | 1 - .evergreen/scripts/run-tests.sh | 1 - .evergreen/scripts/run-with-env.sh | 4 ++- .evergreen/scripts/setup-encryption.sh | 3 --- .evergreen/scripts/teardown-aws.sh | 2 -- 18 files changed, 38 insertions(+), 32 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d75837dfe1..46530aa45b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -228,6 +228,7 @@ functions: params: binary: bash args: + - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh - command: expansions.update params: @@ -267,6 +268,7 @@ functions: working_dir: "src" binary: bash args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mod-wsgi-tests.sh "run mockupdb tests": @@ -276,6 +278,7 @@ functions: working_dir: "src" binary: bash args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mockupdb-tests.sh "run doctests": @@ -285,6 +288,7 @@ functions: working_dir: "src" binary: bash args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-doctests.sh "run tests": @@ -295,6 +299,7 @@ functions: background: true include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/setup-encryption.sh - command: subprocess.exec params: @@ -302,6 +307,7 @@ functions: binary: bash include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh "run enterprise auth tests": @@ -312,6 +318,7 @@ functions: working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-enterprise-auth-tests.sh "run atlas tests": @@ -319,7 +326,7 @@ functions: type: test params: binary: bash - include_expansions_in_env: ["PYTHON_BINARY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" args: - .evergreen/scripts/run-with-env.sh @@ -338,10 +345,11 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["DRIVERS_TOOLS", "PYTHON_BINARY", "skip_EC2_auth_test"] + include_expansions_in_env: ["DRIVERS_TOOLS", "skip_EC2_auth_test"] binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mongodb-aws-test.sh - regular @@ -353,6 +361,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mongodb-aws-test.sh - assume-role @@ -364,6 +373,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mongodb-aws-test.sh - ec2 @@ -376,6 +386,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mongodb-aws-test.sh - web-identity - command: subprocess.exec @@ -385,6 +396,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - AWS_ROLE_SESSION_NAME="test" .evergreen/scripts/run-mongodb-aws-test.sh - web-identity @@ -396,6 +408,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mongodb-aws-test.sh - env-creds @@ -407,6 +420,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-mongodb-aws-test.sh - session-creds @@ -439,6 +453,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-aws-ecs-auth-test.sh "cleanup": @@ -447,6 +462,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/cleanup.sh "fix absolute paths": @@ -483,6 +499,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/install-dependencies.sh "assume ec2 role": @@ -512,6 +529,7 @@ functions: binary: bash working_dir: "src" args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-ocsp-test.sh "run-ocsp-server": @@ -529,6 +547,7 @@ functions: binary: bash include_expansions_in_env: ["MONGODB_URI"] args: + - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/run-load-balancer.sh - command: expansions.update params: @@ -553,6 +572,7 @@ functions: params: binary: bash args: + - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/teardown-aws.sh "teardown atlas": @@ -569,6 +589,7 @@ functions: working_dir: "src" binary: bash args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-perf-tests.sh "attach benchmark test results": diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index a7d8cf5da0..f3b539ffb1 100644 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -1,6 +1,5 @@ #!/bin/bash -. src/.evergreen/scripts/env.sh set -o xtrace # Enable core dumps if enabled on the machine diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index 3c66a13e36..9e583e4f1e 100644 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -1,6 +1,5 @@ #!/bin/bash -. .evergreen/scripts/env.sh if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then . .evergreen/hatch.sh encryption:teardown fi diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index f6b8b65f2c..d088ab3032 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -60,6 +60,13 @@ export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" export TEST_SUITES="${TEST_SUITES:-}" export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}" export SKIP_HATCH="${SKIP_HATCH:-}" +export skip_crypt_shared="${skip_crypt_shared:-}" +export VERSION="${VERSION:-}" +export TOPOLOGY="${TOPOLOGY:-}" +export STORAGE_ENGINE="${STORAGE_ENGINE:-}" +export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" +export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" +export LOAD_BALANCER="${LOAD_BALANCER:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index c7cdbe67cf..ebcc8f3069 100644 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -1,7 +1,6 @@ #!/bin/bash -. .evergreen/scripts/env.sh set -o xtrace file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. -[ -f "$file" ] && bash $file || echo "$file not available, skipping" +[ -f "$file" ] && bash "$file" || echo "$file not available, skipping" diff --git a/.evergreen/scripts/run-aws-ecs-auth-test.sh b/.evergreen/scripts/run-aws-ecs-auth-test.sh index 295ba8ca54..787e0a710b 100644 --- a/.evergreen/scripts/run-aws-ecs-auth-test.sh +++ b/.evergreen/scripts/run-aws-ecs-auth-test.sh @@ -5,9 +5,8 @@ if [ "${skip_ECS_auth_test}" = "true" ]; then echo "This platform does not support the ECS auth test, skipping..." exit 0 fi -. .evergreen/scripts/env.sh set -ex -cd $DRIVERS_TOOLS/.evergreen/auth_aws +cd "$DRIVERS_TOOLS"/.evergreen/auth_aws . ./activate-authawsvenv.sh . aws_setup.sh ecs export MONGODB_BINARIES="$MONGODB_BINARIES" diff --git a/.evergreen/scripts/run-doctests.sh b/.evergreen/scripts/run-doctests.sh index 2e2b6b10b3..f7215ad347 100644 --- a/.evergreen/scripts/run-doctests.sh +++ b/.evergreen/scripts/run-doctests.sh @@ -1,5 +1,4 @@ #!/bin/bash -. .evergreen/scripts/env.sh set -o xtrace -PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test +PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh index 37298a2bdb..31371ead45 100644 --- a/.evergreen/scripts/run-enterprise-auth-tests.sh +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -2,5 +2,5 @@ # Disable xtrace for security reasons (just in case it was accidentally set). set +x -bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth +bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/run-load-balancer.sh b/.evergreen/scripts/run-load-balancer.sh index 9cbc277697..7d431777e5 100644 --- a/.evergreen/scripts/run-load-balancer.sh +++ b/.evergreen/scripts/run-load-balancer.sh @@ -1,4 +1,3 @@ #!/bin/bash -. src/.evergreen/scripts/env.sh MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start diff --git a/.evergreen/scripts/run-mockupdb-tests.sh b/.evergreen/scripts/run-mockupdb-tests.sh index 1148e96fc4..8825a0237d 100644 --- a/.evergreen/scripts/run-mockupdb-tests.sh +++ b/.evergreen/scripts/run-mockupdb-tests.sh @@ -1,6 +1,5 @@ #!/bin/bash -. .evergreen/scripts/env.sh set -o xtrace export PYTHON_BINARY=${PYTHON_BINARY} -bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb +bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb diff --git a/.evergreen/scripts/run-mod-wsgi-tests.sh b/.evergreen/scripts/run-mod-wsgi-tests.sh index 3fcae568e5..e1f5238110 100644 --- a/.evergreen/scripts/run-mod-wsgi-tests.sh +++ b/.evergreen/scripts/run-mod-wsgi-tests.sh @@ -2,8 +2,6 @@ set -o xtrace set -o errexit -. .evergreen/scripts/env.sh - APACHE=$(command -v apache2 || command -v /usr/lib/apache2/mpm-prefork/apache2) || true if [ -n "$APACHE" ]; then APACHE_CONFIG=apache24ubuntu161404.conf diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index 8e15e198a7..ec20bfd06b 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -3,8 +3,6 @@ set -o xtrace set -o errexit # Exit the script with error if any of the commands fail -set -eu - ############################################ # Main Program # ############################################ @@ -21,11 +19,8 @@ if [ "${skip_EC2_auth_test:-}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web exit 0 fi -. .evergreen/scripts/env.sh echo "Running MONGODB-AWS authentication tests for $1" -python -c "import os; print([key for key in os.environ.keys()])" - # Handle credentials and environment setup. . "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1" diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh index ab0ea8e8ef..3c6d3b2b3b 100644 --- a/.evergreen/scripts/run-ocsp-test.sh +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -1,8 +1,5 @@ #!/bin/bash -set -eu -. .evergreen/scripts/env.sh - TEST_OCSP=1 \ PYTHON_BINARY="${PYTHON_BINARY}" \ CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ diff --git a/.evergreen/scripts/run-perf-tests.sh b/.evergreen/scripts/run-perf-tests.sh index ab0cf9fbc1..69a369fee1 100644 --- a/.evergreen/scripts/run-perf-tests.sh +++ b/.evergreen/scripts/run-perf-tests.sh @@ -1,5 +1,4 @@ #!/bin/bash -. .evergreen/scripts/env.sh PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 312a55edee..dd75f41d8c 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -1,6 +1,5 @@ #!/bin/bash -. .evergreen/scripts/env.sh if [ -n "${test_encryption}" ]; then ./.evergreen/hatch.sh encryption:setup & fi diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh index 360941caf4..560f1d197e 100644 --- a/.evergreen/scripts/run-with-env.sh +++ b/.evergreen/scripts/run-with-env.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eu +#!/bin/bash # Example use: bash run-with-env.sh run-tests.sh {args...} @@ -9,5 +9,7 @@ else . src/.evergreen/scripts/env.sh fi +set -eu + # shellcheck source=/dev/null . "$1" "${@:2}" diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh index 4dc3b44f73..c168425f6b 100644 --- a/.evergreen/scripts/setup-encryption.sh +++ b/.evergreen/scripts/setup-encryption.sh @@ -1,8 +1,5 @@ #!/bin/bash -set -eu - -. .evergreen/scripts/env.sh if [ -n "${test_encryption}" ]; then ./.evergreen/hatch.sh encryption:setup & fi diff --git a/.evergreen/scripts/teardown-aws.sh b/.evergreen/scripts/teardown-aws.sh index 2386ee2df2..634d1e5724 100644 --- a/.evergreen/scripts/teardown-aws.sh +++ b/.evergreen/scripts/teardown-aws.sh @@ -1,7 +1,5 @@ #!/bin/bash -. src/.evergreen/scripts/env.sh - cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || exit if [ -f "./aws_e2e_setup.json" ]; then . ./activate-authawsvenv.sh From 8176e6cc3ba02c89303808f6ace84383a266811b Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 12:08:11 -0500 Subject: [PATCH 16/47] Fix client context OS var checks --- .evergreen/config.yml | 2 +- test/asynchronous/test_client_context.py | 8 ++++---- test/test_client_context.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 46530aa45b..bd6bf3b1c6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -264,7 +264,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["PYTHON_BINARY", "MOD_WSGI_VERSION", "MOD_WSGI_EMBEDDED"] + include_expansions_in_env: [MOD_WSGI_VERSION, MOD_WSGI_EMBEDDED] working_dir: "src" binary: bash args: diff --git a/test/asynchronous/test_client_context.py b/test/asynchronous/test_client_context.py index 6d77818436..6a195eb6b8 100644 --- a/test/asynchronous/test_client_context.py +++ b/test/asynchronous/test_client_context.py @@ -25,7 +25,7 @@ class TestAsyncClientContext(AsyncUnitTest): def test_must_connect(self): - if "PYMONGO_MUST_CONNECT" not in os.environ: + if not os.environ.get("PYMONGO_MUST_CONNECT"): raise SkipTest("PYMONGO_MUST_CONNECT is not set") self.assertTrue( @@ -37,7 +37,7 @@ def test_must_connect(self): ) def test_serverless(self): - if "TEST_SERVERLESS" not in os.environ: + if not os.environ.get("TEST_SERVERLESS"): raise SkipTest("TEST_SERVERLESS is not set") self.assertTrue( @@ -47,7 +47,7 @@ def test_serverless(self): ) def test_enableTestCommands_is_disabled(self): - if "PYMONGO_DISABLE_TEST_COMMANDS" not in os.environ: + if not os.environ.get("PYMONGO_DISABLE_TEST_COMMANDS"): raise SkipTest("PYMONGO_DISABLE_TEST_COMMANDS is not set") self.assertFalse( @@ -56,7 +56,7 @@ def test_enableTestCommands_is_disabled(self): ) def test_setdefaultencoding_worked(self): - if "SETDEFAULTENCODING" not in os.environ: + if not os.environ.get("SETDEFAULTENCODING"): raise SkipTest("SETDEFAULTENCODING is not set") self.assertEqual(sys.getdefaultencoding(), os.environ["SETDEFAULTENCODING"]) diff --git a/test/test_client_context.py b/test/test_client_context.py index 5996f9243b..e807ac5f5f 100644 --- a/test/test_client_context.py +++ b/test/test_client_context.py @@ -25,7 +25,7 @@ class TestClientContext(UnitTest): def test_must_connect(self): - if "PYMONGO_MUST_CONNECT" not in os.environ: + if not os.environ.get("PYMONGO_MUST_CONNECT"): raise SkipTest("PYMONGO_MUST_CONNECT is not set") self.assertTrue( @@ -37,7 +37,7 @@ def test_must_connect(self): ) def test_serverless(self): - if "TEST_SERVERLESS" not in os.environ: + if not os.environ.get("TEST_SERVERLESS"): raise SkipTest("TEST_SERVERLESS is not set") self.assertTrue( @@ -47,7 +47,7 @@ def test_serverless(self): ) def test_enableTestCommands_is_disabled(self): - if "PYMONGO_DISABLE_TEST_COMMANDS" not in os.environ: + if not os.environ.get("PYMONGO_DISABLE_TEST_COMMANDS"): raise SkipTest("PYMONGO_DISABLE_TEST_COMMANDS is not set") self.assertFalse( @@ -56,7 +56,7 @@ def test_enableTestCommands_is_disabled(self): ) def test_setdefaultencoding_worked(self): - if "SETDEFAULTENCODING" not in os.environ: + if not os.environ.get("SETDEFAULTENCODING"): raise SkipTest("SETDEFAULTENCODING is not set") self.assertEqual(sys.getdefaultencoding(), os.environ["SETDEFAULTENCODING"]) From c3bcadae362f6e32fa4674783a58236603ea7bdf Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 13:17:07 -0500 Subject: [PATCH 17/47] Fix data lake tests --- .evergreen/run-tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 36fa76e317..784c2dbc26 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -48,18 +48,18 @@ else fi if [ "$AUTH" != "noauth" ]; then - if [ ! -z "$TEST_DATA_LAKE" ]; then + if [ -n "$TEST_DATA_LAKE" ]; then export DB_USER="mhuser" export DB_PASSWORD="pencil" - elif [ ! -z "$TEST_SERVERLESS" ]; then - source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh + elif [ -n "$TEST_SERVERLESS" ]; then + source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh export DB_USER=$SERVERLESS_ATLAS_USER export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD export MONGODB_URI="$SERVERLESS_URI" echo "MONGODB_URI=$MONGODB_URI" export SINGLE_MONGOS_LB_URI=$MONGODB_URI export MULTI_MONGOS_LB_URI=$MONGODB_URI - elif [ ! -z "$TEST_AUTH_OIDC" ]; then + elif [ -n "$TEST_AUTH_OIDC" ]; then export DB_USER=$OIDC_ADMIN_USER export DB_PASSWORD=$OIDC_ADMIN_PWD export DB_IP="$MONGODB_URI" From a49e630d25d7512660f535e1ebc36753889edb30 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 14:39:33 -0500 Subject: [PATCH 18/47] Fix data lake tests --- .evergreen/config.yml | 2 +- .evergreen/scripts/run-mod-wsgi-tests.sh | 2 +- .evergreen/scripts/run-tests.sh | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index bd6bf3b1c6..e7a67d75f3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -305,7 +305,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh diff --git a/.evergreen/scripts/run-mod-wsgi-tests.sh b/.evergreen/scripts/run-mod-wsgi-tests.sh index e1f5238110..607458b8c6 100644 --- a/.evergreen/scripts/run-mod-wsgi-tests.sh +++ b/.evergreen/scripts/run-mod-wsgi-tests.sh @@ -28,7 +28,7 @@ export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_ export PYTHONHOME=/opt/python/$PYTHON_VERSION # If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead # of daemon mode (WSGIDaemonProcess). -if [ -n "$MOD_WSGI_EMBEDDED" ]; then +if [ -n "${MOD_WSGI_EMBEDDED:-}" ]; then export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf else export MOD_WSGI_CONF=mod_wsgi_test.conf diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index dd75f41d8c..0bd503daaa 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -45,6 +45,8 @@ if [ -n "${SKIP_CSOT_TESTS}" ]; then export SKIP_CSOT_TESTS=1 fi +python -c "import os; print([(k, v[:3]) for (k, v) in os.environ.items()])" + GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ PYTHON_BINARY=${PYTHON_BINARY} \ NO_EXT=${NO_EXT} \ From 4a9ddf9e17a0e7b8268e315afbe86c4ab6c2e39f Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 15:15:39 -0500 Subject: [PATCH 19/47] Add expansions to env --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e7a67d75f3..282b84f07a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -40,7 +40,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "PYTHON_BINARY"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "PYTHON_BINARY", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "TEST_INDEX_MANAGEMENT", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_DATA_LAKE", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER"] binary: bash working_dir: "src" args: @@ -305,7 +305,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh From fbc5a0b021ea366815dafdd831dafed6be2a900b Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 8 Nov 2024 15:48:04 -0500 Subject: [PATCH 20/47] debugging --- .evergreen/config.yml | 7 ++++--- .evergreen/scripts/configure-env.sh | 18 +++++++----------- .evergreen/scripts/run-tests.sh | 5 +++-- .evergreen/scripts/run-with-env.sh | 2 +- .evergreen/scripts/setup-encryption.sh | 2 +- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 282b84f07a..43e3800d9f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -40,7 +40,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "PYTHON_BINARY", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "TEST_INDEX_MANAGEMENT", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_DATA_LAKE", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "PYTHON_BINARY", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER"] binary: bash working_dir: "src" args: @@ -85,7 +85,7 @@ functions: working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - - src/.evergreen/scripts/download-and-merge-coverage.sh + - .evergreen/scripts/download-and-merge-coverage.sh - ${bucket_name} - ${revision} - ${version_id} @@ -227,6 +227,7 @@ functions: - command: subprocess.exec params: binary: bash + include_expansions_in_env: ["VERSION", "TOPOLOGY"] args: - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -305,7 +306,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "test_serverless", "CRYPT_SHARED_LIB_PATH"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index d088ab3032..c9759cb244 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -29,7 +29,7 @@ fi export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" -cat < $SCRIPT_DIR/env.sh +cat < "$SCRIPT_DIR"/env.sh set -o errexit export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export CURRENT_VERSION="$CURRENT_VERSION" @@ -41,14 +41,7 @@ export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export AUTH="${AUTH:-noauth}" export SSL="${SSL:-nossl}" export PYTHON_BINARY="${PYTHON_BINARY:-}" -export test_encryption="${test_encryption:-}" -export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" -export test_crypt_shared="${test_crypt_shared:-}" -export test_pyopenssl="${test_pyopenssl:-}" export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" -export test_loadbalancer="${test_loadbalancer:-}" -export test_serverless="${test_serverless:-}" -export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" export MONGODB_STARTED="${MONGODB_STARTED:-}" export DISABLE_TEST_COMMANDS="${DISABLE_TEST_COMMANDS:-}" @@ -56,17 +49,20 @@ export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}" export NO_EXT="${NO_EXT:-}" export COVERAGE="${COVERAGE:-}" export COMPRESSORS="${COMPRESSORS:-}" -export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" export TEST_SUITES="${TEST_SUITES:-}" export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}" export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" -export VERSION="${VERSION:-}" -export TOPOLOGY="${TOPOLOGY:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" export LOAD_BALANCER="${LOAD_BALANCER:-}" +test_encryption="${test_encryption:-}" +test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" +test_crypt_shared="${test_crypt_shared:-}" +test_pyopenssl="${test_pyopenssl:-}" +test_loadbalancer="${test_loadbalancer:-}" +test_serverless="${test_serverless:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 0bd503daaa..7201f1001c 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -1,5 +1,8 @@ #!/bin/bash +TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" +TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" + if [ -n "${test_encryption}" ]; then ./.evergreen/hatch.sh encryption:setup & fi @@ -45,8 +48,6 @@ if [ -n "${SKIP_CSOT_TESTS}" ]; then export SKIP_CSOT_TESTS=1 fi -python -c "import os; print([(k, v[:3]) for (k, v) in os.environ.items()])" - GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ PYTHON_BINARY=${PYTHON_BINARY} \ NO_EXT=${NO_EXT} \ diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh index 560f1d197e..cff5d6d64e 100644 --- a/.evergreen/scripts/run-with-env.sh +++ b/.evergreen/scripts/run-with-env.sh @@ -6,7 +6,7 @@ if [ "${PWD##*/}" == "src" ]; then . .evergreen/scripts/env.sh else - . src/.evergreen/scripts/env.sh + . src/.evergreen/scripts/env.sh fi set -eu diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh index c168425f6b..2f167cd20b 100644 --- a/.evergreen/scripts/setup-encryption.sh +++ b/.evergreen/scripts/setup-encryption.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup & + ./.evergreen/hatch.sh encryption:setup fi From fe40a95769721b50ea66b04016ac390017cfd939 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Mon, 11 Nov 2024 14:23:26 -0500 Subject: [PATCH 21/47] More env fixes --- .evergreen/config.yml | 2 +- .evergreen/scripts/run-tests.sh | 7 ++----- .evergreen/utils.sh | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 43e3800d9f..d09c05b2de 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -306,7 +306,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "test_serverless", "CRYPT_SHARED_LIB_PATH"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 7201f1001c..49e9888758 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -1,8 +1,5 @@ #!/bin/bash -TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" -TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" - if [ -n "${test_encryption}" ]; then ./.evergreen/hatch.sh encryption:setup & fi @@ -41,7 +38,7 @@ fi if [ -n "${test_serverless}" ]; then export TEST_SERVERLESS=1 fi -if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then +if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then export TEST_INDEX_MANAGEMENT=1 fi if [ -n "${SKIP_CSOT_TESTS}" ]; then @@ -55,7 +52,7 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ COMPRESSORS=${COMPRESSORS} \ AUTH=${AUTH} \ SSL=${SSL} \ - TEST_DATA_LAKE=${TEST_DATA_LAKE} \ + TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ TEST_SUITES=${TEST_SUITES} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ SKIP_HATCH=${SKIP_HATCH} \ diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index d44425a905..908cf0564a 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -17,7 +17,7 @@ find_python3() { elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.9" ]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3" fi - elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + elif [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin PYTHON="C:/python/Python39/python.exe" else # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.9+. @@ -56,7 +56,7 @@ createvirtualenv () { # Workaround for bug in older versions of virtualenv. $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH fi - if [ "Windows_NT" = "$OS" ]; then + if [ "Windows_NT" = "${OS:-}" ]; then # Workaround https://bugs.python.org/issue32451: # mongovenv/Scripts/activate: line 3: $'\r': command not found dos2unix $VENVPATH/Scripts/activate || true From 40a99843b6e32b3d23b11e6f7865783f3c3158fc Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Mon, 11 Nov 2024 15:43:18 -0500 Subject: [PATCH 22/47] Add setup-tests.sh --- .evergreen/config.yml | 65 ++++++++++++++++++- .../scripts/bootstrap-mongo-orchestration.sh | 2 +- .evergreen/scripts/configure-env.sh | 7 -- .evergreen/scripts/run-with-env.sh | 8 ++- .evergreen/scripts/setup-tests.sh | 25 +++++++ 5 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 .evergreen/scripts/setup-tests.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d09c05b2de..8534162843 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -227,7 +227,7 @@ functions: - command: subprocess.exec params: binary: bash - include_expansions_in_env: ["VERSION", "TOPOLOGY"] + include_expansions_in_env: ["VERSION", "TOPOLOGY", "ORCHESTRATION_FILE"] args: - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -293,6 +293,13 @@ functions: - .evergreen/scripts/run-doctests.sh "run tests": + - command: subprocess.exec + params: + include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI"] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec params: working_dir: "src" @@ -343,6 +350,13 @@ functions: - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh "run aws auth test with regular aws credentials": + - command: subprocess.exec + params: + include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -355,6 +369,13 @@ functions: - regular "run aws auth test with assume role credentials": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -367,6 +388,13 @@ functions: - assume-role "run aws auth test with aws EC2 credentials": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -379,6 +407,13 @@ functions: - ec2 "run aws auth test with aws web identity credentials": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - # Test with and without AWS_ROLE_SESSION_NAME set. - command: subprocess.exec type: test @@ -402,6 +437,13 @@ functions: - web-identity "run aws auth test with aws credentials as environment variables": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -414,6 +456,13 @@ functions: - env-creds "run aws auth test with aws credentials and session token as environment variables": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -426,6 +475,13 @@ functions: - session-creds "run oidc auth test with test credentials": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: @@ -523,6 +579,13 @@ functions: file: atlas-expansion.yml "run-ocsp-test": + - command: subprocess.exec + params: + include_expansions_in_env: [ "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE" ] + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/setup-tests.sh - command: subprocess.exec type: test params: diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index f3b539ffb1..a431b869e9 100644 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -39,7 +39,7 @@ MONGODB_VERSION=${VERSION} \ SSL=${SSL} \ STORAGE_ENGINE=${STORAGE_ENGINE} \ DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ + ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \ REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ LOAD_BALANCER=${LOAD_BALANCER} \ bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index c9759cb244..b2bf3503b6 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -54,15 +54,8 @@ export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}" export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" -export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" export LOAD_BALANCER="${LOAD_BALANCER:-}" -test_encryption="${test_encryption:-}" -test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" -test_crypt_shared="${test_crypt_shared:-}" -test_pyopenssl="${test_pyopenssl:-}" -test_loadbalancer="${test_loadbalancer:-}" -test_serverless="${test_serverless:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh index cff5d6d64e..59da15851e 100644 --- a/.evergreen/scripts/run-with-env.sh +++ b/.evergreen/scripts/run-with-env.sh @@ -1,12 +1,18 @@ -#!/bin/bash +#!/bin/bash -eu # Example use: bash run-with-env.sh run-tests.sh {args...} # Parameter expansion to get just the current directory's name if [ "${PWD##*/}" == "src" ]; then . .evergreen/scripts/env.sh + if [ -f ".evergreen/scripts/test-env.sh" ]; then + . .evergreen/scripts/test-env.sh + fi else . src/.evergreen/scripts/env.sh + if [ -f "src/.evergreen/scripts/test-env.sh" ]; then + . src/.evergreen/scripts/test-env.sh + fi fi set -eu diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh new file mode 100644 index 0000000000..a996b8559a --- /dev/null +++ b/.evergreen/scripts/setup-tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash -eux + +PROJECT_DIRECTORY="$(pwd)" +SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" + +if [ -f "$SCRIPT_DIR/test-env.sh" ]; then + echo "Reading $SCRIPT_DIR/test-env.sh file" + . "$SCRIPT_DIR/test-env.sh" + exit 0 +fi + +cat < "$SCRIPT_DIR"/test-env.sh +echo "Sourcing test-env.sh now, ORCHESTRATION_FILE: $ORCHESTRATION_FILE" +export test_encryption="${test_encryption:-}" +export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" +export test_crypt_shared="${test_crypt_shared:-}" +export test_pyopenssl="${test_pyopenssl:-}" +export test_loadbalancer="${test_loadbalancer:-}" +export test_serverless="${test_serverless:-}" +export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" +export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" +export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" +EOT + +chmod +x "$SCRIPT_DIR"/test-env.sh From 01704ba4e085ca0ab536a6249cb58c00a3682653 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 12 Nov 2024 10:49:30 -0500 Subject: [PATCH 23/47] Cleanup --- .evergreen/config.yml | 5 +++-- .evergreen/scripts/download-and-merge-coverage.sh | 1 - .evergreen/scripts/run-tests.sh | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8534162843..901f148b9c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -85,6 +85,7 @@ functions: working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: + - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/download-and-merge-coverage.sh - ${bucket_name} - ${revision} @@ -295,7 +296,7 @@ functions: "run tests": - command: subprocess.exec params: - include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI"] + include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"] binary: bash working_dir: "src" args: @@ -313,7 +314,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh diff --git a/.evergreen/scripts/download-and-merge-coverage.sh b/.evergreen/scripts/download-and-merge-coverage.sh index 0083f2899a..a8ea5f52ab 100644 --- a/.evergreen/scripts/download-and-merge-coverage.sh +++ b/.evergreen/scripts/download-and-merge-coverage.sh @@ -1,6 +1,5 @@ #!/bin/bash set +x -. src/.evergreen/scripts/env.sh # Download all the task coverage files. aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/ diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 49e9888758..146fdec24e 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -1,9 +1,5 @@ #!/bin/bash -if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup & -fi - set +x if [ -n "${MONGODB_STARTED}" ]; then export PYMONGO_MUST_CONNECT=true From 57acb8519f976b5261e1024b2e75823392f7ed8c Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 12 Nov 2024 11:13:00 -0500 Subject: [PATCH 24/47] Cleanup --- .evergreen/scripts/setup-tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index a996b8559a..521007e4ee 100644 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -10,7 +10,6 @@ if [ -f "$SCRIPT_DIR/test-env.sh" ]; then fi cat < "$SCRIPT_DIR"/test-env.sh -echo "Sourcing test-env.sh now, ORCHESTRATION_FILE: $ORCHESTRATION_FILE" export test_encryption="${test_encryption:-}" export test_encryption_pyopenssl="${test_encryption_pyopenssl:-}" export test_crypt_shared="${test_crypt_shared:-}" From 707452b405933acab8ad3f0d795da19a29f692ba Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 12 Nov 2024 11:47:49 -0500 Subject: [PATCH 25/47] Remove LOAD_BALANCER from system setup env --- .evergreen/config.yml | 2 +- .evergreen/scripts/configure-env.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 901f148b9c..358106bda3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -228,7 +228,7 @@ functions: - command: subprocess.exec params: binary: bash - include_expansions_in_env: ["VERSION", "TOPOLOGY", "ORCHESTRATION_FILE"] + include_expansions_in_env: ["VERSION", "TOPOLOGY", "ORCHESTRATION_FILE", "LOAD_BALANCER"] args: - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 8181722cc9..9f396c6448 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -55,7 +55,6 @@ export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" -export LOAD_BALANCER="${LOAD_BALANCER:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" From 7617ac656aee74e35cc0bf445e01713169225d83 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 12 Nov 2024 12:32:47 -0500 Subject: [PATCH 26/47] Fix unbound variables --- .../scripts/bootstrap-mongo-orchestration.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index a431b869e9..ccd62fc83b 100644 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -33,14 +33,14 @@ if [ -n "${skip_crypt_shared}" ]; then export SKIP_CRYPT_SHARED=1 fi -MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - STORAGE_ENGINE=${STORAGE_ENGINE} \ - DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ +MONGODB_VERSION=${VERSION:-} \ + TOPOLOGY=${TOPOLOGY:-} \ + AUTH=${AUTH:-noauth} \ + SSL=${SSL:-nossl} \ + STORAGE_ENGINE=${STORAGE_ENGINE:-} \ + DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \ ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - LOAD_BALANCER=${LOAD_BALANCER} \ + REQUIRE_API_VERSION=${REQUIRE_API_VERSION:-} \ + LOAD_BALANCER=${LOAD_BALANCER:-} \ bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh # run-orchestration generates expansion file with the MONGODB_URI for the cluster From 3ff9a41808523855b81475b892fc166c420b79f0 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 12 Nov 2024 14:17:36 -0500 Subject: [PATCH 27/47] Add AUTH + SSL to bootstrap orchestration --- .evergreen/config.yml | 2 +- .evergreen/scripts/bootstrap-mongo-orchestration.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 358106bda3..a4203dec7e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -228,7 +228,7 @@ functions: - command: subprocess.exec params: binary: bash - include_expansions_in_env: ["VERSION", "TOPOLOGY", "ORCHESTRATION_FILE", "LOAD_BALANCER"] + include_expansions_in_env: ["VERSION", "TOPOLOGY", "AUTH", "SSL", "ORCHESTRATION_FILE", "LOAD_BALANCER"] args: - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index ccd62fc83b..20e2b82086 100644 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -33,10 +33,10 @@ if [ -n "${skip_crypt_shared}" ]; then export SKIP_CRYPT_SHARED=1 fi -MONGODB_VERSION=${VERSION:-} \ - TOPOLOGY=${TOPOLOGY:-} \ - AUTH=${AUTH:-noauth} \ - SSL=${SSL:-nossl} \ +MONGODB_VERSION=${VERSION} \ + TOPOLOGY=${TOPOLOGY} \ + AUTH=${AUTH} \ + SSL=${SSL} \ STORAGE_ENGINE=${STORAGE_ENGINE:-} \ DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \ ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \ From 2a3cc4c118d941be782178be78e6ebe57eaa5566 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 12 Nov 2024 16:10:23 -0500 Subject: [PATCH 28/47] Remove AUTH and SSL from system env --- .evergreen/config.yml | 4 ++-- .evergreen/hatch.sh | 2 +- .evergreen/scripts/configure-env.sh | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a4203dec7e..562b396252 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -296,7 +296,7 @@ functions: "run tests": - command: subprocess.exec params: - include_expansions_in_env: ["TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"] + include_expansions_in_env: ["TEST_DATA_LAKE", "AUTH", "SSL", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "test_loadbalancer", "test_serverless", "ORCHESTRATION_FILE"] binary: bash working_dir: "src" args: @@ -314,7 +314,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AUTH", "SSL", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh index 45d5113cd6..98cd9ed734 100644 --- a/.evergreen/hatch.sh +++ b/.evergreen/hatch.sh @@ -29,7 +29,7 @@ else # Set up virtualenv before installing hatch # Ensure hatch does not write to user or global locations. touch hatch_config.toml HATCH_CONFIG=$(pwd)/hatch_config.toml - if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") fi export HATCH_CONFIG diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 9f396c6448..e77a4e87d0 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -38,8 +38,6 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" -export AUTH="${AUTH:-noauth}" -export SSL="${SSL:-nossl}" export PYTHON_BINARY="${PYTHON_BINARY:-}" export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" From 4a83e54a4ec0295f06937da8015af899206b5b1c Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 09:26:10 -0500 Subject: [PATCH 29/47] Fixes --- .evergreen/config.yml | 1 + .evergreen/scripts/archive-mongodb-logs.sh | 1 - .evergreen/setup-encryption.sh | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 562b396252..49791f09a9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -130,6 +130,7 @@ functions: params: binary: bash args: + - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/archive-mongodb-logs.sh - command: archive.targz_pack diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh index 29d7ab62f4..781c818163 100644 --- a/.evergreen/scripts/archive-mongodb-logs.sh +++ b/.evergreen/scripts/archive-mongodb-logs.sh @@ -1,6 +1,5 @@ #!/bin/bash -. src/.evergreen/scripts/env.sh set -o xtrace mkdir out_dir find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x=$1; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' shell {} \; diff --git a/.evergreen/setup-encryption.sh b/.evergreen/setup-encryption.sh index 71231e1732..72e53e7b59 100644 --- a/.evergreen/setup-encryption.sh +++ b/.evergreen/setup-encryption.sh @@ -52,6 +52,6 @@ ls -la libmongocrypt ls -la libmongocrypt/nocrypto if [ -z "${SKIP_SERVERS:-}" ]; then - bash ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh - bash ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh + bash "${DRIVERS_TOOLS}"/.evergreen/csfle/setup-secrets.sh + bash "${DRIVERS_TOOLS}"/.evergreen/csfle/start-servers.sh fi From 9d2212f9376d6e11354ae307bdc4668f6f530012 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 10:22:02 -0500 Subject: [PATCH 30/47] Don't pass PYTHON_BINARY to CSFLE setup --- .evergreen/setup-encryption.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/setup-encryption.sh b/.evergreen/setup-encryption.sh index 72e53e7b59..b403ef9ca8 100644 --- a/.evergreen/setup-encryption.sh +++ b/.evergreen/setup-encryption.sh @@ -52,6 +52,9 @@ ls -la libmongocrypt ls -la libmongocrypt/nocrypto if [ -z "${SKIP_SERVERS:-}" ]; then + PYTHON_BINARY_OLD=${PYTHON_BINARY} + export PYTHON_BINARY="" bash "${DRIVERS_TOOLS}"/.evergreen/csfle/setup-secrets.sh + export PYTHON_BINARY=$PYTHON_BINARY_OLD bash "${DRIVERS_TOOLS}"/.evergreen/csfle/start-servers.sh fi From 9949ac9dbf77e64ba4bf78e848b15c05909f5832 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 14:31:41 -0500 Subject: [PATCH 31/47] Silent coverage --- .evergreen/config.yml | 2 +- .evergreen/scripts/download-and-merge-coverage.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 49791f09a9..6231e1848e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -81,11 +81,11 @@ functions: role_arn: ${assume_role_arn} - command: subprocess.exec params: + silent: true binary: bash working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/download-and-merge-coverage.sh - ${bucket_name} - ${revision} diff --git a/.evergreen/scripts/download-and-merge-coverage.sh b/.evergreen/scripts/download-and-merge-coverage.sh index a8ea5f52ab..808bb957ef 100644 --- a/.evergreen/scripts/download-and-merge-coverage.sh +++ b/.evergreen/scripts/download-and-merge-coverage.sh @@ -1,5 +1,4 @@ #!/bin/bash -set +x # Download all the task coverage files. aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/ From fb1589516735727c671575ea7dea1568ca466858 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 14:35:25 -0500 Subject: [PATCH 32/47] Pass TEST_SUITES to run-tests --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6231e1848e..0c3b88dbb8 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -315,7 +315,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AUTH", "SSL", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AUTH", "SSL", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh From 6f74878a6db34a939d5e7034a01f6010e5b7e013 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 15:13:18 -0500 Subject: [PATCH 33/47] Fix archive-logs --- .evergreen/scripts/archive-mongodb-logs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh index 781c818163..70a337cd11 100644 --- a/.evergreen/scripts/archive-mongodb-logs.sh +++ b/.evergreen/scripts/archive-mongodb-logs.sh @@ -2,6 +2,7 @@ set -o xtrace mkdir out_dir -find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x=$1; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' shell {} \; +# shellcheck disable=SC2156 +find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; tar zcvf mongodb-logs.tar.gz -C out_dir/ . rm -rf out_dir From ae446f60b5dd1291fae0a6c197e0313ef4719316 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 15:45:15 -0500 Subject: [PATCH 34/47] Remove unneeded s3 put --- .evergreen/config.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0c3b88dbb8..f707464cd5 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -132,7 +132,6 @@ functions: args: - src/.evergreen/scripts/run-with-env.sh - src/.evergreen/scripts/archive-mongodb-logs.sh - - command: archive.targz_pack params: target: "mongo-coredumps.tgz" @@ -163,17 +162,6 @@ functions: permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "mongodb-logs.tar.gz" - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: drivers-tools/.evergreen/orchestration/server.log - remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|text/plain} - display_name: "orchestration.log" "upload working dir": - command: ec2.assume_role From 5f0b42963e2d2858659dc57d6002962937e28dc8 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 15:50:43 -0500 Subject: [PATCH 35/47] Add drivers-evergreen-tools teardown --- .evergreen/config.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f707464cd5..13c98b0b26 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -162,6 +162,17 @@ functions: permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "mongodb-logs.tar.gz" + - command: s3.put + params: + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz + bucket: ${aws_bucket} + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: "drivers-tools-logs.tar.gz" "upload working dir": - command: ec2.assume_role @@ -512,6 +523,15 @@ functions: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/cleanup.sh + "teardown": + - command: subprocess.exec + params: + binary: bash + working_dir: "src" + args: + - .evergreen/scripts/run-with-env.sh + - $DRIVERS_TOOLS/.evergreen/teardown.sh + "fix absolute paths": - command: subprocess.exec params: @@ -669,6 +689,7 @@ pre: post: # Disabled, causing timeouts # - func: "upload working dir" + - func: "teardown" - func: "upload coverage" - func: "upload mo artifacts" - func: "upload test results" From 3a3b542894d19ea8b5dc54b3d248c0c1a6dd62f8 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 15:56:24 -0500 Subject: [PATCH 36/47] More upload fixes --- .evergreen/config.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 13c98b0b26..3de05b1e31 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -126,12 +126,6 @@ functions: - command: ec2.assume_role params: role_arn: ${assume_role_arn} - - command: subprocess.exec - params: - binary: bash - args: - - src/.evergreen/scripts/run-with-env.sh - - src/.evergreen/scripts/archive-mongodb-logs.sh - command: archive.targz_pack params: target: "mongo-coredumps.tgz" @@ -151,17 +145,6 @@ functions: content_type: ${content_type|application/gzip} display_name: Core Dumps - Execution optional: true - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: mongodb-logs.tar.gz - remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "mongodb-logs.tar.gz" - command: s3.put params: aws_key: ${AWS_ACCESS_KEY_ID} From d7f79ca7c2a48e0fe123a6fbd88510fd5ec84ff5 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 13 Nov 2024 16:48:25 -0500 Subject: [PATCH 37/47] More fixes --- .evergreen/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3de05b1e31..153291b83e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -512,8 +512,7 @@ functions: binary: bash working_dir: "src" args: - - .evergreen/scripts/run-with-env.sh - - $DRIVERS_TOOLS/.evergreen/teardown.sh + - ${DRIVERS_TOOLS}/.evergreen/teardown.sh "fix absolute paths": - command: subprocess.exec From b8cd9327cce2bbc63486d9baee8a60727b27f568 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 14 Nov 2024 09:23:34 -0500 Subject: [PATCH 38/47] Remove TEST_SUITES from system-setup env, fix bucket --- .evergreen/config.yml | 2 +- .evergreen/scripts/configure-env.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 153291b83e..23b0cc8090 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -152,7 +152,7 @@ functions: aws_session_token: ${AWS_SESSION_TOKEN} local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz - bucket: ${aws_bucket} + bucket: ${bucket_name} permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "drivers-tools-logs.tar.gz" diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index e77a4e87d0..0a35cd92e3 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -47,7 +47,6 @@ export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}" export NO_EXT="${NO_EXT:-}" export COVERAGE="${COVERAGE:-}" export COMPRESSORS="${COMPRESSORS:-}" -export TEST_SUITES="${TEST_SUITES:-}" export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}" export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" From 717c7c0fc64b2faa31d018f1fe7598b16ec62ea2 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 14 Nov 2024 09:58:54 -0500 Subject: [PATCH 39/47] Add default TEST_SUITES to run-tests.sh --- .evergreen/scripts/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 146fdec24e..8674f0e99a 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -49,7 +49,7 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ AUTH=${AUTH} \ SSL=${SSL} \ TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ - TEST_SUITES=${TEST_SUITES} \ + TEST_SUITES=${TEST_SUITES:-} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ SKIP_HATCH=${SKIP_HATCH} \ bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg From f7ad42d5d325646a06d45f1a798e77eb9add872f Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 14 Nov 2024 10:34:38 -0500 Subject: [PATCH 40/47] Fix upload-coverage-report --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 23b0cc8090..0498efbd1b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -104,7 +104,7 @@ functions: working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - - src/.evergreen/scripts/upload-coverage-report.sh + - .evergreen/scripts/upload-coverage-report.sh - ${bucket_name} - ${revision} - ${version_id} From 68c9d29a775d7363be9e01efde4a03ad7f200d72 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 14 Nov 2024 10:49:39 -0500 Subject: [PATCH 41/47] run-with-env.sh should use -u not -eu --- .evergreen/scripts/run-tests.sh | 2 +- .evergreen/scripts/run-with-env.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 8674f0e99a..495db83e70 100644 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -1,5 +1,6 @@ #!/bin/bash +# Disable xtrace set +x if [ -n "${MONGODB_STARTED}" ]; then export PYMONGO_MUST_CONNECT=true @@ -40,7 +41,6 @@ fi if [ -n "${SKIP_CSOT_TESTS}" ]; then export SKIP_CSOT_TESTS=1 fi - GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ PYTHON_BINARY=${PYTHON_BINARY} \ NO_EXT=${NO_EXT} \ diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh index 59da15851e..e3d22786b8 100644 --- a/.evergreen/scripts/run-with-env.sh +++ b/.evergreen/scripts/run-with-env.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eu +#!/bin/bash -u # Example use: bash run-with-env.sh run-tests.sh {args...} @@ -15,7 +15,7 @@ else fi fi -set -eu +set -u # shellcheck source=/dev/null . "$1" "${@:2}" From 637a6cc8e2507ad2e82300937a2d28785ad2b723 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 14 Nov 2024 11:06:02 -0500 Subject: [PATCH 42/47] Add type:test back to run tests --- .evergreen/config.yml | 1 + .evergreen/scripts/run-with-env.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0498efbd1b..4e298cf372 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -294,6 +294,7 @@ functions: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/setup-encryption.sh - command: subprocess.exec + type: test params: working_dir: "src" binary: bash diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh index e3d22786b8..59da15851e 100644 --- a/.evergreen/scripts/run-with-env.sh +++ b/.evergreen/scripts/run-with-env.sh @@ -1,4 +1,4 @@ -#!/bin/bash -u +#!/bin/bash -eu # Example use: bash run-with-env.sh run-tests.sh {args...} @@ -15,7 +15,7 @@ else fi fi -set -u +set -eu # shellcheck source=/dev/null . "$1" "${@:2}" From ed73e7223dd6219abf839bae92b951abe4402254 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 19 Nov 2024 10:10:25 -0500 Subject: [PATCH 43/47] Fix AUTH + SSL --- .evergreen/config.yml | 2 +- .evergreen/scripts/configure-env.sh | 3 +++ .evergreen/scripts/setup-tests.sh | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4e298cf372..0b8e2544f3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -298,7 +298,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AUTH", "SSL", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 0a35cd92e3..d2343aac09 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -52,6 +52,9 @@ export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" +export AUTH="${AUTH:-noauth}" +export SSL="${SSL:-nossl}" + export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index 521007e4ee..56efed33e5 100644 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -19,6 +19,8 @@ export test_serverless="${test_serverless:-}" export TEST_INDEX_MANAGEMENT="${TEST_INDEX_MANAGEMENT:-}" export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" +export AUTH="${AUTH:-noauth}" +export SSL="${SSL:-nossl}" EOT chmod +x "$SCRIPT_DIR"/test-env.sh From b4695c137e66a397bcd3af482f6224f925beaf13 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 19 Nov 2024 10:58:10 -0500 Subject: [PATCH 44/47] Fix bootstrap AUTH + SSL --- .evergreen/scripts/bootstrap-mongo-orchestration.sh | 4 ++-- .evergreen/scripts/configure-env.sh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index 20e2b82086..1d2b145de8 100644 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -35,8 +35,8 @@ fi MONGODB_VERSION=${VERSION} \ TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ + AUTH=${AUTH:-noauth} \ + SSL=${SSL:-nossl} \ STORAGE_ENGINE=${STORAGE_ENGINE:-} \ DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \ ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \ diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index d2343aac09..0a35cd92e3 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -52,9 +52,6 @@ export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" -export AUTH="${AUTH:-noauth}" -export SSL="${SSL:-nossl}" - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" From 765436165166d1bd0ac8e77db43bdb3936110ca5 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Tue, 19 Nov 2024 14:14:35 -0500 Subject: [PATCH 45/47] Only pass PYTHON_BINARY where it was before --- .evergreen/config.yml | 18 +++++++++++------- .evergreen/scripts/configure-env.sh | 1 - 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0b8e2544f3..633067db40 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -40,7 +40,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "PYTHON_BINARY", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER"] binary: bash working_dir: "src" args: @@ -249,7 +249,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: [MOD_WSGI_VERSION, MOD_WSGI_EMBEDDED] + include_expansions_in_env: [MOD_WSGI_VERSION, MOD_WSGI_EMBEDDED, "PYTHON_BINARY"] working_dir: "src" binary: bash args: @@ -260,6 +260,7 @@ functions: - command: subprocess.exec type: test params: + include_expansions_in_env: ["PYTHON_BINARY"] working_dir: "src" binary: bash args: @@ -270,6 +271,7 @@ functions: - command: subprocess.exec type: test params: + include_expansions_in_env: [ "PYTHON_BINARY" ] working_dir: "src" binary: bash args: @@ -298,7 +300,7 @@ functions: params: working_dir: "src" binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY", "TEST_DATA_LAKE", "TEST_INDEX_MANAGEMENT", "CRYPT_SHARED_LIB_PATH", "SINGLE_MONGOS_LB_URI", "MULTI_MONGOS_LB_URI", "TEST_SUITES"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-tests.sh @@ -309,7 +311,7 @@ functions: params: binary: bash working_dir: "src" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY"] args: - .evergreen/scripts/run-with-env.sh - .evergreen/scripts/run-enterprise-auth-tests.sh @@ -319,7 +321,7 @@ functions: type: test params: binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "PYTHON_BINARY"] working_dir: "src" args: - .evergreen/scripts/run-with-env.sh @@ -418,7 +420,8 @@ functions: working_dir: "src" args: - .evergreen/scripts/run-with-env.sh - - AWS_ROLE_SESSION_NAME="test" .evergreen/scripts/run-mongodb-aws-test.sh + - AWS_ROLE_SESSION_NAME="test" + - .evergreen/scripts/run-mongodb-aws-test.sh - web-identity "run aws auth test with aws credentials as environment variables": @@ -582,7 +585,7 @@ functions: - command: subprocess.exec type: test params: - include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED"] + include_expansions_in_env: ["OCSP_ALGORITHM", "OCSP_TLS_SHOULD_SUCCEED", "PYTHON_BINARY"] binary: bash working_dir: "src" args: @@ -1564,6 +1567,7 @@ tasks: - command: subprocess.exec type: test params: + include_expansions_in_env: ["PYTHON_BINARY"] working_dir: "src" binary: "bash" args: diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 0a35cd92e3..0f19f59a1e 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -38,7 +38,6 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" -export PYTHON_BINARY="${PYTHON_BINARY:-}" export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" export MONGODB_STARTED="${MONGODB_STARTED:-}" From 82dcf30f7d8bbee79e9a511a79ab7918fa7f5c4a Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 20 Nov 2024 10:21:01 -0500 Subject: [PATCH 46/47] Cleanup --- .evergreen/scripts/run-with-env.sh | 2 +- .evergreen/scripts/setup-tests.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-with-env.sh b/.evergreen/scripts/run-with-env.sh index 59da15851e..2fd073605d 100644 --- a/.evergreen/scripts/run-with-env.sh +++ b/.evergreen/scripts/run-with-env.sh @@ -18,4 +18,4 @@ fi set -eu # shellcheck source=/dev/null -. "$1" "${@:2}" +. "$@" diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index 56efed33e5..65462b2a68 100644 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -21,6 +21,7 @@ export TEST_DATA_LAKE="${TEST_DATA_LAKE:-}" export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-}" export AUTH="${AUTH:-noauth}" export SSL="${SSL:-nossl}" +export PYTHON_BINARY="${PYTHON_BINARY:-}" EOT chmod +x "$SCRIPT_DIR"/test-env.sh From d5de7648bb8c53e54e13fd8d9c7f8644fa65a3d3 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 20 Nov 2024 10:39:15 -0500 Subject: [PATCH 47/47] AWS auth fix --- .evergreen/config.yml | 5 +++-- .evergreen/scripts/configure-env.sh | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 633067db40..8fee343669 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -40,7 +40,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"] binary: bash working_dir: "src" args: @@ -418,9 +418,10 @@ functions: include_expansions_in_env: [ "DRIVERS_TOOLS", "skip_EC2_auth_test" ] binary: bash working_dir: "src" + env: + AWS_ROLE_SESSION_NAME: test args: - .evergreen/scripts/run-with-env.sh - - AWS_ROLE_SESSION_NAME="test" - .evergreen/scripts/run-mongodb-aws-test.sh - web-identity diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 0f19f59a1e..3c0a0436de 100644 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -51,6 +51,8 @@ export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" +export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}" +export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH"