PYTHON-4721 - Create individualized scripts for all shell.exec commands

.evergreen/scripts/archive-mongodb-logs.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. src/.evergreen/scripts/env.sh
+set -o xtrace
+mkdir out_dir
+find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x=$1; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' shell {} \;
+tar zcvf mongodb-logs.tar.gz -C out_dir/ .
+rm -rf out_dir

.evergreen/scripts/bootstrap-mongo-orchestration.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+. src/.evergreen/scripts/env.sh
+set -o xtrace
+
+# Enable core dumps if enabled on the machine
+# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
+if [ -f /proc/self/coredump_filter ]; then
+    # Set the shell process (and its children processes) to dump ELF headers (bit 4),
+    # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
+    echo 0x13 >/proc/self/coredump_filter
+
+    if [ -f /sbin/sysctl ]; then
+        # Check that the core pattern is set explicitly on our distro image instead
+        # of being the OS's default value. This ensures that coredump names are consistent
+        # across distros and can be picked up by Evergreen.
+        core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
+        if [ "$core_pattern" = "dump_%e.%p.core" ]; then
+            echo "Enabling coredumps"
+            ulimit -c unlimited
+        fi
+    fi
+fi
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
+    if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
+        echo "Enabling coredumps"
+        ulimit -c unlimited
+    fi
+fi
+
+if [ -n "${skip_crypt_shared}" ]; then
+    export SKIP_CRYPT_SHARED=1
+fi
+
+MONGODB_VERSION=${VERSION} \
+    TOPOLOGY=${TOPOLOGY} \
+    AUTH=${AUTH} \
+    SSL=${SSL} \
+    STORAGE_ENGINE=${STORAGE_ENGINE} \
+    DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \
+    ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
+    REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
+    LOAD_BALANCER=${LOAD_BALANCER} \
+    bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
+# run-orchestration generates expansion file with the MONGODB_URI for the cluster

.evergreen/scripts/check-import-time.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+set -x
+export BASE_SHA="$1"
+export HEAD_SHA="$2"
+bash .evergreen/run-import-time-test.sh

.evergreen/scripts/cleanup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
+  . .evergreen/hatch.sh encryption:teardown
+fi
+rm -rf "${DRIVERS_TOOLS}" || true
+rm -f ./secrets-export.sh || true

.evergreen/scripts/download-and-merge-coverage.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# Download all the task coverage files.
+aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/

.evergreen/scripts/fix-absolute-paths.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# shellcheck disable=SC2044
+for filename in $(find $DRIVERS_TOOLS -name \*.json); do
+    perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
+done

.evergreen/scripts/init-test-results.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created"  } ]}' >$PROJECT_DIRECTORY/test-results.json

.evergreen/scripts/install-dependencies.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+set -o xtrace
+file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh"
+# Don't use ${file} syntax here because evergreen treats it as an empty expansion.
+[ -f "$file" ] && bash $file || echo "$file not available, skipping"

.evergreen/scripts/make-files-executable.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# shellcheck disable=SC2044
+for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
+    chmod +x "$i"
+done

.evergreen/scripts/prepare-resources.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. src/.evergreen/scripts/env.sh
+set -o xtrace
+rm -rf $DRIVERS_TOOLS
+if [ "$PROJECT" = "drivers-tools" ]; then
+    # If this was a patch build, doing a fresh clone would not actually test the patch
+    cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS
+else
+    git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
+fi
+echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config

.evergreen/scripts/run-atlas-tests.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+# Disable xtrace for security reasons (just in case it was accidentally set).
+set +x
+set -o errexit
+bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
+TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-aws-ecs-auth-test.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# shellcheck disable=SC2154
+if [ "${skip_ECS_auth_test}" = "true" ]; then
+    echo "This platform does not support the ECS auth test, skipping..."
+    exit 0
+fi
+. .evergreen/scripts/env.sh
+set -ex
+cd $DRIVERS_TOOLS/.evergreen/auth_aws
+. ./activate-authawsvenv.sh
+. aws_setup.sh ecs
+export MONGODB_BINARIES="$MONGODB_BINARIES"
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+python aws_tester.py ecs
+cd -

.evergreen/scripts/run-doctests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+set -o xtrace
+PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test

.evergreen/scripts/run-enterprise-auth-tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Disable xtrace for security reasons (just in case it was accidentally set).
+set +x
+bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth
+TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-gcpkms-fail-test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
+export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
+SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
+SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-getdata.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+. ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true
+get_distro || true
+echo $DISTRO
+echo $MARCH
+echo $OS
+uname -a || true
+ls /etc/*release* || true
+cc --version || true
+gcc --version || true
+clang --version || true
+gcov --version || true
+lcov --version || true
+llvm-cov --version || true
+echo $PATH
+ls -la /usr/local/Cellar/llvm/*/bin/ || true
+ls -la /usr/local/Cellar/ || true
+scan-build --version || true
+genhtml --version || true
+valgrind --version || true

.evergreen/scripts/run-load-balancer.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+. src/.evergreen/scripts/env.sh
+MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start

.evergreen/scripts/run-mockupdb-tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+set -o xtrace
+export PYTHON_BINARY=${PYTHON_BINARY}
+bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb

.evergreen/run-mod-wsgi-tests.sh → .evergreen/scripts/run-mod-wsgi-tests.sh

@@ -2,6 +2,8 @@
 set -o xtrace
 set -o errexit
 
+. .evergreen/scripts/env.sh
+
 APACHE=$(command -v apache2 || command -v /usr/lib/apache2/mpm-prefork/apache2) || true
 if [ -n "$APACHE" ]; then
     APACHE_CONFIG=apache24ubuntu161404.conf

.evergreen/run-mongodb-aws-test.sh → .evergreen/scripts/run-mongodb-aws-test.sh

@@ -13,10 +13,17 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #                 mechanism.
 #  PYTHON_BINARY  The Python version to use.
 
-echo "Running MONGODB-AWS authentication tests"
+# shellcheck disable=SC2154
+if [ "${skip_EC2_auth_test}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then
+   echo "This platform does not support the EC2 auth test, skipping..."
+   exit 0
+fi
+
+. .evergreen/scripts/env.sh
+echo "Running MONGODB-AWS authentication tests for $1"
 
 # Handle credentials and environment setup.
-. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
+. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
 
 # show test output
 set -x

.evergreen/scripts/run-ocsp-test.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+
+TEST_OCSP=1 \
+PYTHON_BINARY="${PYTHON_BINARY}" \
+CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
+OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
+bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
+bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh

.evergreen/scripts/run-perf-tests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+PROJECT_DIRECTORY=${PROJECT_DIRECTORY}
+bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh

.evergreen/scripts/run-tests.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+if [ -n "${test_encryption}" ]; then
+  ./.evergreen/hatch.sh encryption:setup &
+fi
+
+set +x
+if [ -n "${MONGODB_STARTED}" ]; then
+  export PYMONGO_MUST_CONNECT=true
+fi
+if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
+  export PYMONGO_DISABLE_TEST_COMMANDS=1
+fi
+if [ -n "${test_encryption}" ]; then
+  # Disable xtrace (just in case it was accidentally set).
+  set +x
+  bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh
+  export TEST_ENCRYPTION=1
+  if [ -n "${test_encryption_pyopenssl}" ]; then
+    export TEST_ENCRYPTION_PYOPENSSL=1
+  fi
+fi
+if [ -n "${test_crypt_shared}" ]; then
+  export TEST_CRYPT_SHARED=1
+  export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
+fi
+if [ -n "${test_pyopenssl}" ]; then
+  export TEST_PYOPENSSL=1
+fi
+if [ -n "${SETDEFAULTENCODING}" ]; then
+  export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
+fi
+if [ -n "${test_loadbalancer}" ]; then
+  export TEST_LOADBALANCER=1
+  export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
+  export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
+fi
+if [ -n "${test_serverless}" ]; then
+  export TEST_SERVERLESS=1
+fi
+if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then
+  export TEST_INDEX_MANAGEMENT=1
+fi
+if [ -n "${SKIP_CSOT_TESTS}" ]; then
+  export SKIP_CSOT_TESTS=1
+fi
+
+GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
+  PYTHON_BINARY=${PYTHON_BINARY} \
+  NO_EXT=${NO_EXT} \
+  COVERAGE=${COVERAGE} \
+  COMPRESSORS=${COMPRESSORS} \
+  AUTH=${AUTH} \
+  SSL=${SSL} \
+  TEST_DATA_LAKE=${TEST_DATA_LAKE} \
+  TEST_SUITES=${TEST_SUITES} \
+  MONGODB_API_VERSION=${MONGODB_API_VERSION} \
+  SKIP_HATCH=${SKIP_HATCH} \
+  bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/stop-load-balancer.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd "${DRIVERS_TOOLS}"/.evergreen || exit
+DRIVERS_TOOLS=${DRIVERS_TOOLS}
+bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop

.evergreen/scripts/teardown-aws.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+. src/.evergreen/scripts/env.sh
+
+cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" || exit
+if [ -f "./aws_e2e_setup.json" ]; then
+  . ./activate-authawsvenv.sh
+  python ./lib/aws_assign_instance_profile.py
+fi

.evergreen/scripts/teardown-docker.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Remove all Docker images
+DOCKER=$(command -v docker) || true
+if [ -n "$DOCKER" ]; then
+  docker rmi -f "$(docker images -a -q)" &> /dev/null || true
+fi

.evergreen/scripts/upload-coverage-report.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+aws s3 cp htmlcov/ s3://"$1"/coverage/"$2"/"$3"/htmlcov/ --recursive --acl public-read --region us-east-1

.evergreen/scripts/windows-fix.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# shellcheck disable=SC2044
+for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
+    < "$i" tr -d '\r' >"$i".new
+    mv "$i".new "$i"
+done
+# Copy client certificate because symlinks do not work on Windows.
+cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem

test/mod_wsgi_test/README.rst

@@ -107,4 +107,4 @@ Automation
 At MongoDB, Inc. we use a continuous integration job that tests each
 combination in the matrix. The job starts up Apache, starts a single server
 or replica set, and runs ``test_client.py`` with the proper arguments.
-See `run-mod-wsgi-tests.sh <https://github.com/mongodb/mongo-python-driver/blob/master/.evergreen/run-mod-wsgi-tests.sh>`_
+See `run-mod-wsgi-tests.sh <https://github.com/mongodb/mongo-python-driver/blob/master/.evergreen/scripts/run-mod-wsgi-tests.sh>`_