diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e718266efd..7fa6710c6f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -48,19 +48,11 @@ functions: file: src/expansion.yml "prepare resources": - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - rm -rf $DRIVERS_TOOLS - if [ "$PROJECT" = "drivers-tools" ]; then - # If this was a patch build, doing a fresh clone would not actually test the patch - cp -R ${PROJECT_DIRECTORY}/ ${DRIVERS_TOOLS} - else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} - fi - echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config + binary: bash + args: + - src/.evergreen/scripts/prepare-resources.sh "upload coverage" : - command: ec2.assume_role @@ -124,15 +116,12 @@ functions: - command: ec2.assume_role params: role_arn: ${assume_role_arn} - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - mkdir out_dir - find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; - tar zcvf mongodb-logs.tar.gz -C out_dir/ . - rm -rf out_dir + binary: bash + args: + - src/.evergreen/scripts/archive-mongodb-logs.sh + - command: archive.targz_pack params: target: "mongo-coredumps.tgz" @@ -226,54 +215,11 @@ functions: file: "src/xunit-results/TEST-*.xml" "bootstrap mongo-orchestration": - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - - # Enable core dumps if enabled on the machine - # Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml - if [ -f /proc/self/coredump_filter ]; then - # Set the shell process (and its children processes) to dump ELF headers (bit 4), - # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). - echo 0x13 > /proc/self/coredump_filter - - if [ -f /sbin/sysctl ]; then - # Check that the core pattern is set explicitly on our distro image instead - # of being the OS's default value. This ensures that coredump names are consistent - # across distros and can be picked up by Evergreen. - core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") - if [ "$core_pattern" = "dump_%e.%p.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited - fi - fi - fi - - if [ $(uname -s) = "Darwin" ]; then - core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") - if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited - fi - fi - - if [ -n "${skip_crypt_shared}" ]; then - export SKIP_CRYPT_SHARED=1 - fi - - MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - STORAGE_ENGINE=${STORAGE_ENGINE} \ - DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - LOAD_BALANCER=${LOAD_BALANCER} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - # run-orchestration generates expansion file with the MONGODB_URI for the cluster + binary: bash + args: + - src/.evergreen/scripts/bootstrap-mongo-orchestration.sh - command: expansions.update params: file: mo-expansion.yml @@ -284,28 +230,25 @@ functions: value: "1" "bootstrap data lake": - - command: shell.exec + - command: subprocess.exec type: setup params: - script: | - . src/.evergreen/scripts/env.sh - bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - - command: shell.exec + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh + - command: subprocess.exec type: setup params: - script: | - . src/.evergreen/scripts/env.sh - bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh - sleep 1 - docker ps + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh "stop mongo-orchestration": - - command: shell.exec + - command: subprocess.exec params: - script: | - . src/.evergreen/scripts/env.sh - set -o xtrace - bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh "run mod_wsgi tests": - command: shell.exec @@ -355,6 +298,7 @@ functions: - command: shell.exec type: test params: + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] working_dir: "src" shell: bash script: | @@ -536,25 +480,13 @@ functions: .evergreen/run-mongodb-aws-test.sh session-creds "run aws ECS auth test": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: "bash" + binary: "bash" working_dir: "src" - script: | - if [ "${skip_ECS_auth_test}" = "true" ]; then - echo "This platform does not support the ECS auth test, skipping..." - exit 0 - fi - . .evergreen/scripts/env.sh - set -ex - cd ${DRIVERS_TOOLS}/.evergreen/auth_aws - . ./activate-authawsvenv.sh - . aws_setup.sh ecs - export MONGODB_BINARIES="$MONGODB_BINARIES"; - export PROJECT_DIRECTORY="${PROJECT_DIRECTORY}"; - python aws_tester.py ecs - cd - + args: + - .evergreen/scripts/run-aws-ecs-auth-test.sh "cleanup": - command: shell.exec @@ -569,56 +501,35 @@ functions: rm -f ./secrets-export.sh || true "fix absolute paths": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename - done + args: + - src/.evergreen/scripts/fix-absolute-paths.sh "windows fix": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - cat $i | tr -d '\r' > $i.new - mv $i.new $i - done - # Copy client certificate because symlinks do not work on Windows. - cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem + args: + - src/.evergreen/scripts/windows-fix.sh "make files executable": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - chmod +x $i - done + args: + - src/.evergreen/scripts/make-files-executable.sh "init test-results": - - command: shell.exec + - command: subprocess.exec params: - script: | - set +x - . src/.evergreen/scripts/env.sh - echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json + args: + - src/.evergreen/scripts/init-test-results.sh "install dependencies": - - command: shell.exec + - command: subprocess.exec params: working_dir: "src" - script: | - . .evergreen/scripts/env.sh - set -o xtrace - file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" - # Don't use ${file} syntax here because evergreen treats it as an empty expansion. - [ -f "$file" ] && bash $file || echo "$file not available, skipping" + args: + - .evergreen/scripts/install-dependencies.sh "assume ec2 role": - command: ec2.assume_role diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh new file mode 100644 index 0000000000..c25e9b6c0b --- /dev/null +++ b/.evergreen/scripts/archive-mongodb-logs.sh @@ -0,0 +1,8 @@ +#!bin/bash + +. src/.evergreen/scripts/env.sh +set -o xtrace +mkdir out_dir +find $MONGO_ORCHESTRATION_HOME -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; +tar zcvf mongodb-logs.tar.gz -C out_dir/ . +rm -rf out_dir diff --git a/.evergreen/scripts/boostrap-mongo-orchestration.sh b/.evergreen/scripts/boostrap-mongo-orchestration.sh new file mode 100644 index 0000000000..02b635a6d1 --- /dev/null +++ b/.evergreen/scripts/boostrap-mongo-orchestration.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +. src/.evergreen/scripts/env.sh +set -o xtrace + +# Enable core dumps if enabled on the machine +# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml +if [ -f /proc/self/coredump_filter ]; then + # Set the shell process (and its children processes) to dump ELF headers (bit 4), + # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). + echo 0x13 >/proc/self/coredump_filter + + if [ -f /sbin/sysctl ]; then + # Check that the core pattern is set explicitly on our distro image instead + # of being the OS's default value. This ensures that coredump names are consistent + # across distros and can be picked up by Evergreen. + core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") + if [ "$core_pattern" = "dump_%e.%p.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi + fi +fi + +if [ $(uname -s) = "Darwin" ]; then + core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile") + if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi +fi + +if [ -n "${skip_crypt_shared}" ]; then + export SKIP_CRYPT_SHARED=1 +fi + +MONGODB_VERSION=${VERSION} \ + TOPOLOGY=${TOPOLOGY} \ + AUTH=${AUTH} \ + SSL=${SSL} \ + STORAGE_ENGINE=${STORAGE_ENGINE} \ + DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ + ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ + REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ + LOAD_BALANCER=${LOAD_BALANCER} \ + bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh +# run-orchestration generates expansion file with the MONGODB_URI for the cluster diff --git a/.evergreen/scripts/fix-absolute-paths.sh b/.evergreen/scripts/fix-absolute-paths.sh new file mode 100644 index 0000000000..c00c275b60 --- /dev/null +++ b/.evergreen/scripts/fix-absolute-paths.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +for filename in $(find $DRIVERS_TOOLS -name \*.json); do + perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename +done diff --git a/.evergreen/scripts/init-test-results.sh b/.evergreen/scripts/init-test-results.sh new file mode 100644 index 0000000000..666ac60620 --- /dev/null +++ b/.evergreen/scripts/init-test-results.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh new file mode 100644 index 0000000000..c7cdbe67cf --- /dev/null +++ b/.evergreen/scripts/install-dependencies.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. .evergreen/scripts/env.sh +set -o xtrace +file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh" +# Don't use ${file} syntax here because evergreen treats it as an empty expansion. +[ -f "$file" ] && bash $file || echo "$file not available, skipping" diff --git a/.evergreen/scripts/make-files-executable.sh b/.evergreen/scripts/make-files-executable.sh new file mode 100644 index 0000000000..72957e32ef --- /dev/null +++ b/.evergreen/scripts/make-files-executable.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +for i in $(find $DRIVERS_TOOLS/.evergreen $PROJECT_DIRECTORY/.evergreen -name \*.sh); do + chmod +x $i +done diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh new file mode 100644 index 0000000000..33394b55ff --- /dev/null +++ b/.evergreen/scripts/prepare-resources.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +. src/.evergreen/scripts/env.sh +set -o xtrace +rm -rf $DRIVERS_TOOLS +if [ "$PROJECT" = "drivers-tools" ]; then + # If this was a patch build, doing a fresh clone would not actually test the patch + cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS +else + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS +fi +echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config diff --git a/.evergreen/scripts/run-aws-ecs-auth-test.sh b/.evergreen/scripts/run-aws-ecs-auth-test.sh new file mode 100644 index 0000000000..cd692862f6 --- /dev/null +++ b/.evergreen/scripts/run-aws-ecs-auth-test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ "${skip_ECS_auth_test}" = "true" ]; then + echo "This platform does not support the ECS auth test, skipping..." + exit 0 +fi +. .evergreen/scripts/env.sh +set -ex +cd $DRIVERS_TOOLS/.evergreen/auth_aws +. ./activate-authawsvenv.sh +. aws_setup.sh ecs +export MONGODB_BINARIES="$MONGODB_BINARIES" +export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" +python aws_tester.py ecs +cd - diff --git a/.evergreen/scripts/windows-fix.sh b/.evergreen/scripts/windows-fix.sh new file mode 100644 index 0000000000..5e489e9c9c --- /dev/null +++ b/.evergreen/scripts/windows-fix.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set +x +. src/.evergreen/scripts/env.sh +for i in $(find $DRIVERS_TOOLS/.evergreen $PROJECT_DIRECTORY/.evergreen -name \*.sh); do + cat $i | tr -d '\r' >$i.new + mv $i.new $i +done +# Copy client certificate because symlinks do not work on Windows. +cp $DRIVERS_TOOLS/.evergreen/x509gen/client.pem $MONGO_ORCHESTRATION_HOME/lib/client.pem