diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5bfc310600..ed5b5ae242 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -141,61 +141,29 @@ functions: script: | ${PREPARE_SHELL} rm -rf $DRIVERS_TOOLS - if [ "${project}" = "drivers-tools" ]; then - # If this was a patch build, doing a fresh clone would not actually test the patch - cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS - else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS - fi - echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - - command: shell.exec - params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - ${PREPARE_SHELL} - - # initialize submodules - git submodule init - git submodule update - - command: shell.exec - params: - shell: "bash" - working_dir: src/go.mongodb.org/mongo-driver - script: | - . ${DRIVERS_TOOLS}/.evergreen/venv-utils.sh - . ${DRIVERS_TOOLS}/.evergreen/find-python3.sh - - export PYTHON3_BINARY="$(find_python3 2>/dev/null)" - venvcreate "$PYTHON3_BINARY" venv - - echo "PYTHON3_BINARY: $PYTHON3_BINARY" >>expansion.yml - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + - command: subprocess.exec params: - file: src/go.mongodb.org/mongo-driver/expansion.yml + binary: "bash" + include_expansions_in_env: ["PROJECT_DIRECTORY"] + args: + - ${DRIVERS_TOOLS}/.evergreen/setup.sh handle-test-artifacts: - command: gotest.parse_files params: files: - "src/go.mongodb.org/mongo-driver/*.suite" - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: mongodb-logs.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz + local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz + remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} - display_name: "mongodb-logs.tar.gz" + display_name: "drivers-tools-logs.tar.gz" - command: s3.put params: aws_key: ${aws_key} @@ -275,74 +243,23 @@ functions: params: file: mo-expansion.yml - cleanup: - - command: shell.exec + teardown: + - command: subprocess.exec params: - shell: "bash" - script: | + binary: "bash" + args: # Ensure the instance profile is reassigned for aws tests. - cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" - if [ -f "./aws_e2e_setup.json" ]; then - . ./activate-authawsvenv.sh - python ./lib/aws_assign_instance_profile.py - fi - - command: shell.exec - params: - shell: "bash" - script: | - # Attempt to shut down a running load balancer. Ignore any errors that happen if the load - # balancer is not running. - DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop || echo "Ignoring load balancer stop error" - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - # Clean up cse servers - bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - # Stop orchestration and remove drivers tools. - bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh - cd - - rm -rf $DRIVERS_TOOLS || true - - - fix-absolute-paths: - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename - done - - windows-fix: - - command: shell.exec + - ${DRIVERS_TOOLS}/.evergreen/auth_aws/teardown.sh + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY} -name \*.sh); do - cat $i | tr -d '\r' > $i.new - mv $i.new $i - done - # Copy client certificate because symlinks do not work on Windows. Ignore any copy errors. - cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem || echo "Ignoring copy error" - - make-files-executable: - - command: shell.exec + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY} -name \*.sh); do - chmod +x $i - done + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/teardown.sh assume-ec2-role: - command: ec2.assume_role @@ -894,13 +811,10 @@ functions: pre: - func: fetch-source - func: prepare-resources - - func: windows-fix - - func: fix-absolute-paths - - func: make-files-executable post: + - func: teardown - func: handle-test-artifacts - - func: cleanup tasks: - name: static-analysis @@ -1814,10 +1728,6 @@ tasks: echo "Building build-kms-test ... end" source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh echo "Copying files ... begin" - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} tar czf testgcpkms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.* GCPKMS_SRC=testgcpkms.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" @@ -1834,10 +1744,6 @@ tasks: script: | ${PREPARE_SHELL} source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} GCPKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='gcp' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" @@ -1903,9 +1809,7 @@ tasks: echo "Building build-kms-test ... end" echo "Copying files ... begin" - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + source $DRIVERS_TOOLS/.evergreen/csfle/azurekms/secrets-export.sh tar czf testazurekms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.* AZUREKMS_SRC=testazurekms.tgz AZUREKMS_DST=/tmp $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" @@ -1920,10 +1824,7 @@ tasks: working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} - source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + source $DRIVERS_TOOLS/.evergreen/csfle/azurekms/secrets-export.sh AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: "testazurekms-fail-task" @@ -2191,16 +2092,12 @@ task_groups: setup_group: - func: fetch-source - func: prepare-resources - - func: fix-absolute-paths - - func: make-files-executable - func: start-cse-servers - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh ${VAULT_NAME} - bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/serverless/setup.sh - command: expansions.update params: file: serverless-expansion.yml @@ -2231,14 +2128,14 @@ task_groups: file: crypt-expansion.yml teardown_group: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/serverless/teardown.sh + - func: teardown - func: handle-test-artifacts - - func: cleanup + tasks: - ".serverless" @@ -2248,27 +2145,19 @@ task_groups: setup_group: - func: fetch-source - func: prepare-resources - - func: fix-absolute-paths - - func: make-files-executable - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh - # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. - - command: expansions.update + - command: subprocess.exec params: - file: testgcpkms-expansions.yml + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/setup.sh teardown_group: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/teardown.sh + - func: teardown - func: handle-test-artifacts - - func: cleanup tasks: - testgcpkms-task @@ -2279,8 +2168,6 @@ task_groups: setup_group: - func: fetch-source - func: prepare-resources - - func: fix-absolute-paths - - func: make-files-executable - command: shell.exec params: shell: "bash" @@ -2291,24 +2178,15 @@ task_groups: # Get azurekms credentials from the vault. . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh - - command: expansions.update - params: - file: testazurekms-expansions.yml teardown_group: - # setup_group may have failed before updating expansions. Try to update again to clean up resources on setup failure. - - command: expansions.update - params: - file: testazurekms-expansions.yml - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - export AZUREKMS_SCOPE=${AZUREKMS_SCOPE} - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/teardown.sh + - func: teardown - func: handle-test-artifacts - - func: cleanup + tasks: - testazurekms-task @@ -2316,8 +2194,6 @@ task_groups: setup_group: - func: fetch-source - func: prepare-resources - - func: fix-absolute-paths - - func: make-files-executable - func: assume-ec2-role - command: subprocess.exec params: @@ -2333,6 +2209,8 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh + - func: teardown + - func: handle-test-artifacts setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -2342,21 +2220,21 @@ task_groups: setup_group: - func: fetch-source - func: prepare-resources - - func: fix-absolute-paths - - func: make-files-executable - command: subprocess.exec params: binary: bash env: AZUREOIDC_VMNAME_PREFIX: "GO_DRIVER" args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/create-and-setup-vm.sh + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/setup.sh teardown_task: - command: subprocess.exec params: binary: bash args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/teardown.sh + - func: teardown + - func: handle-test-artifacts setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -2366,8 +2244,6 @@ task_groups: setup_group: - func: fetch-source - func: prepare-resources - - func: fix-absolute-paths - - func: make-files-executable - command: subprocess.exec params: binary: bash @@ -2381,6 +2257,8 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh + - func: teardown + - func: handle-test-artifacts setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -2396,11 +2274,8 @@ task_groups: binary: bash env: LAMBDA_STACK_NAME: dbx-go-lambda - AWS_REGION: us-east-1 - task_id: ${task_id} - execution: ${execution} args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup.sh teardown_group: - command: subprocess.exec params: @@ -2410,9 +2285,9 @@ task_groups: LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown.sh + - func: teardown - func: handle-test-artifacts - - func: cleanup setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -2429,11 +2304,8 @@ task_groups: env: MONGODB_VERSION: "7.0" LAMBDA_STACK_NAME: dbx-go-lambda - AWS_REGION: us-east-1 - task_id: ${task_id} - execution: ${execution} args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup.sh - command: expansions.update params: file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml @@ -2452,9 +2324,9 @@ task_groups: working_dir: src/go.mongodb.org/mongo-driver binary: bash args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown.sh + - func: teardown - func: handle-test-artifacts - - func: cleanup setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh old mode 100644 new mode 100755 diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh old mode 100644 new mode 100755 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 733618dead..ec2a89ef6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,3 +54,11 @@ repos: rev: v1.59.1 hooks: - id: golangci-lint + +- repo: local + hooks: + - id: executable-shell + name: executable-shell + entry: chmod +x + language: system + types: [shell] \ No newline at end of file diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh old mode 100644 new mode 100755 diff --git a/etc/gen-ec-certs/gen-ec-certs.sh b/etc/gen-ec-certs/gen-ec-certs.sh old mode 100644 new mode 100755 diff --git a/etc/install-libmongocrypt.sh b/etc/install-libmongocrypt.sh old mode 100644 new mode 100755 diff --git a/etc/run-atlas-test.sh b/etc/run-atlas-test.sh old mode 100644 new mode 100755 diff --git a/etc/run-awskms-test.sh b/etc/run-awskms-test.sh old mode 100644 new mode 100755 diff --git a/etc/run-enterprise-gssapi-test.sh b/etc/run-enterprise-gssapi-test.sh old mode 100644 new mode 100755 diff --git a/etc/run-enterprise-plain-test.sh b/etc/run-enterprise-plain-test.sh old mode 100644 new mode 100755 diff --git a/etc/run-oidc-test.sh b/etc/run-oidc-test.sh old mode 100644 new mode 100755 diff --git a/etc/run-serverless-test.sh b/etc/run-serverless-test.sh old mode 100644 new mode 100755 diff --git a/etc/run_docker.sh b/etc/run_docker.sh old mode 100644 new mode 100755 diff --git a/etc/setup-encryption.sh b/etc/setup-encryption.sh old mode 100644 new mode 100755