Merge branch 'master' into GODRIVER-3241

Author: joyjwang

.evergreen/config.yml

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Source the env.sh file and run the given task
+set -eu
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+PROJECT_DIRECTORY=$(dirname $SCRIPT_DIR)
+pushd ${PROJECT_DIRECTORY} > /dev/null
+
+source env.sh
+task "$@"
+
+popd > /dev/null

.evergreen/run-deployed-lambda-aws-tests.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Set up environment and write env.sh and expansion.yml files.
+set -eu
+
+# Set up default environment variables.
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+PROJECT_DIRECTORY=$(dirname $SCRIPT_DIR)
+ROOT_DIR=$(dirname $PROJECT_DIRECTORY)
+DRIVERS_TOOLS=${DRIVERS_TOOLS:-${ROOT_DIR}/drivers-evergreen-tools}
+MONGO_ORCHESTRATION_HOME="${DRIVERS_TOOLS}/.evergreen/orchestration"
+MONGODB_BINARIES="${DRIVERS_TOOLS}/mongodb/bin"
+OS="${OS:-""}"
+
+# Set Golang environment vars. GOROOT is wherever current Go distribution is, and is set in evergreen config.
+# GOPATH is always 3 directories up from pwd on EVG; GOCACHE is under .cache in the pwd.
+GOROOT=${GOROOT:-$(dirname "$(dirname "$(which go)")")}
+export GOPATH=${GOPATH:-$ROOT_DIR}
+export GOCACHE="${GO_CACHE:-$PROJECT_DIRECTORY/.cache}"
+
+# Handle paths on Windows.
+if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
+  GOPATH=$(cygpath -m $GOPATH)
+  GOCACHE=$(cygpath -w $GOCACHE)
+  DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
+  PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
+  EXTRA_PATH=/cygdrive/c/libmongocrypt/bin
+  MONGO_ORCHESTRATION_HOME=$(cygpath -m $MONGO_ORCHESTRATION_HOME)
+  MONGODB_BINARIES=$(cygpath -m $MONGODB_BINARIES)
+  # Set home variables for Windows, too.
+  USERPROFILE=$(cygpath -w "$ROOT_DIR")
+  HOME=$USERPROFILE
+else
+  EXTRA_PATH=${GCC:-}
+fi
+
+# Add binaries to the path.
+PATH="${GOROOT}/bin:${GOPATH}/bin:${MONGODB_BINARIES}:${EXTRA_PATH}:${PATH}"
+
+# Get the current unique version of this checkout.
+if [ "${IS_PATCH:-}" = "true" ]; then
+    CURRENT_VERSION=$(git describe)-patch-${VERSION_ID}
+else
+    CURRENT_VERSION=latest
+fi
+
+# Ensure a checkout of drivers-tools.
+if [ ! -d "$DRIVERS_TOOLS" ]; then
+  git clone https://github.com/mongodb-labs/drivers-evergreen-tools $DRIVERS_TOOLS
+fi
+
+# Write the .env file for drivers-tools.
+cat <<EOT > ${DRIVERS_TOOLS}/.env
+SKIP_LEGACY_SHELL=1
+DRIVERS_TOOLS="$DRIVERS_TOOLS"
+MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
+MONGODB_BINARIES="$MONGODB_BINARIES"
+TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
+EOT
+
+# Check Go installation.
+go version
+go env
+
+# Install taskfile.
+go install github.com/go-task/task/v3/cmd/[email protected]
+
+# Write our own env file.
+cat <<EOT > env.sh
+export GOROOT="$GOROOT"
+export GOPATH="$GOPATH"
+export GOCACHE="$GOCACHE"
+export DRIVERS_TOOLS="$DRIVERS_TOOLS"
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+export MONGODB_BINARIES="$MONGODB_BINARIES"
+export PATH="$PATH"
+EOT
+
+if [ "Windows_NT" = "$OS" ]; then
+    echo "export USERPROFILE=$USERPROFILE" >> env.sh
+    echo "export HOME=$HOME" >> env.sh
+fi
+
+# source the env.sh file and write the expansion file.
+cat <<EOT > expansion.yml
+CURRENT_VERSION: "$CURRENT_VERSION"
+DRIVERS_TOOLS: "$DRIVERS_TOOLS"
+PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
+RUN_TASK: "$PROJECT_DIRECTORY/.evergreen/run-task.sh"
+EOT
+
+cat env.sh

.evergreen/run-task.sh

@@ -9,6 +9,8 @@ debug
 *.sublime-workspace
 driver-test-data.tar.gz
 perf
+perf.json
+perf.suite
 **mongocryptd.pid
 *.test
 .DS_Store
@@ -20,6 +22,9 @@ libmongocrypt
 venv
 test.suite
 go.work.sum
+.task
+env.sh
+expansion.yml
 
 # AWS SAM-generated files
 internal/cmd/faas/awslambda/.aws-sam
@@ -36,3 +41,4 @@ api-report.txt
 # Ignore secrets files
 secrets-expansion.yml
 secrets-export.sh
+.test.env

.evergreen/run-tests.sh

@@ -34,7 +34,7 @@ repos:
   hooks:
     - id: shellcheck
       name: shellcheck
-      args: ["--severity=error"]
+      args: ["--severity=warning"]
 
 - repo: https://github.com/codespell-project/codespell
   rev: "v2.2.6"

.evergreen/setup-system.sh

@@ -5,6 +5,8 @@ env:
   TEST_TIMEOUT: 1800
   LONG_TEST_TIMEOUT: 3600
 
+dotenv: ['.test.env']
+
 tasks:
 
   ### Utility tasks. ###
@@ -16,7 +18,7 @@ tasks:
   check-license: bash etc/check_license.sh
 
   build:
-    deps: [cross-compile, build-tests, build-compile-check]
+    deps: [cross-compile, build-tests, build-compile-check, install-libmongocrypt]
     cmds:
       - go build ./...
       - go build ${BUILD_TAGS} ./...
@@ -25,6 +27,8 @@ tasks:
 
   build-compile-check: bash etc/compile_check.sh
 
+  build-aws-ecs-test: go build ${BUILD_TAGS} ./internal/cmd/testaws/main.go
+
   cross-compile: 
     - GOOS=linux GOARCH=386 go build ./...
     - GOOS=linux GOARCH=arm go build ./...
@@ -46,12 +50,19 @@ tasks:
 
   api-report: bash etc/api_report.sh
 
-  install-libmongocrypt: bash etc/install-libmongocrypt.sh
+  install-libmongocrypt: 
+    cmds: [bash etc/install-libmongocrypt.sh]
+    status: 
+      - test -d install || test -d /cygdrive/c/libmongocrypt/bin
+
+  run-docker: bash etc/run_docker.sh {{.CLI_ARGS}}
 
-  run-docker: bash etc/run_docker.sh
+  run-fuzz: bash etc/run-fuzz.sh
 
   cherry-picker: bash etc/cherry-picker.sh
 
+  pr-task: bash etc/pr-task.sh
+
   # Lint with various GOOS and GOARCH tasks to catch static analysis failures that may only affect
   # specific operating systems or architectures. For example, staticcheck will only check for 64-bit
   # alignment of atomically accessed variables on 32-bit architectures (see
@@ -80,7 +91,8 @@ tasks:
 
   test-oidc: bash etc/run-oidc-test.sh 'task --silent evg-test-oidc-auth'
 
-  test-atlas: bash etc/run-atlas-test.sh
+  test-atlas-connect: 
+    - go test -v -run ^TestAtlas$ go.mongodb.org/mongo-driver/v2/internal/cmd/testatlas -args "$ATLAS_REPL" "$ATLAS_SHRD" "$ATLAS_FREE" "$ATLAS_TLS11" "$ATLAS_TLS12" "$ATLAS_SERVERLESS" "$ATLAS_SRV_REPL" "$ATLAS_SRV_SHRD" "$ATLAS_SRV_FREE" "$ATLAS_SRV_TLS11" "$ATLAS_SRV_TLS12" "$ATLAS_SRV_SERVERLESS" >> test.suite
 
   test-awskms: bash etc/run-awskms-test.sh
 
@@ -92,7 +104,7 @@ tasks:
       - make -c internal/cmd/faas/awslambda
 
   ### Evergreen specific tasks. ###
-  build-aws-ecs-test: go build ${BUILD_TAGS} ./internal/cmd/testaws/main.go
+  setup-test: bash etc/setup-test.sh {{.CLI_ARGS}}
 
   evg-test:
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=$MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s -p 1 ./... >> test.suite
@@ -101,7 +113,8 @@ tasks:
     - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestUnifiedSpecs/atlas-data-lake-testing >> spec_test.suite
     - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestAtlasDataLake >> spec_test.suite
 
-  evg-test-enterprise-auth: go run -tags gssapi ./internal/cmd/testentauth/main.go
+  evg-test-enterprise-auth: 
+    - go run -tags gssapi ./internal/cmd/testentauth/main.go
 
   evg-test-oidc-auth:
     - go run ./internal/cmd/testoidcauth/main.go
@@ -164,6 +177,12 @@ tasks:
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=${MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s ./internal/integration >> test.suite
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=${MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s ./internal/integration/unified >> test.suite
 
+  evg-test-aws: bash etc/run-mongodb-aws-test.sh {{.CLI_ARGS}}
+
+  evg-test-aws-ecs: bash etc/run-mongodb-aws-ecs-test.sh
+
+  evg-test-deployed-lambda-aws: bash ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh
+
   build-kms-test: go build ${BUILD_TAGS} ./internal/cmd/testkms
 
   ### Benchmark specific tasks and support. ###
@@ -173,16 +192,10 @@ tasks:
       - go test ${BUILD_TAGS} -benchmem -bench=. ./benchmark | test benchmark.suite
 
   driver-benchmark: 
-    deps: [perf-files]
     cmds: 
-      - go run ./internal/cmd/benchmark | tee perf.suite
+      - go test ./internal/cmd/benchmark -v --fullRun | tee perf.suite
 
   ### Internal tasks. ###
-  perf-files: 
-    internal: true
-    cmds:
-      - bash etc/prep-perf.sh
-
   install-lll: 
     internal: true
     cmds:

.gitignore

@@ -183,7 +183,9 @@ However, some of the tests require secrets handling.  Please see the team [Wiki]
 The test suite can be run with or without the secrets as follows:
 
 ```bash
-TASKFILE_TARGET=evg-test-versioned-api bash .evergreen/run-tests.sh
+task setup-env
+task setup-test
+task evg-test-versioned-api
 ```
 
 ### Load Balancer