GODRIVER-2547 Run legacy Client Side Encryption tests on serverless (#1069)

* sync fle2 tests to add load-balanced topology

* add Client Side Encryption tests to evg-test-serverless

* add MAKEFILE_TARGET to run-tests.sh

* use run-tests.sh in run-serverless-tests

* download mongo_crypt shared library and mongocryptd in serverless pre tasks

Author: kevinAlbs

.evergreen/config.yml

@@ -564,18 +564,18 @@ functions:
       type: test
       params:
         working_dir: src/go.mongodb.org/mongo-driver
+        add_expansions_to_env: true
         script: |
           ${PREPARE_SHELL}
 
-          export GOFLAGS=-mod=vendor
           AUTH="auth" \
           SSL="ssl" \
           MONGODB_URI="${SERVERLESS_URI}" \
           SERVERLESS="serverless" \
           SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" \
           SERVERLESS_ATLAS_PASSWORD="${SERVERLESS_ATLAS_PASSWORD}" \
-          MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
-          make evg-test-serverless
+          MAKEFILE_TARGET=evg-test-serverless \
+            sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   run-atlas-data-lake-test:
     - command: shell.exec
@@ -2115,6 +2115,7 @@ task_groups:
       - func: prepare-resources
       - func: fix-absolute-paths
       - func: make-files-executable
+      - func: start-cse-servers
       - command: shell.exec
         params:
           shell: "bash"
@@ -2129,6 +2130,56 @@ task_groups:
       - command: expansions.update
         params:
           file: serverless-expansion.yml
+      - command: shell.exec
+        params:
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+
+            if [ -z "${SERVERLESS_MONGODB_VERSION}" ]; then
+              echo "expected SERVERLESS_MONGODB_VERSION to be set"
+              exit 1
+            fi
+
+            # Download the enterprise server download for current platform to $MONGODB_BINARIES.
+            # This is required for tests that need mongocryptd.
+            # $MONGODB_BINARIES is added to the $PATH in fetch-source.
+            ${PYTHON3_BINARY} $DRIVERS_TOOLS/.evergreen/mongodl.py \
+                --component archive \
+                --version ${SERVERLESS_MONGODB_VERSION} \
+                --edition enterprise \
+                --out $MONGODB_BINARIES \
+                --strip-path-components 2
+
+            # Download the crypt_shared dynamic library for the current platform.
+            ${PYTHON3_BINARY} $DRIVERS_TOOLS/.evergreen/mongodl.py \
+              --component crypt_shared \
+              --version ${SERVERLESS_MONGODB_VERSION} \
+              --edition enterprise \
+              --out . \
+              --only "**/mongo_crypt_v1.*" \
+              --strip-path-components 1
+     
+            # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to
+            # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other
+            # downloaded files.
+            CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \
+              -name 'mongo_crypt_v1.so' -o \
+              -name 'mongo_crypt_v1.dll' -o \
+              -name 'mongo_crypt_v1.dylib')"
+
+            # If we're on Windows, convert the "cygdrive" path to Windows-style paths.
+            if [ "Windows_NT" = "$OS" ]; then
+                CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH)
+            fi
+
+            echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" >> crypt-expansion.yml
+
+      # Load the expansion file to make an evergreen variable with the current unique version
+      - command: expansions.update
+        params:
+          file: crypt-expansion.yml
+
     teardown_group:
       - command: shell.exec
         params:

.evergreen/run-tests.sh

@@ -20,7 +20,7 @@ export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
 export GOFLAGS=-mod=vendor
 
 SSL=${SSL:-nossl}
-if [ "$SSL" != "nossl" ]; then
+if [ "$SSL" != "nossl" -a -z "${SERVERLESS+x}" ]; then
     export MONGO_GO_DRIVER_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
     export MONGO_GO_DRIVER_KEY_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
     export MONGO_GO_DRIVER_PKCS8_ENCRYPTED_KEY_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client-pkcs8-encrypted.pem"
@@ -104,6 +104,10 @@ if [ "Windows_NT" = "$OS" ]; then
   CSFLE_TLS_CERTIFICATE_KEY_FILE=$(cygpath -m $CSFLE_TLS_CERTIFICATE_KEY_FILE)
 fi
 
+if [ -z ${MAKEFILE_TARGET+x} ]; then
+  MAKEFILE_TARGET="evg-test"
+fi
+
 AUTH=${AUTH} \
 SSL=${SSL} \
 MONGO_GO_DRIVER_CA_FILE=${MONGO_GO_DRIVER_CA_FILE} \
@@ -128,6 +132,6 @@ GCP_PRIVATE_KEY="${cse_gcp_private_key}" \
 CSFLE_TLS_CA_FILE="$CSFLE_TLS_CA_FILE" \
 CSFLE_TLS_CERTIFICATE_KEY_FILE="$CSFLE_TLS_CERTIFICATE_KEY_FILE" \
 CRYPT_SHARED_LIB_PATH=$CRYPT_SHARED_LIB_PATH \
-make evg-test \
+make $MAKEFILE_TARGET \
 PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
 LD_LIBRARY_PATH=$LD_LIBRARY_PATH

Makefile

@@ -180,6 +180,8 @@ evg-test-serverless:
 	go test $(BUILD_TAGS) ./mongo/integration -run TestConvenientTransactions -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration -run TestCursor -v -timeout $(TEST_TIMEOUT)s >> test.suite
 	go test $(BUILD_TAGS) ./mongo/integration/unified -run TestUnifiedSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionSpec >> test.suite
+	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse >> test.suite
 
 .PHONY: evg-test-versioned-api
 evg-test-versioned-api:

testdata/client-side-encryption/legacy/fle2-BypassQueryAnalysis.json

@@ -4,7 +4,8 @@
       "minServerVersion": "6.0.0",
       "topology": [
         "replicaset",
-        "sharded"
+        "sharded",
+        "load-balanced"
       ]
     }
   ],

testdata/client-side-encryption/legacy/fle2-BypassQueryAnalysis.yml

@@ -1,7 +1,7 @@
 runOn:
   - minServerVersion: "6.0.0"
     # FLE 2 Encrypted collections are not supported on standalone.
-    topology: [ "replicaset", "sharded" ]
+    topology: [ "replicaset", "sharded", "load-balanced" ]
 database_name: &database_name "default"
 collection_name: &collection_name "default"
 data: []

testdata/client-side-encryption/legacy/fle2-Compact.json

@@ -2,9 +2,10 @@
   "runOn": [
     {
       "minServerVersion": "6.0.0",
-      "topologies": [
+      "topology": [
         "replicaset",
-        "sharded"
+        "sharded",
+        "load-balanced"
       ]
     }
   ],

testdata/client-side-encryption/legacy/fle2-Compact.yml

@@ -1,6 +1,7 @@
 runOn:
   - minServerVersion: "6.0.0"
-    topologies: [ "replicaset", "sharded" ]
+    # FLE 2 Encrypted collections are not supported on standalone.
+    topology: [ "replicaset", "sharded", "load-balanced" ]
 database_name: &database_name "default"
 collection_name: &collection_name "default"
 data: []

testdata/client-side-encryption/legacy/fle2-CreateCollection.json

@@ -4,7 +4,8 @@
       "minServerVersion": "6.0.0",
       "topology": [
         "replicaset",
-        "sharded"
+        "sharded",
+        "load-balanced"
       ]
     }
   ],

testdata/client-side-encryption/legacy/fle2-CreateCollection.yml

@@ -2,7 +2,7 @@
 runOn:
   - minServerVersion: "6.0.0"
     # FLE 2 Encrypted collections are not supported on standalone.
-    topology: [ "replicaset", "sharded" ]
+    topology: [ "replicaset", "sharded", "load-balanced" ]
 
 database_name: &database_name "default"
 collection_name: &collection_name "default"

testdata/client-side-encryption/legacy/fle2-DecryptExistingData.json

@@ -4,7 +4,8 @@
       "minServerVersion": "6.0.0",
       "topology": [
         "replicaset",
-        "sharded"
+        "sharded",
+        "load-balanced"
       ]
     }
   ],