CDRIVER-4427 Test with consistent versions of crypt_shared and server. (#1119)

Author: kevinAlbs

.evergreen/compile-unix.sh

@@ -210,15 +210,6 @@ if [ "darwin" = "$OS" -a "arm64" = "$MARCH" ]; then
 fi
 
 CONFIGURE_FLAGS="$CONFIGURE_FLAGS -DMONGO_SANITIZE=$SANITIZE"
-
-if ! python3 build/mongodl.py --test -C crypt_shared -V 6.0.0-rc8 -o . > /dev/null; then
-   echo "No crypt_shared detected for this platform. Disabling MONGOC_TEST_USE_CRYPT_SHARED."
-   CONFIGURE_FLAGS="$CONFIGURE_FLAGS -DMONGOC_TEST_USE_CRYPT_SHARED=OFF"
-elif [ "$USE_CRYPT_SHARED" = "OFF" ]; then
-   echo "Variant requested disabling csfle. Disabling MONGOC_TEST_USE_CRYPT_SHARED."
-   CONFIGURE_FLAGS="$CONFIGURE_FLAGS -DMONGOC_TEST_USE_CRYPT_SHARED=OFF"
-fi
-
 CONFIGURE_FLAGS="$CONFIGURE_FLAGS $EXTRA_CONFIGURE_FLAGS"
 export MONGOC_TEST_FUTURE_TIMEOUT_MS=30000
 export MONGOC_TEST_SKIP_LIVE=on

.evergreen/config.yml

@@ -408,6 +408,9 @@ functions:
         export OCSP=${OCSP}
         export REQUIRE_API_VERSION=${REQUIRE_API_VERSION}
         sh .evergreen/integration-tests.sh
+  - command: expansions.update
+    params:
+      file: mongoc/mo-expansion.yml
   run tests:
   - command: shell.exec
     type: test
@@ -448,6 +451,7 @@ functions:
         export LOADBALANCED=${LOADBALANCED}
         export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
         export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
+        export CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}"
         set -o errexit
         sh .evergreen/run-tests.sh
   run tests bson:

.evergreen/download-mongodb.sh

@@ -53,12 +53,12 @@ get_mongodb_download_url_for ()
    _VERSION=$2
 
    # Set VERSION_RAPID to the latest rapid release each quarter.
-   VERSION_RAPID="5.3.1"
+   VERSION_RAPID="6.1.0"
    VERSION_60_LATEST="v6.0-latest"
-   VERSION_60="6.0.0"
-   VERSION_50="5.0.9"
-   VERSION_44="4.4.15"
-   VERSION_42="4.2.21"
+   VERSION_60="6.0.2"
+   VERSION_50="5.0.13"
+   VERSION_44="4.4.17"
+   VERSION_42="4.2.23"
    VERSION_40="4.0.28"
    VERSION_36="3.6.23"
    VERSION_34="3.4.24"
@@ -71,7 +71,23 @@ get_mongodb_download_url_for ()
    # getdata matrix on:
    # https://evergreen.mongodb.com/version/5797f0493ff12235e5001f05
    case "$_DISTRO" in
-      darwin*)
+      darwin--arm64)
+         MONGODB_LATEST="http://downloads.10gen.com/osx/mongodb-macos-arm64-enterprise-latest.tgz"
+             MONGODB_RAPID="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-${VERSION_RAPID}.tgz"
+             MONGODB_60_LATEST="http://downloads.10gen.com/osx/mongodb-macos-arm64-enterprise-${VERSION_60_LATEST}.tgz"
+             MONGODB_60="http://downloads.10gen.com/osx/mongodb-macos-arm64-enterprise-${VERSION_60}.tgz"
+             MONGODB_50="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-${VERSION_50}.tgz"
+             MONGODB_44="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-${VERSION_44}.tgz"
+             MONGODB_42="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-${VERSION_42}.tgz"
+             MONGODB_40="http://downloads.10gen.com/osx/mongodb-osx-x86_64-enterprise-${VERSION_40}.tgz"
+             MONGODB_36="http://downloads.10gen.com/osx/mongodb-osx-x86_64-enterprise-${VERSION_36}.tgz"
+             MONGODB_34="http://downloads.10gen.com/osx/mongodb-osx-x86_64-enterprise-${VERSION_34}.tgz"
+             MONGODB_32="http://downloads.10gen.com/osx/mongodb-osx-x86_64-enterprise-${VERSION_32}.tgz"
+             MONGODB_30="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-${VERSION_30}.tgz"
+             MONGODB_26="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-${VERSION_26}.tgz"
+             MONGODB_24="https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-${VERSION_24}.tgz"
+      ;;
+      darwin--x86_64)
          MONGODB_LATEST="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-latest.tgz"
              MONGODB_RAPID="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-${VERSION_RAPID}.tgz"
              MONGODB_60_LATEST="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-${VERSION_60_LATEST}.tgz"
@@ -128,6 +144,9 @@ get_mongodb_download_url_for ()
              MONGODB_60="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel80-${VERSION_60}.tgz"
              MONGODB_50="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel80-${VERSION_50}.tgz"
              MONGODB_44="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel80-${VERSION_44}.tgz"
+             MONGODB_42="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel80-${VERSION_42}.tgz"
+             MONGODB_40="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel80-${VERSION_40}.tgz"
+             MONGODB_36="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel80-${VERSION_36}.tgz"
       ;;
       linux-rhel-7*-s390x)
          MONGODB_LATEST="http://downloads.10gen.com/linux/mongodb-linux-s390x-enterprise-rhel72-latest.tgz"
@@ -267,9 +286,7 @@ get_mongodb_download_url_for ()
       ;;
       linux-debian-9*)
          MONGODB_LATEST="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-latest.tgz"
-             MONGODB_RAPID="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-${VERSION_RAPID}.tgz"
-             MONGODB_60_LATEST="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-${VERSION_60_LATEST}.tgz"
-             MONGODB_60="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-${VERSION_60}.tgz"
+             # SERVER-62308 Removed support for Debian 9 in server version 6.0.0-rc5.
              MONGODB_50="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-${VERSION_50}.tgz"
              MONGODB_44="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-${VERSION_44}.tgz"
              MONGODB_42="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-debian92-${VERSION_42}.tgz"

.evergreen/integration-tests.sh

@@ -183,3 +183,22 @@ echo $MONGO_SHELL_CONNECTION_FLAGS
 
 `pwd`/mongodb/bin/mongo $MONGO_SHELL_CONNECTION_FLAGS --eval 'printjson(db.serverBuildInfo())' admin
 `pwd`/mongodb/bin/mongo $MONGO_SHELL_CONNECTION_FLAGS --eval 'printjson(db.isMaster())' admin
+
+# Create mo-expansion.yml. expansions.update expects the file to exist.
+touch mo-expansion.yml
+
+if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then
+  echo "There is no crypt_shared library for distro='$DISTRO' and version='$MONGODB_VERSION'".
+else
+  echo "Downloading crypt_shared package from $MONGO_CRYPT_SHARED_DOWNLOAD_URL"
+  download_and_extract_crypt_shared "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" "$EXTRACT" "CRYPT_SHARED_LIB_PATH"
+  echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH"
+  if [ -z "$CRYPT_SHARED_LIB_PATH" ]; then
+    echo "CRYPT_SHARED_LIB_PATH must be assigned, but wasn't" 1>&2 # write to stderr"
+    exit 1
+  fi
+cat >>mo-expansion.yml <<EOT 
+CRYPT_SHARED_LIB_PATH: "$CRYPT_SHARED_LIB_PATH"
+EOT
+
+fi

.evergreen/run-tests.sh

@@ -112,7 +112,7 @@ if [ "$CLIENT_SIDE_ENCRYPTION" = "on" ]; then
    echo "Waiting for mock KMS servers to start... done."
    if ! test -d /cygdrive/c; then
       # We have trouble with this test on Windows. only set cryptSharedLibPath on other platforms
-      export MONGOC_TEST_CRYPT_SHARED_LIB_PATH="$(find . -wholename '*src/libmongoc/mongo_crypt_v1.*' -and -regex '.*\(.dll\|.dylib\|.so\)' | head -n1)"
+      export MONGOC_TEST_CRYPT_SHARED_LIB_PATH="$CRYPT_SHARED_LIB_PATH"
       echo "Setting env cryptSharedLibPath: [$MONGOC_TEST_CRYPT_SHARED_LIB_PATH]"
    fi
 fi

build/evergreen_config_lib/functions.py

@@ -282,6 +282,10 @@
         export REQUIRE_API_VERSION=${REQUIRE_API_VERSION}
         sh .evergreen/integration-tests.sh
         ''', test=False),
+        OD([
+            ("command", "expansions.update"),
+            ("params", OD([("file", "mongoc/mo-expansion.yml")]))
+        ])
     )),
     ('run tests', Function(
         shell_mongoc(r'''
@@ -320,6 +324,7 @@
         export LOADBALANCED=${LOADBALANCED}
         export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
         export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
+        export CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}"
         set -o errexit
         sh .evergreen/run-tests.sh
         '''),

mo-expansions.yml

@@ -0,0 +1 @@
+ooar

src/libmongoc/CMakeLists.txt

@@ -856,39 +856,6 @@ if (ENABLE_SHM_COUNTERS STREQUAL "ON")
    )
 endif ()
 
-cmake_dependent_option (
-   MONGOC_TEST_USE_CRYPT_SHARED "If TRUE, download and use a crypt_shared library for client-side-encryption testing. Otherwise will rely on mongocryptd"
-   TRUE ENABLE_TESTS FALSE
-)
-
-if (MONGOC_TEST_USE_CRYPT_SHARED)
-   find_program (_PYTHON3_EXE NAMES python3 py python)
-   mark_as_advanced (_PYTHON3_EXE)
-   if (NOT _PYTHON3_EXE)
-      message (SEND_ERROR
-         "MONGOC_TEST_USE_CRYPT_SHARED is enabled, but we were unable to find a Python 3 interpreter to use mongodl.py. "
-         "Install a Python 3 or set MONGOC_TEST_USE_CRYPT_SHARED to FALSE"
-      )
-   endif ()
-
-   get_filename_component (crypt_shared_dest "${CMAKE_CURRENT_BINARY_DIR}/mongo_crypt_v1${CMAKE_SHARED_LIBRARY_SUFFIX}" ABSOLUTE)
-   # If updating the "--version" for crypt_shared, update the version in compile-unix.sh to match.
-   add_custom_command (
-      OUTPUT "${crypt_shared_dest}"
-      COMMAND
-         "${_PYTHON3_EXE}" -u "${mongo-c-driver_SOURCE_DIR}/build/mongodl.py"
-            --component crypt_shared
-            --version 6.0.0
-            --edition enterprise
-            --out "${CMAKE_CURRENT_BINARY_DIR}"
-            --only "**/mongo_crypt_v1.*"
-            --strip-path-components 1
-      COMMENT "Downloading crypt_shared"
-      VERBATIM
-   )
-   add_custom_target (get-crypt_shared ALL DEPENDS "${crypt_shared_dest}")
-endif ()
-
 function (mongoc_add_test test use_shared)
    if (ENABLE_TESTS)
       add_executable (${test} ${ARGN})