CDRIVER-4062 test load balancer on evg (#837)

Add tasks to the Evergreen configuration based to test against a loadbalanced cluster.
- Tests with auth+SSL and with no auth+SSL on both server version 5.0 and latest.
- Runs tests against: new load balanced tests, all unified tests, retryable reads, retryable writes, change streams, and the C driver specific /loadbalanced/* tests.

Update test-libmongoc to support testing against a load balancer.
- Support the environment variable MONGOC_TEST_LOADBALANCED to enable testing against load balancers in test-libmongoc.
Support the environment variables SINGLE_MONGOS_LB_URI, MULTI_MONGOS_LB_URI to set URIs for test clients.
- Mock the serviceId when MONGOC_TEST_LOADBALANCED is true.
- Support multiple occurrences of the -l flag in test-libmongoc and alias the flag as --match.
- Remove unnecessary constraint of retryable writes tests for replica sets.

CDRIVER-4060 add remaining support to the unified test runner for the new loadbalancer spec tests.
- Iterate the first result of a FindCursor in the createFindCursor operation.
- Support appname as a URI parameter.
- Skip listening or comparing CMAP (Connection Monitoring and Pooling) events, since the C driver does not implement CMAP.

Author: kevinAlbs

.evergreen/config.yml

@@ -440,6 +440,9 @@ functions:
           export MONGOC_TEST_GCP_EMAIL="${client_side_encryption_gcp_email}"
           export MONGOC_TEST_GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
         fi
+        export LOADBALANCED=${LOADBALANCED}
+        export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
+        export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
         set -o errexit
         sh .evergreen/run-tests.sh
   run tests bson:
@@ -26547,6 +26550,114 @@ tasks:
         set -o errexit
         export LD_LIBRARY_PATH=$(pwd)/install-dir/lib
         CERT_TYPE=ecdsa .evergreen/run-ocsp-cache-test.sh
+- name: test-loadbalanced-asan-auth-openssl-5.0
+  tags:
+  - '5.0'
+  - test-asan
+  exec_timeout_secs: 3600
+  depends_on:
+    name: debug-compile-asan-clang-openssl
+  commands:
+  - func: fetch build
+    vars:
+      BUILD_NAME: debug-compile-asan-clang-openssl
+  - func: bootstrap mongo-orchestration
+    vars:
+      AUTH: auth
+      SSL: ssl
+      TOPOLOGY: sharded_cluster
+      VERSION: '5.0'
+  - func: clone drivers-evergreen-tools
+  - func: start load balancer
+    vars:
+      MONGODB_URI: mongodb://localhost:27017,localhost:27018
+  - func: run tests
+    vars:
+      ASAN: 'on'
+      AUTH: auth
+      LOADBALANCED: loadbalanced
+      SSL: ssl
+- name: test-loadbalanced-asan-auth-openssl-latest
+  tags:
+  - latest
+  - test-asan
+  exec_timeout_secs: 3600
+  depends_on:
+    name: debug-compile-asan-clang-openssl
+  commands:
+  - func: fetch build
+    vars:
+      BUILD_NAME: debug-compile-asan-clang-openssl
+  - func: bootstrap mongo-orchestration
+    vars:
+      AUTH: auth
+      SSL: ssl
+      TOPOLOGY: sharded_cluster
+      VERSION: latest
+  - func: clone drivers-evergreen-tools
+  - func: start load balancer
+    vars:
+      MONGODB_URI: mongodb://localhost:27017,localhost:27018
+  - func: run tests
+    vars:
+      ASAN: 'on'
+      AUTH: auth
+      LOADBALANCED: loadbalanced
+      SSL: ssl
+- name: test-loadbalanced-asan-noauth-nossl-5.0
+  tags:
+  - '5.0'
+  - test-asan
+  exec_timeout_secs: 3600
+  depends_on:
+    name: debug-compile-asan-clang-openssl
+  commands:
+  - func: fetch build
+    vars:
+      BUILD_NAME: debug-compile-asan-clang-openssl
+  - func: bootstrap mongo-orchestration
+    vars:
+      AUTH: noauth
+      SSL: nossl
+      TOPOLOGY: sharded_cluster
+      VERSION: '5.0'
+  - func: clone drivers-evergreen-tools
+  - func: start load balancer
+    vars:
+      MONGODB_URI: mongodb://localhost:27017,localhost:27018
+  - func: run tests
+    vars:
+      ASAN: 'on'
+      AUTH: noauth
+      LOADBALANCED: loadbalanced
+      SSL: nossl
+- name: test-loadbalanced-asan-noauth-nossl-latest
+  tags:
+  - latest
+  - test-asan
+  exec_timeout_secs: 3600
+  depends_on:
+    name: debug-compile-asan-clang-openssl
+  commands:
+  - func: fetch build
+    vars:
+      BUILD_NAME: debug-compile-asan-clang-openssl
+  - func: bootstrap mongo-orchestration
+    vars:
+      AUTH: noauth
+      SSL: nossl
+      TOPOLOGY: sharded_cluster
+      VERSION: latest
+  - func: clone drivers-evergreen-tools
+  - func: start load balancer
+    vars:
+      MONGODB_URI: mongodb://localhost:27017,localhost:27018
+  - func: run tests
+    vars:
+      ASAN: 'on'
+      AUTH: noauth
+      LOADBALANCED: loadbalanced
+      SSL: nossl
 buildvariants:
 - name: releng
   display_name: '**Release Archive Creator'

.evergreen/download-mongodb.sh

@@ -51,11 +51,11 @@ get_mongodb_download_url_for ()
    _DISTRO=$1
    _VERSION=$2
 
-   VERSION_50="5.0.0-rc0"
-   VERSION_44="4.4.3"
-   VERSION_42="4.2.11"
-   VERSION_40="4.0.22"
-   VERSION_36="3.6.21"
+   VERSION_50="5.0.0"
+   VERSION_44="4.4.6"
+   VERSION_42="4.2.15"
+   VERSION_40="4.0.25"
+   VERSION_36="3.6.23"
    VERSION_34="3.4.24"
    VERSION_32="3.2.22"
    VERSION_30="3.0.15"

.evergreen/run-tests.sh

@@ -8,6 +8,7 @@ SSL=${SSL:-nossl}
 URI=${URI:-}
 OS=$(uname -s | tr '[:upper:]' '[:lower:]')
 DNS=${DNS:-nodns}
+LOADBALANCED=${LOADBALANCED:-noloadbalanced}
 
 # AddressSanitizer configuration
 export ASAN_OPTIONS="detect_leaks=1 abort_on_error=1 symbolize=1"
@@ -91,6 +92,29 @@ check_mongocryptd() {
 
 export MONGOC_TEST_MONITORING_VERBOSE=on
 
+if [ "$LOADBALANCED" != "noloadbalanced" ]; then
+   if [ -z "$SINGLE_MONGOS_LB_URI" -o -z "$MULTI_MONGOS_LB_URI" ]; then
+      echo "SINGLE_MONGOS_LB_URI and MULTI_MONGOS_LB_URI environment variables required."
+      exit 1
+   fi
+   
+   export MONGOC_TEST_LOADBALANCED=ON
+
+   TEST_ARGS="$TEST_ARGS -l /unified/*"
+   TEST_ARGS="$TEST_ARGS -l /retryable_reads/*"
+   TEST_ARGS="$TEST_ARGS -l /retryable_writes/*"
+   TEST_ARGS="$TEST_ARGS -l /change_streams/*"
+   TEST_ARGS="$TEST_ARGS -l /loadbalanced/*"
+   TEST_ARGS="$TEST_ARGS -l /load_balancers/*"
+   TEST_ARGS="$TEST_ARGS -l /crud/unified/*"
+   TEST_ARGS="$TEST_ARGS -l /transactions/unified/*"
+   TEST_ARGS="$TEST_ARGS -l /collection-management/*"
+   TEST_ARGS="$TEST_ARGS -l /sessions/unified/*"
+   TEST_ARGS="$TEST_ARGS -l /change_streams/unified/*"
+   TEST_ARGS="$TEST_ARGS -l /versioned_api/*"
+   TEST_ARGS="$TEST_ARGS -l /command_monitoring/unified/*"
+fi
+
 case "$OS" in
    cygwin*)
       export PATH=$PATH:/cygdrive/c/mongodb/bin:/cygdrive/c/libmongocrypt/bin

CONTRIBUTING.md

@@ -262,6 +262,12 @@ To test with a declared API version, you can pass the API version using an envir
 
 This will ensure that all tests declare the indicated API version when connecting to a server.
 
+To test a load balanced deployment, set the following environment variables:
+
+* `MONGOC_TEST_LOADBALANCED=on`
+* `SINGLE_MONGOS_LB_URI=<string>` to a MongoDB URI with a host of a load balancer fronting one mongos.
+* `MULTI_MONGOS_LB_URI=<string>` to a MongoDB URI with a host of a load balancer fronting multiple mongos processes.
+
 All tests should pass before submitting a patch.
 
 ## Configuring the test runner

build/evergreen_config_lib/functions.py

@@ -311,6 +311,9 @@
           export MONGOC_TEST_GCP_EMAIL="${client_side_encryption_gcp_email}"
           export MONGOC_TEST_GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
         fi
+        export LOADBALANCED=${LOADBALANCED}
+        export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
+        export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
         set -o errexit
         sh .evergreen/run-tests.sh
         '''),

build/evergreen_config_lib/tasks.py

@@ -1074,4 +1074,71 @@ def _check_allowed(self):
 
 all_tasks = chain(all_tasks, OCSPTask.matrix())
 
+class LoadBalancedTask(MatrixTask):
+    axes = OD([
+        ('asan', [True]),
+        ('build_ssl', ['openssl']), # The SSL library the C driver is built with.
+        ('test_ssl', [True, False]), # Whether tests are run with SSL connections.
+        ('test_auth', [True, False]),
+        ('version', ['5.0', 'latest'])
+    ])
+
+    def _check_allowed (self):
+        # Test with both SSL and auth, or neither.
+        prohibit (self.test_ssl != self.test_auth)
+
+    def __init__(self, *args, **kwargs):
+        super(LoadBalancedTask, self).__init__(*args, **kwargs)
+        if self.asan and self.build_ssl == "openssl":
+            self.add_dependency ('debug-compile-asan-clang-openssl')
+            self.add_tags('test-asan')
+        else:
+            raise RuntimeError ("unimplemented configuration for LoadBalancedTask")
+
+        self.add_tags(self.version)
+        self.options['exec_timeout_secs'] = 3600
+
+    # Return the task name.
+    # Example: test-loadbalanced-asan-auth-openssl-latest
+    @property
+    def name(self):
+        name = "test-loadbalanced"
+        if self.asan:
+            name += "-asan"
+        if self.test_auth:
+            name += "-auth"
+        else:
+            name += "-noauth"
+        if self.test_ssl:
+            name += "-" + self.build_ssl
+        else:
+            name += "-nossl"
+        if self.version:
+            name += "-" + self.version
+        return name
+
+    def to_dict(self):
+        task = super(MatrixTask, self).to_dict()
+        commands = task['commands']
+        commands.append(
+            func('fetch build', BUILD_NAME=self.depends_on['name']))
+
+
+        orchestration = bootstrap(TOPOLOGY='sharded_cluster',
+                                  AUTH='auth' if self.test_auth else 'noauth',
+                                  SSL='ssl' if self.test_ssl else 'nossl',
+                                  VERSION=self.version)
+        commands.append(orchestration)
+        commands.append (func("clone drivers-evergreen-tools"))
+        commands.append (func("start load balancer",
+            MONGODB_URI="mongodb://localhost:27017,localhost:27018"))
+        commands.append(run_tests(ASAN='on' if self.asan else 'off',
+                                  SSL='ssl' if self.test_ssl else 'nossl',
+                                  AUTH='auth' if self.test_auth else 'noauth',
+                                  LOADBALANCED='loadbalanced'))
+
+        return task
+
+all_tasks = chain(all_tasks, LoadBalancedTask.matrix())
+
 all_tasks = list(all_tasks)

build/evergreen_config_lib/variants.py

@@ -657,7 +657,7 @@ def days(n):
     ], {}, batchtime=days(7)),
     Variant ('packaging', 'Linux Distro Packaging', 'ubuntu1804-test', [
         'debian-package-build',
-        OD([('name', 'rpm-package-build'), ('distros', ['rhel80-test'])]),
+        OD([('name', 'rpm-package-build'), ('distros', ['rhel82-arm64-small'])]),
     ], {}, batchtime=days(1)),
     Variant('tsan-ubuntu',
         'Thread Sanitizer (TSAN) Tests (Ubuntu 18.04)',

src/libmongoc/src/mongoc/mongoc-cluster.c

@@ -2321,6 +2321,7 @@ _mongoc_cluster_stream_for_server (mongoc_cluster_t *cluster,
          mongoc_server_stream_cleanup (server_stream);
          mongoc_cluster_disconnect_node (cluster, server_id);
          bson_mutex_unlock (&topology->mutex);
+         _mongoc_bson_init_if_set (reply);
          return NULL;
       }
    }

src/libmongoc/src/mongoc/mongoc-cmd.c

@@ -1022,7 +1022,9 @@ mongoc_cmd_parts_assemble (mongoc_cmd_parts_t *parts,
       }
 
       ret = true;
-   } else if (server_type == MONGOC_SERVER_MONGOS) {
+   } else if (server_type == MONGOC_SERVER_MONGOS ||
+              server_stream->topology_type == MONGOC_TOPOLOGY_LOAD_BALANCED) {
+      /* TODO (CDRIVER-4117) remove the check of the topology description type. */
       _mongoc_cmd_parts_assemble_mongos (parts, server_stream);
       ret = true;
    } else {

src/libmongoc/src/mongoc/mongoc-server-description-private.h

@@ -200,4 +200,10 @@ bool
 mongoc_server_description_has_service_id (
    const mongoc_server_description_t *description);
 
+/* mongoc_global_mock_service_id is only used for testing. The test runner sets
+ * this to true when testing against a load balanced deployment to mock the
+ * presence of a serviceId field in the "hello" response. The purpose of this is
+ * further described in the Load Balancer test README. */
+extern bool mongoc_global_mock_service_id;
+
 #endif