CDRIVER-3954 Run all integration tests in Versioned API evergreen tests (#787)

* CDRIVER-3917 route all test client and client_pool creation through the test framework

* small fixes

* add missing include

* rename pool method to match client method

* Refactor test framework client factories to accept API version

* Revert change in mongoc-cache.c

Despite the #include, the linker does not like this:
tests/test-mongoc-cache.c.o: In function `ping':
src/libmongoc/tests/test-mongoc-cache.c:45: undefined reference to `test_framework_client_new'

* Run all tests with requireApiVersion=true

* Fix large_ismaster test

* Don't set server api when client can't be created

* Don't use test clients in tests specifically testing server API behaviour

* Print default API version in test output

* Set requireApiVersion option earlier

* Consider apiVersion field when assembling large bulk operation

* Fix wrong comment wording

Co-authored-by: Benjamin Rewis <[email protected]>

* Fix inaccurate variable name

* Remove obsolete todo

* Document MONGODB_API_VERSION environment variable in contributing docs

* Document new test methods

* Fix tests with acceptApiVersion2

* Update newly added test

Co-authored-by: samantharitter <[email protected]>
Co-authored-by: Benjamin Rewis <[email protected]>

Author: 3 people

.evergreen/config.yml

@@ -709,8 +709,7 @@ functions:
         export DNS=${DNS}
         export ASAN=${ASAN}
         export MONGODB_API_VERSION=1
-        # TODO: CDRIVER-3954 Run all integration tests
-        ./src/libmongoc/test-libmongoc --no-fork -l /versioned_api/* -d
+        sh .evergreen/run-tests.sh
         unset MONGODB_API_VERSION
   run aws tests:
   - command: shell.exec
@@ -17413,9 +17412,6 @@ tasks:
   depends_on:
     name: debug-compile-nosasl-openssl
   commands:
-  - func: fetch build
-    vars:
-      BUILD_NAME: debug-compile-nosasl-openssl
   - func: fetch build
     vars:
       BUILD_NAME: debug-compile-nosasl-openssl
@@ -17434,25 +17430,22 @@ tasks:
   tags:
   - versioned-api
   depends_on:
-    name: debug-compile-nosasl-openssl
+    name: debug-compile-nosasl-nossl
   commands:
   - func: fetch build
     vars:
-      BUILD_NAME: debug-compile-nosasl-openssl
-  - func: fetch build
-    vars:
-      BUILD_NAME: debug-compile-nosasl-openssl
+      BUILD_NAME: debug-compile-nosasl-nossl
   - func: bootstrap mongo-orchestration
     vars:
-      AUTH: auth
+      AUTH: noauth
       ORCHESTRATION_FILE: versioned-api-testing
-      SSL: ssl
+      SSL: nossl
       TOPOLOGY: server
       VERSION: latest
   - func: test versioned api
     vars:
-      AUTH: auth
-      SSL: ssl
+      AUTH: noauth
+      SSL: nossl
 - name: build-and-run-authentication-tests-openssl-0.9.8
   commands:
   - func: install ssl
@@ -22869,4 +22862,5 @@ buildvariants:
   run_on: ubuntu1804-test
   tasks:
   - debug-compile-nosasl-openssl
+  - debug-compile-nosasl-nossl
   - .versioned-api

.evergreen/integration-tests.sh

@@ -164,12 +164,13 @@ elif [ "$SSL" != "nossl" ]; then
    MONGO_SHELL_CONNECTION_FLAGS="${MONGO_SHELL_CONNECTION_FLAGS} --host localhost --ssl --sslCAFile=$MONGO_ORCHESTRATION_HOME/lib/ca.pem --sslPEMKeyFile=$MONGO_ORCHESTRATION_HOME/lib/client.pem"
 fi
 
+if [ ! -z "$REQUIRE_API_VERSION" ]; then
+  MONGO_SHELL_CONNECTION_FLAGS="${MONGO_SHELL_CONNECTION_FLAGS} --apiVersion=1"
+  # Set the requireApiVersion parameter.
+  `pwd`/mongodb/bin/mongo $MONGO_SHELL_CONNECTION_FLAGS $DIR/require-api-version.js
+fi
+
 echo $MONGO_SHELL_CONNECTION_FLAGS
 
 `pwd`/mongodb/bin/mongo $MONGO_SHELL_CONNECTION_FLAGS --eval 'printjson(db.serverBuildInfo())' admin
 `pwd`/mongodb/bin/mongo $MONGO_SHELL_CONNECTION_FLAGS --eval 'printjson(db.isMaster())' admin
-
-# Set the requireApiVersion parameter.
-if [ ! -z "$REQUIRE_API_VERSION" ]; then
-  `pwd`/mongodb/bin/mongo $MONGO_SHELL_CONNECTION_FLAGS $DIR/require-api-version.js
-fi

CONTRIBUTING.md

@@ -252,6 +252,12 @@ Specification tests may be filtered by their description:
 
 This can be useful in debugging a specific test case in a spec test file with multiple tests.
 
+To test with a declared API version, you can pass the API version using an environment variable:
+
+* `MONGODB_API_VERSION=<string>`
+
+This will ensure that all tests declare the indicated API version when connecting to a server.
+
 All tests should pass before submitting a patch.
 
 ## Configuring the test runner

build/evergreen_config_lib/functions.py

@@ -506,8 +506,7 @@
         export DNS=${DNS}
         export ASAN=${ASAN}
         export MONGODB_API_VERSION=1
-        # TODO: CDRIVER-3954 Run all integration tests
-        ./src/libmongoc/test-libmongoc --no-fork -l /versioned_api/* -d
+        sh .evergreen/run-tests.sh
         unset MONGODB_API_VERSION
 
         '''),

build/evergreen_config_lib/tasks.py

@@ -842,16 +842,14 @@ def __init__(self, *args, **kwargs):
         'test-versioned-api',
         tags=['versioned-api'],
         depends_on='debug-compile-nosasl-openssl',
-        commands=[func('fetch build', BUILD_NAME='debug-compile-nosasl-openssl'),
-                  func('bootstrap mongo-orchestration', TOPOLOGY='server', AUTH='auth', SSL='ssl', VERSION='latest', REQUIRE_API_VERSION='true'),
+        commands=[func('bootstrap mongo-orchestration', TOPOLOGY='server', AUTH='auth', SSL='ssl', VERSION='latest', REQUIRE_API_VERSION='true'),
                   func('test versioned api', AUTH='auth', SSL='ssl')]),
     PostCompileTask(
         'test-versioned-api-accept-version-two',
         tags=['versioned-api'],
-        depends_on='debug-compile-nosasl-openssl',
-        commands=[func('fetch build', BUILD_NAME='debug-compile-nosasl-openssl'),
-                  func('bootstrap mongo-orchestration', TOPOLOGY='server', AUTH='auth', SSL='ssl', VERSION='latest', ORCHESTRATION_FILE='versioned-api-testing'),
-                  func('test versioned api', AUTH='auth', SSL='ssl')]),
+        depends_on='debug-compile-nosasl-nossl',
+        commands=[func('bootstrap mongo-orchestration', TOPOLOGY='server', AUTH='noauth', SSL='nossl', VERSION='latest', ORCHESTRATION_FILE='versioned-api-testing'),
+                  func('test versioned api', AUTH='noauth', SSL='nossl')]),
 ])
 
 

build/evergreen_config_lib/variants.py

@@ -640,6 +640,7 @@ def days(n):
         'Versioned API Tests',
         'ubuntu1804-test',
         ['debug-compile-nosasl-openssl',
+         'debug-compile-nosasl-nossl',
          '.versioned-api'],
         {})
 ]

src/libmongoc/tests/TestSuite.c

@@ -808,7 +808,8 @@ TestSuite_PrintJsonHeader (TestSuite *suite, /* IN */
             "  },\n"
             "  \"options\": {\n"
             "    \"fork\": %s,\n"
-            "    \"tracing\": %s\n"
+            "    \"tracing\": %s,\n"
+            "    \"apiVersion\": %s\n"
             "  },\n"
             "  \"results\": [\n",
             egetenv ("MONGOC_TEST_USER"),
@@ -839,7 +840,8 @@ TestSuite_PrintJsonHeader (TestSuite *suite, /* IN */
                                                                  : "false",
             test_framework_getenv_bool ("MONGOC_CHECK_IPV6") ? "true" : "false",
             (suite->flags & TEST_NOFORK) ? "false" : "true",
-            (suite->flags & TEST_TRACE) ? "true" : "false");
+            (suite->flags & TEST_TRACE) ? "true" : "false",
+            egetenv ("MONGODB_API_VERSION"));
 
    bson_free (hostname);
    bson_free (udspath);

src/libmongoc/tests/json-test-operations.c

@@ -2476,7 +2476,7 @@ json_test_operation (json_test_ctx_t *ctx,
       } else if (!strcmp (op_name, "targetedFailPoint")) {
          mongoc_client_t *client;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          activate_fail_point (
             client, session->server_id, operation, "arguments.failPoint");
@@ -2502,7 +2502,7 @@ json_test_operation (json_test_ctx_t *ctx,
          mongoc_client_t *client;
          bool exists;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          exists = collection_exists (client, operation);
          mongoc_client_destroy (client);
@@ -2512,7 +2512,7 @@ json_test_operation (json_test_ctx_t *ctx,
          mongoc_client_t *client;
          bool exists;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          exists = collection_exists (client, operation);
          mongoc_client_destroy (client);
@@ -2522,7 +2522,7 @@ json_test_operation (json_test_ctx_t *ctx,
          mongoc_client_t *client;
          bool exists;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          exists = index_exists (client, operation);
          mongoc_client_destroy (client);
@@ -2532,7 +2532,7 @@ json_test_operation (json_test_ctx_t *ctx,
          mongoc_client_t *client;
          bool exists;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          exists = index_exists (client, operation);
          mongoc_client_destroy (client);
@@ -2545,7 +2545,7 @@ json_test_operation (json_test_ctx_t *ctx,
       } else if (!strcmp (op_name, "configureFailPoint")) {
          mongoc_client_t *client;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          activate_fail_point (
             client, 0 /* primary */, operation, "arguments.failPoint");
@@ -2564,7 +2564,7 @@ json_test_operation (json_test_ctx_t *ctx,
          mongoc_client_t *client;
          mongoc_database_t *admin_db;
 
-         client = mongoc_client_new_from_uri (ctx->client->uri);
+         client = test_framework_client_new_from_uri (ctx->client->uri, NULL);
          test_framework_set_ssl_opts (client);
          admin_db = mongoc_client_get_database (client, "admin");
          bson_destroy (reply);

src/libmongoc/tests/json-test.c

@@ -1051,7 +1051,7 @@ insert_data (const char *db_name,
    bson_t *majority = tmp_bson ("{'writeConcern': {'w': 'majority'}}");
 
    /* use a fresh client to prepare the collection */
-   client = test_framework_client_new ();
+   client = test_framework_new_default_client ();
 
    db = mongoc_client_get_database (client, db_name);
    collection = mongoc_database_get_collection (db, collection_name);
@@ -1210,7 +1210,7 @@ execute_test (const json_test_config_t *config,
 
    if (bson_has_field (test, "outcome.collection")) {
       /* Use a fresh client to check the outcome collection. */
-      outcome_client = test_framework_client_new ();
+      outcome_client = test_framework_new_default_client ();
       if (bson_has_field (test, "outcome.collection.name")) {
          outcome_coll = mongoc_client_get_collection (
             outcome_client,
@@ -1717,7 +1717,7 @@ run_json_general_test (const json_test_config_t *config)
          set_uri_opts_from_bson (uri, &client_opts);
       }
 
-      client = mongoc_client_new_from_uri (uri);
+      client = test_framework_client_new_from_uri (uri, NULL);
       mongoc_client_set_error_api (client, MONGOC_ERROR_API_VERSION_2);
       test_framework_set_ssl_opts (client);
       /* reconnect right away, if a fail point causes a disconnect */