CDRIVER-3535 implement streamable ismaster

- add backtrace print to evergreen timeout
- add support for awaitable ismaster to mock server
- move server monitor into separate file and model after spec pseudo
code
- fix connection count checks in stepdown tests (resolving CDRIVER-3569)

Author: kevinAlbs

.evergreen/compile-unix.sh

@@ -256,6 +256,8 @@ else
    }
 fi
 
+export MONGOC_TEST_SERVER_LOG=stdout
+
 # Write stderr to error.log and to console. Turn off tracing to avoid spurious
 # log messages that CHECK_LOG considers failures.
 mkfifo pipe || true

.evergreen/compile-windows.sh

@@ -132,4 +132,6 @@ if [ "$SKIP_MOCK_TESTS" = "ON" ]; then
    exit 0
 fi
 
+export MONGOC_TEST_SERVER_LOG=stdout
+
 "$TEST_PATH" --no-fork -d -F test-results.json

.evergreen/config.yml

@@ -726,6 +726,8 @@ post:
 - func: upload mo artifacts
 - func: upload test results
 - func: cleanup
+timeout:
+- func: backtrace
 tasks:
 - name: check-headers
   commands:

.evergreen/debug-core-evergreen.sh

@@ -1,9 +1,27 @@
 #!/usr/bin/env bash
 
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+case "$OS" in
+   cygwin*)
+      echo "debug-core-evergreen.sh is not supported on Windows"
+      exit 0
+   ;;
+esac
+
 echo "Debugging core files"
 
 shopt -s nullglob
 for i in *.core; do
    echo $i
    echo "backtrace full" | gdb -q ./src/libmongoc/test-libmongoc $i
 done
+
+# If there is still a test-libmongoc process running (perhaps due to
+# deadlock, or very slow test) attach a debugger and print stacks.
+TEST_LIBMONGOC_PID="$(pgrep test-libmongoc)"
+if [ -n "$TEST_LIBMONGOC_PID" ]; then
+   echo "test-libmongoc processes still running with PID=$TEST_LIBMONGOC_PID"
+   echo "backtrace full" | gdb -q -p $TEST_LIBMONGOC_PID
+   kill $TEST_LIBMONGOC_PID
+fi

.evergreen/run-tests.sh

@@ -87,6 +87,8 @@ check_mongocryptd() {
    fi
 }
 
+export MONGOC_TEST_MONITORING_VERBOSE=on
+
 case "$OS" in
    cygwin*)
       export PATH=$PATH:/cygdrive/c/mongodb/bin:/cygdrive/c/libmongocrypt/bin

build/generate-evergreen-config.py

@@ -61,6 +61,9 @@
         OD([('func', 'upload test results')]),
         OD([('func', 'cleanup')]),
     ]),
+    ('timeout', [
+        OD([('func', 'backtrace')])
+    ]),
     ('tasks', all_tasks),
     ('buildvariants', all_variants),
 ])

src/libmongoc/CMakeLists.txt

@@ -531,6 +531,7 @@ set (SOURCES ${SOURCES}
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-server-description.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-server-stream.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-client-session.c
+   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-server-monitor.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-set.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-socket.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-buffered.c

src/libmongoc/doc/mongoc_client_pool_set_apm_callbacks.rst

@@ -15,7 +15,7 @@ Synopsis
 
 Register a set of callbacks to receive Application Performance Monitoring events.
 
-The callbacks are copied by the pool and may be destroyed at any time after.
+The ``callbacks`` are copied by the pool and may be destroyed at any time after.  If a ``context`` is passed, it is the application's responsibility to ensure ``context`` remains valid for the lifetime of the pool.
 
 Parameters
 ----------

src/libmongoc/doc/mongoc_client_set_apm_callbacks.rst

@@ -15,7 +15,7 @@ Synopsis
 
 Register a set of callbacks to receive Application Performance Monitoring events.
 
-The callbacks are copied by the client and may be destroyed at any time after.
+The ``callbacks`` are copied by the client and may be destroyed at any time after. If a ``context`` is passed, it is the application's responsibility to ensure ``context`` remains valid for the lifetime of the client.
 
 Parameters
 ----------

src/libmongoc/doc/mongoc_client_set_stream_initiator.rst

@@ -15,6 +15,8 @@ Synopsis
 
 The :symbol:`mongoc_client_set_stream_initiator()` function shall associate a given :symbol:`mongoc_client_t` with a new stream initiator. This will completely replace the default transport (buffered TCP, possibly with TLS). The ``initiator`` should fulfill the :symbol:`mongoc_stream_t` contract. ``user_data`` is passed through to the ``initiator`` callback and may be used for whatever run time customization is necessary.
 
+If ``user_data`` is passed, it is the application's responsibility to ensure ``user_data`` remains valid for the lifetime of the client.
+
 Parameters
 ----------