Allow CTest as a test driver (#870)

vector-of-bool · web-flow · commit 130938ad44a6 · 2021-09-21T15:51:17.000-06:00
* Allow fine-grained test control via CTest Previously, a single test was defined for test-libmongoc. This change allows CTest to see the individual test cases contained within test-libmongoc. These can be selected and executed with the CTest CLI, including with some parallelism. Tag CXX-2301 CXX-2302 * Fix duplicate test registration * Use a signal handler to see abnormal termination. * Modify multi-conf test behavior to match VS behavior
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -63,7 +63,7 @@ else ()
    message (STATUS "No CXX support")
 endif ()
 
-if (NOT CMAKE_BUILD_TYPE)
+if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
    set (CMAKE_BUILD_TYPE "RelWithDebInfo")
    message (
       STATUS "No CMAKE_BUILD_TYPE selected, defaulting to ${CMAKE_BUILD_TYPE}"
@@ -211,6 +211,9 @@ set (BUILD_SOURCE_DIR ${CMAKE_BINARY_DIR})
 
 include (MakeDistFiles)
 
+# Enable CTest
+include (CTest)
+
 # Ensure the default behavior: don't ignore RPATH settings.
 set (CMAKE_SKIP_BUILD_RPATH OFF)
 
@@ -491,3 +494,39 @@ if (CMAKE_GENERATOR_TOOLSET)
    message (STATUS "\tinstance: ${CMAKE_GENERATOR_TOOLSET}")
 endif ()
 
+if (TARGET test-libmongoc)
+   # Generate a file that can be included by CTest to load and enumerate all of the
+   # tests defined by the test-libmongoc executable. Generate one for each
+   # configuration in case of multiconf generators.
+   string (CONFIGURE [=[
+      set (TEST_LIBMONGOC_EXE [[$<TARGET_FILE:test-libmongoc>]])
+      set (SRC_ROOT [[@PROJECT_SOURCE_DIR@]])
+      set (IS_MULTICONF $<BOOL:@CMAKE_CONFIGURATION_TYPES@>)
+      if (NOT IS_MULTICONF OR CTEST_CONFIGURATION_TYPE STREQUAL "$<CONFIG>")
+         # We are not in multi-conf, or the current config matches our config.
+         include ("${SRC_ROOT}/build/cmake/LoadTests.cmake")
+      elseif (NOT CTEST_CONFIGURATION_TYPE)
+         # We are in multi-conf, but no '-C' config was specified
+         message (WARNING "Specify a --build-config when using CTest with a multi-config build")
+      else ()
+         # Do nothing. Not our config.
+      endif ()
+   ]=] code @ONLY)
+   file (GENERATE
+         OUTPUT "${PROJECT_BINARY_DIR}/LoadTests-$<CONFIG>.cmake"
+         CONTENT "${code}")
+   if (CMAKE_CONFIGURATION_TYPES)
+      foreach (conf IN LISTS CMAKE_CONFIGURATION_TYPES)
+         # Direct the generated CTest code to include() the file that loads the tests:
+         set_property (
+            DIRECTORY
+            APPEND PROPERTY
+            TEST_INCLUDE_FILES "${PROJECT_BINARY_DIR}/LoadTests-${conf}.cmake")
+      endforeach ()
+   else ()
+      set_property (
+         DIRECTORY
+         APPEND PROPERTY
+         TEST_INCLUDE_FILES "${PROJECT_BINARY_DIR}/LoadTests-${CMAKE_BUILD_TYPE}.cmake")
+   endif ()
+endif ()
diff --git a/build/cmake/LoadTests.cmake b/build/cmake/LoadTests.cmake
@@ -0,0 +1,48 @@
+# This file is include()'d by CTest. It executes test-libmongoc to get a list
+# of all tests that are registered. Each test is then defined as a CTest test,
+# allowing CTest to control the execution, parallelization, and collection of
+# test results.
+
+if (NOT EXISTS "${TEST_LIBMONGOC_EXE}")
+    # This will fail if 'test-libmongoc' is not compiled yet.
+    message (WARNING "The test executable ${TEST_LIBMONGOC_EXE} is not present. "
+                     "Its tests will not be registered")
+    add_test (mongoc/not-found NOT_FOUND)
+    return ()
+endif ()
+
+# Get the list of tests
+execute_process (
+    COMMAND "${TEST_LIBMONGOC_EXE}" --list-tests --no-fork
+    OUTPUT_VARIABLE tests_out
+    WORKING_DIRECTORY "${SRC_ROOT}"
+    RESULT_VARIABLE retc
+    )
+if (retc)
+    # Failed to list the tests. That's bad.
+    message (FATAL_ERROR "Failed to run test-libmongoc to discover tests [${retc}]:\n${tests_out}")
+endif ()
+
+# Split lines on newlines
+string (REPLACE "\n" ";" lines "${tests_out}")
+
+# Generate the test definitions
+foreach (line IN LISTS lines)
+    if (NOT line MATCHES "^/")
+        # Only generate if the line begins with `/`, which all tests should.
+        continue ()
+    endif ()
+    # The new test name is prefixed with 'mongoc'
+    set (test "mongoc${line}")
+    # Define the test. Use `--ctest-run` to tell it that CTest is in control.
+    add_test ("${test}" "${TEST_LIBMONGOC_EXE}" --ctest-run "${line}")
+    set_tests_properties ("${test}" PROPERTIES
+        # test-libmongoc expects to execute in the root of the source directory
+        WORKING_DIRECTORY "${SRC_ROOT}"
+        # If a test emits '@@ctest-skipped@@', this tells us that the test is
+        # skipped.
+        SKIP_REGULAR_EXPRESSION "@@ctest-skipped@@"
+        # 45 seconds of timeout on each test.
+        TIMEOUT 45
+        )
+endforeach ()
diff --git a/src/libbson/src/bson/bson-macros.h b/src/libbson/src/bson/bson-macros.h
@@ -306,6 +306,11 @@
 #define BSON_GNUC_DEPRECATED_FOR(f) BSON_GNUC_DEPRECATED
 #endif
 
+/**
+ * @brief String-ify the given argument
+ */
+#define BSON_STR(...) #__VA_ARGS__
+
 /**
  * @brief Mark the attached declared entity as "possibly-unused."
  *
diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt
@@ -1042,12 +1042,6 @@ mongoc_add_test (test-mongoc-gssapi FALSE ${PROJECT_SOURCE_DIR}/tests/test-mongo
 mongoc_add_test (test-mongoc-cache FALSE ${PROJECT_SOURCE_DIR}/tests/test-mongoc-cache.c)
 
 if (ENABLE_TESTS)
-   enable_testing ()
-   add_test (NAME test-libmongoc
-      COMMAND test-libmongoc
-      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../..
-   )
-
    # "make test" doesn't compile tests, so we create "make check" which compiles
    # and runs tests: https://gitlab.kitware.com/cmake/cmake/issues/8774
    add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} -V
diff --git a/src/libmongoc/tests/TestSuite.c b/src/libmongoc/tests/TestSuite.c
@@ -25,6 +25,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <signal.h>
 #include <mongoc/mongoc-util-private.h>
 #if !defined(_WIN32)
 #include <sys/types.h>
@@ -112,6 +113,43 @@ static BSON_ONCE_FUN (_test_suite_ensure_mutex_once)
 }
 
 
+static void
+_handle_signal (int signum)
+{
+   const char *s = "\nProcess was interrupted by the delivery of a signal.\n";
+   const char *sigstr;
+   switch (signum) {
+   case SIGABRT:
+      sigstr = "SIGABRT - Abnormal termination";
+      break;
+   case SIGINT:
+      sigstr = "SIGINT - Interrupted";
+      break;
+   case SIGTERM:
+      sigstr = "SIGTERM - Termination requested";
+      break;
+   case SIGSEGV:
+      sigstr = "SIGSEGV - Access violation";
+      break;
+   default:
+      sigstr = "(Unknown signal delivered)";
+   }
+#ifdef BSON_OS_UNIX
+   /* On POSIX these APIs are signal-safe */
+   write (STDERR_FILENO, s, strlen (s));
+   write (STDERR_FILENO, "  ", 2);
+   write (STDERR_FILENO, sigstr, strlen (sigstr));
+   write (STDERR_FILENO, "\n", 1);
+   fsync (STDERR_FILENO);
+#else
+   /* On Windows these APIs are signal-safe */
+   fprintf (stderr, "\n%s\n  %s\n", s, sigstr);
+   fflush (stderr);
+#endif
+   _Exit (signum);
+}
+
+
 void
 TestSuite_Init (TestSuite *suite, const char *name, int argc, char **argv)
 {
@@ -125,8 +163,14 @@ TestSuite_Init (TestSuite *suite, const char *name, int argc, char **argv)
    suite->flags = 0;
    suite->prgname = bson_strdup (argv[0]);
    suite->silent = false;
+   suite->ctest_run = NULL;
    _mongoc_array_init (&suite->match_patterns, sizeof (char *));
 
+   suite->prev_sigabrt = signal (SIGABRT, _handle_signal);
+   suite->prev_sigint = signal (SIGINT, _handle_signal);
+   suite->prev_sigterm = signal (SIGTERM, _handle_signal);
+   suite->prev_sigsegv = signal (SIGSEGV, _handle_signal);
+
    for (i = 1; i < argc; i++) {
       if (0 == strcmp ("-d", argv[i])) {
          suite->flags |= TEST_DEBUGOUTPUT;
@@ -165,6 +209,16 @@ TestSuite_Init (TestSuite *suite, const char *name, int argc, char **argv)
       } else if ((0 == strcmp ("-s", argv[i])) ||
                  (0 == strcmp ("--silent", argv[i]))) {
          suite->silent = true;
+      } else if ((0 == strcmp ("--ctest-run", argv[i]))) {
+         if (suite->ctest_run) {
+            test_error ("'--ctest-run' can only be specified once");
+         }
+         if (argc - 1 == i) {
+            test_error ("'--ctest-run' requires an argument");
+         }
+         suite->flags |= TEST_NOFORK;
+         suite->silent = true;
+         suite->ctest_run = bson_strdup (argv[++i]);
       } else if ((0 == strcmp ("-l", argv[i])) ||
                  (0 == strcmp ("--match", argv[i]))) {
          char *val;
@@ -180,6 +234,10 @@ TestSuite_Init (TestSuite *suite, const char *name, int argc, char **argv)
       }
    }
 
+   if (suite->match_patterns.len != 0 && suite->ctest_run != NULL) {
+      test_error ("'--ctest-run' cannot be specified with '-l' or '--match'");
+   }
+
    if (test_framework_getenv_bool ("MONGOC_TEST_VALGRIND")) {
       suite->flags |= TEST_VALGRIND;
    }
@@ -559,6 +617,10 @@ TestSuite_RunTest (TestSuite *suite, /* IN */
 
    for (i = 0; i < test->num_checks; i++) {
       if (!test->checks[i]()) {
+         if (suite->ctest_run) {
+            /* Write a marker that tells CTest that we are skipping this test */
+            test_msg ("@@ctest-skipped@@");
+         }
          if (!suite->silent) {
             bson_string_append_printf (
                buf,
@@ -667,6 +729,8 @@ TestSuite_PrintHelp (TestSuite *suite) /* IN */
       "first error).\n"
       "    -l, --match PATTERN   Run test by name, e.g. \"/Client/command\" or "
       "\"/Client/*\". May be repeated.\n"
+      "    --ctest-run TEST      Run only the named TEST for CTest\n"
+      "                          integration.\n"
       "    -s, --silent          Suppress all output.\n"
       "    -F FILENAME           Write test results (JSON) to FILENAME.\n"
       "    -d                    Print debug output (useful if a test hangs).\n"
@@ -878,6 +942,11 @@ test_matches (TestSuite *suite, Test *test)
 {
    int i;
 
+   if (suite->ctest_run) {
+      /* We only want exactly the named test */
+      return strcmp (test->name, suite->ctest_run) == 0;
+   }
+
    /* If no match patterns were provided, then assume all match. */
    if (suite->match_patterns.len == 0) {
       return true;
@@ -909,6 +978,14 @@ TestSuite_RunAll (TestSuite *suite /* IN */)
       }
    }
 
+   if (suite->ctest_run) {
+      /* We should have matched *at most* one test */
+      ASSERT (count <= 1);
+      if (count == 0) {
+         test_error ("No such test '%s'", suite->ctest_run);
+      }
+   }
+
    for (test = suite->tests; test; test = test->next) {
       if (test_matches (suite, test)) {
          status += TestSuite_RunTest (suite, test, &count);
@@ -1001,12 +1078,18 @@ TestSuite_Destroy (TestSuite *suite)
 
    free (suite->name);
    free (suite->prgname);
+   free (suite->ctest_run);
    for (i = 0; i < suite->match_patterns.len; i++) {
       char *val = _mongoc_array_index (&suite->match_patterns, char *, i);
       bson_free (val);
    }
 
    _mongoc_array_destroy (&suite->match_patterns);
+
+   signal (SIGABRT, suite->prev_sigabrt);
+   signal (SIGINT, suite->prev_sigint);
+   signal (SIGTERM, suite->prev_sigterm);
+   signal (SIGSEGV, suite->prev_sigsegv);
 }
 
 
diff --git a/src/libmongoc/tests/TestSuite.h b/src/libmongoc/tests/TestSuite.h
diff --git a/src/libmongoc/tests/test-mongoc-server-selection.c b/src/libmongoc/tests/test-mongoc-server-selection.c
