Stop using ctest (#5169)

Author: CaseyCarter

CMakeLists.txt

@@ -118,7 +118,6 @@ set(VCLIBS_DEBUG_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/Od>")
 set(VCLIBS_RELEASE_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:/O2;/Os>")
 
 if(BUILD_TESTING)
-    enable_testing()
     add_subdirectory(tests)
 endif()
 

README.md

@@ -269,34 +269,33 @@ C:\Users\username\Desktop>dumpbin /DEPENDENTS .\example.exe | findstr msvcp
     * Otherwise, use [LLVM's installer][] and choose to add LLVM to your `PATH` during installation.
 4. Follow the instructions below.
 
-## Running All The Tests
+## Running The Tests
 
-After configuring and building the project, running `ctest` from the build output directory will run all the tests.
-CTest will only display the standard error output of tests that failed. In order to get more details from CTest's
-`lit` invocations, run the tests with `ctest -V`.
+Our tests are currently split across three test suites that are located at `tests\std`, `tests\tr1`, and
+`llvm-project\libcxx\test\std`. The test runner `${PROJECT_BINARY_DIR}\tests\utils\stl-lit\stl-lit.py` accepts paths to
+directories in the test suites and runs all tests located in the subtree rooted at those paths. This can mean executing
+the entirety of a single test suite, running all tests under a category in `libcxx`, or running a single test in `std`
+and `tr1`.
 
-## Running A Subset Of The Tests
-
-`${PROJECT_BINARY_DIR}\tests\utils\stl-lit\stl-lit.py` can be invoked on a subdirectory of a test suite and will execute
-all the tests under that subdirectory. This can mean executing the entirety of a single test suite, running all tests
-under a category in libcxx, or running a single test in `std` and `tr1`.
+Some useful `stl-lit.py` options:
+* `-v` (verbose) tells `stl-lit.py` to show us output from failed test cases.
+* `-Dnotags=ASAN` disables the "extra ASan configs" that we typically run only in CI. This is useful to limit runtime
+  for full validation runs, but often omitted when running just a few test cases to enable the extra ASan coverage.
 
 ## Examples
 
 These examples assume that your current directory is `C:\Dev\STL\out\x64`.
 
-* This command will run all of the test suites with verbose output.
-  + `ctest -V`
-* This command will also run all of the test suites.
-  + `python tests\utils\stl-lit\stl-lit.py ..\..\llvm-project\libcxx\test ..\..\tests\std ..\..\tests\tr1`
-* This command will run all of the std test suite.
-  + `python tests\utils\stl-lit\stl-lit.py ..\..\tests\std`
+* This command will run all of the test suites:
+  + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\llvm-project\libcxx\test ..\..\tests\std ..\..\tests\tr1`
+* This command will run only the std test suite.
+  + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\tests\std`
 * If you want to run a subset of a test suite, you need to point it to the right place in the sources. The following
-will run the single test found under VSO_0000000_any_calling_conventions.
-  + `python tests\utils\stl-lit\stl-lit.py ..\..\tests\std\tests\VSO_0000000_any_calling_conventions`
+will run the single test found under `VSO_0000000_any_calling_conventions`.
+  + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\tests\std\tests\VSO_0000000_any_calling_conventions`
 * You can invoke `stl-lit` with any arbitrary subdirectory of a test suite. In libcxx this allows you to have finer
 control over what category of tests you would like to run. The following will run all the libcxx map tests.
-  + `python tests\utils\stl-lit\stl-lit.py ..\..\llvm-project\libcxx\test\std\containers\associative\map`
+  + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\llvm-project\libcxx\test\std\containers\associative\map`
 * You can also use the `--filter` option to include tests whose names match a regular expression. The following
   command will run tests with "atomic_wait" in their names in both the std and libcxx test suites.
   + `python tests\utils\stl-lit\stl-lit.py ..\..\llvm-project\libcxx\test ..\..\tests\std --filter=atomic_wait`
@@ -305,31 +304,8 @@ control over what category of tests you would like to run. The following will ru
 
 ## Interpreting The Results Of Tests
 
-### CTest
-
-When running the tests via CTest, all of the test suites are considered to be a single test. If any single test in a
-test suite fails, CTest will simply report that the `stl` test failed.
-
-Example:
-```
-0% tests passed, 1 tests failed out of 1
-
-Total Test time (real) = 2441.55 sec
-
-The following tests FAILED:
-      1 - stl (Failed)
-```
-
-The primary utility of CTest in this case is to conveniently invoke `stl-lit.py` with the correct set of arguments.
-
-CTest will output everything that was sent to stderr for each of the failed test suites, which can be used to identify
-which individual test within the test suite failed. It can sometimes be helpful to run CTest with the `-V` option in
-order to see the stdout of the tests.
-
-### stl-lit
-
-When running the tests directly via the generated `stl-lit.py` script the result of each test will be printed. The
-format of each result is `{Result Code}: {Test Suite Name} :: {Test Name}:{Configuration Number}`.
+`stl-lit.py` prints the result of each test. The format of each result is
+`{Result Code}: {Test Suite Name} :: {Test Name}:{Configuration Number}`.
 
 Example:
 ```

azure-devops/asan-pipeline.yml

@@ -27,7 +27,7 @@ stages:
           hostArch: x64
           targetArch: x64
           asanBuild: true
-          ctestOptions: '--tests-regex stlasan'
+          testTargets: STL-ASan-CI
 
   - stage: Build_And_Test_x86
     displayName: 'Build and Test x86'
@@ -41,6 +41,6 @@ stages:
           hostArch: x86
           targetArch: x86
           asanBuild: true
-          ctestOptions: '--tests-regex stlasan'
+          testTargets: STL-ASan-CI
 
   # no coverage for ARM and ARM64

azure-devops/build-and-test.yml

@@ -15,9 +15,9 @@ parameters:
 - name: buildBenchmarks
   type: boolean
   default: false
-- name: ctestOptions
+- name: testTargets
   type: string
-  default: '--exclude-regex stlasan'
+  default: 'STL-CI'
 - name: numShards
   type: number
   default: 8
@@ -61,5 +61,5 @@ jobs:
     parameters:
       hostArch: ${{ parameters.hostArch }}
       targetArch: ${{ parameters.targetArch }}
-      ctestOptions: ${{ parameters.ctestOptions }}
+      testTargets: ${{ parameters.testTargets }}
       skipTesting: ${{ parameters.skipTesting }}

azure-devops/run-tests.yml

@@ -6,15 +6,15 @@ parameters:
   type: string
 - name: targetArch
   type: string
-- name: ctestOptions
+- name: testTargets
   type: string
 - name: skipTesting
   type: boolean
 steps:
 - script: |
     call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
     -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
-    ctest --verbose ${{ parameters.ctestOptions }}
+    ninja --verbose -k 0 ${{ parameters.testTargets }}
   displayName: 'Build and Run Tests'
   timeoutInMinutes: 30
   condition: and(succeeded(), not(${{ parameters.skipTesting }}))

tests/CMakeLists.txt

@@ -26,19 +26,11 @@ add_subdirectory(utils/stl-lit)
 find_package(Python "3.13" REQUIRED COMPONENTS Interpreter)
 
 if(NOT DEFINED LIT_FLAGS)
-    list(APPEND LIT_FLAGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/test_results.json")
+    set(LIT_FLAGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/test_results.json")
 endif()
 
+set(STL_LIT_COMMAND ${Python_EXECUTABLE} ${STL_LIT_OUTPUT} ${LIT_FLAGS})
 get_property(STL_LIT_TEST_DIRS GLOBAL PROPERTY STL_LIT_TEST_DIRS)
-list(APPEND STL_LIT_COMMAND "${STL_LIT_OUTPUT}"
-                            "${LIT_FLAGS}"
-                            "-D" "notags=ASAN"
-                            "${STL_LIT_TEST_DIRS}")
-list(APPEND STLASAN_LIT_COMMAND "${STL_LIT_OUTPUT}"
-                                "${LIT_FLAGS}"
-                                "-D" "tags=ASAN"
-                                "${STL_LIT_TEST_DIRS}")
-
-add_test(NAME stl COMMAND ${Python_EXECUTABLE} ${STL_LIT_COMMAND} COMMAND_EXPAND_LISTS)
-add_test(NAME stlasan COMMAND ${Python_EXECUTABLE} ${STLASAN_LIT_COMMAND} COMMAND_EXPAND_LISTS)
-set_tests_properties(stl stlasan PROPERTIES RUN_SERIAL ON)
+
+add_custom_target(STL-CI COMMAND ${STL_LIT_COMMAND} -Dnotags=ASAN ${STL_LIT_TEST_DIRS} USES_TERMINAL)
+add_custom_target(STL-ASan-CI COMMAND ${STL_LIT_COMMAND} -Dtags=ASAN ${STL_LIT_TEST_DIRS} USES_TERMINAL)

tests/utils/stl/test/params.py

@@ -49,8 +49,10 @@ def beNice(prio: str) -> list[ConfigAction]:
     }
     psutil.Process().nice(priority_map[prio])
   except ImportError:
-    import sys
-    print(f'NOTE: Module "psutil" is not installed, so the priority setting "{prio}" has no effect.', file=sys.stderr)
+    if not hasattr(beNice, 'suppress'):
+      import sys
+      print(f'NOTE: Module "psutil" is not installed, so the priority setting "{prio}" has no effect.', file=sys.stderr)
+      beNice.suppress = True
   return []