Add end to end test devoted to ctest

.github/workflows/ci-main-linux.yml

@@ -74,6 +74,12 @@ jobs:
         with:
           run: yarn endToEndTestsSingleRoot
 
+      - name: Run single root test for ctest
+        uses: GabrielBB/[email protected]
+        with:
+          run: yarn endToEndTestsSingleRootCTest
+
+
       - name: Run multi root test
         uses: GabrielBB/[email protected]
         with:

.github/workflows/ci-main-mac.yml

@@ -64,6 +64,9 @@ jobs:
       - name: Run single root test
         run: yarn endToEndTestsSingleRoot
 
+      - name: Run single root test for ctest
+        run: yarn endToEndTestsSingleRootCTest
+
       - name: Run multi root test
         run: yarn endToEndTestsMultiRoot
 

.github/workflows/ci-main.win.yml

@@ -66,6 +66,9 @@ jobs:
       - name: Run single root test
         run: yarn endToEndTestsSingleRoot
 
+      - name: Run single root test for ctest
+        run: yarn endToEndTestsSingleRootCTest
+
       - name: Run multi root test
         run: yarn endToEndTestsMultiRoot
 

.vscode/launch.json

@@ -118,6 +118,32 @@
                 "TEST_FILTER": ".*"
             },
         },
+        {
+            "name": "Run single-root-ctest Tests",
+            "type": "extensionHost",
+            "request": "launch",
+            "runtimeExecutable": "${execPath}",
+            "args": [
+                "${workspaceFolder}/test/end-to-end-tests/single-root-ctest/project-folder",
+                "--extensionDevelopmentPath=${workspaceFolder}",
+                "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/index"
+            ],
+            "sourceMaps": true,
+            "outFiles": [
+                "${workspaceFolder}/dist/**/*.js",
+                "${workspaceFolder}/out/*",
+                "${workspaceFolder}/out/src/**",
+                "${workspaceFolder}/out/test/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/test/*"
+            ],
+            "preLaunchTask": "Pretest",
+            "env": {
+                "CMT_TESTING": "1",
+                "CMT_QUIET_CONSOLE": "1",
+                "TEST_FILTER": ".*"
+            },
+        },
         {
             "name": "Run multi-root-UI Tests",
             "type": "extensionHost",

package.json

@@ -3793,6 +3793,7 @@
     "integrationTests": "yarn run pretest && node ./out/test/integration-tests/runTest.js",
     "endToEndTestsSuccessfulBuild": "yarn run pretest && node ./out/test/end-to-end-tests/successful-build/runTest.js",
     "endToEndTestsSingleRoot": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-UI/runTest.js",
+    "endToEndTestsSingleRootCTest": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-ctest/runTest.js",
     "endToEndTestsMultiRoot": "yarn run pretest && node ./out/test/end-to-end-tests/multi-root-UI/runTest.js",
     "backendTests": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 999999 --colors -r ts-node/register -r tsconfig-paths/register ./test/unit-tests/backend/**/*.test.ts",
     "build-product-icon-font": "yarn --cwd ./tools/product-icon-font-generator/ install && yarn --cwd ./tools/product-icon-font-generator/ build && node ./tools/product-icon-font-generator/dist/index.js --source-directory ./res/product-icons/ --output-directory ./res/ --woff2"

src/cmakeProject.ts

@@ -2391,7 +2391,7 @@ export class CMakeProject {
     async ctest(fromWorkflow: boolean = false): Promise<number> {
         const drv = await this.preTest(fromWorkflow);
         const retc = await this.cTestController.runCTest(drv);
-        return (retc) ? 0 : -1;
+        return retc;
     }
 
     async cpack(fromWorkflow: boolean = false): Promise<number> {

test/end-to-end-tests/single-root-ctest/index.ts

@@ -0,0 +1,54 @@
+// eslint-disable-next-line import/no-unassigned-import
+import 'module-alias/register';
+
+import * as path from 'path';
+import * as Mocha from 'mocha';
+import * as glob from 'glob';
+import { Logger } from '@cmt/logging';
+
+export function run(): Promise<void> {
+    // Create the mocha test
+    const mocha = new Mocha({
+        ui: 'tdd',
+        color: true
+    });
+
+    const testsRoot = __dirname;
+
+    return new Promise((c, e) => {
+        glob('**/*.test.js', { cwd: testsRoot }, (err, files) => {
+            if (err) {
+                return e(err);
+            }
+
+            // Add files to the test suite
+            const regex = process.env.TEST_FILTER ? new RegExp(process.env.TEST_FILTER) : /.*/;
+            files.forEach(f => {
+                if (regex.test(f)) {
+                    mocha.addFile(path.resolve(testsRoot, f));
+                }
+            });
+
+            try {
+                // Run the mocha test
+                mocha.timeout(100000);
+
+                // Log the name of each test before it starts.
+                const beforeEach: Mocha.Func = function (this: Mocha.Context, done: Mocha.Done) {
+                    Logger.logTestName(this.currentTest?.parent?.title, this.currentTest?.title);
+                    done();
+                };
+                mocha.rootHooks({beforeEach});
+                mocha.run(failures => {
+                    if (failures > 0) {
+                        e(new Error(`${failures} tests failed.`));
+                    } else {
+                        c();
+                    }
+                });
+            } catch (err) {
+                e(err);
+            }
+        });
+    });
+}

test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cmake.buildDirectory": "${workspaceFolder}/build",
+    "cmake.useCMakePresets": "always",
+    "cmake.configureOnOpen": false,
+    "cmake.ctest.allowParallelJobs": false
+}

test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.6.0)
+project(TestCTestProcess VERSION 0.1.0)
+
+add_library(TestUtils SHARED)
+target_sources(TestUtils PRIVATE test_utils.cpp)
+target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED
+                                                           ON)
+
+set(TESTS_DIR
+    ""
+    CACHE STRING "Directory where the files generated by tests will be written")
+# Will generate a file get_test_dir.h with the content of get_test_dir.h.in It's
+# goal is to provide a function that returns the directory where the tests will
+# be executed
+configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY)
+add_library(GetTestDir INTERFACE)
+target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR})
+
+# Adds an executable that will generate an output file by concatenating the
+# content of each file in the test directory
+add_executable(GenerateOutputFile generate_output_file.cpp)
+target_link_libraries(GenerateOutputFile PRIVATE GetTestDir)
+set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17
+                                                    CXX_STANDARD_REQUIRED ON)
+
+enable_testing()
+
+# The generation of json readable output file must occur after each test run
+# Declares a fixture that will be executed after each test run
+add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile)
+set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+include(RegisterTest)
+
+set(TESTS_OUTPUT_FILES
+    ""
+    CACHE STRING "Output files generated by tests")
+set(TESTS_NAMES
+    ""
+    CACHE STRING "Names of the tests")
+set(TESTS_SUCCESS
+    ""
+    CACHE BOOL "Success of the tests")
+
+register_tests(
+  TEST_DIRECTORY
+  ${TESTS_DIR}
+  TEST_NAME_LIST
+  ${TESTS_NAMES}
+  TEST_OUTPUT_FILE_LIST
+  ${TESTS_OUTPUT_FILES}
+  TEST_SUCCESS_LIST
+  ${TESTS_SUCCESS})

test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json

@@ -0,0 +1,91 @@
+{
+  "version": 9,
+  "configurePresets": [
+    {
+      "name": "LinuxBase",
+      "description": "Sets compilers for linux based systems",
+      "hidden": true,
+      "generator": "Ninja",
+      "binaryDir": "${workspaceFolder}/build",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++"
+      }
+    },
+    {
+      "name": "WindowsBase",
+      "description": "Sets compilers for linux based systems",
+      "hidden": true,
+      "generator": "Visual Studio 16 2019",
+      "binaryDir": "${workspaceFolder}/build",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "cl.exe",
+        "CMAKE_CXX_COMPILER": "cl.exe"
+      }
+    },
+    {
+      "name": "2Successes",
+      "description": "Configures 2 tests that end successfully",
+      "hidden": true,
+      "cacheVariables": {
+        "TESTS_DIR": "${workspaceFolder}/build/vscode-cmake-tools-tests",
+        "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt",
+        "TESTS_NAMES": "Suite1.TestA;Suite2.TestB",
+        "TESTS_SUCCESS": "true;true"
+      }
+    },
+    {
+      "name": "2Successes1Failure",
+      "description": "Configures 2 tests that end successfully and 1 that fails",
+      "hidden": true,
+      "cacheVariables": {
+        "TESTS_DIR": "${workspaceFolder}/build/vscode-cmake-tools-tests",
+        "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt",
+        "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC",
+        "TESTS_SUCCESS": "true;false;true"
+      }
+    },
+    {
+      "name": "3Failures",
+      "description": "Configures 3 tests that fail",
+      "hidden": true,
+      "cacheVariables": {
+        "TESTS_DIR": "${workspaceFolder}/build/vscode-cmake-tools-tests",
+        "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt",
+        "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC",
+        "TESTS_SUCCESS": "false;false;false"
+      }
+    },
+    {
+      "name": "Linux-2Successes",
+      "inherits": ["LinuxBase", "2Successes"],
+      "description": "Configures 2 tests that end successfully under linux"
+    },
+    {
+      "name": "Linux-2Successes1Failure",
+      "inherits": ["LinuxBase", "2Successes1Failure"],
+      "description": "Configures 2 tests that end successfully and 1 that fails under linux"
+    },
+    {
+      "name": "Linux-3Failures",
+      "inherits": ["LinuxBase", "3Failures"],
+      "description": "Configures 3 tests that fail under linux"
+    },
+    {
+      "name": "Windows-2Successes",
+      "inherits": ["WindowsBase", "2Successes"],
+      "description": "Configures 2 tests that end successfully under windows"
+    },
+    {
+      "name": "Windows-2Successes1Failure",
+      "inherits": ["WindowsBase", "2Successes1Failure"],
+      "description": "Configures 2 tests that end successfully and 1 that fails under windows"
+    },
+    {
+      "name": "Windows-3Failures",
+      "inherits": ["WindowsBase", "3Failures"],
+      "description": "Configures 3 tests that fail under windows"
+    }
+  ]
+}