Add end to end test devoted to ctest #4186
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -2331,7 +2331,7 @@ export class CMakeProject { | |||||
| async ctest(fromWorkflow: boolean = false): Promise<number> { | ||||||
| const drv = await this.preTest(fromWorkflow); | ||||||
| const retc = await this.cTestController.runCTest(drv); | ||||||
| return (retc === 0) ? 0 : -1; | ||||||
|
||||||
| return (retc === 0) ? 0 : -1; | |
| return retc; |
We should keep the integrity of the return code, the runCTest method doesn't return number | undefined it only returns a number so we should trust that.
Collaborator
There was a problem hiding this comment.
We should likely also update that in the below cpack method.
Contributor
Author
There was a problem hiding this comment.
@gcampbell-msft i did change to keep integrity of the return code in the ctest method.
I am not confident to do the same change in the cpack method because i do not know how to check the result.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| // eslint-disable-next-line import/no-unassigned-import | ||
| import 'module-alias/register'; | ||
|
|
||
| import * as path from 'path'; | ||
| import * as Mocha from 'mocha'; | ||
| import * as glob from 'glob'; | ||
| import { Logger } from '@cmt/logging'; | ||
|
|
||
| export function run(): Promise<void> { | ||
| // Create the mocha test | ||
| const mocha = new Mocha({ | ||
| ui: 'tdd', | ||
| color: true | ||
| }); | ||
|
|
||
| const testsRoot = __dirname; | ||
|
|
||
| return new Promise((c, e) => { | ||
| glob('**/*.test.js', { cwd: testsRoot }, (err, files) => { | ||
| if (err) { | ||
| return e(err); | ||
| } | ||
|
|
||
| // Add files to the test suite | ||
| const regex = process.env.TEST_FILTER ? new RegExp(process.env.TEST_FILTER) : /.*/; | ||
| files.forEach(f => { | ||
| if (regex.test(f)) { | ||
| mocha.addFile(path.resolve(testsRoot, f)); | ||
| } | ||
| }); | ||
|
|
||
| try { | ||
| // Run the mocha test | ||
| mocha.timeout(100000); | ||
|
|
||
| // Log the name of each test before it starts. | ||
| const beforeEach: Mocha.Func = function (this: Mocha.Context, done: Mocha.Done) { | ||
| Logger.logTestName(this.currentTest?.parent?.title, this.currentTest?.title); | ||
| done(); | ||
| }; | ||
| mocha.rootHooks({beforeEach}); | ||
| mocha.run(failures => { | ||
| if (failures > 0) { | ||
| e(new Error(`${failures} tests failed.`)); | ||
| } else { | ||
| c(); | ||
| } | ||
| }); | ||
| } catch (err) { | ||
| e(err); | ||
| } | ||
| }); | ||
| }); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| { | ||
| "cmake.buildDirectory": "${workspaceFolder}/build", | ||
| "cmake.useCMakePresets": "always", | ||
| "cmake.configureOnOpen": false, | ||
| "cmake.ctest.allowParallelJobs": false | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| cmake_minimum_required(VERSION 3.6.0) | ||
| project(TestCTestProcess VERSION 0.1.0) | ||
|
|
||
| add_library(TestUtils SHARED) | ||
| target_sources(TestUtils PRIVATE test_utils.cpp) | ||
| target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) | ||
| set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED | ||
| ON) | ||
|
|
||
| set(TESTS_DIR | ||
| "" | ||
| CACHE STRING "Directory where the files generated by tests will be written") | ||
| # Will generate a file get_test_dir.h with the content of get_test_dir.h.in It's | ||
| # goal is to provide a function that returns the directory where the tests will | ||
| # be executed | ||
| configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY) | ||
| add_library(GetTestDir INTERFACE) | ||
| target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) | ||
|
|
||
| # Adds an executable that will generate an output file by concatenating the | ||
| # content of each file in the test directory | ||
| add_executable(GenerateOutputFile generate_output_file.cpp) | ||
| target_link_libraries(GenerateOutputFile PRIVATE GetTestDir) | ||
| set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 | ||
| CXX_STANDARD_REQUIRED ON) | ||
|
|
||
| enable_testing() | ||
|
|
||
| # The generation of json readable output file must occur after each test run | ||
| # Declares a fixture that will be executed after each test run | ||
| add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile) | ||
| set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT) | ||
|
|
||
| list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") | ||
| include(RegisterTest) | ||
|
|
||
| set(TESTS_OUTPUT_FILES | ||
| "" | ||
| CACHE STRING "Output files generated by tests") | ||
| set(TESTS_NAMES | ||
| "" | ||
| CACHE STRING "Names of the tests") | ||
| set(TESTS_SUCCESS | ||
| "" | ||
| CACHE BOOL "Success of the tests") | ||
|
|
||
| register_tests( | ||
| TEST_DIRECTORY | ||
| ${TESTS_DIR} | ||
| TEST_NAME_LIST | ||
| ${TESTS_NAMES} | ||
| TEST_OUTPUT_FILE_LIST | ||
| ${TESTS_OUTPUT_FILES} | ||
| TEST_SUCCESS_LIST | ||
| ${TESTS_SUCCESS}) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| { | ||
| "version": 2, | ||
|
||
| "configurePresets": [ | ||
| { | ||
| "name": "2Successes", | ||
| "description": "Sets generator, build and install directory, vcpkg", | ||
| "generator": "Ninja", | ||
| "binaryDir": "${workspaceFolder}/build", | ||
| "cacheVariables": { | ||
|
Collaborator
There was a problem hiding this comment. We need to ensure that we define the CMAKE_C_COMPILER and CMAKE_CXX_COMPILER, depending on what OS this is run on.
Contributor
Author
There was a problem hiding this comment. @gcampbell-msft i added hidden configure preset that hold compiler specifications for linux and windows systems. These presets are then mixed up with those setting up the tests to have all cases covered. |
||
| "CMAKE_BUILD_TYPE": "Debug", | ||
| "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", | ||
| "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt", | ||
| "TESTS_NAMES": "Suite1.TestA;Suite2.TestB", | ||
| "TESTS_SUCCESS": "true;true" | ||
| } | ||
| }, | ||
| { | ||
| "name": "2Successes1Failure", | ||
| "description": "Sets generator, build and install directory, vcpkg", | ||
| "generator": "Ninja", | ||
| "binaryDir": "${workspaceFolder}/build", | ||
| "cacheVariables": { | ||
| "CMAKE_BUILD_TYPE": "Debug", | ||
| "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", | ||
| "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt", | ||
| "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", | ||
| "TESTS_SUCCESS": "true;false;true" | ||
| } | ||
| }, | ||
| { | ||
| "name": "3Failures", | ||
| "description": "Sets generator, build and install directory, vcpkg", | ||
| "generator": "Ninja", | ||
| "binaryDir": "${workspaceFolder}/build", | ||
| "cacheVariables": { | ||
| "CMAKE_BUILD_TYPE": "Debug", | ||
| "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", | ||
| "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt", | ||
| "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", | ||
| "TESTS_SUCCESS": "false;false;false" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| #-------------------------------------------------------------------- | ||
| # Generate the source file of the test in argument | ||
| # | ||
| # Parameters: | ||
| # output_file_path: path to the file, the test will write | ||
| # test_success: true if the test end successfully. False otherwise. | ||
| # Returns: | ||
| # test_source: name of the source file that will be generated | ||
| #-------------------------------------------------------------------- | ||
| function(generate_test_source_file output_file_path test_success) | ||
| get_filename_component(output_file_name ${output_file_path} NAME_WE) | ||
| # Declare variables used in the template file (test.cpp.in) | ||
| set(test_filename ${output_file_path}) | ||
| set(success ${test_success}) | ||
| # Generate test source file | ||
| set(test_source "${output_file_name}.cpp") | ||
| configure_file(test.cpp.in ${test_source} @ONLY) | ||
| set(test_source "${test_source}" PARENT_SCOPE) | ||
| endfunction() | ||
|
|
||
| #-------------------------------------------------------------------- | ||
| # Build the name of the test executable from the name of the test source file | ||
| # | ||
| # The name of the executable will be the one of the test source file without '_' | ||
| # and in CamelCase. | ||
| # | ||
| # Parameters: | ||
| # test_source: name of the test source file | ||
| # Returns: | ||
| # test_exe: name of the test executable | ||
| #-------------------------------------------------------------------- | ||
| function(build_test_exe_name test_source) | ||
| get_filename_component(test_name ${test_source} NAME_WE) | ||
| # Replace _ with ; so that the name becomes a list of sub-words | ||
| string(REPLACE "_" ";" splitted_test_name ${test_name}) | ||
| set(test_exe) | ||
| # For each of the sub-word, extract the first letter | ||
| # from the rest of the word (radical) | ||
| foreach(word ${splitted_test_name}) | ||
| string(SUBSTRING ${word} 0 1 first_letter) | ||
| string(SUBSTRING ${word} 1 -1 radical) | ||
| # Turns first sub-word letter into upper case | ||
| string(TOUPPER ${first_letter} up_first_letter) | ||
| # Concat uppercase first letter and radical | ||
| set(test_exe "${test_exe}${up_first_letter}${radical}") | ||
| endforeach() | ||
| # Returns test_exe | ||
| set(test_exe ${test_exe} PARENT_SCOPE) | ||
| endfunction() | ||
|
|
||
| #-------------------------------------------------------------------- | ||
| # Create and register a test | ||
| # | ||
| # Usage: | ||
| # register_test(TEST_DIR <dir> TEST_NAME <name> TEST_OUTPUT_FILE <path> TEST_SUCCESS <success>) | ||
| # Parameters: | ||
| # TEST_DIR: path to the directory where the test will write its output file | ||
| # TEST_NAME: name of the test | ||
| # TEST_OUTPUT_FILE: name of the file the test should generate | ||
| # TEST_SUCCESS: whether or not the test should end successfully | ||
| #-------------------------------------------------------------------- | ||
| function(register_test) | ||
| set(options) | ||
| set(oneValueArgs "TEST_DIR;TEST_NAME;TEST_OUTPUT_FILE;TEST_SUCCESS") | ||
| ### PARSING ARGUMENTS | ||
| cmake_parse_arguments(register_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) | ||
| if(DEFINED register_test_KEYWORDS_MISSING_VALUES) | ||
| message( | ||
| FATAL_ERROR | ||
| "In the call to register_test function, the keywords ${register_test_KEYWORDS_MISSING_VALUES} are awaiting for at least one value" | ||
| ) | ||
| endif() | ||
| if(DEFINED register_test_UNPARSED_ARGUMENTS) | ||
| message( | ||
| FATAL_ERROR | ||
| "Following arguments are unknown to register_test function: ${register_test_UNPARSED_ARGUMENTS}" | ||
| ) | ||
| endif() | ||
| if(NOT DEFINED register_test_TEST_DIR) | ||
| message(FATAL_ERROR "The function register_test is awaiting for TEST_DIR keyword") | ||
| endif() | ||
| if(NOT DEFINED register_test_TEST_NAME) | ||
| message(FATAL_ERROR "The function register_test is awaiting for TEST_NAME keyword") | ||
| endif() | ||
| if(NOT DEFINED register_test_TEST_OUTPUT_FILE) | ||
| message(FATAL_ERROR "The function register_test is awaiting for TEST_OUTPUT_FILE keyword") | ||
| endif() | ||
| if(NOT DEFINED register_test_TEST_SUCCESS) | ||
| message(FATAL_ERROR "The function register_test is awaiting for TEST_SUCCESS keyword") | ||
| endif() | ||
|
|
||
| set(test_output_file_path "${register_test_TEST_DIR}/${register_test_TEST_OUTPUT_FILE}") | ||
| message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${test_output_file_path} returning as success: ${register_test_TEST_SUCCESS}") | ||
| ### GENERATE TEST | ||
| generate_test_source_file(${test_output_file_path} ${register_test_TEST_SUCCESS}) # => returns test_source | ||
| build_test_exe_name(${test_source}) # => returns test_exe | ||
| message(STATUS "--> Creating test executable ${test_exe} with source ${test_source}") | ||
| add_executable(${test_exe} ${test_source}) | ||
| target_link_libraries(${test_exe} PRIVATE TestUtils GetTestDir) | ||
| add_test(NAME "${register_test_TEST_NAME}" COMMAND "${test_exe}") | ||
| set_tests_properties("${register_test_TEST_NAME}" PROPERTIES FIXTURES_REQUIRED GENOUT) | ||
| endfunction() | ||
|
|
||
| #-------------------------------------------------------------------- | ||
| # Create and register tests in arguments | ||
| # | ||
| # Usage: | ||
| # register_tests(TEST_DIRECTORY <dir> TEST_NAME_LIST <names> TEST_OUTPUT_FILE_LIST <paths> TEST_SUCCESS_LIST <successes>) | ||
| # Parameters: | ||
| # TEST_DIRECTORY: path to the directory where the tests will write their output files | ||
| # TEST_NAME_LIST: list of test names | ||
| # TEST_OUTPUT_FILE_LIST: list of file names the tests should generate | ||
| # TEST_SUCCESS_LIST: list of boolean values indicating whether or not the tests should end successfully | ||
| #-------------------------------------------------------------------- | ||
| function(register_tests) | ||
| set(options) | ||
| set(oneValueArgs "TEST_DIRECTORY") | ||
| set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_LIST;TEST_SUCCESS_LIST") | ||
| ### PARSING ARGUMENTS | ||
| cmake_parse_arguments(register_tests "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) | ||
| if(DEFINED register_tests_KEYWORDS_MISSING_VALUES) | ||
| message( | ||
| FATAL_ERROR | ||
| "In the call to register_tests function, the keywords ${register_tests_KEYWORDS_MISSING_VALUES} are awaiting for at least one value" | ||
| ) | ||
| endif() | ||
| if(DEFINED register_tests_UNPARSED_ARGUMENTS) | ||
| message( | ||
| FATAL_ERROR | ||
| "Following arguments are unknown to register_tests function: ${register_tests_UNPARSED_ARGUMENTS}" | ||
| ) | ||
| endif() | ||
| if(NOT DEFINED register_tests_TEST_DIRECTORY) | ||
| message(FATAL_ERROR "The function register_tests is awaiting for TEST_DIRECTORY keyword") | ||
| endif() | ||
| if(NOT DEFINED register_tests_TEST_NAME_LIST) | ||
| message(FATAL_ERROR "The function register_tests is awaiting for TEST_NAME_LIST keyword") | ||
| endif() | ||
| if(NOT DEFINED register_tests_TEST_OUTPUT_FILE_LIST) | ||
| message(FATAL_ERROR "The function register_tests is awaiting for TEST_OUTPUT_FILE_LIST keyword") | ||
| endif() | ||
| if(NOT DEFINED register_tests_TEST_SUCCESS_LIST) | ||
| message(FATAL_ERROR "The function register_tests is awaiting for TEST_SUCCESS_LIST keyword") | ||
| endif() | ||
|
|
||
| list(LENGTH register_tests_TEST_NAME_LIST NB_TESTS) | ||
| math(EXPR MAX_INDEX "${NB_TESTS}-1") | ||
| foreach(test_index RANGE ${MAX_INDEX}) | ||
| list(GET register_tests_TEST_OUTPUT_FILE_LIST ${test_index} test_output) | ||
| list(GET register_tests_TEST_NAME_LIST ${test_index} test_name) | ||
| list(GET register_tests_TEST_SUCCESS_LIST ${test_index} test_success) | ||
| register_test(TEST_DIR ${register_tests_TEST_DIRECTORY} TEST_NAME ${test_name} TEST_OUTPUT_FILE ${test_output} TEST_SUCCESS ${test_success}) | ||
| endforeach() | ||
| endfunction() |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@hippo91 Is this a necessary change? It doesn't have to do with your other changes so I'm wondering if it was on purpose.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@gcampbell-msft you are right it was an error.