Skip to content

Commit d3baa96

Browse files
mjcaisse-intelmjcaisse-intel
authored
Merge pull request intel#98 from elbeno/add-mull
✨ 👷 Add mull tests
2 parents 589e1c4 + 89165d2 commit d3baa96

File tree

11 files changed

+215
-0
lines changed

11 files changed

+215
-0
lines changed

‎.github/workflows/test.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ env:
1919
DEBIAN_FRONTEND: noninteractive
2020
CMAKE_GENERATOR: Ninja
2121
TARGET_LLVM_VERSION: 18
22+
MULL_LLVM_VERSION: 15
2223

2324
concurrency:
2425
group: ${{ github.head_ref || github.run_id }}
@@ -201,6 +202,56 @@ jobs:
201202
working-directory: ${{github.workspace}}/test/application
202203
run: cmake --build build -t cpp_tests
203204

205+
mutate:
206+
runs-on: ${{ github.repository_owner == 'intel' && 'intel-' || '' }}ubuntu-22.04
207+
steps:
208+
- name: Install build tools
209+
run: |
210+
wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh ${{env.MULL_LLVM_VERSION}}
211+
sudo apt install -y python3-pip ninja-build
212+
213+
- name: Install mull
214+
env:
215+
MULL_VERSION: 0.23.0
216+
run: |
217+
wget https://github.com/mull-project/mull/releases/download/${{env.MULL_VERSION}}/Mull-${{env.MULL_LLVM_VERSION}}-${{env.MULL_VERSION}}-LLVM-${{env.MULL_LLVM_VERSION}}.0-ubuntu-22.04.deb
218+
sudo dpkg -i Mull-${{env.MULL_LLVM_VERSION}}-${{env.MULL_VERSION}}-LLVM-${{env.MULL_LLVM_VERSION}}.0-ubuntu-22.04.deb
219+
220+
- name: Checkout PR branch
221+
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
222+
223+
- name: Restore CPM cache
224+
env:
225+
cache-name: cpm-cache-0
226+
id: cpm-cache-restore
227+
uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
228+
with:
229+
path: ~/cpm-cache
230+
key: ${{runner.os}}-${{env.cache-name}}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
231+
restore-keys: |
232+
${{runner.os}}-${{env.cache-name}}-
233+
234+
- name: Configure cmake for app
235+
env:
236+
CC: "/usr/lib/llvm-${{env.MULL_LLVM_VERSION}}/bin/clang"
237+
CXX: "/usr/lib/llvm-${{env.MULL_LLVM_VERSION}}/bin/clang++"
238+
working-directory: ${{github.workspace}}/test/application
239+
run: cmake -Bbuild -DCPM_SOURCE_CACHE=~/cpm-cache
240+
241+
- name: Save CPM cache
242+
env:
243+
cache-name: cpm-cache-0
244+
if: steps.cpm-cache-restore.outputs.cache-hit != 'true'
245+
uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
246+
with:
247+
path: ~/cpm-cache
248+
key: ${{runner.os}}-${{env.cache-name}}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
249+
250+
- name: Build app and run mull tests
251+
working-directory: ${{github.workspace}}/test/application
252+
run: |
253+
cmake --build build -t mull_tests
254+
204255
merge_ok:
205256
runs-on: ${{ github.repository_owner == 'intel' && 'intel-' || '' }}ubuntu-22.04
206257
needs: [test, sanitize]

‎cmake/main.cmake‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ option(INFRA_PROVIDE_CLANG_FORMAT "Provide .clang-format file" ON)
2525
option(INFRA_PROVIDE_CLANG_TIDY "Provide .clang-tidy file" ON)
2626
option(INFRA_PROVIDE_CMAKE_FORMAT "Provide .cmake-format.yaml file" ON)
2727
option(INFRA_PROVIDE_PRESETS "Provide cmake presets and toolchains" ON)
28+
option(INFRA_PROVIDE_MULL "Provide mull.yml file" ON)
2829
option(INFRA_PROVIDE_GITIGNORE "Add provided things to .gitignore" ON)
2930
option(INFRA_USE_SYMLINKS "Use symlinks to provide common files" ON)
3031

@@ -34,6 +35,7 @@ if(${PROJECT_SOURCE_DIR}/cmake STREQUAL CMAKE_CURRENT_LIST_DIR)
3435
set(INFRA_PROVIDE_CLANG_TIDY OFF)
3536
set(INFRA_PROVIDE_CMAKE_FORMAT OFF)
3637
set(INFRA_PROVIDE_PRESETS OFF)
38+
set(INFRA_PROVIDE_MULL OFF)
3739
set(INFRA_PROVIDE_GITIGNORE OFF)
3840
endif()
3941

‎cmake/mull.cmake‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
add_custom_target(mull_tests)
2+
3+
function(add_mull_test name)
4+
message(
5+
STATUS
6+
"add_mull_test(${name}) is disabled because CMAKE_CXX_COMPILER_ID is ${CMAKE_CXX_COMPILER_ID}."
7+
)
8+
endfunction()
9+
10+
if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
11+
return()
12+
endif()
13+
14+
function(find_mull)
15+
string(REGEX MATCH "[0-9]+" version ${CMAKE_CXX_COMPILER_VERSION})
16+
find_program(MULL_RUNNER_PROGRAM "mull-runner-${version}")
17+
if(MULL_RUNNER_PROGRAM)
18+
message(
19+
STATUS "mull-runner-${version} found at ${MULL_RUNNER_PROGRAM}.")
20+
else()
21+
message(
22+
STATUS
23+
"mull-runner-${version} not found. add_mull_test() will fail unless a valid RUNNER_DIR is provided."
24+
)
25+
endif()
26+
find_file(MULL_PLUGIN_LIBRARY "mull-ir-frontend-${version}"
27+
HINTS "/usr/lib")
28+
if(MULL_PLUGIN_LIBRARY)
29+
message(
30+
STATUS
31+
"mull-ir-frontend-${version} found at ${MULL_PLUGIN_LIBRARY}.")
32+
else()
33+
message(
34+
STATUS
35+
"mull-ir-frontend-${version} not found. add_mull_test() will fail unless a valid PLUGIN_DIR is provided."
36+
)
37+
endif()
38+
return(PROPAGATE MULL_RUNNER_PROGRAM MULL_PLUGIN_LIBRARY)
39+
endfunction()
40+
41+
find_mull()
42+
43+
function(add_mull_test name)
44+
set(options EXCLUDE_CTEST)
45+
set(singleValueArgs PLUGIN_DIR RUNNER_DIR)
46+
set(multiValueArgs RUNNER_ARGS)
47+
cmake_parse_arguments(MULL "${options}" "${singleValueArgs}"
48+
"${multiValueArgs}" ${ARGN})
49+
50+
string(REGEX MATCH "[0-9]+" version ${CMAKE_CXX_COMPILER_VERSION})
51+
52+
set(MULL_RUNNER_NOT_FOUND_COMMAND
53+
COMMAND ${CMAKE_COMMAND} -E echo
54+
"Cannot run mull_${name} because mull-runner-${version} not found."
55+
COMMAND ${CMAKE_COMMAND} -E false)
56+
set(MULL_PLUGIN_NOT_FOUND_COMMAND
57+
COMMAND ${CMAKE_COMMAND} -E echo
58+
"Cannot build mull_${name} because mull-ir-frontend-${version} not found."
59+
COMMAND ${CMAKE_COMMAND} -E false)
60+
61+
if(MULL_RUNNER_DIR)
62+
find_program(MULL_RUNNER "mull-runner-${version}"
63+
HINTS ${MULL_RUNNER_DIR})
64+
else()
65+
set(MULL_RUNNER ${MULL_RUNNER_PROGRAM})
66+
set(MULL_RUNNER_DIR "<RUNNER_DIR not provided>")
67+
endif()
68+
if(NOT MULL_RUNNER)
69+
message(
70+
WARNING
71+
"mull-runner-${version} not found at ${MULL_RUNNER_DIR}. mull_${name} is a failing test."
72+
)
73+
add_custom_target(mull_${name} ${MULL_RUNNER_NOT_FOUND_COMMAND})
74+
add_dependencies(mull_tests mull_${name})
75+
return()
76+
endif()
77+
78+
if(MULL_PLUGIN_DIR)
79+
find_file(MULL_PLUGIN "mull-ir-frontend-${version}"
80+
HINTS ${MULL_PLUGIN_DIR})
81+
else()
82+
set(MULL_PLUGIN ${MULL_PLUGIN_LIBRARY})
83+
set(MULL_PLUGIN_DIR "<PLUGIN_DIR not provided>")
84+
endif()
85+
if(NOT MULL_PLUGIN)
86+
message(
87+
WARNING
88+
"mull-ir-frontend-${version} not found at ${MULL_PLUGIN_DIR}. mull_${name} is a failing test."
89+
)
90+
add_custom_target(mull_${name} ${MULL_PLUGIN_NOT_FOUND_COMMAND})
91+
add_dependencies(mull_tests mull_${name})
92+
return()
93+
endif()
94+
95+
target_compile_options(${name} PRIVATE -fpass-plugin=${MULL_PLUGIN} -O0 -g
96+
-grecord-command-line)
97+
target_link_libraries(${name} PRIVATE coverage)
98+
99+
set(mull_test_command $<TARGET_FILE:${name}>)
100+
add_custom_target(mull_${name} DEPENDS ${name}.mull)
101+
add_custom_command(
102+
OUTPUT ${name}.mull
103+
COMMAND ${MULL_RUNNER} ${MULL_RUNNER_ARGS} ${mull_test_command}
104+
COMMAND ${CMAKE_COMMAND} "-E" "touch" "${name}.mull"
105+
DEPENDS ${name}
106+
COMMAND_EXPAND_LISTS)
107+
108+
if(NOT MULL_EXCLUDE_CTEST)
109+
add_test(NAME MULL.${name}
110+
COMMAND ${MULL_RUNNER} ${MULL_RUNNER_ARGS}
111+
${mull_test_command} COMMAND_EXPAND_LISTS)
112+
endif()
113+
114+
add_dependencies(mull_tests mull_${name})
115+
endfunction()

‎cmake/quality.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ if(PROJECT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
1818
include(${CMAKE_CURRENT_LIST_DIR}/profile.cmake)
1919
include(${CMAKE_CURRENT_LIST_DIR}/diagnostics.cmake)
2020
include(${CMAKE_CURRENT_LIST_DIR}/coverage.cmake)
21+
include(${CMAKE_CURRENT_LIST_DIR}/mull.cmake)
2122

2223
include(${CMAKE_CURRENT_LIST_DIR}/format.cmake)
2324
include(${CMAKE_CURRENT_LIST_DIR}/clang-tidy.cmake)

‎cmake/setup.cmake‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ function(make_gitignore)
3636
list(APPEND GITIGNORE_CONTENTS "/toolchains")
3737
endif()
3838
endif()
39+
if(INFRA_PROVIDE_MULL)
40+
if(NOT EXISTS "${CMAKE_SOURCE_DIR}/mull.yml")
41+
list(APPEND GITIGNORE_CONTENTS "mull.yml")
42+
endif()
43+
endif()
3944
endif()
4045

4146
string(REPLACE ";" "\n" GITIGNORE_CONTENTS "${GITIGNORE_CONTENTS}")
@@ -104,6 +109,9 @@ if(PROJECT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
104109
put_in_project_dir("CMakePresets.json")
105110
put_in_project_dir("toolchains")
106111
endif()
112+
if(INFRA_PROVIDE_MULL)
113+
put_in_project_dir("mull.yml")
114+
endif()
107115

108116
if(INFRA_PROVIDE_GITHUB_WORKFLOWS)
109117
execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory

‎docs/options.adoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ When `ON`, this repository provides a `.cmake-format.yaml` file at the root leve
2626
When `ON`, this repository provides a `CMakePresets.json` file at the root
2727
level, and a `toolchains` directory.
2828

29+
*`INFRA_PROVIDE_MULL`*
30+
31+
When `ON`, this repository provides a `mull.yml` file at the root level.
32+
2933
*`INFRA_PROVIDE_GITIGNORE`*
3034

3135
When `ON`, this repository provides a `.gitignore` file at the root level. It

‎docs/testing.adoc‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,35 @@ add_feature_test(
101101

102102
`add_feature_test` takes the same optional arguments as `add_unit_test`.
103103

104+
=== Mutation tests
105+
106+
When compiling with clang, any C++ test executable can be used to create a
107+
mutation test using https://github.com/mull-project/mull[mull].
108+
109+
[source,cmake]
110+
----
111+
add_unit_test(
112+
my_test
113+
CATCH2
114+
FILES my_test.cpp
115+
LIBRARIES my_lib)
116+
117+
add_mull_test(my_test)
118+
----
119+
120+
The version of mull used must match the version of clang used. Arguments to `add_mull_test` are:
121+
122+
- `EXCLUDE_CTEST` - to optionally exclude the test from the ctest suite
123+
- `PLUGIN_DIR` - to provide the location of `mull-ir-frontend-<version>`
124+
- `RUNNER_DIR` - to provide the location of `mull-runner-<version>`
125+
- `RUNNER_ARGS` - for any additional arguments to be passed to `mull-runner-<version>`
126+
127+
`PLUGIN_DIR` and `RUNNER_DIR` may be omitted if they reside in `/usr/lib` and on
128+
the path respectively. In that case, cmake will find them and report that
129+
it has.
130+
131+
For more information on mutation tests, see https://mull.readthedocs.io[the mull documentation].
132+
104133
=== Compilation failure tests
105134

106135
When writing compile-time code, it's often useful to check compilation failures

‎mull.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
mutators:
2+
- cxx_all

‎test/application/clean.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ set -euo pipefail
55
rm -f .clang-format
66
rm -f .clang-tidy
77
rm -f .cmake-format.yaml
8+
rm -f mull.yml
89
rm -f .gitignore
910
rm -f CMakePresets.json
1011
rm -f toolchains

‎test/application/test/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ add_unit_test(
77
LIBRARIES
88
diagnostics
99
warnings)
10+
add_mull_test(catch2_test EXCLUDE_CTEST RUNNER_ARGS --allow-surviving)
1011

1112
add_unit_test(
1213
gtest_test

0 commit comments

Comments
 (0)