Skip to content

Commit 86bf15b

Browse files
authored
feat: coverage reporting (#496)
- [x] Fix Linux/Clang build - [x] Exclude tests/ directory from results - [x] Why is CTest reporting 0-4% on passed builds? - [x] Script to clean old coverage output - [ ] Spurious error codes from CTest
2 parents ac181a1 + 300745f commit 86bf15b

File tree

8 files changed

+68
-157
lines changed

8 files changed

+68
-157
lines changed

.github/workflows/build_and_test.yml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,12 +91,13 @@ jobs:
9191
sudo apt-get update
9292
sudo apt-get install -y libc++-dev libc++abi-dev
9393
94-
# TODO: ensure install of runtime libraries for sanitizers, set LD_LIBRARY_PATH
94+
# TODO: ensure install of runtime libraries for sanitizers, set LD_LIBRARY_PATH
9595
- name: Ensure Clang version
9696
if: runner.os == 'Linux' && matrix.preset == 'clang'
97-
uses: egor-tensin/setup-clang@471a6f8ef1d449dba8e1a51780e7f943572a3f99 # v2.1
97+
uses: aminya/setup-cpp@1f17f92d6a52bfcb1a25348e2c526c2e5cbb1134 # v1.8.0
9898
with:
99-
version: 21
99+
llvm: 21
100+
clang: 21
100101

101102
- name: Ensure CMake 4.0
102103
uses: lukka/get-cmake@f176ccd3f28bda569c43aae4894f06b2435a3375 # v4.2.3
@@ -112,7 +113,7 @@ jobs:
112113
env:
113114
BUILD_NAME: ${{ github.head_ref || github.ref_name }}.${{ matrix.os }}.${{ matrix.preset }}.${{ matrix.config }}
114115
CMAKE_OSX_ARCHITECTURES: arm64;x86_64
115-
run: cmake --preset ${{ matrix.preset }} -D BENBOT_DOCS=OFF -D BUILDNAME=${{ env.BUILD_NAME }}
116+
run: cmake --preset ${{ matrix.preset }} -D BENBOT_DOCS=OFF -D BUILDNAME=${{ env.BUILD_NAME }} --log-level=VERBOSE
116117

117118
- name: Build and test
118119
run: |
@@ -149,7 +150,7 @@ jobs:
149150

150151
steps:
151152
- name: Download JUnit results
152-
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8
153+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
153154
with:
154155
path: junit
155156
pattern: junit-*

.github/workflows/tag_and_release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ jobs:
207207

208208
- name: Download artifacts
209209
id: download
210-
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8
210+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
211211
with:
212212
path: artifacts
213213

config/cmake/All.cmake

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010
#
1111
# ======================================================================================
1212

13+
include_guard (GLOBAL)
14+
1315
include ("${CMAKE_CURRENT_LIST_DIR}/Sanitizers.cmake")
14-
# include ("${CMAKE_CURRENT_LIST_DIR}/Coverage.cmake")
16+
include ("${CMAKE_CURRENT_LIST_DIR}/Coverage.cmake")
1517
include ("${CMAKE_CURRENT_LIST_DIR}/Warnings.cmake")
1618

1719
# General settings
@@ -34,3 +36,12 @@ list (JOIN debug_configs "," debug_configs)
3436

3537
set (CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:${debug_configs}>:Debug>" CACHE STRING "")
3638
endblock ()
39+
40+
# Enhance error reporting and compiler messages
41+
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
42+
add_compile_options (-fcolor-diagnostics)
43+
elseif (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
44+
add_compile_options (-fdiagnostics-color=always)
45+
elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC" AND MSVC_VERSION GREATER 1900)
46+
add_compile_options (/diagnostics:column)
47+
endif ()

config/cmake/Coverage.cmake

Lines changed: 29 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -10,163 +10,48 @@
1010
#
1111
# ======================================================================================
1212

13-
#[=======================================================================[.rst:
14-
Coverage.cmake
15-
----------------------
16-
17-
Including this module enables support for code coverage reports.
18-
19-
This module sets ``CTEST_COVERAGE_COMMAND`` to ``gcov``'s location, if it can be found.
20-
21-
Variables
22-
^^^^^^^^^^
23-
24-
.. variable:: GCOV_PROGRAM
25-
26-
Path to the ``gcov`` executable.
27-
28-
.. variable:: GENINFO_PROGRAM
29-
30-
Path to the ``geninfo`` executable.
31-
32-
.. variable:: GENHTML_PROGRAM
33-
34-
Path to the ``genhtml`` executable.
35-
36-
Targets
37-
^^^^^^^^^
38-
39-
Each target will only exist if all required tools could be found.
40-
41-
.. target:: coverage-clean
42-
43-
Executes a script to remove all previously generated coverage files from the build tree.
44-
This can be useful if you get test output containing errors mentioning failure to merge
45-
previous coverage output, etc.
46-
47-
.. target:: coverage-report
48-
49-
Builds a coverage report from coverage files found in the build tree. Requires that gcov,
50-
geninfo, and genhtml could all be found.
51-
52-
.. target:: open-coverage
53-
54-
First builds ``coverage-report``, then opens the generated HTML in your default browser.
55-
56-
#]=======================================================================]
13+
# Including this module globally enables coverage flags.
5714

5815
include_guard (GLOBAL)
5916

6017
include (FeatureSummary)
6118

62-
# set up compiler flags to generate coverage output
63-
64-
get_cmake_property (debug_configs DEBUG_CONFIGURATIONS)
65-
66-
if (NOT debug_configs)
67-
set (debug_configs Debug)
68-
endif ()
69-
70-
list (JOIN debug_configs "," debug_configs)
71-
72-
set (config_debug "$<CONFIG:${debug_configs}>")
73-
74-
if (MSVC)
75-
add_compile_options ("$<${config_debug}:/fsanitize-coverage=edge>")
76-
return ()
77-
endif ()
78-
79-
add_compile_options ("$<${config_debug}:--coverage>")
80-
add_link_options ("$<${config_debug}:--coverage>")
81-
82-
if (APPLE)
83-
add_compile_options ("$<${config_debug}:-fprofile-arcs>")
84-
add_link_options ("$<${config_debug}:-fprofile-arcs>")
85-
endif ()
86-
87-
if (CMAKE_C_COMPILER_ID MATCHES "Clang")
88-
add_compile_options (
89-
"$<${config_debug}:-ftest-coverage;-fprofile-instr-generate;-fcoverage-mapping>"
90-
)
91-
endif ()
92-
93-
# add custom target to clean old coverage output
94-
95-
set (COVERAGE_OUTPUT_DIR "${CMAKE_BINARY_DIR}/coverage/$<CONFIG>")
96-
set (COVERAGE_INFO_FILE "${COVERAGE_OUTPUT_DIR}/coverage.info")
97-
98-
add_custom_target (
99-
coverage-clean
100-
COMMAND
101-
"${CMAKE_COMMAND}" -D "COVERAGE_OUTPUT_DIR=${COVERAGE_OUTPUT_DIR}" -D
102-
"CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}" -D "CMAKE_SOURCE_DIR=${CMAKE_SOURCE_DIR}" -P
103-
"${CMAKE_CURRENT_LIST_DIR}/detail/DeleteOldCoverageOutput.cmake"
104-
COMMENT "[coverage] - Cleaning old coverage output files..."
105-
VERBATIM
106-
)
107-
108-
# set up coverage reports
109-
110-
find_program (GCOV_PROGRAM gcov DOC "gcov executable")
111-
find_program (GENINFO_PROGRAM geninfo DOC "geninfo executable")
112-
find_program (GENHTML_PROGRAM genhtml DOC "genhtml executable")
113-
114-
mark_as_advanced (GCOV_PROGRAM GENINFO_PROGRAM GENHTML_PROGRAM)
115-
11619
add_feature_info (
117-
CoverageReports "GENINFO_PROGRAM AND GENHTML_PROGRAM"
118-
"Custom targets to generate/open coverage report (requires geninfo & genhtml)"
20+
coverage [[CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" AND NOT WIN32]]
21+
"Enabled coverage reporting flags for debug configurations"
11922
)
12023

121-
if (NOT (GENINFO_PROGRAM AND GENHTML_PROGRAM))
122-
return ()
123-
endif ()
24+
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" AND NOT WIN32)
25+
get_cmake_property (debug_configs DEBUG_CONFIGURATIONS)
12426

125-
include (FetchContent)
27+
if (NOT debug_configs)
28+
set (debug_configs Debug)
29+
endif ()
12630

127-
add_custom_command (
128-
OUTPUT "${COVERAGE_INFO_FILE}"
129-
COMMAND
130-
"${GENINFO_PROGRAM}" "${CMAKE_SOURCE_DIR}" --exclude "${FETCHCONTENT_BASE_DIR}" --gcov-tool
131-
"${GCOV_PROGRAM}" --branch-coverage --no-external --follow --forget-test-names
132-
--demangle-cpp --keep-going --rc derive_function_end_line=0 -o "${COVERAGE_INFO_FILE}"
133-
--ignore-errors empty,inconsistent,format,unsupported,category,range,source,unused
134-
COMMENT "[coverage] - Running geninfo on .gcda coverage output files..."
135-
VERBATIM
136-
)
31+
if (APPLE)
32+
# On MacOS, UBSAN seems to interfere with coverage collection, it erroneously reports 0-4%,
33+
# so just disable it
34+
list (REMOVE_ITEM debug_configs UBSAN)
35+
endif ()
13736

138-
set (index_html "${COVERAGE_OUTPUT_DIR}/index.html")
37+
list (JOIN debug_configs "," debug_configs)
13938

140-
add_custom_command (
141-
OUTPUT "${index_html}"
142-
COMMAND
143-
"${GENHTML_PROGRAM}" "${COVERAGE_INFO_FILE}" --header-title
144-
"${CMAKE_PROJECT_NAME} coverage report" --prefix "${CMAKE_SOURCE_DIR}" --precision 1
145-
--filter "brace,blank,range,function" --show-zero-columns --suppress-aliases
146-
--simplified-colors --elide-path-mismatch --sort --dark-mode --synthesize-missing
147-
--show-navigation --legend --function-coverage --branch-coverage --demangle-cpp
148-
--forget-test-names --keep-going -o "${COVERAGE_OUTPUT_DIR}" --ignore-errors
149-
empty,inconsistent,format,unsupported,category,range,source
150-
COMMAND "${CMAKE_COMMAND}" -E echo
151-
"Coverage report generated. Open ${index_html} in your browser."
152-
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
153-
DEPENDS "${COVERAGE_INFO_FILE}"
154-
COMMENT "[coverage] - Running genhtml to create coverage report..."
155-
VERBATIM
156-
)
39+
set (config_debug "$<CONFIG:${debug_configs}>")
15740

158-
add_custom_target (coverage-report DEPENDS "${index_html}")
41+
add_compile_options ("$<${config_debug}:-g;-O0;--coverage>")
42+
add_link_options ("$<${config_debug}:--coverage>")
15943

160-
set_property (DIRECTORY APPEND PROPERTY ADDITIONAL_CLEAN_FILES "${COVERAGE_OUTPUT_DIR}")
44+
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
45+
link_libraries ("$<${config_debug}:gcov>")
46+
endif ()
16147

162-
if (WIN32)
163-
set (open_cmd start)
164-
else ()
165-
set (open_cmd open)
166-
endif ()
167-
168-
add_custom_target (open-coverage COMMAND "${open_cmd}" "${index_html}")
169-
170-
add_dependencies (open-coverage coverage-report)
48+
set (clean_script "${CMAKE_SOURCE_DIR}/scripts/CleanOldCoverageOutput.cmake")
17149

172-
set_target_properties (coverage-report coverage-clean open-coverage PROPERTIES FOLDER coverage)
50+
add_custom_target (
51+
coverage-clean
52+
COMMAND "${CMAKE_COMMAND}" -D "BUILD_DIR=${CMAKE_BINARY_DIR}" -P "${clean_script}"
53+
COMMENT "Cleaning old coverage output files..."
54+
VERBATIM USES_TERMINAL
55+
SOURCES "${clean_script}"
56+
)
57+
endif ()

config/cmake/Sanitizers.cmake

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ function (__add_build_config name flags linkerFlags)
6262
CMAKE_CXX_FLAGS_${name} CMAKE_C_FLAGS_${name} CMAKE_EXE_LINKER_FLAGS_${name}
6363
CMAKE_SHARED_LINKER_FLAGS_${name} CMAKE_MODULE_LINKER_FLAGS_${name}
6464
)
65+
66+
message (VERBOSE "Added build configuration ${name}")
6567
endfunction ()
6668

6769
# ASAN

config/cmake/Warnings.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
# Including this module enables some default warnings at directory scope.
1414

15-
include_guard (DIRECTORY)
15+
include_guard (GLOBAL)
1616

1717
if (MSVC)
1818
add_compile_options (
@@ -35,7 +35,7 @@ if (MSVC)
3535
return ()
3636
endif ()
3737

38-
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
38+
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
3939
add_compile_options (
4040
-pedantic
4141
-pedantic-errors

config/cmake/detail/DeleteOldCoverageOutput.cmake renamed to scripts/CleanOldCoverageOutput.cmake

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,27 @@
1010
#
1111
# ======================================================================================
1212

13+
#[[
14+
This script can be run to remove all old coverage files in a build directory.
15+
16+
Example usage (from repo root):
17+
cmake -D BUILD_DIR=$(pwd)/Builds/clang -P scripts/CleanOldCoverageOutput.cmake
18+
]]
19+
1320
cmake_minimum_required (VERSION 4.0.0 FATAL_ERROR)
1421

15-
file (REMOVE_RECURSE "${COVERAGE_OUTPUT_DIR}")
22+
if (NOT DEFINED BUILD_DIR)
23+
message (FATAL_ERROR "BUILD_DIR not defined!")
24+
endif ()
1625

1726
file (GLOB_RECURSE coverage_files LIST_DIRECTORIES false
18-
"${CMAKE_BINARY_DIR}/*.gcda" "${CMAKE_BINARY_DIR}/*.gcov" "${CMAKE_BINARY_DIR}/*.profraw"
19-
"${CMAKE_SOURCE_DIR}/*.gcda" "${CMAKE_SOURCE_DIR}/*.gcov" "${CMAKE_SOURCE_DIR}/*.profraw"
27+
"${BUILD_DIR}/*.gcda" "${BUILD_DIR}/*.gcov" "${BUILD_DIR}/*.profraw"
2028
)
2129

2230
if (coverage_files)
2331
file (REMOVE ${coverage_files})
32+
list (LENGTH coverage_files num_files)
33+
message (STATUS "Removed ${num_files} files.")
34+
else ()
35+
message (STATUS "No coverage files found.")
2436
endif ()

tests/CTestCustom.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ set (CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 100)
2727
set (slash "[/\\]")
2828

2929
list (APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE @CMAKE_PREFIX_PATH@ "@FETCHCONTENT_BASE_DIR@"
30-
"${slash}_deps${slash}"
30+
"${slash}_deps${slash}" "tests${slash}"
3131
)
3232

3333
list (

0 commit comments

Comments
 (0)