Skip to content

Commit ded8ce4

Browse files
committed
cmake: update code coverage module
The patch make the following changes: - GCC-based code coverage implementation and Gcovr tool were replaced by SanitizerCoverage-based implementation [1] and Clang-based tools for code coverage analysis. Gcovr is a third-party tools and it has no advantages in comparison to Clang-based tools. - Two compiler flags (-fprofile-arcs and -ftest-coverage) were removed, because these flags are GCC-specific. These flags are supreseded by the following flags: `-fprofile-instr-generate` and `-fcoverage-mapping`. - The CMake variable CODE_COVERAGE_FLAGS is now defined in CodeCoverage.cmake. - MC/DC code coverage is enabled. Clang has added MC/DC support as of its 18.1.0 release. MC/DC is a fine-grained coverage metric required by many automotive and aviation industrial standards for certifying mission-critical software. - Disabled ENABLE_COV CMake option in GHA because code coverage may conflict with fuzzing testing. The patch follows up the commit 66be4bd ("cmake: support building code coverage report"). 1. https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
1 parent db19449 commit ded8ce4

File tree

9 files changed

+85
-79
lines changed

9 files changed

+85
-79
lines changed

.github/workflows/test.yaml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ jobs:
6868
cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
6969
-DUSE_LUA=ON -DENABLE_BUILD_PROTOBUF=OFF \
7070
-DENABLE_INTERNAL_TESTS=ON -DENABLE_LAPI_TESTS=ON \
71-
-DENABLE_COV=ON \
7271
-G Ninja -S . -B build
7372
if: ${{ matrix.LUA == 'lua' }}
7473

@@ -77,7 +76,7 @@ jobs:
7776
cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
7877
-DUSE_LUAJIT=ON -DENABLE_BUILD_PROTOBUF=OFF \
7978
-DENABLE_INTERNAL_TESTS=ON -DENABLE_LAPI_TESTS=ON \
80-
-DENABLE_COV=ON -DENABLE_LUAJIT_RANDOM_RA=ON \
79+
-DENABLE_LUAJIT_RANDOM_RA=ON \
8180
-G Ninja -S . -B build
8281
if: ${{ matrix.LUA == 'luajit' }}
8382

CMakeLists.txt

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,16 @@ set(CMAKE_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_INCLUDE_PATH}
2929
include(utils)
3030
include(SetBuildParallelLevel)
3131
include(SetHardwareArch)
32+
if(ENABLE_COV)
33+
include(CodeCoverage)
34+
endif()
3235

33-
if (ENABLE_LAPI_TESTS)
36+
# Code coverage tools can conflict with LibFuzzer, typically
37+
# resulting in drastically slower execution speeds, crash
38+
# reproduction failures, or inaccurate coverage reports.
39+
# Disable libFuzzer static linkage when code coverage is enabled.
40+
if (ENABLE_LAPI_TESTS AND NOT ENABLE_COV)
41+
message(STATUS "libFuzzer static linkage is enabled")
3442
include(LibFuzzer)
3543
SetLibFuzzerPath(FUZZER_NO_MAIN_LIBRARY)
3644
SetLibFuzzerObjDir(LibFuzzerObjDir)
@@ -106,10 +114,6 @@ endif()
106114

107115
SetBuildParallelLevel(CMAKE_BUILD_PARALLEL_LEVEL)
108116

109-
if(ENABLE_COV)
110-
include(CodeCoverage)
111-
endif()
112-
113117
add_subdirectory(extra)
114118
add_subdirectory(libluamut)
115119
add_subdirectory(tests)

cmake/BuildLua.cmake

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,6 @@ macro(build_lua LUA_VERSION)
6161
endif()
6262

6363
if (ENABLE_COV)
64-
string(JOIN " " CODE_COVERAGE_FLAGS
65-
-fcoverage-mapping
66-
-fprofile-arcs
67-
-fprofile-instr-generate
68-
-ftest-coverage
69-
)
7064
AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
7165
AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
7266
endif()

cmake/BuildLuaJIT.cmake

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,6 @@ macro(build_luajit LJ_VERSION)
8080
endif()
8181

8282
if (ENABLE_COV)
83-
string(JOIN " " CODE_COVERAGE_FLAGS
84-
-fcoverage-mapping
85-
-fprofile-arcs
86-
-fprofile-instr-generate
87-
-ftest-coverage
88-
)
8983
AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
9084
AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
9185
endif()

cmake/CodeCoverage.cmake

Lines changed: 52 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,71 +1,71 @@
1-
find_program(GCOVR gcovr)
21
find_program(LLVM_COV llvm-cov)
3-
4-
set(COVERAGE_DIR "${PROJECT_BINARY_DIR}/coverage")
5-
set(COVERAGE_HTML_REPORT "${COVERAGE_DIR}/report.html")
6-
set(COVERAGE_XML_REPORT "${COVERAGE_DIR}/report.xml")
2+
find_program(LLVM_PROFDATA llvm-profdata)
73

84
set(target_name "coverage-report")
9-
if(NOT GCOVR OR NOT LLVM_COV)
10-
set(MSG "${target_name} is a dummy target")
5+
if(NOT LLVM_PROFDATA AND NOT LLVM_COV)
6+
set(MESSAGE "${target_name} is a dummy target")
117
add_custom_target(${target_name}
12-
COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --red ${MSG}
8+
COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --red ${MESSAGE}
9+
COMMENT ${MESSAGE}
1310
)
14-
message(WARNING "Either `gcovr' or `llvm-cov` not found, "
15-
"so ${target_name} target is dummy.")
11+
message(WARNING "Either `llvm-profdata' or `llvm-cov` not found, "
12+
"so target ${target_name} is dummy.")
1613
return()
1714
endif()
1815

19-
# See https://gcovr.com/en/stable/manpage.html.
20-
set(GCOVR_OPTIONS
21-
--txt-metric branch
22-
--cobertura ${COVERAGE_XML_REPORT}
23-
--decisions
24-
--gcov-executable "llvm-cov gcov"
25-
--html
26-
--html-details
27-
--html-title "Code Coverage Report"
28-
-j ${CMAKE_BUILD_PARALLEL_LEVEL}
29-
--output ${COVERAGE_HTML_REPORT}
30-
--print-summary
31-
--root ${LUA_SOURCE_DIR}
32-
--sort-key uncovered-percent
16+
set(CODE_COVERAGE_DIR "${PROJECT_BINARY_DIR}/coverage")
17+
set(CODE_COVERAGE_HTML_REPORT ${CODE_COVERAGE_DIR}/index.html)
18+
list(APPEND CODE_COVERAGE_FLAGS
19+
-fcoverage-mapping
20+
-fprofile-instr-generate
3321
)
3422

35-
if(USE_LUA)
36-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --object-directory ${LUA_SOURCE_DIR})
37-
endif ()
23+
# Clang Version 18.1.0 was the first release with full, native
24+
# support for MC/DC coverage analysis using the source-based code
25+
# coverage feature.
26+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND
27+
CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "18.1")
28+
list(APPEND CODE_COVERAGE_FLAGS
29+
-fcoverage-mcdc
30+
)
31+
endif()
3832

39-
if(IS_LUAJIT)
40-
# Exclude DynASM files, that contain a low-level VM code for CPUs.
41-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --exclude ".*\.dasc")
42-
# Exclude buildvm source code, it's a project's build infrastructure.
43-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --exclude ".*/host/")
44-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --object-directory ${LUA_SOURCE_DIR}/src)
45-
endif ()
33+
file(MAKE_DIRECTORY ${CODE_COVERAGE_DIR})
4634

47-
file(MAKE_DIRECTORY ${COVERAGE_DIR})
48-
add_custom_target(${target_name})
49-
add_custom_command(TARGET ${target_name}
50-
COMMENT "Building coverage report"
51-
COMMAND ${GCOVR} ${GCOVR_OPTIONS}
52-
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
35+
list(APPEND LLVM_COV_PROFRAW_MASK
36+
${PROJECT_BINARY_DIR}/tests/capi/*.profraw
37+
${PROJECT_BINARY_DIR}/tests/lapi/*.profraw
38+
)
39+
set(LLVM_COV_PROFDATA ${PROJECT_BINARY_DIR}/tests/default.profdata)
40+
41+
list(APPEND LLVM_COV_FLAGS
42+
-instr-profile=${LLVM_COV_PROFDATA}
43+
-output-dir=${CODE_COVERAGE_DIR}
44+
--show-branches=count
45+
--show-expansions
46+
--show-mcdc
47+
--show-mcdc-summary
5348
)
5449

55-
# The .gcda count data file is generated when a program containing
56-
# object files built with the GCC -fprofile-arcs option is executed.
57-
# https://gcc.gnu.org/onlinedocs/gcc/Gcov-Data-Files.html
58-
set(GCDA_FILES "${LUA_SOURCE_DIR}/*.gcda")
59-
if(IS_LUAJIT)
60-
# Files 'src/host/*.gcda' are not removed, because
61-
# CMake cannot remove recursively by globbing.
62-
# Files 'src/host/*.gcda' are not used for building coverage report.
63-
set(GCDA_FILES "${LUA_SOURCE_DIR}/src/*.gcda")
50+
# XXX: This variable is defined in BuildLua.cmake and
51+
# BuildLuaJIT.cmake. However, these modules are included after
52+
# CodeCoverage.cmake, so not available.
53+
set(LUA_EXECUTABLE ${PROJECT_BINARY_DIR}/luajit-v2.1/source/src/luajit)
54+
if(USE_LUA)
55+
set(LUA_EXECUTABLE ${PROJECT_BINARY_DIR}/lua-master/source/lua)
6456
endif()
57+
58+
add_custom_target(${target_name}
59+
COMMAND ${LLVM_PROFDATA} merge -sparse ${LLVM_COV_PROFRAW_MASK}
60+
-o ${LLVM_COV_PROFDATA}
61+
COMMAND ${LLVM_COV} show --format=html ${LLVM_COV_FLAGS} ${LUA_EXECUTABLE}
62+
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
63+
COMMENT "Generating HTML code coverage report in ${CODE_COVERAGE_DIR}"
64+
)
65+
6566
add_custom_target(coverage-reset
6667
COMMENT "Reset code coverage counters"
67-
COMMAND ${CMAKE_COMMAND} -E rm -f ${GCDA_FILES}
68+
COMMAND ${CMAKE_COMMAND} -E rm -f ${LLVM_COV_PROFRAW_MASK} ${LLVM_COV_PROFDATA}
6869
)
6970

70-
message(STATUS "Code coverage HTML report: ${COVERAGE_HTML_REPORT}")
71-
message(STATUS "Code coverage XML report: ${COVERAGE_XML_REPORT}")
71+
message(STATUS "Code coverage HTML report: ${CODE_COVERAGE_HTML_REPORT}")

libluamut/CMakeLists.txt

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
set(CFLAGS -Wall -Wextra -Wpedantic -Wno-unused-parameter)
22

33
if (ENABLE_COV)
4-
string(JOIN " " CODE_COVERAGE_FLAGS
5-
-fcoverage-mapping
6-
-fprofile-arcs
7-
-fprofile-instr-generate
8-
-ftest-coverage
9-
)
104
AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
115
AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
126
endif()

tests/CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,16 @@ string(JOIN " " LIBFUZZER_OPTS
1212
-workers=${CMAKE_BUILD_PARALLEL_LEVEL}
1313
)
1414

15+
if(ENABLE_COV)
16+
string(JOIN " " LIBFUZZER_OPTS
17+
${LIBFUZZER_OPTS}
18+
-ignore_ooms=1
19+
-ignore_timeouts=1
20+
-ignore_crashes=1
21+
-timeout=1
22+
)
23+
endif()
24+
1525
set(CORPUS_BASE_PATH ${PROJECT_SOURCE_DIR}/corpus)
1626
if(IS_LUAJIT)
1727
set(CORPUS_BASE_PATH ${CORPUS_BASE_PATH}/corpus)

tests/capi/CMakeLists.txt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,19 @@ function(create_test)
9595
add_test(NAME ${test_name}
9696
COMMAND ${SHELL} -c "$<TARGET_FILE:${test_name}> ${LIBFUZZER_OPTS}"
9797
)
98+
if(ENABLE_COV)
99+
list(APPEND TEST_ENV
100+
LLVM_PROFILE_FILE=${test_name}.%1m.%p.profraw
101+
)
102+
endif()
98103
if (USE_LUA)
99-
set_tests_properties(${test_name} PROPERTIES
100-
ENVIRONMENT "ASAN_OPTIONS='detect_invalid_pointer_pairs=2'"
104+
list(APPEND TEST_ENV
105+
ASAN_OPTIONS='detect_invalid_pointer_pairs=2'
101106
)
102107
endif()
103108
set_tests_properties(${test_name} PROPERTIES
104109
LABELS capi
110+
ENVIRONMENT "${TEST_ENV}"
105111
)
106112

107113
if (IS_LUAJIT)

tests/lapi/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ function(create_test)
5050
COMMAND ${SHELL} -c "${LUA_EXECUTABLE} ${FUZZ_FILENAME} ${LIBFUZZER_OPTS}"
5151
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
5252
)
53+
if(ENABLE_COV)
54+
list(APPEND TEST_ENV
55+
LLVM_PROFILE_FILE=${test_name}.%1m.%p.profraw
56+
)
57+
endif()
5358
set_tests_properties(${test_name} PROPERTIES
5459
LABELS "lapi"
5560
ENVIRONMENT "${TEST_ENV}"

0 commit comments

Comments
 (0)