Skip to content

Commit 7470d21

Browse files
committed
cmake: update code coverage module
The patch make the following changes: - Two compiler flags (-fprofile-arcs and -ftest-coverage) were removed, because these flags are GCC-specific. - CODE_COVERAGE_FLAGS is defined in CodeCoverage.cmake. - Use Clang tools for code coverage instead of Gcovr. - Enabled MC/DC code coverage. - Disabled ENABLE_COV CMake option in GHA The newly MCDC coverage complements the existing gcov implementation. Gcov works at the object code level which may better reflect actual execution. However, Gcov lacks the necessary information to correlate coverage measurement with source code location when compiler optimization level is non-zero (which is the default when building the kernel). In addition, gcov reports are occasionally ambiguous when attempting to compare with source code level developer intent. 4.1.6. Tension Between Fuzz Testing And 100% MC/DC Testing [4] 1. https://clang.llvm.org/docs/SourceBasedCodeCoverage.html 2. https://llvm.org/docs/LibFuzzer.html#fork-mode 2. https://lwn.net/Articles/989075/ 3. https://maskray.me/blog/2024-01-28-mc-dc-and-compiler-implementations 4. https://www.sqlite.org/testing.html
1 parent 6166b9f commit 7470d21

File tree

9 files changed

+85
-79
lines changed

9 files changed

+85
-79
lines changed

.github/workflows/test.yaml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ jobs:
6868
cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
6969
-DUSE_LUA=ON -DENABLE_BUILD_PROTOBUF=OFF \
7070
-DENABLE_INTERNAL_TESTS=ON -DENABLE_LAPI_TESTS=ON \
71-
-DENABLE_COV=ON \
7271
-G Ninja -S . -B build
7372
if: ${{ matrix.LUA == 'lua' }}
7473

@@ -77,7 +76,7 @@ jobs:
7776
cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
7877
-DUSE_LUAJIT=ON -DENABLE_BUILD_PROTOBUF=OFF \
7978
-DENABLE_INTERNAL_TESTS=ON -DENABLE_LAPI_TESTS=ON \
80-
-DENABLE_COV=ON -DENABLE_LUAJIT_RANDOM_RA=ON \
79+
-DENABLE_LUAJIT_RANDOM_RA=ON \
8180
-G Ninja -S . -B build
8281
if: ${{ matrix.LUA == 'luajit' }}
8382

CMakeLists.txt

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,16 @@ set(CMAKE_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_INCLUDE_PATH}
2929
include(utils)
3030
include(SetBuildParallelLevel)
3131
include(SetHardwareArch)
32+
if(ENABLE_COV)
33+
include(CodeCoverage)
34+
endif()
3235

33-
if (ENABLE_LAPI_TESTS)
36+
# Code coverage tools can conflict with LibFuzzer, typically
37+
# resulting in drastically slower execution speeds, crash
38+
# reproduction failures, or inaccurate coverage reports.
39+
# Disable libFuzzer static linkage when code coverage is enabled.
40+
if (ENABLE_LAPI_TESTS AND NOT ENABLE_COV)
41+
message(STATUS "libFuzzer static linkage is enabled")
3442
include(LibFuzzer)
3543
SetLibFuzzerPath(FUZZER_NO_MAIN_LIBRARY)
3644
SetLibFuzzerObjDir(LibFuzzerObjDir)
@@ -106,10 +114,6 @@ endif()
106114

107115
SetBuildParallelLevel(CMAKE_BUILD_PARALLEL_LEVEL)
108116

109-
if(ENABLE_COV)
110-
include(CodeCoverage)
111-
endif()
112-
113117
add_subdirectory(extra)
114118
add_subdirectory(libluamut)
115119
add_subdirectory(tests)

cmake/BuildLua.cmake

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,6 @@ macro(build_lua LUA_VERSION)
6161
endif()
6262

6363
if (ENABLE_COV)
64-
string(JOIN " " CODE_COVERAGE_FLAGS
65-
-fcoverage-mapping
66-
-fprofile-arcs
67-
-fprofile-instr-generate
68-
-ftest-coverage
69-
)
7064
AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
7165
AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
7266
endif()

cmake/BuildLuaJIT.cmake

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,6 @@ macro(build_luajit LJ_VERSION)
8080
endif()
8181

8282
if (ENABLE_COV)
83-
string(JOIN " " CODE_COVERAGE_FLAGS
84-
-fcoverage-mapping
85-
-fprofile-arcs
86-
-fprofile-instr-generate
87-
-ftest-coverage
88-
)
8983
AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
9084
AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
9185
endif()

cmake/CodeCoverage.cmake

Lines changed: 52 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,71 +1,71 @@
1-
find_program(GCOVR gcovr)
21
find_program(LLVM_COV llvm-cov)
3-
4-
set(COVERAGE_DIR "${PROJECT_BINARY_DIR}/coverage")
5-
set(COVERAGE_HTML_REPORT "${COVERAGE_DIR}/report.html")
6-
set(COVERAGE_XML_REPORT "${COVERAGE_DIR}/report.xml")
2+
find_program(LLVM_PROFDATA llvm-profdata)
73

84
set(target_name "coverage-report")
9-
if(NOT GCOVR OR NOT LLVM_COV)
10-
set(MSG "${target_name} is a dummy target")
5+
if(NOT LLVM_PROFDATA AND NOT LLVM_COV)
6+
set(MESSAGE "${target_name} is a dummy target")
117
add_custom_target(${target_name}
12-
COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --red ${MSG}
8+
COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --red ${MESSAGE}
9+
COMMENT ${MESSAGE}
1310
)
14-
message(WARNING "Either `gcovr' or `llvm-cov` not found, "
15-
"so ${target_name} target is dummy.")
11+
message(WARNING "Either `llvm-profdata' or `llvm-cov` not found, "
12+
"so target ${target_name} is dummy.")
1613
return()
1714
endif()
1815

19-
# See https://gcovr.com/en/stable/manpage.html.
20-
set(GCOVR_OPTIONS
21-
--txt-metric branch
22-
--cobertura ${COVERAGE_XML_REPORT}
23-
--decisions
24-
--gcov-executable "llvm-cov gcov"
25-
--html
26-
--html-details
27-
--html-title "Code Coverage Report"
28-
-j ${CMAKE_BUILD_PARALLEL_LEVEL}
29-
--output ${COVERAGE_HTML_REPORT}
30-
--print-summary
31-
--root ${LUA_SOURCE_DIR}
32-
--sort-key uncovered-percent
16+
set(CODE_COVERAGE_DIR "${PROJECT_BINARY_DIR}/coverage")
17+
set(CODE_COVERAGE_HTML_REPORT ${CODE_COVERAGE_DIR}/index.html)
18+
list(APPEND CODE_COVERAGE_FLAGS
19+
-fcoverage-mapping
20+
-fprofile-instr-generate
3321
)
3422

35-
if(USE_LUA)
36-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --object-directory ${LUA_SOURCE_DIR})
37-
endif ()
23+
# Clang Version 18.1.0 was the first release with full, native
24+
# support for MC/DC coverage analysis using the source-based code
25+
# coverage feature.
26+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND
27+
CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "18.1")
28+
list(APPEND CODE_COVERAGE_FLAGS
29+
-fcoverage-mcdc
30+
)
31+
endif()
3832

39-
if(IS_LUAJIT)
40-
# Exclude DynASM files, that contain a low-level VM code for CPUs.
41-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --exclude ".*\.dasc")
42-
# Exclude buildvm source code, it's a project's build infrastructure.
43-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --exclude ".*/host/")
44-
set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --object-directory ${LUA_SOURCE_DIR}/src)
45-
endif ()
33+
file(MAKE_DIRECTORY ${CODE_COVERAGE_DIR})
4634

47-
file(MAKE_DIRECTORY ${COVERAGE_DIR})
48-
add_custom_target(${target_name})
49-
add_custom_command(TARGET ${target_name}
50-
COMMENT "Building coverage report"
51-
COMMAND ${GCOVR} ${GCOVR_OPTIONS}
52-
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
35+
list(APPEND LLVM_COV_PROFRAW_MASK
36+
${PROJECT_BINARY_DIR}/tests/capi/*.profraw
37+
${PROJECT_BINARY_DIR}/tests/lapi/*.profraw
38+
)
39+
set(LLVM_COV_PROFDATA ${PROJECT_BINARY_DIR}/tests/default.profdata)
40+
41+
list(APPEND LLVM_COV_FLAGS
42+
-instr-profile=${LLVM_COV_PROFDATA}
43+
-output-dir=${CODE_COVERAGE_DIR}
44+
--show-branches=count
45+
--show-expansions
46+
--show-mcdc
47+
--show-mcdc-summary
5348
)
5449

55-
# The .gcda count data file is generated when a program containing
56-
# object files built with the GCC -fprofile-arcs option is executed.
57-
# https://gcc.gnu.org/onlinedocs/gcc/Gcov-Data-Files.html
58-
set(GCDA_FILES "${LUA_SOURCE_DIR}/*.gcda")
59-
if(IS_LUAJIT)
60-
# Files 'src/host/*.gcda' are not removed, because
61-
# CMake cannot remove recursively by globbing.
62-
# Files 'src/host/*.gcda' are not used for building coverage report.
63-
set(GCDA_FILES "${LUA_SOURCE_DIR}/src/*.gcda")
50+
# XXX: This variable is defined in BuildLua.cmake and
51+
# BuildLuaJIT.cmake. However, these modules are included after
52+
# CodeCoverage.cmake, so not available.
53+
set(LUA_EXECUTABLE ${PROJECT_BINARY_DIR}/luajit-v2.1/source/src/luajit)
54+
if(USE_LUA)
55+
set(LUA_EXECUTABLE ${PROJECT_BINARY_DIR}/lua-master/source/lua)
6456
endif()
57+
58+
add_custom_target(${target_name}
59+
COMMAND ${LLVM_PROFDATA} merge -sparse ${LLVM_COV_PROFRAW_MASK}
60+
-o ${LLVM_COV_PROFDATA}
61+
COMMAND ${LLVM_COV} show --format=html ${LLVM_COV_FLAGS} ${LUA_EXECUTABLE}
62+
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
63+
COMMENT "Generating HTML code coverage report in ${CODE_COVERAGE_DIR}"
64+
)
65+
6566
add_custom_target(coverage-reset
6667
COMMENT "Reset code coverage counters"
67-
COMMAND ${CMAKE_COMMAND} -E rm -f ${GCDA_FILES}
68+
COMMAND ${CMAKE_COMMAND} -E rm -f ${LLVM_COV_PROFRAW_MASK} ${LLVM_COV_PROFDATA}
6869
)
6970

70-
message(STATUS "Code coverage HTML report: ${COVERAGE_HTML_REPORT}")
71-
message(STATUS "Code coverage XML report: ${COVERAGE_XML_REPORT}")
71+
message(STATUS "Code coverage HTML report: ${CODE_COVERAGE_HTML_REPORT}")

libluamut/CMakeLists.txt

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
set(CFLAGS -Wall -Wextra -Wpedantic -Wno-unused-parameter)
22

33
if (ENABLE_COV)
4-
string(JOIN " " CODE_COVERAGE_FLAGS
5-
-fcoverage-mapping
6-
-fprofile-arcs
7-
-fprofile-instr-generate
8-
-ftest-coverage
9-
)
104
AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
115
AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
126
endif()

tests/CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,16 @@ string(JOIN " " LIBFUZZER_OPTS
1212
-workers=${CMAKE_BUILD_PARALLEL_LEVEL}
1313
)
1414

15+
if(ENABLE_COV)
16+
string(JOIN " " LIBFUZZER_OPTS
17+
${LIBFUZZER_OPTS}
18+
-fork=${CMAKE_BUILD_PARALLEL_LEVEL}
19+
-ignore_ooms=1
20+
-ignore_timeouts=1
21+
-ignore_crashes=1
22+
)
23+
endif()
24+
1525
set(CORPUS_BASE_PATH ${PROJECT_SOURCE_DIR}/corpus)
1626
if(IS_LUAJIT)
1727
set(CORPUS_BASE_PATH ${CORPUS_BASE_PATH}/corpus)

tests/capi/CMakeLists.txt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,19 @@ function(create_test)
9595
add_test(NAME ${test_name}
9696
COMMAND ${SHELL} -c "$<TARGET_FILE:${test_name}> ${LIBFUZZER_OPTS}"
9797
)
98+
if(ENABLE_COV)
99+
list(APPEND TEST_ENV
100+
LLVM_PROFILE_FILE=${test_name}.%1m.%p.profraw
101+
)
102+
endif()
98103
if (USE_LUA)
99-
set_tests_properties(${test_name} PROPERTIES
100-
ENVIRONMENT "ASAN_OPTIONS='detect_invalid_pointer_pairs=2'"
104+
list(APPEND TEST_ENV
105+
ASAN_OPTIONS='detect_invalid_pointer_pairs=2'
101106
)
102107
endif()
103108
set_tests_properties(${test_name} PROPERTIES
104109
LABELS capi
110+
ENVIRONMENT "${TEST_ENV}"
105111
)
106112

107113
if (IS_LUAJIT)

tests/lapi/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ function(create_test)
5050
COMMAND ${SHELL} -c "${LUA_EXECUTABLE} ${FUZZ_FILENAME} ${LIBFUZZER_OPTS}"
5151
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
5252
)
53+
if(ENABLE_COV)
54+
list(APPEND TEST_ENV
55+
LLVM_PROFILE_FILE=${test_name}.%1m.%p.profraw
56+
)
57+
endif()
5358
set_tests_properties(${test_name} PROPERTIES
5459
LABELS "lapi"
5560
ENVIRONMENT "${TEST_ENV}"

0 commit comments

Comments
 (0)