cmake: update code coverage module

The newly MCDC coverage complements the existing gcov
implementation. Gcov works at the object code level which may better
reflect actual execution. However, Gcov lacks the necessary information to
correlate coverage measurement with source code location when compiler
optimization level is non-zero (which is the default when building the
kernel). In addition, gcov reports are occasionally ambiguous when
attempting to compare with source code level developer intent.

Two compiler flags (-fprofile-arcs and -ftest-coverage) were
removed, because these flags are GCC-specific.

4.1.6. Tension Between Fuzz Testing And 100% MC/DC Testing [4]

1. https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
2. https://llvm.org/docs/LibFuzzer.html#fork-mode
2. https://lwn.net/Articles/989075/
3. https://maskray.me/blog/2024-01-28-mc-dc-and-compiler-implementations
4. https://www.sqlite.org/testing.html

Author: ligurio

CMakeLists.txt

@@ -29,6 +29,9 @@ set(CMAKE_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_INCLUDE_PATH}
 include(utils)
 include(SetBuildParallelLevel)
 include(SetHardwareArch)
+if(ENABLE_COV)
+  include(CodeCoverage)
+endif()
 
 if (ENABLE_LAPI_TESTS)
   include(LibFuzzer)
@@ -106,10 +109,6 @@ endif()
 
 SetBuildParallelLevel(CMAKE_BUILD_PARALLEL_LEVEL)
 
-if(ENABLE_COV)
-    include(CodeCoverage)
-endif()
-
 add_subdirectory(extra)
 add_subdirectory(libluamut)
 add_subdirectory(tests)

cmake/BuildLua.cmake

@@ -59,12 +59,6 @@ macro(build_lua LUA_VERSION)
     endif()
 
     if (ENABLE_COV)
-        string(JOIN " " CODE_COVERAGE_FLAGS
-          -fcoverage-mapping
-          -fprofile-arcs
-          -fprofile-instr-generate
-          -ftest-coverage
-        )
         AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
         AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
     endif()

cmake/BuildLuaJIT.cmake

@@ -80,12 +80,6 @@ macro(build_luajit LJ_VERSION)
     endif()
 
     if (ENABLE_COV)
-        string(JOIN " " CODE_COVERAGE_FLAGS
-            -fcoverage-mapping
-            -fprofile-arcs
-            -fprofile-instr-generate
-            -ftest-coverage
-        )
         AppendFlags(CFLAGS ${CODE_COVERAGE_FLAGS})
         AppendFlags(LDFLAGS ${CODE_COVERAGE_FLAGS})
     endif()

cmake/CodeCoverage.cmake

@@ -1,71 +1,63 @@
-find_program(GCOVR gcovr)
 find_program(LLVM_COV llvm-cov)
+find_program(LLVM_PROFDATA llvm-profdata)
 
-set(COVERAGE_DIR "${PROJECT_BINARY_DIR}/coverage")
-set(COVERAGE_HTML_REPORT "${COVERAGE_DIR}/report.html")
-set(COVERAGE_XML_REPORT "${COVERAGE_DIR}/report.xml")
+if(NOT LLVM_COV OR NOT LLVM_PROFDATA)
+  message(FATAL_ERROR "Required LLVM coverage tools not found.")
+endif()
+
+set(CODE_COVERAGE_DIR "${PROJECT_BINARY_DIR}/coverage")
+set(CODE_COVERAGE_HTML_REPORT ${CODE_COVERAGE_DIR}/index.html)
+list(APPEND CODE_COVERAGE_FLAGS
+  -fcoverage-mapping
+  -fprofile-instr-generate
+)
 
-set(target_name "coverage-report")
-if(NOT GCOVR OR NOT LLVM_COV)
-  set(MSG "${target_name} is a dummy target")
-  add_custom_target(${target_name}
-    COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --red ${MSG}
+# Clang Version 18.1.0 was the first release with full, native
+# support for MC/DC coverage analysis using the source-based code
+# coverage feature.
+if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND
+   CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "18.1")
+  list(APPEND CODE_COVERAGE_FLAGS
+    -fcoverage-mcdc
   )
-  message(WARNING "Either `gcovr' or `llvm-cov` not found, "
-                  "so ${target_name} target is dummy.")
-  return()
 endif()
 
-# See https://gcovr.com/en/stable/manpage.html.
-set(GCOVR_OPTIONS
-  --txt-metric branch
-  --cobertura ${COVERAGE_XML_REPORT}
-  --decisions
-  --gcov-executable "llvm-cov gcov"
-  --html
-  --html-details
-  --html-title "Code Coverage Report"
-  -j ${CMAKE_BUILD_PARALLEL_LEVEL}
-  --output ${COVERAGE_HTML_REPORT}
-  --print-summary
-  --root ${LUA_SOURCE_DIR}
-  --sort-key uncovered-percent
+file(MAKE_DIRECTORY ${CODE_COVERAGE_DIR})
+
+list(APPEND LLVM_COV_PROFRAW_MASK
+  ${PROJECT_BINARY_DIR}/tests/capi/*.profraw
+  ${PROJECT_BINARY_DIR}/tests/lapi/*.profraw
+)
+set(LLVM_COV_PROFDATA ${PROJECT_BINARY_DIR}/tests/default.profdata)
+
+list(APPEND LLVM_COV_FLAGS
+  -instr-profile=${LLVM_COV_PROFDATA}
+  -output-dir=${CODE_COVERAGE_DIR}
+  --show-branches=count
+  --show-expansions
+  --show-mcdc
+  --show-mcdc-summary
 )
 
+# XXX: This variable is defined in BuildLua.cmake and
+# BuildLuaJIT.cmake. However, these modules are included after
+# CodeCoverage.cmake, so not available.
+set(LUA_EXECUTABLE ${PROJECT_BINARY_DIR}/luajit-v2.1/source/src/luajit)
 if(USE_LUA)
-  set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --object-directory ${LUA_SOURCE_DIR})
-endif ()
-
-if(IS_LUAJIT)
-  # Exclude DynASM files, that contain a low-level VM code for CPUs.
-  set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --exclude ".*\.dasc")
-  # Exclude buildvm source code, it's a project's build infrastructure.
-  set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --exclude ".*/host/")
-  set(GCOVR_OPTIONS ${GCOVR_OPTIONS} --object-directory ${LUA_SOURCE_DIR}/src)
-endif ()
+  set(LUA_EXECUTABLE ${PROJECT_BINARY_DIR}/lua-master/source/lua)
+endif()
 
-file(MAKE_DIRECTORY ${COVERAGE_DIR})
-add_custom_target(${target_name})
-add_custom_command(TARGET ${target_name}
-  COMMENT "Building coverage report"
-  COMMAND ${GCOVR} ${GCOVR_OPTIONS}
+add_custom_target(coverage-report
+  COMMAND ${LLVM_PROFDATA} merge -sparse ${LLVM_COV_PROFRAW_MASK}
+    -o ${LLVM_COV_PROFDATA}
+  COMMAND ${LLVM_COV} show --format=html ${LLVM_COV_FLAGS} ${LUA_EXECUTABLE}
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+  COMMENT "Generating HTML code coverage report in ${CODE_COVERAGE_DIR}"
 )
 
-# The .gcda count data file is generated when a program containing
-# object files built with the GCC -fprofile-arcs option is executed.
-# https://gcc.gnu.org/onlinedocs/gcc/Gcov-Data-Files.html
-set(GCDA_FILES "${LUA_SOURCE_DIR}/*.gcda")
-if(IS_LUAJIT)
-  # Files 'src/host/*.gcda' are not removed, because
-  # CMake cannot remove recursively by globbing.
-  # Files 'src/host/*.gcda' are not used for building coverage report.
-  set(GCDA_FILES "${LUA_SOURCE_DIR}/src/*.gcda")
-endif()
 add_custom_target(coverage-reset
   COMMENT "Reset code coverage counters"
-  COMMAND ${CMAKE_COMMAND} -E rm -f ${GCDA_FILES}
+  COMMAND ${CMAKE_COMMAND} -E rm -f ${LLVM_COV_PROFRAW_MASK} ${LLVM_COV_PROFDATA}
 )
 
-message(STATUS "Code coverage HTML report: ${COVERAGE_HTML_REPORT}")
-message(STATUS "Code coverage XML report: ${COVERAGE_XML_REPORT}")
+message(STATUS "Code coverage HTML report: ${CODE_COVERAGE_HTML_REPORT}")

libluamut/CMakeLists.txt

@@ -1,10 +1,8 @@
 set(CFLAGS -Wall -Wextra -Wpedantic -Wno-unused-parameter)
 
 if (ENABLE_COV)
-  set(CFLAGS ${CFLAGS} -fprofile-instr-generate -fprofile-arcs
-             -fcoverage-mapping -ftest-coverage)
-  set(LDFLAGS ${LDFLAGS} -fprofile-instr-generate -fprofile-arcs
-              -fcoverage-mapping -ftest-coverage)
+  set(CFLAGS ${CFLAGS} "${CODE_COVERAGE_FLAGS}")
+  set(LDFLAGS ${LDFLAGS} "${CODE_COVERAGE_FLAGS}")
 endif()
 
 set(LIB_LUA_MUTATE lua_mutate)

tests/CMakeLists.txt

@@ -12,6 +12,15 @@ string(JOIN " " LIBFUZZER_OPTS
   -workers=${CMAKE_BUILD_PARALLEL_LEVEL}
 )
 
+if(ENABLE_COV)
+  string(JOIN " " LIBFUZZER_OPTS
+    -fork=${CMAKE_BUILD_PARALLEL_LEVEL}
+    -ignore_ooms=1
+    -ignore_timeouts=1
+    -ignore_crashes=1
+  )
+endif()
+
 set(CORPUS_BASE_PATH ${PROJECT_SOURCE_DIR}/corpus)
 if(IS_LUAJIT)
   set(CORPUS_BASE_PATH ${CORPUS_BASE_PATH}/corpus)

tests/capi/CMakeLists.txt

@@ -95,13 +95,17 @@ function(create_test)
   add_test(NAME ${test_name}
            COMMAND ${SHELL} -c "$<TARGET_FILE:${test_name}> ${LIBFUZZER_OPTS}"
   )
+  list(APPEND TEST_ENV
+    LLVM_PROFILE_FILE=${test_name}.%1m.%p.profraw
+  )
   if (USE_LUA)
-    set_tests_properties(${test_name} PROPERTIES
-      ENVIRONMENT "ASAN_OPTIONS='detect_invalid_pointer_pairs=2'"
+    list(APPEND TEST_ENV
+      asan_options='detect_invalid_pointer_pairs=2'
     )
   endif()
   set_tests_properties(${test_name} PROPERTIES
     LABELS capi
+    ENVIRONMENT "${TEST_ENV}"
   )
 
   if (IS_LUAJIT)

tests/lapi/CMakeLists.txt

@@ -50,6 +50,9 @@ function(create_test)
     COMMAND ${SHELL} -c "${LUA_EXECUTABLE} ${FUZZ_FILENAME} ${LIBFUZZER_OPTS}"
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   )
+  list(APPEND TEST_ENV
+    LLVM_PROFILE_FILE=${test_name}.%1m.%p.profraw
+  )
   set_tests_properties(${test_name} PROPERTIES
     LABELS "lapi"
     ENVIRONMENT "${TEST_ENV}"