feat(test): support coverage (#1205)

ken-matsui · web-flow · commit 391a250e1185 · 2025-08-01T04:59:12.000Z
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -13,12 +13,13 @@ permissions:
 
 jobs:
   build-and-test:
-    name: "build & test (${{ matrix.cxx.name }} - ${{ matrix.build }})"
+    name: "build & test (${{ matrix.cxx.name }} - ${{ matrix.build }}${{ matrix.coverage && ' - coverage' || '' }})"
     runs-on: ${{ matrix.cxx.os }}
     strategy:
       fail-fast: false
       matrix:
         build: [dev, release]
+        coverage: [false]
         cxx:
           - name: "Linux - Clang 16"
             cmd: clang++-16
@@ -53,6 +54,14 @@ jobs:
           - name: "macOS 15 - Apple Clang"
             cmd: c++
             os: macos-15
+        include:
+          # Coverage testing with GCC 14
+          - build: dev
+            coverage: true
+            cxx:
+              name: "Linux - GCC 14"
+              cmd: g++-14
+              os: ubuntu-24.04
     env:
       CXX: ${{ matrix.cxx.cmd }}
     steps:
@@ -69,6 +78,10 @@ jobs:
         if: runner.os == 'Linux'
         uses: ./.github/actions/setup-ubuntu-deps
 
+      - name: Install lcov
+        if: matrix.coverage
+        run: sudo apt-get install -y lcov
+
       - name: Setup macOS dependencies
         if: runner.os == 'macOS'
         uses: ./.github/actions/setup-macos-deps
@@ -100,7 +113,10 @@ jobs:
 
       - name: Stage 2 - Build & Test
         run: |
-          ./build/cabin --verbose run ${{ matrix.build == 'release' && '--release' || '' }} test --verbose
+          [[ '${{ matrix.build }}' == 'release' ]] && RELEASE='--release'
+          [[ '${{ matrix.coverage }}' == 'true' ]] && COVERAGE='--coverage'
+          # shellcheck disable=SC2086
+          ./build/cabin --verbose run $RELEASE test --verbose $COVERAGE
 
       - name: Stage 2 - Print version
         run: ./cabin-out/${{ matrix.build }}/cabin version --verbose
@@ -111,14 +127,13 @@ jobs:
           CABIN: ${{ github.workspace }}/cabin-out/${{ matrix.build }}/cabin
           CABIN_TERM_COLOR: auto
 
-      # - name: Print coverage
-      #   if: success() && matrix.coverage == 'on'
-      #   run: |
-      #     lcov --directory . --capture --output-file coverage.info --gcov-tool "${CC_PATH/gcc/gcov}"
-      #     lcov --remove coverage.info '/usr/*' "${HOME}"'/.cache/*' --output-file coverage.info
-      #     lcov --list coverage.info
-      #   env:
-      #     CC_PATH: /usr/bin/${{ env.CC }}
+      - name: Print coverage
+        if: success() && matrix.coverage
+        run: |
+          lcov --capture --directory . --no-external \
+               --gcov-tool "gcov-${CXX##*-}"         \
+               --output-file coverage.info
+          lcov --list coverage.info
 
   format:
     needs: build-and-test
diff --git a/src/BuildConfig.cc b/src/BuildConfig.cc
@@ -746,16 +746,27 @@ Result<void> BuildConfig::configureBuild() {
   return Ok();
 }
 
+void BuildConfig::enableCoverage() {
+  project.compilerOpts.cFlags.others.emplace_back("--coverage");
+  project.compilerOpts.ldFlags.others.emplace_back("--coverage");
+}
+
 Result<BuildConfig> emitMakefile(const Manifest& manifest,
                                  const BuildProfile& buildProfile,
-                                 const bool includeDevDeps) {
+                                 const bool includeDevDeps,
+                                 const bool enableCoverage) {
   const Profile& profile = manifest.profiles.at(buildProfile);
   auto config = Try(BuildConfig::init(manifest, buildProfile));
 
   // When emitting Makefile, we also build the project.  So, we need to
   // make sure the dependencies are installed.
   Try(config.installDeps(includeDevDeps));
 
+  // Add coverage flags if requested
+  if (enableCoverage) {
+    config.enableCoverage();
+  }
+
   bool buildProj = false;
   bool buildCompDb = false;
   if (config.makefileIsUpToDate()) {
diff --git a/src/BuildConfig.hpp b/src/BuildConfig.hpp
@@ -182,11 +182,13 @@ class BuildConfig {
                      tbb::spin_mutex* mtx = nullptr);
 
   Result<void> configureBuild();
+  void enableCoverage();
 };
 
 Result<BuildConfig> emitMakefile(const Manifest& manifest,
                                  const BuildProfile& buildProfile,
-                                 bool includeDevDeps);
+                                 bool includeDevDeps,
+                                 bool enableCoverage = false);
 Result<std::string> emitCompdb(const Manifest& manifest,
                                const BuildProfile& buildProfile,
                                bool includeDevDeps);
diff --git a/src/Cmd/Test.cc b/src/Cmd/Test.cc
@@ -28,6 +28,7 @@ class Test {
   Manifest manifest;
   std::string unittestTargetPrefix;
   std::vector<std::string> unittestTargets;
+  bool enableCoverage = false;
 
   explicit Test(Manifest manifest) : manifest(std::move(manifest)) {}
 
@@ -43,14 +44,15 @@ const Subcmd TEST_CMD = //
         .setShort("t")
         .setDesc("Run the tests of a local package")
         .addOpt(OPT_JOBS)
+        .addOpt(Opt{ "--coverage" }.setDesc("Enable code coverage analysis"))
         .setMainFn(Test::exec);
 
 Result<void> Test::compileTestTargets() {
   const auto start = std::chrono::steady_clock::now();
 
   const BuildProfile buildProfile = BuildProfile::Test;
-  const BuildConfig config =
-      Try(emitMakefile(manifest, buildProfile, /*includeDevDeps=*/true));
+  const BuildConfig config = Try(emitMakefile(
+      manifest, buildProfile, /*includeDevDeps=*/true, enableCoverage));
 
   // Collect test targets from the generated Makefile.
   unittestTargetPrefix = (config.outBasePath / "unittests").string() + '/';
@@ -156,6 +158,8 @@ Result<void> Test::runTestTargets() {
 }
 
 Result<void> Test::exec(const CliArgsView cliArgs) {
+  bool enableCoverage = false;
+
   for (auto itr = cliArgs.begin(); itr != cliArgs.end(); ++itr) {
     const std::string_view arg = *itr;
 
@@ -175,13 +179,16 @@ Result<void> Test::exec(const CliArgsView cliArgs) {
           nextArg.data(), nextArg.data() + nextArg.size(), numThreads);
       Ensure(ec == std::errc(), "invalid number of threads: {}", nextArg);
       setParallelism(numThreads);
+    } else if (arg == "--coverage") {
+      enableCoverage = true;
     } else {
       return TEST_CMD.noSuchArg(arg);
     }
   }
 
   Manifest manifest = Try(Manifest::tryParse());
   Test cmd(std::move(manifest));
+  cmd.enableCoverage = enableCoverage;
 
   Try(cmd.compileTestTargets());
   if (cmd.unittestTargets.empty()) {
diff --git a/tests/08-test.sh b/tests/08-test.sh
@@ -0,0 +1,173 @@
+#!/bin/sh
+
+test_description='Test the test command'
+
+WHEREAMI=$(dirname "$(realpath "$0")")
+. $WHEREAMI/setup.sh
+
+test_expect_success 'cabin test basic functionality' '
+    OUT=$(mktemp -d) &&
+    test_when_finished "rm -rf $OUT" &&
+    cd $OUT &&
+    "$CABIN" new test_project &&
+    cd test_project &&
+
+    # Add a simple test to the main.cc file
+    cat >src/main.cc <<-EOF &&
+#include <iostream>
+
+#ifdef CABIN_TEST
+void test_addition() {
+    int result = 2 + 2;
+    if (result != 4) {
+        std::cerr << "Test failed: 2 + 2 = " << result << ", expected 4" << std::endl;
+        std::exit(1);
+    }
+    std::cout << "test test addition ... ok" << std::endl;
+}
+
+int main() {
+    test_addition();
+    return 0;
+}
+#else
+int main() {
+    std::cout << "Hello, world!" << std::endl;
+    return 0;
+}
+#endif
+EOF
+
+    "$CABIN" test 1>stdout 2>stderr &&
+    (
+        test -d cabin-out &&
+        test -d cabin-out/test &&
+        test -d cabin-out/test/unittests
+    ) &&
+    grep -q "test addition.*ok" stdout &&
+    grep -q "1 passed; 0 failed" stderr
+'
+
+test_expect_success 'cabin test --help shows coverage option' '
+    OUT=$(mktemp -d) &&
+    test_when_finished "rm -rf $OUT" &&
+    cd $OUT &&
+    "$CABIN" new test_project &&
+    cd test_project &&
+    "$CABIN" test --help >help_output 2>&1 &&
+    grep -q -- "--coverage" help_output &&
+    grep -q "Enable code coverage analysis" help_output
+'
+
+test_expect_success 'cabin test --coverage generates coverage files' '
+    OUT=$(mktemp -d) &&
+    test_when_finished "rm -rf $OUT" &&
+    cd $OUT &&
+    "$CABIN" new coverage_project &&
+    cd coverage_project &&
+
+    # Add a simple test
+    cat >src/main.cc <<-EOF &&
+#include <iostream>
+
+#ifdef CABIN_TEST
+void test_function() {
+    std::cout << "test coverage function ... ok" << std::endl;
+}
+
+int main() {
+    test_function();
+    return 0;
+}
+#else
+int main() {
+    std::cout << "Hello, world!" << std::endl;
+    return 0;
+}
+#endif
+EOF
+
+    "$CABIN" test --coverage 1>stdout 2>stderr &&
+
+    # Check that coverage files were generated
+    find cabin-out/test -name "*.gcda" | head -1 | grep -q "\.gcda$" &&
+    find cabin-out/test -name "*.gcno" | head -1 | grep -q "\.gcno$" &&
+
+    # Check test output
+    grep -q "coverage function.*ok" stdout &&
+    grep -q "1 passed; 0 failed" stderr
+'
+
+test_expect_success 'cabin test --coverage uses coverage flags in compilation' '
+    OUT=$(mktemp -d) &&
+    test_when_finished "rm -rf $OUT" &&
+    cd $OUT &&
+    "$CABIN" new verbose_project &&
+    cd verbose_project &&
+
+    # Add a simple test
+    cat >src/main.cc <<-EOF &&
+#include <iostream>
+
+#ifdef CABIN_TEST
+int main() {
+    std::cout << "test verbose compilation ... ok" << std::endl;
+    return 0;
+}
+#else
+int main() {
+    std::cout << "Hello, world!" << std::endl;
+    return 0;
+}
+#endif
+EOF
+
+    # Clear any existing build artifacts to force recompilation
+    rm -rf cabin-out &&
+
+    "$CABIN" test --coverage -vv 1>stdout 2>stderr &&
+
+    # Check that --coverage flag appears in compilation commands
+    grep -q -- "--coverage" stdout &&
+
+    # Check test passes
+    grep -q "verbose compilation.*ok" stdout &&
+    grep -q "1 passed; 0 failed" stderr
+'
+
+test_expect_success 'cabin test without --coverage does not generate coverage files' '
+    OUT=$(mktemp -d) &&
+    test_when_finished "rm -rf $OUT" &&
+    cd $OUT &&
+    "$CABIN" new no_coverage_project &&
+    cd no_coverage_project &&
+
+    # Add a simple test
+    cat >src/main.cc <<-EOF &&
+#include <iostream>
+
+#ifdef CABIN_TEST
+int main() {
+    std::cout << "test no coverage ... ok" << std::endl;
+    return 0;
+}
+#else
+int main() {
+    std::cout << "Hello, world!" << std::endl;
+    return 0;
+}
+#endif
+EOF
+
+    "$CABIN" test 1>stdout 2>stderr &&
+
+    # Check that no coverage files were generated in a clean test
+    # (Note: there might be some from previous tests, so we check that coverage files are not created)
+    test $(find cabin-out/test -name "*.gcda" | wc -l) -eq 0 &&
+
+    # Check test passes
+    grep -q "no coverage.*ok" stdout &&
+    grep -q "1 passed; 0 failed" stderr
+'
+
+test_done
