Add --gcov-report flag to spin test (#159)

* ENH: Added `--generate-gcov-report` flag to `spin test`

* TST: Added testcases for html coverage reports

* MAINT: Rename flag to `gcov-report`

* MAINT: Fix doc string for `gcov-report`

Co-authored-by: Jarrod Millman <[email protected]>

* TST: Added UT for `gcov-report`

* ENH, TST: Added checks for missing debug files

* Simplify coverage logic, and build on demand

- Like the existing coverage option, `spin test --gcov` is now enough
  to generate gcov reports.
- The format of the report is set with `--gcov-format`.
- Coverage builds are triggered when `--gcov` is added (necessary
  to generate the coverage report), but no rebuild is done when the
  flag is removed (too expensive).

* Fix test command in README

* Fix tests, now that coverage build is triggered automatically

---------

Co-authored-by: Jarrod Millman <[email protected]>
Co-authored-by: Stefan van der Walt <[email protected]>

Author: 3 people

.github/workflows/test.yml

@@ -29,7 +29,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get update
-          sudo apt-get install -y gdb
+          sudo apt-get install -y gdb lcov
       - name: Tests PyTest
         run: |
           pipx run nox --forcecolor -s test

README.md

@@ -247,7 +247,7 @@ fc-cache -f -v
 `spin` tests are invoked using:
 
 ```
-nox -s tests
+nox -s test
 ```
 
 ## History

noxfile.py

@@ -3,5 +3,5 @@
 
 @nox.session
 def test(session: nox.Session) -> None:
-    session.install(".", "pytest", "build", "meson-python", "ninja")
+    session.install(".", "pytest", "build", "meson-python", "ninja", "gcovr")
     session.run("pytest", "spin", *session.posargs)

spin/cmds/meson.py

@@ -2,15 +2,26 @@
 import copy
 import json
 import os
+import re
 import shutil
 import sys
+from enum import Enum
+from pathlib import Path
 
 import click
 
 from .util import get_commands, get_config
 from .util import run as _run
 
 install_dir = "build-install"
+build_dir = "build"
+
+
+class GcovReportFormat(str, Enum):
+    html = "html"
+    xml = "xml"
+    text = "text"
+    sonarqube = "sonarqube"
 
 
 # Allow specification of meson binary in configuration
@@ -124,6 +135,54 @@ def _meson_version_configured():
         pass
 
 
+def _meson_coverage_configured() -> bool:
+    try:
+        build_options_fn = os.path.join(
+            "build", "meson-info", "intro-buildoptions.json"
+        )
+        with open(build_options_fn) as f:
+            build_options = json.load(f)
+        for b in build_options:
+            if (b["name"] == "b_coverage") and (b["value"] is True):
+                return True
+    except:
+        pass
+
+    return False
+
+
+def _check_coverage_tool_installation(coverage_type: GcovReportFormat):
+    requirements = {  # https://github.com/mesonbuild/meson/blob/6e381714c7cb15877e2bcaa304b93c212252ada3/docs/markdown/Unit-tests.md?plain=1#L49-L62
+        GcovReportFormat.html: ["Gcovr/GenHTML", "lcov"],
+        GcovReportFormat.xml: ["Gcovr (version 3.3 or higher)"],
+        GcovReportFormat.text: ["Gcovr (version 3.3 or higher)"],
+        GcovReportFormat.sonarqube: ["Gcovr (version 4.2 or higher)"],
+    }
+
+    # First check the presence of a valid build
+    if not (os.path.exists(build_dir)):
+        raise click.ClickException(
+            "`build` folder not found, cannot generate coverage reports. "
+            "Generate coverage artefacts by running `spin test --gcov`"
+        )
+
+    debug_files = Path(build_dir).rglob("*.gcno")
+    if len(list(debug_files)) == 0:
+        raise click.ClickException(
+            "Debug build not found, cannot generate coverage reports.\n\n"
+            "Please rebuild using `spin build --clean --gcov` first."
+        )
+
+    # Verify the tools are installed prior to the build
+    p = _run(["ninja", "-C", build_dir, "-t", "targets", "all"], output=False)
+    if f"coverage-{coverage_type.value}" not in p.stdout.decode("ascii"):
+        raise click.ClickException(
+            f"coverage-{coverage_type.value} is not supported... "
+            f"Ensure the following are installed: {', '.join(requirements[coverage_type])} "
+            "and rerun `spin test --gcov`"
+        )
+
+
 @click.command()
 @click.option("-j", "--jobs", help="Number of parallel tasks to launch", type=int)
 @click.option("--clean", is_flag=True, help="Clean build directory before build")
@@ -133,18 +192,7 @@ def _meson_version_configured():
 @click.option(
     "--gcov",
     is_flag=True,
-    help="""Enable C code coverage via `gcov`.
-
-    The meson-generated `build/build.ninja` has targets for compiling
-    coverage reports.
-
-    E.g., to build an HTML report, in the `build` directory run
-     `ninja coverage-html`.
-
-    To see a list all supported formats, run
-    `ninja -t targets | grep coverage-`.
-
-    Also see https://mesonbuild.com/howtox.html#producing-a-coverage-report.""",
+    help="Enable C code coverage using `gcov`. Use `spin test --gcov` to generate reports.",
 )
 @click.argument("meson_args", nargs=-1)
 def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=False):
@@ -165,7 +213,6 @@ def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=F
 
       CFLAGS="-O0 -g" spin build
     """
-    build_dir = "build"
     meson_args = list(meson_args)
 
     if gcov:
@@ -191,7 +238,9 @@ def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=F
         # Build dir has been configured; check if it was configured by
         # current version of Meson
 
-        if _meson_version() != _meson_version_configured():
+        if (_meson_version() != _meson_version_configured()) or (
+            gcov and not _meson_coverage_configured()
+        ):
             _run(setup_cmd + ["--reconfigure"], output=not quiet)
 
         # Any other conditions that warrant a reconfigure?
@@ -255,17 +304,30 @@ def _get_configured_command(command_name):
     "-c",
     "--coverage",
     is_flag=True,
-    help="Generate a coverage report of executed tests. An HTML copy of the report is written to `build/coverage`.",
+    help="Generate a Python coverage report of executed tests. An HTML copy of the report is written to `build/coverage`.",
 )
 @click.option(
     "--gcov",
     is_flag=True,
-    help="Enable C code coverage via `gcov`. `gcov` output goes to `build/**/*.gc*`. "
-    "Reports can be generated using `ninja coverage*` commands. "
-    "See https://mesonbuild.com/howtox.html#producing-a-coverage-report",
+    help="Generate a C coverage report in `build/meson-logs/coveragereport`.",
+)
+@click.option(
+    "--gcov-format",
+    type=click.Choice(GcovReportFormat),
+    default="html",
+    help=f"Format of the gcov report. Can be one of {', '.join(e.value for e in GcovReportFormat)}.",
 )
 @click.pass_context
-def test(ctx, pytest_args, n_jobs, tests, verbose, coverage=False, gcov=False):
+def test(
+    ctx,
+    pytest_args,
+    n_jobs,
+    tests,
+    verbose,
+    coverage=False,
+    gcov=None,
+    gcov_format=None,
+):
     """🔧 Run tests
 
     PYTEST_ARGS are passed through directly to pytest, e.g.:
@@ -315,7 +377,7 @@ def test(ctx, pytest_args, n_jobs, tests, verbose, coverage=False, gcov=False):
         click.secho(
             "Invoking `build` prior to running tests:", bold=True, fg="bright_green"
         )
-        ctx.invoke(build_cmd, gcov=gcov)
+        ctx.invoke(build_cmd, gcov=bool(gcov))
 
     package = cfg.get("tool.spin.package", None)
     if (not pytest_args) and (not tests):
@@ -372,10 +434,44 @@ def test(ctx, pytest_args, n_jobs, tests, verbose, coverage=False, gcov=False):
         cmd = [sys.executable, "-P", "-m", "pytest"]
     else:
         cmd = ["pytest"]
-    _run(
-        cmd + list(pytest_args),
-        replace=True,
-    )
+    p = _run(cmd + list(pytest_args))
+
+    if gcov:
+        # Verify the tools are present
+        click.secho(
+            "Verifying gcov dependencies...",
+            bold=True,
+            fg="bright_yellow",
+        )
+        _check_coverage_tool_installation(gcov_format)
+
+        # Generate report
+        click.secho(
+            f"Generating {gcov_format.value} coverage report...",
+            bold=True,
+            fg="bright_yellow",
+        )
+        p = _run(
+            [
+                "ninja",
+                "-C",
+                build_dir,
+                f"coverage-{gcov_format.value.lower()}",
+            ],
+            output=False,
+        )
+        coverage_folder = click.style(
+            re.search(r"file://(.*)", p.stdout.decode("utf-8")).group(1),
+            bold=True,
+            fg="bright_yellow",
+        )
+        click.secho(
+            f"Coverage report generated successfully and written to {coverage_folder}",
+            bold=True,
+            fg="bright_green",
+        )
+
+        raise SystemExit(p.returncode)
 
 
 @click.command()

spin/tests/test_build_cmds.py

@@ -4,6 +4,7 @@
 import tempfile
 from pathlib import Path
 
+import pytest
 from testutil import skip_on_windows, skip_unless_linux, spin, stdout
 
 from spin.cmds.util import run
@@ -27,6 +28,33 @@ def test_debug_builds():
     assert len(list(debug_files)) != 0, "debug files not generated for gcov build"
 
 
+def test_coverage_builds():
+    """Does gcov test generate coverage files?"""
+    spin("test", "--gcov")
+
+    coverage_files = Path(".").rglob("*.gcda")
+    assert len(list(coverage_files)) != 0, "coverage files not generated for gcov build"
+
+
+@pytest.mark.parametrize(
+    "report_type,output_file",
+    [
+        ("html", Path("coveragereport/index.html")),
+        ("xml", Path("coverage.xml")),
+        ("text", Path("coverage.txt")),
+        ("sonarqube", Path("sonarqube.xml")),
+    ],
+)
+def test_coverage_reports(report_type, output_file):
+    """Does gcov test generate coverage reports?"""
+    spin("test", "--gcov", f"--gcov-format={report_type}")
+
+    coverage_report = Path("./build/meson-logs", output_file)
+    assert (
+        coverage_report.exists()
+    ), f"coverage report not generated for gcov build ({report_type})"
+
+
 def test_expand_pythonpath():
     """Does an $ENV_VAR get expanded in `spin run`?"""
     output = spin("run", "echo $PYTHONPATH")