Skip to content

Commit 529fe22

Browse files
committed
coverage: Local workflow
1 parent aff22e4 commit 529fe22

File tree

9 files changed

+382
-9
lines changed

9 files changed

+382
-9
lines changed

.coverage

52 KB
Binary file not shown.

Justfile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,25 @@ py-test:
4747
test-pure:
4848
uv run --group dev --group test pytest codetracer-pure-python-recorder
4949

50+
# Generate combined coverage artefacts for both crates
51+
alias cov := coverage
52+
coverage:
53+
just coverage-rust
54+
just coverage-python
55+
56+
coverage-rust:
57+
mkdir -p codetracer-python-recorder/target/coverage/rust
58+
LLVM_COV="$(command -v llvm-cov)" LLVM_PROFDATA="$(command -v llvm-profdata)" \
59+
uv run cargo llvm-cov nextest --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features --lcov --output-path codetracer-python-recorder/target/coverage/rust/lcov.info
60+
LLVM_COV="$(command -v llvm-cov)" LLVM_PROFDATA="$(command -v llvm-profdata)" \
61+
uv run cargo llvm-cov report --summary-only --json --manifest-path codetracer-python-recorder/Cargo.toml --output-path codetracer-python-recorder/target/coverage/rust/summary.json
62+
python3 codetracer-python-recorder/scripts/render_rust_coverage_summary.py \
63+
codetracer-python-recorder/target/coverage/rust/summary.json --root "$(pwd)"
64+
65+
coverage-python:
66+
mkdir -p codetracer-python-recorder/target/coverage/python
67+
uv run --group dev --group test pytest --cov=codetracer_python_recorder --cov-report=term --cov-report=xml:codetracer-python-recorder/target/coverage/python/coverage.xml codetracer-python-recorder/tests/python
68+
5069
# Build the module in release mode
5170
build:
5271
just venv \
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"""Render a concise Rust coverage summary table from cargo-llvm-cov JSON."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
import json
7+
import pathlib
8+
import sys
9+
from typing import Iterable, List, Tuple
10+
11+
12+
def parse_args(argv: Iterable[str] | None = None) -> argparse.Namespace:
13+
parser = argparse.ArgumentParser(description=__doc__ or "")
14+
parser.add_argument(
15+
"summary_path",
16+
type=pathlib.Path,
17+
help="Path to cargo-llvm-cov JSON summary (e.g. summary.json)",
18+
)
19+
parser.add_argument(
20+
"--root",
21+
type=pathlib.Path,
22+
default=pathlib.Path.cwd(),
23+
help="Repository root used to relativise file paths (default: current working directory)",
24+
)
25+
return parser.parse_args(argv)
26+
27+
28+
def load_rows(summary_path: pathlib.Path, repo_root: pathlib.Path) -> List[Tuple[str, int, int, float]]:
29+
try:
30+
payload = json.loads(summary_path.read_text(encoding="utf-8"))
31+
except FileNotFoundError as exc:
32+
raise SystemExit(f"Rust coverage summary not found: {summary_path}") from exc
33+
34+
repo_root = repo_root.resolve()
35+
rows: List[Tuple[str, int, int, float]] = []
36+
37+
for dataset in payload.get("data", []):
38+
for entry in dataset.get("files", []):
39+
filename = entry.get("filename")
40+
if not filename:
41+
continue
42+
path = pathlib.Path(filename)
43+
try:
44+
rel_path = path.resolve().relative_to(repo_root)
45+
except Exception:
46+
# Skip entries outside the repository (stdlib, third-party deps, etc.).
47+
continue
48+
49+
line_summary = (entry.get("summary") or {}).get("lines") or {}
50+
total = int(line_summary.get("count", 0))
51+
covered = int(line_summary.get("covered", 0))
52+
missed = max(total - covered, 0)
53+
percent = float(line_summary.get("percent", 0.0))
54+
rows.append((rel_path.as_posix(), total, missed, percent))
55+
56+
rows.sort(key=lambda item: item[0])
57+
return rows
58+
59+
60+
def render(rows: List[Tuple[str, int, int, float]]) -> str:
61+
if not rows:
62+
return "Rust coverage summary: no project files found"
63+
64+
name_width = max(len(name) for name, *_ in rows)
65+
lines = ["Rust coverage summary (lines):", f"{'Name'.ljust(name_width)} Lines Miss Cover"]
66+
67+
for name, total, missed, percent in rows:
68+
lines.append(f"{name.ljust(name_width)} {total:5d} {missed:4d} {percent:5.1f}%")
69+
70+
return "\n".join(lines)
71+
72+
73+
def main(argv: Iterable[str] | None = None) -> int:
74+
args = parse_args(argv)
75+
rows = load_rows(args.summary_path, args.root)
76+
output = render(rows)
77+
print(output)
78+
return 0
79+
80+
81+
if __name__ == "__main__":
82+
sys.exit(main())

codetracer-python-recorder/tests/README.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,25 @@ developers can see which harness they are touching at a glance.
1515
For unit tests that do not require the FFI boundary, prefer `#[cfg(test)]`
1616
modules co-located with the Rust source, or Python module-level tests inside the
1717
`codetracer_python_recorder` package.
18+
19+
## Coverage Workflow
20+
21+
When you need coverage artefacts, prefer the Just helpers so local runs match the
22+
CI configuration:
23+
24+
- `just coverage-rust` runs `cargo llvm-cov nextest` with the same flags as the
25+
Rust test job and writes `lcov.info` under
26+
`codetracer-python-recorder/target/coverage/rust/`. The helper wires in
27+
`LLVM_COV`/`LLVM_PROFDATA` from the dev shell (provided by
28+
`llvmPackages_latest.llvm`), so make sure you re-enter `nix develop` after
29+
pulling updates. It also captures `summary.json` and prints a per-file table
30+
so the console mirrors the pytest coverage output. (Run
31+
`cargo llvm-cov nextest --html` manually if you need a browsable report.)
32+
- `just coverage-python` executes pytest with `pytest-cov`, limiting collection to
33+
`codetracer_python_recorder` and emitting `coverage.xml` in
34+
`codetracer-python-recorder/target/coverage/python/`.
35+
- `just coverage` is a convenience wrapper that invokes both commands in sequence.
36+
37+
All commands create their output directories on first run, so no manual setup is
38+
required beyond entering the Nix shell (`nix develop`) or syncing the UV virtual
39+
environment (`just venv`).

design-docs/test-suite-coverage-plan.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,14 @@
1010
- **Python:** Use `pytest --cov` with the `coverage` plugin. Restrict collection to the `codetracer_python_recorder` package to avoid noise from site-packages. Generate both terminal summaries and Cobertura XML for upload.
1111

1212
## Prerequisites & Dependencies
13-
- Add `cargo-llvm-cov` to the dev environment so the Just targets and CI runners share the same binary. In the Nix shell, include the package and ensure the Rust toolchain exposes `llvm-tools-preview`.
13+
- Add `cargo-llvm-cov` to the dev environment so the Just targets and CI runners share the same binary. In the Nix shell, include the package and ensure the Rust toolchain exposes `llvm-tools-preview` or equivalent `llvm` binaries. The current dev shell ships `llvmPackages_latest.llvm`, making `llvm-cov`/`llvm-profdata` available without rustup components.
1414
- Extend the UV `dev` dependency group with `pytest-cov` and `coverage[toml]` so Python coverage instrumentation is reproducible locally and in CI.
1515
- Standardise coverage outputs under `codetracer-python-recorder/target/coverage` to keep artefacts inside the Rust crate. Use `target/coverage/{rust,python}` for per-language assets and a top-level `index.txt` to note the run metadata if needed later.
1616

1717
## Execution Strategy
1818
1. **Local Workflow**
1919
- Add convenience Just targets that mirror the default test steps:
20-
- `just coverage-rust``uv run cargo llvm-cov --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features --nextest --lcov --output-path codetracer-python-recorder/target/coverage/rust/lcov.info --html codetracer-python-recorder/target/coverage/rust/html`.
20+
- `just coverage-rust``LLVM_COV=$(command -v llvm-cov) LLVM_PROFDATA=$(command -v llvm-profdata) uv run cargo llvm-cov --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features --nextest --lcov --output-path codetracer-python-recorder/target/coverage/rust/lcov.info`, followed by `cargo llvm-cov report --summary-only --json` to generate `summary.json` and a Python helper that prints a table mirroring the pytest coverage output. Document that contributors can run a second `cargo llvm-cov … --html --output-dir …` invocation when they need browsable reports because the CLI disallows combining `--lcov` and `--html` in a single run.
2121
- `just coverage-python``uv run --group dev --group test pytest --cov=codetracer_python_recorder --cov-report=term --cov-report=xml:codetracer-python-recorder/target/coverage/python/coverage.xml codetracer-python-recorder/tests/python`.
2222
- `just coverage` wrapper → runs the Rust step followed by the Python step so developers get both artefacts with one command, matching the eventual CI flow.
2323
- Ensure the commands create their output directories (`target/coverage/rust` and `target/coverage/python`) before writing results to avoid failures on first use.
@@ -27,7 +27,7 @@
2727
- Extend `.github/workflows/ci.yml` with optional `coverage-rust` and `coverage-python` jobs that depend on the primary test jobs and only run when `matrix.python-version == '3.12'` and `matrix.os == 'ubuntu-latest'` to avoid duplicate collection.
2828
- Reuse the Just targets so CI mirrors local behaviour. Inject `RUSTFLAGS`/`RUSTDOCFLAGS` from the test jobs’ cache to avoid rebuilding dependencies.
2929
- Publish artefacts via `actions/upload-artifact`:
30-
- Rust: `codetracer-python-recorder/target/coverage/rust/lcov.info` plus a gzipped archive of the HTML folder.
30+
- Rust: `codetracer-python-recorder/target/coverage/rust/lcov.info`, the machine-readable `summary.json`, and optionally a gzipped HTML folder produced via a follow-up `cargo llvm-cov nextest --html --output-dir …` run in the same job.
3131
- Python: `codetracer-python-recorder/target/coverage/python/coverage.xml` for future parsing.
3232
- Mark coverage steps with `continue-on-error: true` during the stabilisation phase and note the run IDs in the job summary for quick retrieval.
3333

design-docs/test-suite-coverage-plan.status.md

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,11 @@
22

33
## Current Status
44
- ✅ Plan doc expanded with prerequisites, detailed Just targets, CI strategy, and an implementation checklist (see `design-docs/test-suite-coverage-plan.md`).
5-
- 🚧 Implementation: add coverage tooling dependencies to the dev shell and UV groups so local + CI runners share the same setup.
6-
- Implementation: land `just coverage-*` helpers and update developer documentation.
5+
- Implementation: coverage dependencies added to the dev shell (`flake.nix`) and UV groups (`pyproject.toml`).
6+
- Implementation: `just coverage-*` helpers landed with matching documentation in `codetracer-python-recorder/tests/README.md`.
77
- ⏳ Implementation: wire optional coverage jobs into `.github/workflows/ci.yml` with artefact uploads.
88
- ⏳ Assessment: capture baseline coverage numbers before proposing enforcement thresholds.
99

1010
## Next Steps
11-
1. Update environment dependencies for coverage (`cargo-llvm-cov`, `pytest-cov`, `coverage[toml]`).
12-
2. Introduce the Just coverage commands and document how to use them.
13-
3. Extend CI with non-blocking coverage collection and review the initial artefacts.
11+
1. Extend CI with non-blocking coverage collection and review the initial artefacts.
12+
2. Capture baseline coverage numbers ahead of enforcement proposals.

flake.nix

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
clippy
3434
rust-analyzer
3535
cargo-nextest
36+
cargo-llvm-cov
37+
llvmPackages_latest.llvm
3638

3739
# Build tooling for Python extensions
3840
maturin

pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ typeCheckingMode = "basic"
2626
[dependency-groups]
2727
dev = [
2828
"pytest>=8.3.5",
29+
"pytest-cov>=5.0.0",
30+
"coverage[toml]>=7.6.0",
2931
]
3032

3133
test = [

0 commit comments

Comments
 (0)