Skip to content

Commit 26e5dca

Browse files
authored
Tests and coverage (#43)
Added more tests and started computing coverage statistics - ✅ Stage 0 – Baseline captured by ADR 0003 and the initial improvement plan. - ✅ Stage 1 – Layout Consolidation: directory moves completed, test commands updated, README added, and `just test` now runs the Rust and Python harnesses separately. - ✅ Stage 2 – Bootstrap & Output Coverage: unit tests now exercise `TraceSessionBootstrap` helpers (directory/format/argv handling) and `TraceOutputPaths::configure_writer`, with `just test` covering the new cases. - ✅ Stage 3 – Activation Guard Rails: added unit tests around `ActivationController` covering activation start, non-matching frames, and deactivation behaviour; existing runtime integration tests continue to pass. - ✅ Stage 4 – Python Unit Coverage: added `tests/python/unit/test_session_helpers.py` for facade utilities and introduced `tests/python/support` for shared fixtures; updated monitoring tests to use the helper directory builder. - ✅ Stage 5 – CI & Coverage Instrumentation: CI now runs the split Rust/Python test jobs plus a non-blocking coverage job that reuses `just coverage`, uploads LCOV/XML/JSON artefacts, and posts a per-PR summary comment. - ✅ Stage 6 – Cleanup & Documentation: ADR 0003 is now Accepted, top-level docs describe the testing/coverage workflow, and the tests README references the CI coverage comment for contributors.
2 parents efea1b5 + 954d3e3 commit 26e5dca

30 files changed

+1561
-9
lines changed

.coverage

52 KB
Binary file not shown.

.github/workflows/ci.yml

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,84 @@ jobs:
2222
nix_path: nixpkgs=channel:nixos-25.05
2323
extra_nix_config: |
2424
experimental-features = nix-command flakes
25-
- name: Build Rust module and run tests via Nix
26-
run: nix develop --command bash -lc 'just venv ${{matrix.python-version}} dev test'
25+
- name: Prepare dev environment
26+
run: nix develop --command bash -lc 'just venv ${{matrix.python-version}} dev'
27+
28+
- name: Rust tests
29+
run: nix develop --command bash -lc 'just cargo-test'
30+
31+
- name: Python tests
32+
run: nix develop --command bash -lc 'just py-test'
33+
34+
coverage:
35+
name: Coverage (Python 3.12)
36+
needs: nix-tests
37+
runs-on: ubuntu-latest
38+
continue-on-error: true
39+
permissions:
40+
contents: read
41+
issues: write
42+
pull-requests: write
43+
steps:
44+
- uses: actions/checkout@v4
45+
- uses: cachix/install-nix-action@v27
46+
with:
47+
nix_path: nixpkgs=channel:nixos-25.05
48+
extra_nix_config: |
49+
experimental-features = nix-command flakes
50+
51+
- name: Prepare dev environment (Python 3.12)
52+
run: nix develop --command bash -lc 'just venv 3.12 dev'
53+
54+
- name: Collect coverage
55+
id: coverage-run
56+
run: nix develop --command bash -lc 'just coverage'
57+
58+
- name: Generate coverage comment
59+
if: steps.coverage-run.outcome == 'success'
60+
run: |
61+
ROOT="$(pwd)"
62+
nix develop --command bash -lc "python3 codetracer-python-recorder/scripts/generate_coverage_comment.py \
63+
--rust-summary codetracer-python-recorder/target/coverage/rust/summary.json \
64+
--python-json codetracer-python-recorder/target/coverage/python/coverage.json \
65+
--output codetracer-python-recorder/target/coverage/coverage-comment.md \
66+
--repo-root \"${ROOT}\""
67+
68+
- name: Upload Rust coverage artefacts
69+
if: steps.coverage-run.outcome == 'success'
70+
uses: actions/upload-artifact@v4
71+
with:
72+
name: rust-coverage
73+
path: |
74+
codetracer-python-recorder/target/coverage/rust/lcov.info
75+
codetracer-python-recorder/target/coverage/rust/summary.json
76+
if-no-files-found: warn
77+
78+
- name: Upload Python coverage artefacts
79+
if: steps.coverage-run.outcome == 'success'
80+
uses: actions/upload-artifact@v4
81+
with:
82+
name: python-coverage
83+
path: |
84+
codetracer-python-recorder/target/coverage/python/coverage.xml
85+
codetracer-python-recorder/target/coverage/python/coverage.json
86+
if-no-files-found: warn
87+
88+
- name: Upload coverage comment
89+
if: steps.coverage-run.outcome == 'success'
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: coverage-comment
93+
path: codetracer-python-recorder/target/coverage/coverage-comment.md
94+
if-no-files-found: warn
95+
96+
- name: Comment coverage summary
97+
if: github.event_name == 'pull_request' && steps.coverage-run.outcome == 'success'
98+
uses: peter-evans/create-or-update-comment@v4
99+
with:
100+
issue-number: ${{ github.event.pull_request.number }}
101+
identifier: coverage-summary
102+
body-path: codetracer-python-recorder/target/coverage/coverage-comment.md
27103

28104
# rust-tests:
29105
# name: Rust module test on ${{ matrix.os }} (Python ${{ matrix.python-version }})

Justfile

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,30 @@ cargo-test:
4141
uv run cargo nextest run --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features
4242

4343
py-test:
44-
uv run --group dev --group test pytest
44+
uv run --group dev --group test pytest codetracer-python-recorder/tests/python codetracer-pure-python-recorder
4545

4646
# Run tests only on the pure recorder
4747
test-pure:
4848
uv run --group dev --group test pytest codetracer-pure-python-recorder
4949

50+
# Generate combined coverage artefacts for both crates
51+
coverage:
52+
just coverage-rust
53+
just coverage-python
54+
55+
coverage-rust:
56+
mkdir -p codetracer-python-recorder/target/coverage/rust
57+
LLVM_COV="$(command -v llvm-cov)" LLVM_PROFDATA="$(command -v llvm-profdata)" \
58+
uv run cargo llvm-cov nextest --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features --lcov --output-path codetracer-python-recorder/target/coverage/rust/lcov.info
59+
LLVM_COV="$(command -v llvm-cov)" LLVM_PROFDATA="$(command -v llvm-profdata)" \
60+
uv run cargo llvm-cov report --summary-only --json --manifest-path codetracer-python-recorder/Cargo.toml --output-path codetracer-python-recorder/target/coverage/rust/summary.json
61+
python3 codetracer-python-recorder/scripts/render_rust_coverage_summary.py \
62+
codetracer-python-recorder/target/coverage/rust/summary.json --root "$(pwd)"
63+
64+
coverage-python:
65+
mkdir -p codetracer-python-recorder/target/coverage/python
66+
uv run --group dev --group test pytest --cov=codetracer_python_recorder --cov-report=term --cov-report=xml:codetracer-python-recorder/target/coverage/python/coverage.xml --cov-report=json:codetracer-python-recorder/target/coverage/python/coverage.json codetracer-python-recorder/tests/python
67+
5068
# Build the module in release mode
5169
build:
5270
just venv \
@@ -64,4 +82,3 @@ test-all:
6482
file="${file[0]}"; \
6583
uv run -p "python3.$v" --with "${file}" --with pytest -- pytest -q; \
6684
done
67-

README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,15 @@ Basic workflow:
4545
- from codetracer_python_recorder import hello
4646
- hello()
4747

48+
#### Testing & Coverage
49+
50+
- Run the full split test suite (Rust nextest + Python pytest): `just test`
51+
- Run only Rust integration/unit tests: `just cargo-test`
52+
- Run only Python tests (including the pure-Python recorder to guard regressions): `just py-test`
53+
- Collect coverage artefacts locally (LCOV + Cobertura/JSON): `just coverage`
54+
55+
The CI workflow mirrors these commands. Pull requests get an automated comment with the latest Rust/Python coverage tables and downloadable artefacts (`lcov.info`, `coverage.xml`, `coverage.json`).
56+
4857
### Future directions
4958

5059
The current Python support is an unfinished prototype. We can finish it. In the future, it may be expanded to function in a way to similar to the more complete implementations, e.g. [Noir](https://github.com/blocksense-network/noir/tree/blocksense/tooling/tracer).
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
#!/usr/bin/env python3
2+
"""Generate a Markdown coverage summary comment for CI."""
3+
4+
from __future__ import annotations
5+
6+
import argparse
7+
import json
8+
import pathlib
9+
from typing import Dict, Iterable, List, Tuple
10+
11+
from render_rust_coverage_summary import load_summary as load_rust_summary
12+
13+
Row = Tuple[str, int, int, float]
14+
15+
16+
def parse_args(argv: Iterable[str] | None = None) -> argparse.Namespace:
17+
parser = argparse.ArgumentParser(description=__doc__ or "")
18+
parser.add_argument(
19+
"--rust-summary",
20+
type=pathlib.Path,
21+
required=True,
22+
help="Path to cargo-llvm-cov JSON summary",
23+
)
24+
parser.add_argument(
25+
"--python-json",
26+
type=pathlib.Path,
27+
required=True,
28+
help="Path to coverage.py JSON report",
29+
)
30+
parser.add_argument(
31+
"--output",
32+
type=pathlib.Path,
33+
required=True,
34+
help="Output Markdown file for the PR comment",
35+
)
36+
parser.add_argument(
37+
"--repo-root",
38+
type=pathlib.Path,
39+
default=pathlib.Path.cwd(),
40+
help="Repository root used to relativise file paths (default: current working directory)",
41+
)
42+
parser.add_argument(
43+
"--max-rows",
44+
type=int,
45+
default=20,
46+
help="Maximum number of per-file rows to display for each language (default: 20).",
47+
)
48+
return parser.parse_args(argv)
49+
50+
51+
def _select_rows(rows: List[Row], max_rows: int) -> Tuple[List[Row], bool]:
52+
if max_rows <= 0 or len(rows) <= max_rows:
53+
return sorted(rows, key=lambda item: item[0]), False
54+
55+
priority_sorted = sorted(rows, key=lambda item: (-item[2], item[0]))
56+
trimmed = priority_sorted[:max_rows]
57+
return sorted(trimmed, key=lambda item: item[0]), True
58+
59+
60+
def _format_table(rows: List[Row], headers: Tuple[str, str, str, str]) -> List[str]:
61+
lines = [
62+
f"| {headers[0]} | {headers[1]} | {headers[2]} | {headers[3]} |",
63+
"| --- | ---: | ---: | ---: |",
64+
]
65+
for name, total, missed, percent in rows:
66+
lines.append(
67+
f"| `{name}` | {total:,} | {missed:,} | {percent:5.1f}% |"
68+
)
69+
return lines
70+
71+
72+
def _load_python_rows(
73+
report_path: pathlib.Path,
74+
repo_root: pathlib.Path,
75+
) -> Tuple[List[Row], Dict[str, float]]:
76+
try:
77+
payload = json.loads(report_path.read_text(encoding="utf-8"))
78+
except FileNotFoundError as exc:
79+
raise SystemExit(f"Python coverage JSON not found: {report_path}") from exc
80+
81+
repo_root = repo_root.resolve()
82+
rows: List[Row] = []
83+
84+
for path_str, details in payload.get("files", {}).items():
85+
summary = (details.get("summary") or {})
86+
total = int(summary.get("num_statements", 0))
87+
missed = int(summary.get("missing_lines", 0))
88+
percent = float(summary.get("percent_covered", 0.0))
89+
90+
if total == 0 and missed == 0:
91+
continue
92+
93+
file_path = pathlib.Path(path_str)
94+
if not file_path.is_absolute():
95+
file_path = (repo_root / file_path).resolve()
96+
else:
97+
file_path = file_path.resolve()
98+
99+
try:
100+
rel_path = file_path.relative_to(repo_root)
101+
except ValueError:
102+
continue
103+
104+
rows.append((rel_path.as_posix(), total, missed, percent))
105+
106+
rows.sort(key=lambda item: item[0])
107+
108+
totals = payload.get("totals", {})
109+
return rows, {
110+
"total": float(totals.get("num_statements", 0)),
111+
"covered": float(totals.get("covered_lines", 0)),
112+
"missed": float(totals.get("missing_lines", 0)),
113+
"percent": float(totals.get("percent_covered", 0.0)),
114+
}
115+
116+
117+
def _load_rust_rows(
118+
summary_path: pathlib.Path,
119+
repo_root: pathlib.Path,
120+
) -> Tuple[List[Row], Dict[str, float]]:
121+
rows, totals = load_rust_summary(summary_path, repo_root)
122+
# Normalise totals dict to expected keys
123+
total = float(totals.get("count", 0))
124+
covered = float(totals.get("covered", 0))
125+
missed = float(totals.get("notcovered", total - covered))
126+
return rows, {
127+
"total": total,
128+
"covered": covered,
129+
"missed": missed,
130+
"percent": float(totals.get("percent", 0.0)),
131+
}
132+
133+
134+
def _format_summary_block(
135+
heading: str,
136+
column_label: str,
137+
totals: Dict[str, float],
138+
rows: List[Row],
139+
max_rows: int,
140+
) -> List[str]:
141+
display_rows, truncated = _select_rows(rows, max_rows)
142+
lines = [heading]
143+
total = int(totals.get("total", 0))
144+
covered = int(totals.get("covered", 0))
145+
missed = int(totals.get("missed", total - covered))
146+
percent = totals.get("percent", 0.0)
147+
lines.append(
148+
f"**{percent:0.1f}%** covered ({covered:,} / {total:,} | {missed:,} missed)"
149+
)
150+
lines.extend(
151+
_format_table(display_rows, ("File", column_label, "Miss", "Cover"))
152+
)
153+
if truncated:
154+
lines.append(
155+
f"_Showing top {max_rows} entries by missed lines (of {len(rows)} total)._"
156+
)
157+
return lines
158+
159+
160+
def main(argv: Iterable[str] | None = None) -> int:
161+
args = parse_args(argv)
162+
repo_root = args.repo_root.resolve()
163+
164+
rust_rows, rust_totals = _load_rust_rows(args.rust_summary, repo_root)
165+
python_rows, python_totals = _load_python_rows(args.python_json, repo_root)
166+
167+
output_lines: List[str] = ["### Coverage Summary", ""]
168+
169+
output_lines.extend(
170+
_format_summary_block(
171+
"**Rust (lines)**", "Lines", rust_totals, rust_rows, args.max_rows
172+
)
173+
)
174+
output_lines.extend(["", ""])
175+
output_lines.extend(
176+
_format_summary_block(
177+
"**Python (statements)**", "Stmts", python_totals, python_rows, args.max_rows
178+
)
179+
)
180+
output_lines.append("")
181+
output_lines.append(
182+
"_Generated automatically via `generate_coverage_comment.py`._"
183+
)
184+
185+
args.output.parent.mkdir(parents=True, exist_ok=True)
186+
args.output.write_text("\n".join(output_lines) + "\n", encoding="utf-8")
187+
return 0
188+
189+
190+
if __name__ == "__main__":
191+
raise SystemExit(main())

0 commit comments

Comments
 (0)