Tests and coverage

.github/workflows/ci.yml

@@ -22,8 +22,84 @@ jobs:
           nix_path: nixpkgs=channel:nixos-25.05
           extra_nix_config: |
             experimental-features = nix-command flakes
-      - name: Build Rust module and run tests via Nix
-        run: nix develop --command bash -lc 'just venv ${{matrix.python-version}} dev test'
+      - name: Prepare dev environment
+        run: nix develop --command bash -lc 'just venv ${{matrix.python-version}} dev'
+
+      - name: Rust tests
+        run: nix develop --command bash -lc 'just cargo-test'
+
+      - name: Python tests
+        run: nix develop --command bash -lc 'just py-test'
+
+  coverage:
+    name: Coverage (Python 3.12)
+    needs: nix-tests
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cachix/install-nix-action@v27
+        with:
+          nix_path: nixpkgs=channel:nixos-25.05
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: Prepare dev environment (Python 3.12)
+        run: nix develop --command bash -lc 'just venv 3.12 dev'
+
+      - name: Collect coverage
+        id: coverage-run
+        run: nix develop --command bash -lc 'just coverage'
+
+      - name: Generate coverage comment
+        if: steps.coverage-run.outcome == 'success'
+        run: |
+          ROOT="$(pwd)"
+          nix develop --command bash -lc "python3 codetracer-python-recorder/scripts/generate_coverage_comment.py \
+            --rust-summary codetracer-python-recorder/target/coverage/rust/summary.json \
+            --python-json codetracer-python-recorder/target/coverage/python/coverage.json \
+            --output codetracer-python-recorder/target/coverage/coverage-comment.md \
+            --repo-root \"${ROOT}\""
+
+      - name: Upload Rust coverage artefacts
+        if: steps.coverage-run.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-coverage
+          path: |
+            codetracer-python-recorder/target/coverage/rust/lcov.info
+            codetracer-python-recorder/target/coverage/rust/summary.json
+          if-no-files-found: warn
+
+      - name: Upload Python coverage artefacts
+        if: steps.coverage-run.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-coverage
+          path: |
+            codetracer-python-recorder/target/coverage/python/coverage.xml
+            codetracer-python-recorder/target/coverage/python/coverage.json
+          if-no-files-found: warn
+
+      - name: Upload coverage comment
+        if: steps.coverage-run.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-comment
+          path: codetracer-python-recorder/target/coverage/coverage-comment.md
+          if-no-files-found: warn
+
+      - name: Comment coverage summary
+        if: github.event_name == 'pull_request' && steps.coverage-run.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          identifier: coverage-summary
+          body-path: codetracer-python-recorder/target/coverage/coverage-comment.md
 
   # rust-tests:
   #   name: Rust module test on ${{ matrix.os }} (Python ${{ matrix.python-version }})

Justfile

@@ -41,12 +41,30 @@ cargo-test:
     uv run cargo nextest run --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features
 
 py-test:
-    uv run --group dev --group test pytest
+    uv run --group dev --group test pytest codetracer-python-recorder/tests/python codetracer-pure-python-recorder
 
 # Run tests only on the pure recorder
 test-pure:
     uv run --group dev --group test pytest codetracer-pure-python-recorder
 
+# Generate combined coverage artefacts for both crates
+coverage:
+    just coverage-rust
+    just coverage-python
+
+coverage-rust:
+    mkdir -p codetracer-python-recorder/target/coverage/rust
+    LLVM_COV="$(command -v llvm-cov)" LLVM_PROFDATA="$(command -v llvm-profdata)" \
+        uv run cargo llvm-cov nextest --manifest-path codetracer-python-recorder/Cargo.toml --no-default-features --lcov --output-path codetracer-python-recorder/target/coverage/rust/lcov.info
+    LLVM_COV="$(command -v llvm-cov)" LLVM_PROFDATA="$(command -v llvm-profdata)" \
+        uv run cargo llvm-cov report --summary-only --json --manifest-path codetracer-python-recorder/Cargo.toml --output-path codetracer-python-recorder/target/coverage/rust/summary.json
+    python3 codetracer-python-recorder/scripts/render_rust_coverage_summary.py \
+        codetracer-python-recorder/target/coverage/rust/summary.json --root "$(pwd)"
+
+coverage-python:
+    mkdir -p codetracer-python-recorder/target/coverage/python
+    uv run --group dev --group test pytest --cov=codetracer_python_recorder --cov-report=term --cov-report=xml:codetracer-python-recorder/target/coverage/python/coverage.xml --cov-report=json:codetracer-python-recorder/target/coverage/python/coverage.json codetracer-python-recorder/tests/python
+
 # Build the module in release mode
 build:
     just venv \
@@ -64,4 +82,3 @@ test-all:
         file="${file[0]}"; \
         uv run -p "python3.$v" --with "${file}" --with pytest -- pytest -q; \
     done
-

README.md

@@ -45,6 +45,15 @@ Basic workflow:
   - from codetracer_python_recorder import hello
   - hello()
 
+#### Testing & Coverage
+
+- Run the full split test suite (Rust nextest + Python pytest): `just test`
+- Run only Rust integration/unit tests: `just cargo-test`
+- Run only Python tests (including the pure-Python recorder to guard regressions): `just py-test`
+- Collect coverage artefacts locally (LCOV + Cobertura/JSON): `just coverage`
+
+The CI workflow mirrors these commands. Pull requests get an automated comment with the latest Rust/Python coverage tables and downloadable artefacts (`lcov.info`, `coverage.xml`, `coverage.json`).
+
 ### Future directions
 
 The current Python support is an unfinished prototype. We can finish it. In the future, it may be expanded to function in a way to similar to the more complete implementations, e.g. [Noir](https://github.com/blocksense-network/noir/tree/blocksense/tooling/tracer).

codetracer-python-recorder/scripts/generate_coverage_comment.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Generate a Markdown coverage summary comment for CI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+from typing import Dict, Iterable, List, Tuple
+
+from render_rust_coverage_summary import load_summary as load_rust_summary
+
+Row = Tuple[str, int, int, float]
+
+
+def parse_args(argv: Iterable[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__ or "")
+    parser.add_argument(
+        "--rust-summary",
+        type=pathlib.Path,
+        required=True,
+        help="Path to cargo-llvm-cov JSON summary",
+    )
+    parser.add_argument(
+        "--python-json",
+        type=pathlib.Path,
+        required=True,
+        help="Path to coverage.py JSON report",
+    )
+    parser.add_argument(
+        "--output",
+        type=pathlib.Path,
+        required=True,
+        help="Output Markdown file for the PR comment",
+    )
+    parser.add_argument(
+        "--repo-root",
+        type=pathlib.Path,
+        default=pathlib.Path.cwd(),
+        help="Repository root used to relativise file paths (default: current working directory)",
+    )
+    parser.add_argument(
+        "--max-rows",
+        type=int,
+        default=20,
+        help="Maximum number of per-file rows to display for each language (default: 20).",
+    )
+    return parser.parse_args(argv)
+
+
+def _select_rows(rows: List[Row], max_rows: int) -> Tuple[List[Row], bool]:
+    if max_rows <= 0 or len(rows) <= max_rows:
+        return sorted(rows, key=lambda item: item[0]), False
+
+    priority_sorted = sorted(rows, key=lambda item: (-item[2], item[0]))
+    trimmed = priority_sorted[:max_rows]
+    return sorted(trimmed, key=lambda item: item[0]), True
+
+
+def _format_table(rows: List[Row], headers: Tuple[str, str, str, str]) -> List[str]:
+    lines = [
+        f"| {headers[0]} | {headers[1]} | {headers[2]} | {headers[3]} |",
+        "| --- | ---: | ---: | ---: |",
+    ]
+    for name, total, missed, percent in rows:
+        lines.append(
+            f"| `{name}` | {total:,} | {missed:,} | {percent:5.1f}% |"
+        )
+    return lines
+
+
+def _load_python_rows(
+    report_path: pathlib.Path,
+    repo_root: pathlib.Path,
+) -> Tuple[List[Row], Dict[str, float]]:
+    try:
+        payload = json.loads(report_path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise SystemExit(f"Python coverage JSON not found: {report_path}") from exc
+
+    repo_root = repo_root.resolve()
+    rows: List[Row] = []
+
+    for path_str, details in payload.get("files", {}).items():
+        summary = (details.get("summary") or {})
+        total = int(summary.get("num_statements", 0))
+        missed = int(summary.get("missing_lines", 0))
+        percent = float(summary.get("percent_covered", 0.0))
+
+        if total == 0 and missed == 0:
+            continue
+
+        file_path = pathlib.Path(path_str)
+        if not file_path.is_absolute():
+            file_path = (repo_root / file_path).resolve()
+        else:
+            file_path = file_path.resolve()
+
+        try:
+            rel_path = file_path.relative_to(repo_root)
+        except ValueError:
+            continue
+
+        rows.append((rel_path.as_posix(), total, missed, percent))
+
+    rows.sort(key=lambda item: item[0])
+
+    totals = payload.get("totals", {})
+    return rows, {
+        "total": float(totals.get("num_statements", 0)),
+        "covered": float(totals.get("covered_lines", 0)),
+        "missed": float(totals.get("missing_lines", 0)),
+        "percent": float(totals.get("percent_covered", 0.0)),
+    }
+
+
+def _load_rust_rows(
+    summary_path: pathlib.Path,
+    repo_root: pathlib.Path,
+) -> Tuple[List[Row], Dict[str, float]]:
+    rows, totals = load_rust_summary(summary_path, repo_root)
+    # Normalise totals dict to expected keys
+    total = float(totals.get("count", 0))
+    covered = float(totals.get("covered", 0))
+    missed = float(totals.get("notcovered", total - covered))
+    return rows, {
+        "total": total,
+        "covered": covered,
+        "missed": missed,
+        "percent": float(totals.get("percent", 0.0)),
+    }
+
+
+def _format_summary_block(
+    heading: str,
+    column_label: str,
+    totals: Dict[str, float],
+    rows: List[Row],
+    max_rows: int,
+) -> List[str]:
+    display_rows, truncated = _select_rows(rows, max_rows)
+    lines = [heading]
+    total = int(totals.get("total", 0))
+    covered = int(totals.get("covered", 0))
+    missed = int(totals.get("missed", total - covered))
+    percent = totals.get("percent", 0.0)
+    lines.append(
+        f"**{percent:0.1f}%** covered ({covered:,} / {total:,} | {missed:,} missed)"
+    )
+    lines.extend(
+        _format_table(display_rows, ("File", column_label, "Miss", "Cover"))
+    )
+    if truncated:
+        lines.append(
+            f"_Showing top {max_rows} entries by missed lines (of {len(rows)} total)._"
+        )
+    return lines
+
+
+def main(argv: Iterable[str] | None = None) -> int:
+    args = parse_args(argv)
+    repo_root = args.repo_root.resolve()
+
+    rust_rows, rust_totals = _load_rust_rows(args.rust_summary, repo_root)
+    python_rows, python_totals = _load_python_rows(args.python_json, repo_root)
+
+    output_lines: List[str] = ["### Coverage Summary", ""]
+
+    output_lines.extend(
+        _format_summary_block(
+            "**Rust (lines)**", "Lines", rust_totals, rust_rows, args.max_rows
+        )
+    )
+    output_lines.extend(["", ""])
+    output_lines.extend(
+        _format_summary_block(
+            "**Python (statements)**", "Stmts", python_totals, python_rows, args.max_rows
+        )
+    )
+    output_lines.append("")
+    output_lines.append(
+        "_Generated automatically via `generate_coverage_comment.py`._"
+    )
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text("\n".join(output_lines) + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())