diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 4659b47..d9a6461 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -28,16 +28,117 @@ jobs: run: | # Fetch main branch git fetch origin main - # Try to get baseline from main, if it doesn't exist use empty baseline + # Get baseline from main (for comparison - must not decrease) git show origin/main:coverage-baseline.json > baseline-from-main.json 2>/dev/null || echo '{"lines":"0","functions":"0","branches":"0","statements":"0"}' > baseline-from-main.json - echo "📊 Baseline from main:" + echo "📊 Baseline from main branch:" cat baseline-from-main.json + - name: Get baseline from PR + run: | + # Get baseline from current PR branch (what developer committed) + if [ -f coverage-baseline.json ]; then + # Validate JSON format + if jq empty coverage-baseline.json 2>/dev/null; then + cp coverage-baseline.json baseline-from-pr.json + echo "📊 Baseline from PR (committed by developer):" + cat baseline-from-pr.json + else + echo "❌ ERROR: coverage-baseline.json is not valid JSON!" + echo "Please run: npm run coverage:update-baseline" + exit 1 + fi + else + echo "❌ ERROR: No coverage-baseline.json found in PR!" + echo "" + echo "You must run coverage locally and commit the baseline file." + echo "" + echo "📝 To fix: Run these commands locally and commit the result:" + echo " npm run coverage" + echo " npm run coverage:update-baseline" + echo " git add coverage-baseline.json" + echo " git commit -m 'chore: update coverage baseline'" + echo "" + exit 1 + fi + - name: Run coverage - run: npm run coverage + id: run_coverage + timeout-minutes: 15 + run: | + set -e # Exit immediately if coverage fails + npm run coverage + echo "✅ Coverage completed successfully" + + - name: Validate coverage + run: | + echo "==================================================" + echo "🔍 COVERAGE VALIDATION" + echo "==================================================" + + # Parse CI-generated coverage using dedicated script + CI_LINES=$(npx ts-node --files scripts/get-coverage-percentage.ts) + + # Get baselines + PR_LINES=$(jq -r .lines baseline-from-pr.json) + MAIN_LINES=$(jq -r .lines baseline-from-main.json) + + echo "" + echo "📊 Coverage Results:" + echo " CI (actual): $CI_LINES%" + echo " PR baseline: $PR_LINES%" + echo " Main baseline: $MAIN_LINES%" + echo "" + + # Check 1: CI must match PR baseline (developer ran coverage correctly) + echo "Check 1: Did developer run coverage locally?" + if [ "$CI_LINES" = "$PR_LINES" ]; then + echo " ✅ PASS - CI coverage matches PR baseline ($CI_LINES% == $PR_LINES%)" + else + echo " ❌ FAIL - CI coverage doesn't match PR baseline!" + echo "" + echo " Expected: $PR_LINES% (from your committed coverage-baseline.json)" + echo " Actual: $CI_LINES% (from fresh CI coverage run)" + echo "" + echo "💡 This means either:" + echo " 1. You forgot to run 'npm run coverage:update-baseline' locally" + echo " 2. You modified coverage-baseline.json manually (cheating)" + echo " 3. Your local coverage differs from CI (check .env setup)" + echo "" + echo "📝 To fix: Run these commands locally and commit the result:" + echo " npm run coverage" + echo " npm run coverage:update-baseline" + echo " git add coverage-baseline.json" + echo " git commit -m 'chore: update coverage baseline'" + echo "" + exit 1 + fi + + echo "" + + # Check 2: CI must be >= main baseline (coverage didn't decrease) + echo "Check 2: Did coverage decrease?" + if awk "BEGIN {exit !($CI_LINES >= $MAIN_LINES)}"; then + if awk "BEGIN {exit !($CI_LINES > $MAIN_LINES)}"; then + echo " ✅ PASS - Coverage improved! ($MAIN_LINES% → $CI_LINES%)" + else + echo " ✅ PASS - Coverage maintained ($CI_LINES%)" + fi + else + echo " ❌ FAIL - Coverage decreased!" + echo "" + echo " Main baseline: $MAIN_LINES%" + echo " Your PR: $CI_LINES%" + echo " Decrease: $(awk "BEGIN {print $MAIN_LINES - $CI_LINES}")%" + echo "" + echo "💡 Please add tests to maintain or improve coverage." + echo "" + exit 1 + fi - - name: Check coverage against main baseline - run: npm run coverage:check -- --baseline=baseline-from-main.json + echo "" + echo "==================================================" + echo "✅ ALL CHECKS PASSED" + echo "==================================================" - name: Upload coverage report (optional) if: always() diff --git a/.github/workflows/update-baseline.yml b/.github/workflows/update-baseline.yml deleted file mode 100644 index df89d01..0000000 --- a/.github/workflows/update-baseline.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Update Coverage Baseline - -# Trigger: Runs after a PR is merged to main -on: - push: - branches: [main] - -jobs: - update-baseline: - runs-on: ubuntu-latest - - steps: - - name: Checkout main - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Setup environment variables - run: cp .env.example .env - - - name: Run coverage - run: npm run coverage - - - name: Update baseline - run: npm run coverage:update-baseline - - - name: Check if baseline changed - id: check_changes - run: | - if git diff --quiet coverage-baseline.json; then - echo "changed=false" >> $GITHUB_OUTPUT - echo "📊 Coverage baseline unchanged" - else - echo "changed=true" >> $GITHUB_OUTPUT - COVERAGE=$(jq -r .lines coverage-baseline.json) - echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT - echo "📊 Coverage baseline will be updated to: $COVERAGE%" - fi - - - name: Commit and push new baseline - if: steps.check_changes.outputs.changed == 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add coverage-baseline.json - git commit -m "chore: update coverage baseline to ${{ steps.check_changes.outputs.coverage }}% - - Lines: ${{ steps.check_changes.outputs.coverage }}% - - [automated update after merge]" - git push diff --git a/COVERAGE.md b/COVERAGE.md index b2f1ea2..f6634eb 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -2,38 +2,38 @@ This project uses automated coverage checks to prevent test coverage from decreasing. -## How It Works: Two-Phase Automated Protection +## How It Works: Dual Validation -This system uses two GitHub Actions workflows that work together to enforce coverage without manual maintenance: +Developers run coverage locally and commit the baseline file. CI validates both that the developer ran coverage correctly AND that coverage didn't decrease. -### Phase 1: PR Check (`.github/workflows/coverage.yml`) +### Coverage Workflow (`.github/workflows/coverage.yml`) **Triggers:** Every pull request to main **What it does:** -1. Fetches `coverage-baseline.json` from **main branch** (not PR branch) -2. Runs `npm run coverage` on PR code to generate fresh coverage -3. Compares PR coverage against main's baseline -4. **Result:** - - ❌ Blocks merge if coverage drops below baseline - - ✅ Passes if coverage maintained or improved - - Shows detailed comparison in CI output - -**Security:** Developers cannot cheat by modifying the baseline file in their PR because CI always fetches the baseline from the main branch. - -### Phase 2: Auto-Update (`.github/workflows/update-baseline.yml`) -**Triggers:** Every push to main (after PR merge) - -**What it does:** -1. Runs `npm run coverage` on the new main branch code -2. Updates `coverage-baseline.json` with the new coverage values -3. Commits the updated baseline automatically (only if coverage changed) -4. Uses `github-actions[bot]` for the commit - -**Result:** The baseline automatically tracks the current coverage on main, requiring zero manual maintenance. +1. **Fetches baseline from main branch** - the current production baseline +2. **Reads baseline from PR branch** - the baseline you committed +3. **Runs coverage fresh in CI** - generates actual coverage from your code +4. **Performs two validations:** + + **Validation 1: Did you run coverage locally?** + - ✅ **PASS** if `CI coverage === PR baseline` (you ran coverage correctly) + - ❌ **FAIL** if `CI coverage !== PR baseline` (you forgot to run coverage or tampered with file) + + **Validation 2: Did coverage decrease?** + - ✅ **PASS** if `CI coverage >= main baseline` (coverage maintained or improved) + - ❌ **FAIL** if `CI coverage < main baseline` (coverage decreased) + +**Security Model:** +- ✅ **Can't skip running coverage** - CI checks if your committed baseline matches actual coverage +- ✅ **Can't decrease coverage** - CI checks if your coverage is below main's baseline +- ✅ **Can't cheat** - CI regenerates coverage fresh and validates against both baselines +- ✅ **Can't commit invalid baseline** - CI validates JSON format before processing +- ✅ **Can't skip baseline file** - CI fails immediately if baseline file is missing +- ✅ **Visible in PR** - Baseline changes are visible in the PR diff --- -**Combined Effect:** Coverage can only go up or stay the same, never down! This creates a "ratchet effect" where quality continuously improves. ✅ +**Effect:** Coverage can only go up or stay the same, never down! This creates a "ratchet effect" where quality continuously improves. ✅ ## Commands @@ -52,25 +52,40 @@ npm run coverage:update-baseline ### For Developers -When working on a PR: -1. Run `npm run coverage` to generate coverage report -2. Run `npm run coverage:check` to verify you maintained coverage -3. If check fails, add more tests until it passes +**IMPORTANT:** You must run coverage locally and commit the baseline file with your PR. + +**Step-by-step:** +1. Make your code changes +2. Run coverage locally: + ```bash + npm run coverage + ``` +3. Update the baseline file: + ```bash + npm run coverage:update-baseline + ``` +4. Commit the baseline file: + ```bash + git add coverage-baseline.json + git commit -m "chore: update coverage baseline" + ``` +5. Push your PR + +**What CI validates:** +- ✅ **Check 1:** Your committed baseline matches CI coverage (proves you ran coverage) +- ✅ **Check 2:** Your coverage is >= main's baseline (proves coverage didn't drop) + +**If CI fails:** +- **"No coverage-baseline.json found in PR"** → You forgot to commit the baseline file. Run steps 2-4 above and push. +- **"coverage-baseline.json is not valid JSON"** → The baseline file is corrupted. Run `npm run coverage:update-baseline` and commit. +- **"CI coverage doesn't match PR baseline"** → You forgot to update the baseline. Run steps 2-3 above and push. +- **"Coverage decreased"** → Add more tests to maintain or improve coverage. ### For Maintainers -**No manual work needed!** Phase 2 automatically: -- Runs coverage after each merge to main -- Updates `coverage-baseline.json` -- Commits the new baseline +**No special maintenance needed!** Developers commit their own baseline files. -You can manually update baseline if needed: -```bash -npm run coverage:update-baseline -git add coverage-baseline.json -git commit -m "chore: update coverage baseline" -git push -``` +The workflow only validates - it doesn't modify anything. When a PR merges, the updated baseline goes to main automatically. ## Current Coverage @@ -86,10 +101,12 @@ Current baseline (as of initial setup): - Uses Hardhat's built-in coverage tool (generates `coverage/lcov.info`) - Parses LCOV format to extract: lines, functions, branches, statements - Stores baseline in `coverage-baseline.json` at repository root -- Script: `scripts/check-coverage.ts` +- Scripts: + - `scripts/check-coverage.ts` - Local validation (compares coverage against baseline) + - `scripts/get-coverage-percentage.ts` - Extracts coverage percentage from lcov.info (used by CI) ### Environment Setup for CI -Both workflows copy `.env.example` to `.env` to enable fork tests with public RPC endpoints during coverage runs. +The workflow copies `.env.example` to `.env` to enable fork tests with public RPC endpoints during coverage runs. ### Branch Protection To enforce coverage checks, enable branch protection on main: diff --git a/coverage-baseline.json b/coverage-baseline.json index fa925e6..0778ddc 100644 --- a/coverage-baseline.json +++ b/coverage-baseline.json @@ -1,6 +1,6 @@ { - "lines": "96.03", - "functions": "98.27", - "branches": "86.19", - "statements": "96.03" + "lines": "96.88", + "functions": "98.57", + "branches": "87.76", + "statements": "96.88" } \ No newline at end of file diff --git a/scripts/get-coverage-percentage.ts b/scripts/get-coverage-percentage.ts new file mode 100644 index 0000000..fb33150 --- /dev/null +++ b/scripts/get-coverage-percentage.ts @@ -0,0 +1,38 @@ +#!/usr/bin/env ts-node + +import fs from "fs"; +import path from "path"; + +/** + * Extracts line coverage percentage from lcov.info + * Outputs just the percentage number (e.g., "96.03") + */ + +const lcovPath = path.join(__dirname, "..", "coverage", "lcov.info"); + +// Check if file exists +if (!fs.existsSync(lcovPath)) { + console.error("Error: coverage/lcov.info not found"); + process.exit(1); +} + +// Read and parse lcov file +const content = fs.readFileSync(lcovPath, "utf8"); +const lines = content.split("\n"); + +let linesFound = 0; +let linesHit = 0; + +for (const line of lines) { + if (line.startsWith("LF:")) { + linesFound += parseInt(line.substring(3), 10); + } else if (line.startsWith("LH:")) { + linesHit += parseInt(line.substring(3), 10); + } +} + +// Calculate percentage +const percentage = linesFound > 0 ? (linesHit / linesFound * 100).toFixed(2) : "0"; + +// Output only the percentage (no extra text) +console.log(percentage);