diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..4659b47 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,47 @@ +name: Coverage Check + +on: + pull_request: + branches: [main, master] + +jobs: + coverage: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Setup environment variables + run: cp .env.example .env + + - name: Get baseline from main branch + run: | + # Fetch main branch + git fetch origin main + # Try to get baseline from main, if it doesn't exist use empty baseline + git show origin/main:coverage-baseline.json > baseline-from-main.json 2>/dev/null || echo '{"lines":"0","functions":"0","branches":"0","statements":"0"}' > baseline-from-main.json + echo "šŸ“Š Baseline from main:" + cat baseline-from-main.json + + - name: Run coverage + run: npm run coverage + + - name: Check coverage against main baseline + run: npm run coverage:check -- --baseline=baseline-from-main.json + + - name: Upload coverage report (optional) + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: coverage/ diff --git a/.github/workflows/update-baseline.yml b/.github/workflows/update-baseline.yml new file mode 100644 index 0000000..df89d01 --- /dev/null +++ b/.github/workflows/update-baseline.yml @@ -0,0 +1,60 @@ +name: Update Coverage Baseline + +# Trigger: Runs after a PR is merged to main +on: + push: + branches: [main] + +jobs: + update-baseline: + runs-on: ubuntu-latest + + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Setup environment variables + run: cp .env.example .env + + - name: Run coverage + run: npm run coverage + + - name: Update baseline + run: npm run coverage:update-baseline + + - name: Check if baseline changed + id: check_changes + run: | + if git diff --quiet coverage-baseline.json; then + echo "changed=false" >> $GITHUB_OUTPUT + echo "šŸ“Š Coverage baseline unchanged" + else + echo "changed=true" >> $GITHUB_OUTPUT + COVERAGE=$(jq -r .lines coverage-baseline.json) + echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT + echo "šŸ“Š Coverage baseline will be updated to: $COVERAGE%" + fi + + - name: Commit and push new baseline + if: steps.check_changes.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add coverage-baseline.json + git commit -m "chore: update coverage baseline to ${{ steps.check_changes.outputs.coverage }}% + + Lines: ${{ steps.check_changes.outputs.coverage }}% + + [automated update after merge]" + git push diff --git a/COVERAGE.md b/COVERAGE.md new file mode 100644 index 0000000..b2f1ea2 --- /dev/null +++ b/COVERAGE.md @@ -0,0 +1,99 @@ +# Coverage Gate System + +This project uses automated coverage checks to prevent test coverage from decreasing. + +## How It Works: Two-Phase Automated Protection + +This system uses two GitHub Actions workflows that work together to enforce coverage without manual maintenance: + +### Phase 1: PR Check (`.github/workflows/coverage.yml`) +**Triggers:** Every pull request to main + +**What it does:** +1. Fetches `coverage-baseline.json` from **main branch** (not PR branch) +2. Runs `npm run coverage` on PR code to generate fresh coverage +3. Compares PR coverage against main's baseline +4. **Result:** + - āŒ Blocks merge if coverage drops below baseline + - āœ… Passes if coverage maintained or improved + - Shows detailed comparison in CI output + +**Security:** Developers cannot cheat by modifying the baseline file in their PR because CI always fetches the baseline from the main branch. + +### Phase 2: Auto-Update (`.github/workflows/update-baseline.yml`) +**Triggers:** Every push to main (after PR merge) + +**What it does:** +1. Runs `npm run coverage` on the new main branch code +2. Updates `coverage-baseline.json` with the new coverage values +3. Commits the updated baseline automatically (only if coverage changed) +4. Uses `github-actions[bot]` for the commit + +**Result:** The baseline automatically tracks the current coverage on main, requiring zero manual maintenance. + +--- + +**Combined Effect:** Coverage can only go up or stay the same, never down! This creates a "ratchet effect" where quality continuously improves. āœ… + +## Commands + +```bash +# Run coverage +npm run coverage + +# Check if coverage maintained (compares against baseline) +npm run coverage:check + +# Update baseline (after improving coverage) +npm run coverage:update-baseline +``` + +## Workflow + +### For Developers + +When working on a PR: +1. Run `npm run coverage` to generate coverage report +2. Run `npm run coverage:check` to verify you maintained coverage +3. If check fails, add more tests until it passes + +### For Maintainers + +**No manual work needed!** Phase 2 automatically: +- Runs coverage after each merge to main +- Updates `coverage-baseline.json` +- Commits the new baseline + +You can manually update baseline if needed: +```bash +npm run coverage:update-baseline +git add coverage-baseline.json +git commit -m "chore: update coverage baseline" +git push +``` + +## Current Coverage + +Current baseline (as of initial setup): +- **Lines:** 96.03% +- **Functions:** 98.27% +- **Branches:** 86.19% +- **Statements:** 96.03% + +## Technical Details + +### Coverage Calculation +- Uses Hardhat's built-in coverage tool (generates `coverage/lcov.info`) +- Parses LCOV format to extract: lines, functions, branches, statements +- Stores baseline in `coverage-baseline.json` at repository root +- Script: `scripts/check-coverage.ts` + +### Environment Setup for CI +Both workflows copy `.env.example` to `.env` to enable fork tests with public RPC endpoints during coverage runs. + +### Branch Protection +To enforce coverage checks, enable branch protection on main: +1. GitHub Settings ā†’ Branches ā†’ Branch protection rules +2. Add rule for `main` branch +3. Enable "Require status checks to pass before merging" +4. Select "coverage" as required check diff --git a/coverage-baseline.json b/coverage-baseline.json new file mode 100644 index 0000000..fa925e6 --- /dev/null +++ b/coverage-baseline.json @@ -0,0 +1,6 @@ +{ + "lines": "96.03", + "functions": "98.27", + "branches": "86.19", + "statements": "96.03" +} \ No newline at end of file diff --git a/package.json b/package.json index 43d1f48..fa3cf11 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,9 @@ "test:deploy": "ts-node --files ./scripts/deploy.ts", "test:ethereum": "FORK_TEST=ETHEREUM hardhat test --typecheck ./specific-fork-test/ethereum/*.ts", "test:scripts": "SCRIPT_ENV=CI DEPLOY_ID=CI ts-node --files ./scripts/test.ts", - "coverage": "hardhat coverage" + "coverage": "hardhat coverage", + "coverage:check": "ts-node --files scripts/check-coverage.ts", + "coverage:update-baseline": "ts-node --files scripts/check-coverage.ts --update-baseline" }, "repository": { "type": "git", diff --git a/scripts/check-coverage.ts b/scripts/check-coverage.ts new file mode 100644 index 0000000..3c2bf27 --- /dev/null +++ b/scripts/check-coverage.ts @@ -0,0 +1,119 @@ +#!/usr/bin/env ts-node + +import fs from "fs"; +import path from "path"; + +interface CoverageData { + lines: string; + functions: string; + branches: string; + statements: string; +} + +/** + * Parses coverage from lcov.info file + */ +function parseLcovCoverage(lcovPath: string): CoverageData { + const content = fs.readFileSync(lcovPath, "utf8"); + + let linesFound = 0; + let linesHit = 0; + let functionsFound = 0; + let functionsHit = 0; + let branchesFound = 0; + let branchesHit = 0; + + const lines = content.split("\n"); + for (const line of lines) { + if (line.startsWith("LF:")) { + linesFound += parseInt(line.substring(3), 10); + } else if (line.startsWith("LH:")) { + linesHit += parseInt(line.substring(3), 10); + } else if (line.startsWith("BRF:")) { + branchesFound += parseInt(line.substring(4), 10); + } else if (line.startsWith("BRH:")) { + branchesHit += parseInt(line.substring(4), 10); + } else if (line.startsWith("FNF:")) { + functionsFound += parseInt(line.substring(4), 10); + } else if (line.startsWith("FNH:")) { + functionsHit += parseInt(line.substring(4), 10); + } + } + + return { + lines: linesFound > 0 ? (linesHit / linesFound * 100).toFixed(2) : "0", + functions: functionsFound > 0 ? (functionsHit / functionsFound * 100).toFixed(2) : "0", + branches: branchesFound > 0 ? (branchesHit / branchesFound * 100).toFixed(2) : "0", + statements: linesFound > 0 ? (linesHit / linesFound * 100).toFixed(2) : "0" + }; +} + +// Main +const lcovPath = path.join(__dirname, "..", "coverage", "lcov.info"); + +// Check if custom baseline path provided (for CI to compare against main) +const baselineArg = process.argv.find(arg => arg.startsWith("--baseline=")); +const baselinePath = baselineArg + ? baselineArg.split("=")[1] + : path.join(__dirname, "..", "coverage-baseline.json"); + +// Check if we're updating baseline +const isUpdatingBaseline = process.argv.includes("--update-baseline"); + +if (!fs.existsSync(lcovPath)) { + console.error("āŒ Coverage file not found. Run: npm run coverage"); + process.exit(1); +} + +const current = parseLcovCoverage(lcovPath); + +// If updating baseline, save and exit +if (isUpdatingBaseline) { + fs.writeFileSync(baselinePath, JSON.stringify(current, null, 2)); + console.log("\nāœ… Coverage baseline updated:"); + console.log(` Lines: ${current.lines}%`); + console.log(` Functions: ${current.functions}%`); + console.log(` Branches: ${current.branches}%`); + console.log(` Statements: ${current.statements}%\n`); + process.exit(0); +} + +// Load baseline +let baseline: CoverageData = {lines: "0", functions: "0", branches: "0", statements: "0"}; +if (fs.existsSync(baselinePath)) { + baseline = JSON.parse(fs.readFileSync(baselinePath, "utf8")) as CoverageData; +} + +// Display comparison +console.log("\nšŸ“Š Coverage Comparison:"); +console.log("ā”€".repeat(50)); +console.log(`Lines: ${baseline.lines}% ā†’ ${current.lines}%`); +console.log(`Functions: ${baseline.functions}% ā†’ ${current.functions}%`); +console.log(`Branches: ${baseline.branches}% ā†’ ${current.branches}%`); +console.log(`Statements: ${baseline.statements}% ā†’ ${current.statements}%`); +console.log("ā”€".repeat(50)); + +// Check for drops +const drops: string[] = []; +if (parseFloat(current.lines) < parseFloat(baseline.lines)) { + drops.push(`Lines dropped: ${baseline.lines}% ā†’ ${current.lines}%`); +} +if (parseFloat(current.functions) < parseFloat(baseline.functions)) { + drops.push(`Functions dropped: ${baseline.functions}% ā†’ ${current.functions}%`); +} +if (parseFloat(current.branches) < parseFloat(baseline.branches)) { + drops.push(`Branches dropped: ${baseline.branches}% ā†’ ${current.branches}%`); +} +if (parseFloat(current.statements) < parseFloat(baseline.statements)) { + drops.push(`Statements dropped: ${baseline.statements}% ā†’ ${current.statements}%`); +} + +if (drops.length > 0) { + console.log("\nāŒ Coverage decreased:\n"); + drops.forEach((drop: string) => console.log(` ā€¢ ${drop}`)); + console.log("\nšŸ’” Please add tests to maintain or improve coverage.\n"); + process.exit(1); +} + +console.log("\nāœ… Coverage maintained or improved!\n"); +process.exit(0);