Skip to content

Commit 9cb29ba

Browse files
LyonSsSLyonSsS
authored
Add coverage gating (#150)
* enhanced hardhat coverage for coverage gating * adding signature * testing git commti signature 2 * update github actions to latest versions * added .env.exmaples to .env file * updated coverage gate logic * updated COVERAGE.md explination
1 parent 7187ffa commit 9cb29ba

File tree

6 files changed

+334
-1
lines changed

6 files changed

+334
-1
lines changed

‎.github/workflows/coverage.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Coverage Check
2+
3+
on:
4+
pull_request:
5+
branches: [main, master]
6+
7+
jobs:
8+
coverage:
9+
runs-on: ubuntu-latest
10+
11+
steps:
12+
- name: Checkout code
13+
uses: actions/checkout@v4
14+
15+
- name: Setup Node.js
16+
uses: actions/setup-node@v4
17+
with:
18+
node-version: 22
19+
cache: 'npm'
20+
21+
- name: Install dependencies
22+
run: npm ci
23+
24+
- name: Setup environment variables
25+
run: cp .env.example .env
26+
27+
- name: Get baseline from main branch
28+
run: |
29+
# Fetch main branch
30+
git fetch origin main
31+
# Try to get baseline from main, if it doesn't exist use empty baseline
32+
git show origin/main:coverage-baseline.json > baseline-from-main.json 2>/dev/null || echo '{"lines":"0","functions":"0","branches":"0","statements":"0"}' > baseline-from-main.json
33+
echo "📊 Baseline from main:"
34+
cat baseline-from-main.json
35+
36+
- name: Run coverage
37+
run: npm run coverage
38+
39+
- name: Check coverage against main baseline
40+
run: npm run coverage:check -- --baseline=baseline-from-main.json
41+
42+
- name: Upload coverage report (optional)
43+
if: always()
44+
uses: actions/upload-artifact@v4
45+
with:
46+
name: coverage-report
47+
path: coverage/

‎.github/workflows/update-baseline.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Update Coverage Baseline
2+
3+
# Trigger: Runs after a PR is merged to main
4+
on:
5+
push:
6+
branches: [main]
7+
8+
jobs:
9+
update-baseline:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Checkout main
14+
uses: actions/checkout@v4
15+
with:
16+
token: ${{ secrets.GITHUB_TOKEN }}
17+
18+
- name: Setup Node.js
19+
uses: actions/setup-node@v4
20+
with:
21+
node-version: 22
22+
cache: 'npm'
23+
24+
- name: Install dependencies
25+
run: npm ci
26+
27+
- name: Setup environment variables
28+
run: cp .env.example .env
29+
30+
- name: Run coverage
31+
run: npm run coverage
32+
33+
- name: Update baseline
34+
run: npm run coverage:update-baseline
35+
36+
- name: Check if baseline changed
37+
id: check_changes
38+
run: |
39+
if git diff --quiet coverage-baseline.json; then
40+
echo "changed=false" >> $GITHUB_OUTPUT
41+
echo "📊 Coverage baseline unchanged"
42+
else
43+
echo "changed=true" >> $GITHUB_OUTPUT
44+
COVERAGE=$(jq -r .lines coverage-baseline.json)
45+
echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT
46+
echo "📊 Coverage baseline will be updated to: $COVERAGE%"
47+
fi
48+
49+
- name: Commit and push new baseline
50+
if: steps.check_changes.outputs.changed == 'true'
51+
run: |
52+
git config user.name "github-actions[bot]"
53+
git config user.email "github-actions[bot]@users.noreply.github.com"
54+
git add coverage-baseline.json
55+
git commit -m "chore: update coverage baseline to ${{ steps.check_changes.outputs.coverage }}%
56+
57+
Lines: ${{ steps.check_changes.outputs.coverage }}%
58+
59+
[automated update after merge]"
60+
git push

‎COVERAGE.md‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Coverage Gate System
2+
3+
This project uses automated coverage checks to prevent test coverage from decreasing.
4+
5+
## How It Works: Two-Phase Automated Protection
6+
7+
This system uses two GitHub Actions workflows that work together to enforce coverage without manual maintenance:
8+
9+
### Phase 1: PR Check (`.github/workflows/coverage.yml`)
10+
**Triggers:** Every pull request to main
11+
12+
**What it does:**
13+
1. Fetches `coverage-baseline.json` from **main branch** (not PR branch)
14+
2. Runs `npm run coverage` on PR code to generate fresh coverage
15+
3. Compares PR coverage against main's baseline
16+
4. **Result:**
17+
- ❌ Blocks merge if coverage drops below baseline
18+
- ✅ Passes if coverage maintained or improved
19+
- Shows detailed comparison in CI output
20+
21+
**Security:** Developers cannot cheat by modifying the baseline file in their PR because CI always fetches the baseline from the main branch.
22+
23+
### Phase 2: Auto-Update (`.github/workflows/update-baseline.yml`)
24+
**Triggers:** Every push to main (after PR merge)
25+
26+
**What it does:**
27+
1. Runs `npm run coverage` on the new main branch code
28+
2. Updates `coverage-baseline.json` with the new coverage values
29+
3. Commits the updated baseline automatically (only if coverage changed)
30+
4. Uses `github-actions[bot]` for the commit
31+
32+
**Result:** The baseline automatically tracks the current coverage on main, requiring zero manual maintenance.
33+
34+
---
35+
36+
**Combined Effect:** Coverage can only go up or stay the same, never down! This creates a "ratchet effect" where quality continuously improves. ✅
37+
38+
## Commands
39+
40+
```bash
41+
# Run coverage
42+
npm run coverage
43+
44+
# Check if coverage maintained (compares against baseline)
45+
npm run coverage:check
46+
47+
# Update baseline (after improving coverage)
48+
npm run coverage:update-baseline
49+
```
50+
51+
## Workflow
52+
53+
### For Developers
54+
55+
When working on a PR:
56+
1. Run `npm run coverage` to generate coverage report
57+
2. Run `npm run coverage:check` to verify you maintained coverage
58+
3. If check fails, add more tests until it passes
59+
60+
### For Maintainers
61+
62+
**No manual work needed!** Phase 2 automatically:
63+
- Runs coverage after each merge to main
64+
- Updates `coverage-baseline.json`
65+
- Commits the new baseline
66+
67+
You can manually update baseline if needed:
68+
```bash
69+
npm run coverage:update-baseline
70+
git add coverage-baseline.json
71+
git commit -m "chore: update coverage baseline"
72+
git push
73+
```
74+
75+
## Current Coverage
76+
77+
Current baseline (as of initial setup):
78+
- **Lines:** 96.03%
79+
- **Functions:** 98.27%
80+
- **Branches:** 86.19%
81+
- **Statements:** 96.03%
82+
83+
## Technical Details
84+
85+
### Coverage Calculation
86+
- Uses Hardhat's built-in coverage tool (generates `coverage/lcov.info`)
87+
- Parses LCOV format to extract: lines, functions, branches, statements
88+
- Stores baseline in `coverage-baseline.json` at repository root
89+
- Script: `scripts/check-coverage.ts`
90+
91+
### Environment Setup for CI
92+
Both workflows copy `.env.example` to `.env` to enable fork tests with public RPC endpoints during coverage runs.
93+
94+
### Branch Protection
95+
To enforce coverage checks, enable branch protection on main:
96+
1. GitHub Settings → Branches → Branch protection rules
97+
2. Add rule for `main` branch
98+
3. Enable "Require status checks to pass before merging"
99+
4. Select "coverage" as required check

‎coverage-baseline.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"lines": "96.03",
3+
"functions": "98.27",
4+
"branches": "86.19",
5+
"statements": "96.03"
6+
}

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,9 @@
139139
"test:deploy": "ts-node --files ./scripts/deploy.ts",
140140
"test:ethereum": "FORK_TEST=ETHEREUM hardhat test --typecheck ./specific-fork-test/ethereum/*.ts",
141141
"test:scripts": "SCRIPT_ENV=CI DEPLOY_ID=CI ts-node --files ./scripts/test.ts",
142-
"coverage": "hardhat coverage"
142+
"coverage": "hardhat coverage",
143+
"coverage:check": "ts-node --files scripts/check-coverage.ts",
144+
"coverage:update-baseline": "ts-node --files scripts/check-coverage.ts --update-baseline"
143145
},
144146
"repository": {
145147
"type": "git",

‎scripts/check-coverage.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env ts-node
2+
3+
import fs from "fs";
4+
import path from "path";
5+
6+
interface CoverageData {
7+
lines: string;
8+
functions: string;
9+
branches: string;
10+
statements: string;
11+
}
12+
13+
/**
14+
* Parses coverage from lcov.info file
15+
*/
16+
function parseLcovCoverage(lcovPath: string): CoverageData {
17+
const content = fs.readFileSync(lcovPath, "utf8");
18+
19+
let linesFound = 0;
20+
let linesHit = 0;
21+
let functionsFound = 0;
22+
let functionsHit = 0;
23+
let branchesFound = 0;
24+
let branchesHit = 0;
25+
26+
const lines = content.split("\n");
27+
for (const line of lines) {
28+
if (line.startsWith("LF:")) {
29+
linesFound += parseInt(line.substring(3), 10);
30+
} else if (line.startsWith("LH:")) {
31+
linesHit += parseInt(line.substring(3), 10);
32+
} else if (line.startsWith("BRF:")) {
33+
branchesFound += parseInt(line.substring(4), 10);
34+
} else if (line.startsWith("BRH:")) {
35+
branchesHit += parseInt(line.substring(4), 10);
36+
} else if (line.startsWith("FNF:")) {
37+
functionsFound += parseInt(line.substring(4), 10);
38+
} else if (line.startsWith("FNH:")) {
39+
functionsHit += parseInt(line.substring(4), 10);
40+
}
41+
}
42+
43+
return {
44+
lines: linesFound > 0 ? (linesHit / linesFound * 100).toFixed(2) : "0",
45+
functions: functionsFound > 0 ? (functionsHit / functionsFound * 100).toFixed(2) : "0",
46+
branches: branchesFound > 0 ? (branchesHit / branchesFound * 100).toFixed(2) : "0",
47+
statements: linesFound > 0 ? (linesHit / linesFound * 100).toFixed(2) : "0"
48+
};
49+
}
50+
51+
// Main
52+
const lcovPath = path.join(__dirname, "..", "coverage", "lcov.info");
53+
54+
// Check if custom baseline path provided (for CI to compare against main)
55+
const baselineArg = process.argv.find(arg => arg.startsWith("--baseline="));
56+
const baselinePath = baselineArg
57+
? baselineArg.split("=")[1]
58+
: path.join(__dirname, "..", "coverage-baseline.json");
59+
60+
// Check if we're updating baseline
61+
const isUpdatingBaseline = process.argv.includes("--update-baseline");
62+
63+
if (!fs.existsSync(lcovPath)) {
64+
console.error("❌ Coverage file not found. Run: npm run coverage");
65+
process.exit(1);
66+
}
67+
68+
const current = parseLcovCoverage(lcovPath);
69+
70+
// If updating baseline, save and exit
71+
if (isUpdatingBaseline) {
72+
fs.writeFileSync(baselinePath, JSON.stringify(current, null, 2));
73+
console.log("\n✅ Coverage baseline updated:");
74+
console.log(` Lines: ${current.lines}%`);
75+
console.log(` Functions: ${current.functions}%`);
76+
console.log(` Branches: ${current.branches}%`);
77+
console.log(` Statements: ${current.statements}%\n`);
78+
process.exit(0);
79+
}
80+
81+
// Load baseline
82+
let baseline: CoverageData = {lines: "0", functions: "0", branches: "0", statements: "0"};
83+
if (fs.existsSync(baselinePath)) {
84+
baseline = JSON.parse(fs.readFileSync(baselinePath, "utf8")) as CoverageData;
85+
}
86+
87+
// Display comparison
88+
console.log("\n📊 Coverage Comparison:");
89+
console.log("─".repeat(50));
90+
console.log(`Lines: ${baseline.lines}% → ${current.lines}%`);
91+
console.log(`Functions: ${baseline.functions}% → ${current.functions}%`);
92+
console.log(`Branches: ${baseline.branches}% → ${current.branches}%`);
93+
console.log(`Statements: ${baseline.statements}% → ${current.statements}%`);
94+
console.log("─".repeat(50));
95+
96+
// Check for drops
97+
const drops: string[] = [];
98+
if (parseFloat(current.lines) < parseFloat(baseline.lines)) {
99+
drops.push(`Lines dropped: ${baseline.lines}% → ${current.lines}%`);
100+
}
101+
if (parseFloat(current.functions) < parseFloat(baseline.functions)) {
102+
drops.push(`Functions dropped: ${baseline.functions}% → ${current.functions}%`);
103+
}
104+
if (parseFloat(current.branches) < parseFloat(baseline.branches)) {
105+
drops.push(`Branches dropped: ${baseline.branches}% → ${current.branches}%`);
106+
}
107+
if (parseFloat(current.statements) < parseFloat(baseline.statements)) {
108+
drops.push(`Statements dropped: ${baseline.statements}% → ${current.statements}%`);
109+
}
110+
111+
if (drops.length > 0) {
112+
console.log("\n❌ Coverage decreased:\n");
113+
drops.forEach((drop: string) => console.log(` • ${drop}`));
114+
console.log("\n💡 Please add tests to maintain or improve coverage.\n");
115+
process.exit(1);
116+
}
117+
118+
console.log("\n✅ Coverage maintained or improved!\n");
119+
process.exit(0);

0 commit comments

Comments
 (0)