From 19cf177b825f9656e231b3403cd97e9ba19970f5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Jul 2025 12:41:41 +0000 Subject: [PATCH 1/2] Add code coverage integration with Makefile targets and nightly CI Co-authored-by: dannywillems <6018454+dannywillems@users.noreply.github.com> --- .github/workflows/coverage.yaml | 109 +++++++++++ .gitignore | 3 + Makefile | 102 ++++++++++ codecov.yml | 51 +++++ website/docs/developers/code-coverage.md | 237 +++++++++++++++++++++++ 5 files changed, 502 insertions(+) create mode 100644 .github/workflows/coverage.yaml create mode 100644 codecov.yml create mode 100644 website/docs/developers/code-coverage.md diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 000000000..b6a82b5bc --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,109 @@ +name: Code Coverage (Nightly) + +on: + schedule: + # Run at 2:00 AM UTC every day + - cron: '0 2 * * *' + workflow_dispatch: + inputs: + coverage_type: + description: 'Type of coverage to run' + required: false + default: 'comprehensive' + type: choice + options: + - 'basic' + - 'comprehensive' + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: full + OPENMINA_PANIC_ON_BUG: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + code-coverage: + runs-on: ubuntu-22.04 + + steps: + - name: Git checkout + uses: actions/checkout@v4 + + - name: Setup build dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, llvm-tools-preview + toolchain: 1.84 + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: "coverage-v0" + + - name: Install grcov + run: | + cargo install grcov + + - name: Determine coverage type + id: coverage-type + run: | + if [ "${{ github.event.inputs.coverage_type }}" = "basic" ]; then + echo "target=test-coverage" >> $GITHUB_OUTPUT + echo "type=basic" >> $GITHUB_OUTPUT + else + echo "target=test-with-coverage" >> $GITHUB_OUTPUT + echo "type=comprehensive" >> $GITHUB_OUTPUT + fi + + - name: Run tests with coverage + run: | + make ${{ steps.coverage-type.outputs.target }} + + - name: Generate LCOV report + run: | + make coverage-lcov + + - name: Generate HTML report + run: | + make coverage-report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: target/coverage/lcov.info + flags: ${{ steps.coverage-type.outputs.type }} + name: openmina-coverage + fail_ci_if_error: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload coverage reports as artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-reports-${{ steps.coverage-type.outputs.type }} + path: | + target/coverage/lcov.info + target/coverage/html/ + retention-days: 30 + + - name: Coverage Summary + run: | + echo "## Coverage Summary" >> $GITHUB_STEP_SUMMARY + echo "Coverage report type: ${{ steps.coverage-type.outputs.type }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Coverage Report Files" >> $GITHUB_STEP_SUMMARY + echo "- LCOV: \`target/coverage/lcov.info\`" >> $GITHUB_STEP_SUMMARY + echo "- HTML Report: \`target/coverage/html/index.html\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Coverage Summary" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + make coverage-summary >> $GITHUB_STEP_SUMMARY || echo "Coverage summary generation failed" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.gitignore b/.gitignore index 991de41f0..307039ea6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ pkg/ cargo-build-test.json tests.tsv +# Coverage +target/coverage/ + # Generated API docs should not be committed website/static/api-docs/ diff --git a/Makefile b/Makefile index 685a7c92a..1900de64f 100644 --- a/Makefile +++ b/Makefile @@ -227,6 +227,108 @@ test-release: ## Run tests in release mode test-vrf: ## Run VRF tests, requires nightly Rust @cd vrf && cargo +nightly test --release -- -Z unstable-options --report-time +# Coverage targets + +.PHONY: setup-coverage-tools +setup-coverage-tools: ## Install tools required for code coverage + @echo "Installing coverage tools..." + @rustup component add llvm-tools-preview + @cargo install grcov || echo "grcov already installed" + @echo "Coverage tools installed successfully" + +.PHONY: test-coverage +test-coverage: ## Run tests with code coverage (basic, fast) + @echo "Running tests with code coverage..." + @mkdir -p target/coverage + @CARGO_INCREMENTAL=0 \ + RUSTFLAGS="-Cinstrument-coverage" \ + LLVM_PROFILE_FILE="target/coverage/cargo-test-%p-%m.profraw" \ + cargo test --workspace \ + --exclude fuzzer \ + --exclude heartbeats-processor \ + --lib \ + --tests + @echo "Coverage data collected in target/coverage/" + +.PHONY: test-with-coverage +test-with-coverage: ## Run comprehensive tests with code coverage (slower, more complete) + @echo "Running comprehensive tests with code coverage..." + @mkdir -p target/coverage + @CARGO_INCREMENTAL=0 \ + RUSTFLAGS="-Cinstrument-coverage" \ + LLVM_PROFILE_FILE="target/coverage/cargo-test-%p-%m.profraw" \ + cargo test --workspace \ + --exclude fuzzer \ + --exclude heartbeats-processor \ + --lib \ + --tests \ + --bins + @echo "Coverage data collected in target/coverage/" + +.PHONY: coverage-report +coverage-report: ## Generate HTML coverage report from collected data + @echo "Generating HTML coverage report..." + @mkdir -p target/coverage/html + @grcov target/coverage \ + --binary-path target/debug/deps/ \ + --source-dir . \ + --output-types html \ + --branch \ + --ignore-not-existing \ + --ignore "/*" \ + --ignore "target/*" \ + --ignore "tests/*" \ + --ignore "**/tests.rs" \ + --ignore "**/test_*.rs" \ + --ignore "**/bench_*.rs" \ + --ignore "**/benches/*" \ + --output-path target/coverage/html + @echo "HTML coverage report generated in target/coverage/html/" + @echo "Open target/coverage/html/index.html in your browser to view the report" + +.PHONY: coverage-lcov +coverage-lcov: ## Generate LCOV coverage report for CI/codecov + @echo "Generating LCOV coverage report..." + @mkdir -p target/coverage + @grcov target/coverage \ + --binary-path target/debug/deps/ \ + --source-dir . \ + --output-types lcov \ + --branch \ + --ignore-not-existing \ + --ignore "/*" \ + --ignore "target/*" \ + --ignore "tests/*" \ + --ignore "**/tests.rs" \ + --ignore "**/test_*.rs" \ + --ignore "**/bench_*.rs" \ + --ignore "**/benches/*" \ + --output-path target/coverage/lcov.info + @echo "LCOV coverage report generated at target/coverage/lcov.info" + +.PHONY: coverage-clean +coverage-clean: ## Clean coverage data and reports + @echo "Cleaning coverage data..." + @rm -rf target/coverage + @echo "Coverage data cleaned" + +.PHONY: coverage-summary +coverage-summary: ## Display coverage summary from collected data + @echo "Generating coverage summary..." + @grcov target/coverage \ + --binary-path target/debug/deps/ \ + --source-dir . \ + --output-types markdown \ + --branch \ + --ignore-not-existing \ + --ignore "/*" \ + --ignore "target/*" \ + --ignore "tests/*" \ + --ignore "**/tests.rs" \ + --ignore "**/test_*.rs" \ + --ignore "**/bench_*.rs" \ + --ignore "**/benches/*" + # Docker build targets .PHONY: docker-build-all diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..35a880af6 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,51 @@ +codecov: + require_ci_to_pass: yes + notify: + after_n_builds: 1 + +coverage: + precision: 2 + round: down + range: "70...95" + + status: + project: + default: + target: auto + threshold: 1% + if_not_found: success + if_ci_failed: error + patch: + default: + target: auto + threshold: 1% + if_not_found: success + if_ci_failed: error + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +ignore: + - "**/*test*.rs" + - "**/tests.rs" + - "**/benches/**" + - "**/target/**" + - "**/fuzzer/**" + - "**/tools/fuzzing/**" + - "**/examples/**" + - "**/node/testing/**" + +comment: + layout: "reach,diff,flags,tree" + behavior: default + require_changes: no + require_base: no + require_head: yes + +github_checks: + annotations: true \ No newline at end of file diff --git a/website/docs/developers/code-coverage.md b/website/docs/developers/code-coverage.md new file mode 100644 index 000000000..d68402408 --- /dev/null +++ b/website/docs/developers/code-coverage.md @@ -0,0 +1,237 @@ +--- +sidebar_position: 6 +--- + +# Code Coverage + +This guide explains how to generate, analyze, and use code coverage reports in +the OpenMina project. + +## Overview + +OpenMina uses Rust's built-in LLVM-based code coverage instrumentation to +generate coverage reports. This provides accurate line-level and branch-level +coverage information for all Rust code in the workspace. + +## Quick Start + +### Setup Coverage Tools + +First, install the required coverage tools: + +```bash +make setup-coverage-tools +``` + +This installs: + +- `llvm-tools-preview` Rust component for LLVM coverage tools +- `grcov` for processing coverage data and generating reports + +### Generate Coverage Reports + +#### Basic Coverage (Fast) + +For quick coverage checks during development: + +```bash +make test-coverage +make coverage-report +``` + +This runs tests for libraries and basic test files, then generates an HTML +report you can view in your browser. + +#### Comprehensive Coverage (Complete) + +For complete coverage analysis: + +```bash +make test-with-coverage +make coverage-report +``` + +This runs all tests including integration tests and binaries, providing more +complete coverage data. + +### View Reports + +After generating coverage, open the HTML report: + +```bash +# On Linux +xdg-open target/coverage/html/index.html + +# On macOS +open target/coverage/html/index.html +``` + +## Coverage Commands Reference + +| Command | Description | +| --------------------------- | ------------------------------------------------ | +| `make setup-coverage-tools` | Install required coverage tools | +| `make test-coverage` | Run basic tests with coverage (fast) | +| `make test-with-coverage` | Run comprehensive tests with coverage (complete) | +| `make coverage-report` | Generate HTML coverage report | +| `make coverage-lcov` | Generate LCOV report for CI/codecov | +| `make coverage-summary` | Display coverage summary in terminal | +| `make coverage-clean` | Clean all coverage data and reports | + +## Understanding Coverage Reports + +### HTML Reports + +The HTML report (`target/coverage/html/index.html`) provides: + +- **Overview**: Overall coverage percentage for the project +- **File List**: Coverage breakdown by source file +- **Line Coverage**: Which lines are covered (green) or uncovered (red) +- **Branch Coverage**: Which code branches are taken during tests +- **Function Coverage**: Which functions are called during tests + +### Coverage Metrics + +- **Line Coverage**: Percentage of executable lines that are executed by tests +- **Branch Coverage**: Percentage of conditional branches that are taken during + tests +- **Function Coverage**: Percentage of functions that are called during tests + +### Coverage Thresholds + +The project aims for: + +- **Minimum**: 70% line coverage +- **Target**: 85%+ line coverage +- **Excellent**: 95%+ line coverage + +## CI Integration + +### Nightly Coverage + +Coverage reports are automatically generated nightly and uploaded to +[Codecov](https://app.codecov.io/gh/o1-labs/openmina): + +- **Schedule**: Daily at 2:00 AM UTC +- **Types**: Both basic and comprehensive coverage +- **Reports**: Uploaded to Codecov and stored as artifacts + +### Manual Trigger + +You can manually trigger coverage analysis through GitHub Actions: + +1. Go to the "Actions" tab in the GitHub repository +2. Select "Code Coverage (Nightly)" workflow +3. Click "Run workflow" +4. Choose coverage type (basic or comprehensive) + +## Best Practices + +### Writing Testable Code + +1. **Write Unit Tests**: Test individual functions and modules +2. **Test Edge Cases**: Include tests for error conditions and boundary cases +3. **Mock Dependencies**: Use mocks for external dependencies in tests +4. **Keep Functions Small**: Smaller functions are easier to test completely + +### Improving Coverage + +1. **Identify Gaps**: Use coverage reports to find untested code +2. **Prioritize Critical Code**: Focus on testing important business logic first +3. **Test Error Paths**: Ensure error handling code is covered +4. **Add Integration Tests**: Test how components work together + +### Excluding Code from Coverage + +Some code should be excluded from coverage analysis: + +- **Test Code**: Files ending in `*test*.rs`, `tests.rs` +- **Generated Code**: Auto-generated code that doesn't need testing +- **Platform-Specific Code**: Code that only runs on specific platforms +- **Debug/Development Code**: Code only used for debugging + +Exclusions are configured in `codecov.yml`. + +## Troubleshooting + +### Coverage Tools Not Found + +If you get errors about missing tools: + +```bash +# Reinstall coverage tools +make setup-coverage-tools + +# Verify installation +rustup component list --installed | grep llvm-tools +cargo install --list | grep grcov +``` + +### Empty Coverage Reports + +If reports show no coverage: + +1. Ensure tests are actually running: `cargo test --workspace --lib` +2. Check that `RUSTFLAGS` environment variable isn't overriding coverage flags +3. Verify coverage data files exist: `ls target/coverage/*.profraw` + +### Build Errors with Coverage + +Coverage instrumentation can sometimes cause build issues: + +```bash +# Clean and rebuild +make coverage-clean +cargo clean +make test-coverage +``` + +### Slow Coverage Generation + +Coverage builds are slower than regular builds due to instrumentation: + +- Use `test-coverage` (basic) for faster iteration during development +- Use `test-with-coverage` (comprehensive) for complete analysis +- Consider running coverage on specific packages: + `cargo test -p --lib` + +## Example Workflow + +Here's a typical development workflow using coverage: + +```bash +# 1. Install tools (one time) +make setup-coverage-tools + +# 2. During development - quick coverage check +make test-coverage +make coverage-summary + +# 3. Before committing - comprehensive coverage +make test-with-coverage +make coverage-report + +# 4. View detailed report +open target/coverage/html/index.html + +# 5. Clean up when done +make coverage-clean +``` + +## Integration with IDEs + +### VS Code + +Install the "Coverage Gutters" extension to view coverage directly in the +editor: + +1. Install the extension +2. Generate LCOV report: `make coverage-lcov` +3. Open the command palette (Ctrl+Shift+P) +4. Run "Coverage Gutters: Display Coverage" +5. Point to `target/coverage/lcov.info` + +### Other IDEs + +Most IDEs support LCOV format coverage files. Generate the LCOV report with +`make coverage-lcov` and configure your IDE to read `target/coverage/lcov.info`. From bdc2742f90785e8c474756a55010c90a7edd93d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 25 Jul 2025 12:45:50 +0000 Subject: [PATCH 2/2] Add coverage documentation to developer sidebar and fix rustfmt requirement Co-authored-by: dannywillems <6018454+dannywillems@users.noreply.github.com> --- website/sidebars.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/website/sidebars.ts b/website/sidebars.ts index e0cc29321..52adc134a 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -68,6 +68,13 @@ const sidebars: SidebarsConfig = { 'developers/updating-ocaml-node', ], }, + { + type: 'category', + label: 'Development', + items: [ + 'developers/code-coverage', + ], + }, { type: 'category', label: 'Architecture',