Skip to content

Integrate make coverage into make test #5015

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jtraglia merged 2 commits into from
Mar 24, 2026
+53 −86
Merged

Integrate make coverage into make test #5015

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd924f2
restructure: make coverage
brech1 Mar 23, 2026
d951c60
fix: lint
brech1 Mar 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 36 additions & 63 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \
.PHONY: \
_sync \
clean \
coverage \
help \
lint \
serve_docs \
Expand All @@ -29,8 +28,8 @@ ALL_EXECUTABLE_SPEC_NAMES = \
# Help
###############################################################################

BOLD = $(shell tput bold)
NORM = $(shell tput sgr0)
BOLD := $(shell tput bold)
NORM := $(shell tput sgr0)

# Print help.
help:
Expand All @@ -44,7 +43,6 @@ endif
help-nonverbose:
@echo "make $(BOLD)clean$(NORM) -- delete all untracked files"
@echo "make $(BOLD)comptests$(NORM) -- generate compliance tests"
@echo "make $(BOLD)coverage$(NORM) -- run pyspec tests with coverage"
@echo "make $(BOLD)lint$(NORM) -- run linters and checks"
@echo "make $(BOLD)serve_docs$(NORM) -- start a local docs web server"
@echo "make $(BOLD)test$(NORM) -- run pyspec tests"
Expand All @@ -63,40 +61,33 @@ help-verbose:
@echo " Runs pyspec tests with various configuration options. Tests run in parallel"
@echo " by default using pytest with the minimal preset and fastest BLS library."
@echo ""
@echo " Parameters:"
@echo " k=<test> Run specific test by name"
@echo " fork=<fork> Test specific fork (phase0, altair, bellatrix, capella, etc.)"
@echo " preset=<preset> Use specific preset (mainnet or minimal; default: minimal)"
@echo " bls=<type> BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
@echo " kzg=<type> KZG library type (spec, ckzg; default: ckzg)"
@echo " component=<value> Test component: (all, pyspec, fw; default: all)"
@echo " reftests=<bool> Generate reference tests (default: false)"
@echo " Filtering:"
@echo " k=<name> Run only tests matching this name"
@echo " fork=<fork> Run only tests for this fork (phase0, altair, bellatrix, capella, etc.)"
@echo " preset=<preset> Preset to use: mainnet, minimal (default: minimal)"
@echo " component=<comp> What to test: all, pyspec, fw (default: all)"
@echo ""
@echo " Libraries:"
@echo " bls=<type> BLS library: py_ecc, milagro, arkworks, fastest (default: fastest)"
@echo " kzg=<type> KZG library: spec, ckzg (default: ckzg)"
@echo ""
@echo " Output:"
@echo " reftests=true Generate reference test vectors"
@echo " coverage=true Enable code coverage tracking"
@echo ""
@echo " Examples:"
@echo " make test"
@echo " make test k=test_verify_kzg_proof"
@echo " make test fork=deneb"
@echo " make test preset=mainnet"
@echo " make test preset=mainnet fork=deneb k=test_verify_kzg_proof"
@echo " make test bls=arkworks"
@echo " make test component=fw"
@echo " make test bls=arkworks"
Copy link
Member

@jihoonsong jihoonsong Mar 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

@echo " make test reftests=true"
@echo " make test reftests=true fork=fulu"
@echo " make test reftests=true preset=mainnet fork=fulu k=invalid_committee_index"
@echo ""
@echo "$(BOLD)make coverage$(NORM)"
@echo ""
@echo " Runs tests with code coverage tracking and generates an HTML report."
@echo " Automatically opens the coverage report in your browser after completion."
@echo ""
@echo " Parameters:"
@echo " k=<test> Run specific test by name"
@echo " fork=<fork> Test specific fork"
@echo ""
@echo " Examples:"
@echo " make coverage"
@echo " make coverage k=test_process_attestation"
@echo " make coverage fork=electra"
@echo " make test coverage=true k=test_process_attestation"
@echo " make test coverage=true fork=electra"
@echo ""
@echo "$(BOLD)CODE QUALITY$(NORM)"
@echo "$(BOLD)--------------------------------------------------------------------------------$(NORM)"
Expand Down Expand Up @@ -165,8 +156,6 @@ help-verbose:
# Virtual Environment
###############################################################################

VENV = .venv

# Use non-editable installs for compliance test generators.
# More details: ethereum/consensus-specs#4633.
UV_RUN = uv run
Expand Down Expand Up @@ -200,19 +189,30 @@ _pyspec: _sync

TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
REFTESTS_DIR = $(CURDIR)/reftests
COV_REPORT_DIR = $(PYSPEC_DIR)/.htmlcov

# Run pyspec tests.
test: MAYBE_TEST := $(if $(k),-k=$(k))
# Disable parallelism when running a specific test.
# Parallelism makes debugging difficult (print doesn't work).
test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal)
#
# Filtering
test: MAYBE_TEST := $(if $(k),-k "$(k)")
test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
test: PRESET := $(if $(filter fw,$(component)),,$(if $(preset),--preset=$(preset),))
test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg))
# Disable parallelism when running a specific test. Makes debugging difficult (print doesn't work).
test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal)
test: MAYBE_SPEC := $(if $(filter fw,$(component)),,$(PYSPEC_DIR)/eth_consensus_specs)
test: MAYBE_INFRA := $(if $(filter pyspec,$(component)),,$(CURDIR)/tests/infra)
#
# Libraries
test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg))
#
# Output
test: MAYBE_REFTESTS := $(if $(filter true,$(reftests)),--reftests --reftests-output=$(REFTESTS_DIR))
test: COVERAGE_PRESETS := $(if $(preset),$(preset),$(if $(filter true,$(reftests)),minimal mainnet,minimal))
test: COV_SCOPE_SINGLE := $(foreach P,$(COVERAGE_PRESETS), --cov=eth_consensus_specs.$(fork).$P)
test: COV_SCOPE_ALL := $(foreach P,$(COVERAGE_PRESETS),$(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$P))
test: COV_SCOPE := $(if $(filter true,$(coverage)),$(if $(fork),$(COV_SCOPE_SINGLE),$(COV_SCOPE_ALL)))
test: COVERAGE := $(if $(filter true,$(coverage)),--coverage $(COV_SCOPE) --cov-report="html:$(COV_REPORT_DIR)" --cov-report="json:$(COV_REPORT_DIR)/coverage.json" --cov-branch --no-cov-on-fail)
test: _pyspec
@mkdir -p $(TEST_REPORT_DIR)
@$(UV_RUN) pytest \
Expand All @@ -227,37 +227,10 @@ test: _pyspec
--html=$(TEST_REPORT_DIR)/test_results.html \
--self-contained-html \
$(MAYBE_REFTESTS) \
$(COVERAGE) \
$(MAYBE_INFRA) \
$(MAYBE_SPEC)

###############################################################################
# Coverage
###############################################################################

TEST_PRESET_TYPE ?= minimal
COV_HTML_OUT=$(PYSPEC_DIR)/.htmlcov
COV_INDEX_FILE=$(COV_HTML_OUT)/index.html
COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$(TEST_PRESET_TYPE))

# Run pytest with coverage tracking
_test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k))
_test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork))
_test_with_coverage: _pyspec
@$(UV_RUN) pytest \
-n auto \
$(MAYBE_TEST) \
$(MAYBE_FORK) \
--disable-bls \
$(COVERAGE_SCOPE) \
--cov-report="html:$(COV_HTML_OUT)" \
--cov-branch \
$(PYSPEC_DIR)/eth_consensus_specs

# Run tests with coverage then open the coverage report.
# See `make test` for a list of options.
coverage: _test_with_coverage
@echo "Opening result: $(COV_INDEX_FILE)"
@((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &

###############################################################################
# Documentation
Expand Down
16 changes: 14 additions & 2 deletions tests/core/pyspec/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,21 @@ Note: these options can be used together, like:
make test preset=minimal k=test_verify_kzg_proof fork=deneb
```

### How to view code coverage report
### How to generate coverage reports

Run `make coverage` to run all tests and open the html code coverage report.
Run `make test coverage=true` to enable coverage tracking and generate reports.

Reports are saved at:

- **HTML report**: `tests/core/pyspec/.htmlcov/index.html`
- **JSON report**: `tests/core/pyspec/.htmlcov/coverage.json`

To open the HTML report in a browser:

```shell
xdg-open tests/core/pyspec/.htmlcov/index.html # Linux
open tests/core/pyspec/.htmlcov/index.html # macOS
```

## Contributing

Expand Down
11 changes: 2 additions & 9 deletions tests/core/pyspec/eth_consensus_specs/test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ def pytest_addoption(parser):
),
)
parser.addoption(
"--disable-bls",
"--coverage",
action="store_true",
default=False,
help="bls-default: make tests that are not dependent on BLS run without BLS",
help="coverage: enable code coverage tracking",
)
parser.addoption(
"--bls-type",
Expand Down Expand Up @@ -131,13 +131,6 @@ def run_phases(request):
context.DEFAULT_PYTEST_FORKS = ALL_PHASES


@fixture(autouse=True)
def bls_default(request):
disable_bls = request.config.getoption("--disable-bls")
if disable_bls:
context.DEFAULT_BLS_ACTIVE = False


@fixture(autouse=True)
def bls_type(request):
bls_type = request.config.getoption("--bls-type")
Expand Down
13 changes: 1 addition & 12 deletions tests/core/pyspec/eth_consensus_specs/test/context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,17 +290,6 @@ def entry(*args, **kw):
return entry


# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`.
# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*.
# The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON).
# - Some tests are marked as BLS-requiring, and ignore this setting.
# (tests that express differences caused by BLS, e.g. invalid signatures being rejected)
# - Some other tests are marked as BLS-ignoring, and ignore this setting.
# (tests that are heavily performance impacted / require unsigned state transitions)
# - Most tests respect the BLS setting.
DEFAULT_BLS_ACTIVE = True


is_pytest = True
is_generator = False

Expand Down Expand Up @@ -436,7 +425,7 @@ def bls_switch(fn):

def entry(*args, **kw):
old_state = bls.bls_active
bls.bls_active = kw.pop("bls_active", DEFAULT_BLS_ACTIVE)
bls.bls_active = kw.pop("bls_active", True)
res = fn(*args, **kw)
if res is not None:
yield from res
Expand Down