Skip to content

Commit 74912ed

Browse files
brech1brech1
authored
Integrate make coverage into make test (#5015)
This PR removes the `make coverage` command and integrates it into `make test` through an optional `coverage` parameter. The coverage report now supports all optionalities the `make test` command has. It's not compatible with framework only test runs. The disable BLS option has been removed from the codebase. An issue has been created to track the removal of the `@always_bls` decorator. - #5032 A new `json` formatted coverage report is now generated, saved on the same directory as the HTML one.
1 parent a0dba8b commit 74912ed

File tree

4 files changed

+53
-86
lines changed

4 files changed

+53
-86
lines changed

‎Makefile‎

Lines changed: 36 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \
1919
.PHONY: \
2020
_sync \
2121
clean \
22-
coverage \
2322
help \
2423
lint \
2524
serve_docs \
@@ -29,8 +28,8 @@ ALL_EXECUTABLE_SPEC_NAMES = \
2928
# Help
3029
###############################################################################
3130

32-
BOLD = $(shell tput bold)
33-
NORM = $(shell tput sgr0)
31+
BOLD := $(shell tput bold)
32+
NORM := $(shell tput sgr0)
3433

3534
# Print help.
3635
help:
@@ -44,7 +43,6 @@ endif
4443
help-nonverbose:
4544
@echo "make $(BOLD)clean$(NORM) -- delete all untracked files"
4645
@echo "make $(BOLD)comptests$(NORM) -- generate compliance tests"
47-
@echo "make $(BOLD)coverage$(NORM) -- run pyspec tests with coverage"
4846
@echo "make $(BOLD)lint$(NORM) -- run linters and checks"
4947
@echo "make $(BOLD)serve_docs$(NORM) -- start a local docs web server"
5048
@echo "make $(BOLD)test$(NORM) -- run pyspec tests"
@@ -63,40 +61,33 @@ help-verbose:
6361
@echo " Runs pyspec tests with various configuration options. Tests run in parallel"
6462
@echo " by default using pytest with the minimal preset and fastest BLS library."
6563
@echo ""
66-
@echo " Parameters:"
67-
@echo " k=<test> Run specific test by name"
68-
@echo " fork=<fork> Test specific fork (phase0, altair, bellatrix, capella, etc.)"
69-
@echo " preset=<preset> Use specific preset (mainnet or minimal; default: minimal)"
70-
@echo " bls=<type> BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
71-
@echo " kzg=<type> KZG library type (spec, ckzg; default: ckzg)"
72-
@echo " component=<value> Test component: (all, pyspec, fw; default: all)"
73-
@echo " reftests=<bool> Generate reference tests (default: false)"
64+
@echo " Filtering:"
65+
@echo " k=<name> Run only tests matching this name"
66+
@echo " fork=<fork> Run only tests for this fork (phase0, altair, bellatrix, capella, etc.)"
67+
@echo " preset=<preset> Preset to use: mainnet, minimal (default: minimal)"
68+
@echo " component=<comp> What to test: all, pyspec, fw (default: all)"
69+
@echo ""
70+
@echo " Libraries:"
71+
@echo " bls=<type> BLS library: py_ecc, milagro, arkworks, fastest (default: fastest)"
72+
@echo " kzg=<type> KZG library: spec, ckzg (default: ckzg)"
73+
@echo ""
74+
@echo " Output:"
75+
@echo " reftests=true Generate reference test vectors"
76+
@echo " coverage=true Enable code coverage tracking"
7477
@echo ""
7578
@echo " Examples:"
7679
@echo " make test"
7780
@echo " make test k=test_verify_kzg_proof"
7881
@echo " make test fork=deneb"
7982
@echo " make test preset=mainnet"
8083
@echo " make test preset=mainnet fork=deneb k=test_verify_kzg_proof"
81-
@echo " make test bls=arkworks"
8284
@echo " make test component=fw"
85+
@echo " make test bls=arkworks"
8386
@echo " make test reftests=true"
8487
@echo " make test reftests=true fork=fulu"
8588
@echo " make test reftests=true preset=mainnet fork=fulu k=invalid_committee_index"
86-
@echo ""
87-
@echo "$(BOLD)make coverage$(NORM)"
88-
@echo ""
89-
@echo " Runs tests with code coverage tracking and generates an HTML report."
90-
@echo " Automatically opens the coverage report in your browser after completion."
91-
@echo ""
92-
@echo " Parameters:"
93-
@echo " k=<test> Run specific test by name"
94-
@echo " fork=<fork> Test specific fork"
95-
@echo ""
96-
@echo " Examples:"
97-
@echo " make coverage"
98-
@echo " make coverage k=test_process_attestation"
99-
@echo " make coverage fork=electra"
89+
@echo " make test coverage=true k=test_process_attestation"
90+
@echo " make test coverage=true fork=electra"
10091
@echo ""
10192
@echo "$(BOLD)CODE QUALITY$(NORM)"
10293
@echo "$(BOLD)--------------------------------------------------------------------------------$(NORM)"
@@ -165,8 +156,6 @@ help-verbose:
165156
# Virtual Environment
166157
###############################################################################
167158

168-
VENV = .venv
169-
170159
# Use non-editable installs for compliance test generators.
171160
# More details: ethereum/consensus-specs#4633.
172161
UV_RUN = uv run
@@ -200,19 +189,30 @@ _pyspec: _sync
200189

201190
TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
202191
REFTESTS_DIR = $(CURDIR)/reftests
192+
COV_REPORT_DIR = $(PYSPEC_DIR)/.htmlcov
203193

204194
# Run pyspec tests.
205-
test: MAYBE_TEST := $(if $(k),-k=$(k))
206-
# Disable parallelism when running a specific test.
207-
# Parallelism makes debugging difficult (print doesn't work).
208-
test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal)
195+
#
196+
# Filtering
197+
test: MAYBE_TEST := $(if $(k),-k "$(k)")
209198
test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
210199
test: PRESET := $(if $(filter fw,$(component)),,$(if $(preset),--preset=$(preset),))
211-
test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
212-
test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg))
200+
# Disable parallelism when running a specific test. Makes debugging difficult (print doesn't work).
201+
test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal)
213202
test: MAYBE_SPEC := $(if $(filter fw,$(component)),,$(PYSPEC_DIR)/eth_consensus_specs)
214203
test: MAYBE_INFRA := $(if $(filter pyspec,$(component)),,$(CURDIR)/tests/infra)
204+
#
205+
# Libraries
206+
test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
207+
test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg))
208+
#
209+
# Output
215210
test: MAYBE_REFTESTS := $(if $(filter true,$(reftests)),--reftests --reftests-output=$(REFTESTS_DIR))
211+
test: COVERAGE_PRESETS := $(if $(preset),$(preset),$(if $(filter true,$(reftests)),minimal mainnet,minimal))
212+
test: COV_SCOPE_SINGLE := $(foreach P,$(COVERAGE_PRESETS), --cov=eth_consensus_specs.$(fork).$P)
213+
test: COV_SCOPE_ALL := $(foreach P,$(COVERAGE_PRESETS),$(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$P))
214+
test: COV_SCOPE := $(if $(filter true,$(coverage)),$(if $(fork),$(COV_SCOPE_SINGLE),$(COV_SCOPE_ALL)))
215+
test: COVERAGE := $(if $(filter true,$(coverage)),--coverage $(COV_SCOPE) --cov-report="html:$(COV_REPORT_DIR)" --cov-report="json:$(COV_REPORT_DIR)/coverage.json" --cov-branch --no-cov-on-fail)
216216
test: _pyspec
217217
@mkdir -p $(TEST_REPORT_DIR)
218218
@$(UV_RUN) pytest \
@@ -227,37 +227,10 @@ test: _pyspec
227227
--html=$(TEST_REPORT_DIR)/test_results.html \
228228
--self-contained-html \
229229
$(MAYBE_REFTESTS) \
230+
$(COVERAGE) \
230231
$(MAYBE_INFRA) \
231232
$(MAYBE_SPEC)
232233

233-
###############################################################################
234-
# Coverage
235-
###############################################################################
236-
237-
TEST_PRESET_TYPE ?= minimal
238-
COV_HTML_OUT=$(PYSPEC_DIR)/.htmlcov
239-
COV_INDEX_FILE=$(COV_HTML_OUT)/index.html
240-
COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$(TEST_PRESET_TYPE))
241-
242-
# Run pytest with coverage tracking
243-
_test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k))
244-
_test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork))
245-
_test_with_coverage: _pyspec
246-
@$(UV_RUN) pytest \
247-
-n auto \
248-
$(MAYBE_TEST) \
249-
$(MAYBE_FORK) \
250-
--disable-bls \
251-
$(COVERAGE_SCOPE) \
252-
--cov-report="html:$(COV_HTML_OUT)" \
253-
--cov-branch \
254-
$(PYSPEC_DIR)/eth_consensus_specs
255-
256-
# Run tests with coverage then open the coverage report.
257-
# See `make test` for a list of options.
258-
coverage: _test_with_coverage
259-
@echo "Opening result: $(COV_INDEX_FILE)"
260-
@((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
261234

262235
###############################################################################
263236
# Documentation

‎tests/core/pyspec/README.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,21 @@ Note: these options can be used together, like:
4646
make test preset=minimal k=test_verify_kzg_proof fork=deneb
4747
```
4848

49-
### How to view code coverage report
49+
### How to generate coverage reports
5050

51-
Run `make coverage` to run all tests and open the html code coverage report.
51+
Run `make test coverage=true` to enable coverage tracking and generate reports.
52+
53+
Reports are saved at:
54+
55+
- **HTML report**: `tests/core/pyspec/.htmlcov/index.html`
56+
- **JSON report**: `tests/core/pyspec/.htmlcov/coverage.json`
57+
58+
To open the HTML report in a browser:
59+
60+
```shell
61+
xdg-open tests/core/pyspec/.htmlcov/index.html # Linux
62+
open tests/core/pyspec/.htmlcov/index.html # macOS
63+
```
5264

5365
## Contributing
5466

‎tests/core/pyspec/eth_consensus_specs/test/conftest.py‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ def pytest_addoption(parser):
5050
),
5151
)
5252
parser.addoption(
53-
"--disable-bls",
53+
"--coverage",
5454
action="store_true",
5555
default=False,
56-
help="bls-default: make tests that are not dependent on BLS run without BLS",
56+
help="coverage: enable code coverage tracking",
5757
)
5858
parser.addoption(
5959
"--bls-type",
@@ -131,13 +131,6 @@ def run_phases(request):
131131
context.DEFAULT_PYTEST_FORKS = ALL_PHASES
132132

133133

134-
@fixture(autouse=True)
135-
def bls_default(request):
136-
disable_bls = request.config.getoption("--disable-bls")
137-
if disable_bls:
138-
context.DEFAULT_BLS_ACTIVE = False
139-
140-
141134
@fixture(autouse=True)
142135
def bls_type(request):
143136
bls_type = request.config.getoption("--bls-type")

‎tests/core/pyspec/eth_consensus_specs/test/context.py‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -290,17 +290,6 @@ def entry(*args, **kw):
290290
return entry
291291

292292

293-
# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`.
294-
# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*.
295-
# The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON).
296-
# - Some tests are marked as BLS-requiring, and ignore this setting.
297-
# (tests that express differences caused by BLS, e.g. invalid signatures being rejected)
298-
# - Some other tests are marked as BLS-ignoring, and ignore this setting.
299-
# (tests that are heavily performance impacted / require unsigned state transitions)
300-
# - Most tests respect the BLS setting.
301-
DEFAULT_BLS_ACTIVE = True
302-
303-
304293
is_pytest = True
305294
is_generator = False
306295

@@ -436,7 +425,7 @@ def bls_switch(fn):
436425

437426
def entry(*args, **kw):
438427
old_state = bls.bls_active
439-
bls.bls_active = kw.pop("bls_active", DEFAULT_BLS_ACTIVE)
428+
bls.bls_active = kw.pop("bls_active", True)
440429
res = fn(*args, **kw)
441430
if res is not None:
442431
yield from res

0 commit comments

Comments
 (0)