Skip to content

Commit 6ec5392

Browse files
brech1brech1
committed
restructure: make coverage
1 parent cd1d26c commit 6ec5392

File tree

4 files changed

+55
-84
lines changed

4 files changed

+55
-84
lines changed

‎Makefile‎

Lines changed: 36 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \
1919
.PHONY: \
2020
_sync \
2121
clean \
22-
coverage \
2322
help \
2423
lint \
2524
serve_docs \
@@ -29,8 +28,8 @@ ALL_EXECUTABLE_SPEC_NAMES = \
2928
# Help
3029
###############################################################################
3130

32-
BOLD = $(shell tput bold)
33-
NORM = $(shell tput sgr0)
31+
BOLD := $(shell tput bold)
32+
NORM := $(shell tput sgr0)
3433

3534
# Print help.
3635
help:
@@ -44,7 +43,6 @@ endif
4443
help-nonverbose:
4544
@echo "make $(BOLD)clean$(NORM) -- delete all untracked files"
4645
@echo "make $(BOLD)comptests$(NORM) -- generate compliance tests"
47-
@echo "make $(BOLD)coverage$(NORM) -- run pyspec tests with coverage"
4846
@echo "make $(BOLD)lint$(NORM) -- run linters and checks"
4947
@echo "make $(BOLD)serve_docs$(NORM) -- start a local docs web server"
5048
@echo "make $(BOLD)test$(NORM) -- run pyspec tests"
@@ -63,40 +61,33 @@ help-verbose:
6361
@echo " Runs pyspec tests with various configuration options. Tests run in parallel"
6462
@echo " by default using pytest with the minimal preset and fastest BLS library."
6563
@echo ""
66-
@echo " Parameters:"
67-
@echo " k=<test> Run specific test by name"
68-
@echo " fork=<fork> Test specific fork (phase0, altair, bellatrix, capella, etc.)"
69-
@echo " preset=<preset> Use specific preset (mainnet or minimal; default: minimal)"
70-
@echo " bls=<type> BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
71-
@echo " kzg=<type> KZG library type (spec, ckzg; default: ckzg)"
72-
@echo " component=<value> Test component: (all, pyspec, fw; default: all)"
73-
@echo " reftests=<bool> Generate reference tests (default: false)"
64+
@echo " Filtering:"
65+
@echo " k=<name> Run only tests matching this name"
66+
@echo " fork=<fork> Run only tests for this fork (phase0, altair, bellatrix, capella, etc.)"
67+
@echo " preset=<preset> Preset to use: mainnet, minimal (default: minimal)"
68+
@echo " component=<comp> What to test: all, pyspec, fw (default: all)"
69+
@echo ""
70+
@echo " Libraries:"
71+
@echo " bls=<type> BLS library: py_ecc, milagro, arkworks, fastest (default: fastest)"
72+
@echo " kzg=<type> KZG library: spec, ckzg (default: ckzg)"
73+
@echo ""
74+
@echo " Output:"
75+
@echo " reftests=true Generate reference test vectors"
76+
@echo " coverage=true Enable code coverage tracking"
7477
@echo ""
7578
@echo " Examples:"
7679
@echo " make test"
7780
@echo " make test k=test_verify_kzg_proof"
7881
@echo " make test fork=deneb"
7982
@echo " make test preset=mainnet"
8083
@echo " make test preset=mainnet fork=deneb k=test_verify_kzg_proof"
81-
@echo " make test bls=arkworks"
8284
@echo " make test component=fw"
85+
@echo " make test bls=arkworks"
8386
@echo " make test reftests=true"
8487
@echo " make test reftests=true fork=fulu"
8588
@echo " make test reftests=true preset=mainnet fork=fulu k=invalid_committee_index"
86-
@echo ""
87-
@echo "$(BOLD)make coverage$(NORM)"
88-
@echo ""
89-
@echo " Runs tests with code coverage tracking and generates an HTML report."
90-
@echo " Automatically opens the coverage report in your browser after completion."
91-
@echo ""
92-
@echo " Parameters:"
93-
@echo " k=<test> Run specific test by name"
94-
@echo " fork=<fork> Test specific fork"
95-
@echo ""
96-
@echo " Examples:"
97-
@echo " make coverage"
98-
@echo " make coverage k=test_process_attestation"
99-
@echo " make coverage fork=electra"
89+
@echo " make test coverage=true k=test_process_attestation"
90+
@echo " make test coverage=true fork=electra"
10091
@echo ""
10192
@echo "$(BOLD)CODE QUALITY$(NORM)"
10293
@echo "$(BOLD)--------------------------------------------------------------------------------$(NORM)"
@@ -165,8 +156,6 @@ help-verbose:
165156
# Virtual Environment
166157
###############################################################################
167158

168-
VENV = .venv
169-
170159
# Use non-editable installs for compliance test generators.
171160
# More details: ethereum/consensus-specs#4633.
172161
UV_RUN = uv run
@@ -200,19 +189,30 @@ _pyspec: _sync
200189

201190
TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
202191
REFTESTS_DIR = $(CURDIR)/reftests
192+
COV_REPORT_DIR = $(PYSPEC_DIR)/.htmlcov
203193

204194
# Run pyspec tests.
205-
test: MAYBE_TEST := $(if $(k),-k=$(k))
206-
# Disable parallelism when running a specific test.
207-
# Parallelism makes debugging difficult (print doesn't work).
208-
test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal)
195+
#
196+
# Filtering
197+
test: MAYBE_TEST := $(if $(k),-k "$(k)")
209198
test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
210199
test: PRESET := $(if $(filter fw,$(component)),,$(if $(preset),--preset=$(preset),))
211-
test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
212-
test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg))
200+
# Disable parallelism when running a specific test. Makes debugging difficult (print doesn't work).
201+
test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal)
213202
test: MAYBE_SPEC := $(if $(filter fw,$(component)),,$(PYSPEC_DIR)/eth_consensus_specs)
214203
test: MAYBE_INFRA := $(if $(filter pyspec,$(component)),,$(CURDIR)/tests/infra)
204+
#
205+
# Libraries
206+
test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
207+
test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg))
208+
#
209+
# Output
215210
test: MAYBE_REFTESTS := $(if $(filter true,$(reftests)),--reftests --reftests-output=$(REFTESTS_DIR))
211+
test: COVERAGE_PRESETS := $(if $(preset),$(preset),$(if $(filter true,$(reftests)),minimal mainnet,minimal))
212+
test: COV_SCOPE_SINGLE := $(foreach P,$(COVERAGE_PRESETS), --cov=eth_consensus_specs.$(fork).$P)
213+
test: COV_SCOPE_ALL := $(foreach P,$(COVERAGE_PRESETS),$(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$P))
214+
test: COV_SCOPE := $(if $(filter true,$(coverage)),$(if $(fork),$(COV_SCOPE_SINGLE),$(COV_SCOPE_ALL)))
215+
test: COVERAGE := $(if $(filter true,$(coverage)),--coverage $(COV_SCOPE) --cov-report="html:$(COV_REPORT_DIR)" --cov-report="json:$(COV_REPORT_DIR)/coverage.json" --cov-branch --no-cov-on-fail)
216216
test: _pyspec
217217
@mkdir -p $(TEST_REPORT_DIR)
218218
@$(UV_RUN) pytest \
@@ -227,37 +227,10 @@ test: _pyspec
227227
--html=$(TEST_REPORT_DIR)/test_results.html \
228228
--self-contained-html \
229229
$(MAYBE_REFTESTS) \
230+
$(COVERAGE) \
230231
$(MAYBE_INFRA) \
231232
$(MAYBE_SPEC)
232233

233-
###############################################################################
234-
# Coverage
235-
###############################################################################
236-
237-
TEST_PRESET_TYPE ?= minimal
238-
COV_HTML_OUT=$(PYSPEC_DIR)/.htmlcov
239-
COV_INDEX_FILE=$(COV_HTML_OUT)/index.html
240-
COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$(TEST_PRESET_TYPE))
241-
242-
# Run pytest with coverage tracking
243-
_test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k))
244-
_test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork))
245-
_test_with_coverage: _pyspec
246-
@$(UV_RUN) pytest \
247-
-n auto \
248-
$(MAYBE_TEST) \
249-
$(MAYBE_FORK) \
250-
--disable-bls \
251-
$(COVERAGE_SCOPE) \
252-
--cov-report="html:$(COV_HTML_OUT)" \
253-
--cov-branch \
254-
$(PYSPEC_DIR)/eth_consensus_specs
255-
256-
# Run tests with coverage then open the coverage report.
257-
# See `make test` for a list of options.
258-
coverage: _test_with_coverage
259-
@echo "Opening result: $(COV_INDEX_FILE)"
260-
@((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
261234

262235
###############################################################################
263236
# Documentation

‎tests/core/pyspec/README.md‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,23 @@ Note: these options can be used together, like:
4646
make test preset=minimal k=test_verify_kzg_proof fork=deneb
4747
```
4848

49-
### How to view code coverage report
49+
### How to generate coverage reports
50+
51+
Run `make test coverage=true` to enable coverage tracking and generate
52+
reports.
53+
54+
Reports are saved at:
55+
56+
- **HTML report**: `tests/core/pyspec/.htmlcov/index.html`
57+
- **JSON report**: `tests/core/pyspec/.htmlcov/coverage.json`
58+
59+
To open the HTML report in a browser:
60+
61+
```shell
62+
xdg-open tests/core/pyspec/.htmlcov/index.html # Linux
63+
open tests/core/pyspec/.htmlcov/index.html # macOS
64+
```
5065

51-
Run `make coverage` to run all tests and open the html code coverage report.
5266

5367
## Contributing
5468

‎tests/core/pyspec/eth_consensus_specs/test/conftest.py‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ def pytest_addoption(parser):
5050
),
5151
)
5252
parser.addoption(
53-
"--disable-bls",
53+
"--coverage",
5454
action="store_true",
5555
default=False,
56-
help="bls-default: make tests that are not dependent on BLS run without BLS",
56+
help="coverage: enable code coverage tracking",
5757
)
5858
parser.addoption(
5959
"--bls-type",
@@ -131,13 +131,6 @@ def run_phases(request):
131131
context.DEFAULT_PYTEST_FORKS = ALL_PHASES
132132

133133

134-
@fixture(autouse=True)
135-
def bls_default(request):
136-
disable_bls = request.config.getoption("--disable-bls")
137-
if disable_bls:
138-
context.DEFAULT_BLS_ACTIVE = False
139-
140-
141134
@fixture(autouse=True)
142135
def bls_type(request):
143136
bls_type = request.config.getoption("--bls-type")

‎tests/core/pyspec/eth_consensus_specs/test/context.py‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -290,15 +290,6 @@ def entry(*args, **kw):
290290
return entry
291291

292292

293-
# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`.
294-
# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*.
295-
# The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON).
296-
# - Some tests are marked as BLS-requiring, and ignore this setting.
297-
# (tests that express differences caused by BLS, e.g. invalid signatures being rejected)
298-
# - Some other tests are marked as BLS-ignoring, and ignore this setting.
299-
# (tests that are heavily performance impacted / require unsigned state transitions)
300-
# - Most tests respect the BLS setting.
301-
DEFAULT_BLS_ACTIVE = True
302293

303294

304295
is_pytest = True
@@ -436,7 +427,7 @@ def bls_switch(fn):
436427

437428
def entry(*args, **kw):
438429
old_state = bls.bls_active
439-
bls.bls_active = kw.pop("bls_active", DEFAULT_BLS_ACTIVE)
430+
bls.bls_active = kw.pop("bls_active", True)
440431
res = fn(*args, **kw)
441432
if res is not None:
442433
yield from res

0 commit comments

Comments
 (0)