diff --git a/Makefile b/Makefile index 92eeb48dbb..2c2bcce166 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \ .PHONY: \ _sync \ clean \ - coverage \ help \ lint \ serve_docs \ @@ -29,8 +28,8 @@ ALL_EXECUTABLE_SPEC_NAMES = \ # Help ############################################################################### -BOLD = $(shell tput bold) -NORM = $(shell tput sgr0) +BOLD := $(shell tput bold) +NORM := $(shell tput sgr0) # Print help. help: @@ -44,7 +43,6 @@ endif help-nonverbose: @echo "make $(BOLD)clean$(NORM) -- delete all untracked files" @echo "make $(BOLD)comptests$(NORM) -- generate compliance tests" - @echo "make $(BOLD)coverage$(NORM) -- run pyspec tests with coverage" @echo "make $(BOLD)lint$(NORM) -- run linters and checks" @echo "make $(BOLD)serve_docs$(NORM) -- start a local docs web server" @echo "make $(BOLD)test$(NORM) -- run pyspec tests" @@ -63,14 +61,19 @@ help-verbose: @echo " Runs pyspec tests with various configuration options. Tests run in parallel" @echo " by default using pytest with the minimal preset and fastest BLS library." @echo "" - @echo " Parameters:" - @echo " k= Run specific test by name" - @echo " fork= Test specific fork (phase0, altair, bellatrix, capella, etc.)" - @echo " preset= Use specific preset (mainnet or minimal; default: minimal)" - @echo " bls= BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)" - @echo " kzg= KZG library type (spec, ckzg; default: ckzg)" - @echo " component= Test component: (all, pyspec, fw; default: all)" - @echo " reftests= Generate reference tests (default: false)" + @echo " Filtering:" + @echo " k= Run only tests matching this name" + @echo " fork= Run only tests for this fork (phase0, altair, bellatrix, capella, etc.)" + @echo " preset= Preset to use: mainnet, minimal (default: minimal)" + @echo " component= What to test: all, pyspec, fw (default: all)" + @echo "" + @echo " Libraries:" + @echo " bls= BLS library: py_ecc, milagro, arkworks, fastest (default: fastest)" + @echo " kzg= KZG library: spec, ckzg (default: ckzg)" + @echo "" + @echo " Output:" + @echo " reftests=true Generate reference test vectors" + @echo " coverage=true Enable code coverage tracking" @echo "" @echo " Examples:" @echo " make test" @@ -78,25 +81,13 @@ help-verbose: @echo " make test fork=deneb" @echo " make test preset=mainnet" @echo " make test preset=mainnet fork=deneb k=test_verify_kzg_proof" - @echo " make test bls=arkworks" @echo " make test component=fw" + @echo " make test bls=arkworks" @echo " make test reftests=true" @echo " make test reftests=true fork=fulu" @echo " make test reftests=true preset=mainnet fork=fulu k=invalid_committee_index" - @echo "" - @echo "$(BOLD)make coverage$(NORM)" - @echo "" - @echo " Runs tests with code coverage tracking and generates an HTML report." - @echo " Automatically opens the coverage report in your browser after completion." - @echo "" - @echo " Parameters:" - @echo " k= Run specific test by name" - @echo " fork= Test specific fork" - @echo "" - @echo " Examples:" - @echo " make coverage" - @echo " make coverage k=test_process_attestation" - @echo " make coverage fork=electra" + @echo " make test coverage=true k=test_process_attestation" + @echo " make test coverage=true fork=electra" @echo "" @echo "$(BOLD)CODE QUALITY$(NORM)" @echo "$(BOLD)--------------------------------------------------------------------------------$(NORM)" @@ -165,8 +156,6 @@ help-verbose: # Virtual Environment ############################################################################### -VENV = .venv - # Use non-editable installs for compliance test generators. # More details: ethereum/consensus-specs#4633. UV_RUN = uv run @@ -200,19 +189,30 @@ _pyspec: _sync TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports REFTESTS_DIR = $(CURDIR)/reftests +COV_REPORT_DIR = $(PYSPEC_DIR)/.htmlcov # Run pyspec tests. -test: MAYBE_TEST := $(if $(k),-k=$(k)) -# Disable parallelism when running a specific test. -# Parallelism makes debugging difficult (print doesn't work). -test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal) +# +# Filtering +test: MAYBE_TEST := $(if $(k),-k "$(k)") test: MAYBE_FORK := $(if $(fork),--fork=$(fork)) test: PRESET := $(if $(filter fw,$(component)),,$(if $(preset),--preset=$(preset),)) -test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest)) -test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg)) +# Disable parallelism when running a specific test. Makes debugging difficult (print doesn't work). +test: MAYBE_PARALLEL := $(if $(k),,-n logical --dist=worksteal) test: MAYBE_SPEC := $(if $(filter fw,$(component)),,$(PYSPEC_DIR)/eth_consensus_specs) test: MAYBE_INFRA := $(if $(filter pyspec,$(component)),,$(CURDIR)/tests/infra) +# +# Libraries +test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest)) +test: KZG := $(if $(filter fw,$(component)),,--kzg-type=$(if $(kzg),$(kzg),ckzg)) +# +# Output test: MAYBE_REFTESTS := $(if $(filter true,$(reftests)),--reftests --reftests-output=$(REFTESTS_DIR)) +test: COVERAGE_PRESETS := $(if $(preset),$(preset),$(if $(filter true,$(reftests)),minimal mainnet,minimal)) +test: COV_SCOPE_SINGLE := $(foreach P,$(COVERAGE_PRESETS), --cov=eth_consensus_specs.$(fork).$P) +test: COV_SCOPE_ALL := $(foreach P,$(COVERAGE_PRESETS),$(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$P)) +test: COV_SCOPE := $(if $(filter true,$(coverage)),$(if $(fork),$(COV_SCOPE_SINGLE),$(COV_SCOPE_ALL))) +test: COVERAGE := $(if $(filter true,$(coverage)),--coverage $(COV_SCOPE) --cov-report="html:$(COV_REPORT_DIR)" --cov-report="json:$(COV_REPORT_DIR)/coverage.json" --cov-branch --no-cov-on-fail) test: _pyspec @mkdir -p $(TEST_REPORT_DIR) @$(UV_RUN) pytest \ @@ -227,37 +227,10 @@ test: _pyspec --html=$(TEST_REPORT_DIR)/test_results.html \ --self-contained-html \ $(MAYBE_REFTESTS) \ + $(COVERAGE) \ $(MAYBE_INFRA) \ $(MAYBE_SPEC) -############################################################################### -# Coverage -############################################################################### - -TEST_PRESET_TYPE ?= minimal -COV_HTML_OUT=$(PYSPEC_DIR)/.htmlcov -COV_INDEX_FILE=$(COV_HTML_OUT)/index.html -COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth_consensus_specs.$S.$(TEST_PRESET_TYPE)) - -# Run pytest with coverage tracking -_test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k)) -_test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork)) -_test_with_coverage: _pyspec - @$(UV_RUN) pytest \ - -n auto \ - $(MAYBE_TEST) \ - $(MAYBE_FORK) \ - --disable-bls \ - $(COVERAGE_SCOPE) \ - --cov-report="html:$(COV_HTML_OUT)" \ - --cov-branch \ - $(PYSPEC_DIR)/eth_consensus_specs - -# Run tests with coverage then open the coverage report. -# See `make test` for a list of options. -coverage: _test_with_coverage - @echo "Opening result: $(COV_INDEX_FILE)" - @((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & ############################################################################### # Documentation diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index 7b4b434f30..7aff9d4c33 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -46,9 +46,21 @@ Note: these options can be used together, like: make test preset=minimal k=test_verify_kzg_proof fork=deneb ``` -### How to view code coverage report +### How to generate coverage reports -Run `make coverage` to run all tests and open the html code coverage report. +Run `make test coverage=true` to enable coverage tracking and generate reports. + +Reports are saved at: + +- **HTML report**: `tests/core/pyspec/.htmlcov/index.html` +- **JSON report**: `tests/core/pyspec/.htmlcov/coverage.json` + +To open the HTML report in a browser: + +```shell +xdg-open tests/core/pyspec/.htmlcov/index.html # Linux +open tests/core/pyspec/.htmlcov/index.html # macOS +``` ## Contributing diff --git a/tests/core/pyspec/eth_consensus_specs/test/conftest.py b/tests/core/pyspec/eth_consensus_specs/test/conftest.py index 9dee37d803..fcfd1372e7 100644 --- a/tests/core/pyspec/eth_consensus_specs/test/conftest.py +++ b/tests/core/pyspec/eth_consensus_specs/test/conftest.py @@ -50,10 +50,10 @@ def pytest_addoption(parser): ), ) parser.addoption( - "--disable-bls", + "--coverage", action="store_true", default=False, - help="bls-default: make tests that are not dependent on BLS run without BLS", + help="coverage: enable code coverage tracking", ) parser.addoption( "--bls-type", @@ -131,13 +131,6 @@ def run_phases(request): context.DEFAULT_PYTEST_FORKS = ALL_PHASES -@fixture(autouse=True) -def bls_default(request): - disable_bls = request.config.getoption("--disable-bls") - if disable_bls: - context.DEFAULT_BLS_ACTIVE = False - - @fixture(autouse=True) def bls_type(request): bls_type = request.config.getoption("--bls-type") diff --git a/tests/core/pyspec/eth_consensus_specs/test/context.py b/tests/core/pyspec/eth_consensus_specs/test/context.py index 41059a8092..99804ea7ba 100644 --- a/tests/core/pyspec/eth_consensus_specs/test/context.py +++ b/tests/core/pyspec/eth_consensus_specs/test/context.py @@ -290,17 +290,6 @@ def entry(*args, **kw): return entry -# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`. -# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*. -# The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). -# - Some tests are marked as BLS-requiring, and ignore this setting. -# (tests that express differences caused by BLS, e.g. invalid signatures being rejected) -# - Some other tests are marked as BLS-ignoring, and ignore this setting. -# (tests that are heavily performance impacted / require unsigned state transitions) -# - Most tests respect the BLS setting. -DEFAULT_BLS_ACTIVE = True - - is_pytest = True is_generator = False @@ -436,7 +425,7 @@ def bls_switch(fn): def entry(*args, **kw): old_state = bls.bls_active - bls.bls_active = kw.pop("bls_active", DEFAULT_BLS_ACTIVE) + bls.bls_active = kw.pop("bls_active", True) res = fn(*args, **kw) if res is not None: yield from res