Skip to content

Commit 179cde6

Browse files
authored
Add new check_mods function for generators (#3970)
* Add new check_mods function for generators * Use single qoutes for consistency * Add [ERROR] to the exception & update readme * Fix typos * Fix lint
1 parent 5fa7791 commit 179cde6

File tree

14 files changed

+92
-12
lines changed

14 files changed

+92
-12
lines changed

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from importlib import import_module
22
from inspect import getmembers, isfunction
3+
from pkgutil import walk_packages
34
from typing import Any, Callable, Dict, Iterable, Optional, List, Union
45

56
from eth2spec.utils import bls
@@ -134,3 +135,69 @@ def combine_mods(dict_1, dict_2):
134135
dict_3[key].append(dict_1[key])
135136

136137
return dict_3
138+
139+
140+
def check_mods(all_mods, pkg):
141+
"""
142+
Raise an exception if there is a missing/unexpected module in all_mods.
143+
"""
144+
def get_expected_modules(package, absolute=False):
145+
"""
146+
Return all modules (which are not packages) inside the given package.
147+
"""
148+
modules = []
149+
eth2spec = import_module('eth2spec')
150+
prefix = eth2spec.__name__ + '.'
151+
for _, modname, ispkg in walk_packages(eth2spec.__path__, prefix):
152+
s = package if absolute else f'.{package}.'
153+
if s in modname and not ispkg:
154+
modules.append(modname)
155+
return modules
156+
157+
mods = []
158+
for fork in all_mods:
159+
for mod in all_mods[fork].values():
160+
# If this key has a single value, normalize to list.
161+
if isinstance(mod, str):
162+
mod = [mod]
163+
164+
# For each submodule, check if it is package.
165+
# This is a "trick" we do to reuse a test format.
166+
for sub in mod:
167+
is_package = '.test_' not in sub
168+
if is_package:
169+
mods.extend(get_expected_modules(sub, absolute=True))
170+
else:
171+
mods.append(sub)
172+
173+
problems = []
174+
expected_mods = get_expected_modules(pkg)
175+
if mods != expected_mods:
176+
for e in expected_mods:
177+
# Skip forks which are not in all_mods.
178+
# The fork name is the 3rd item in the path.
179+
fork = e.split('.')[2]
180+
if fork not in all_mods:
181+
continue
182+
# Skip modules in the unittests package.
183+
# These are not associated with generators.
184+
if '.unittests.' in e:
185+
continue
186+
# The expected module is not in our list of modules.
187+
# Add it to our list of problems.
188+
if e not in mods:
189+
problems.append('missing: ' + e)
190+
191+
for t in mods:
192+
# Skip helper modules.
193+
# These do not define test functions.
194+
if t.startswith('eth2spec.test.helpers'):
195+
continue
196+
# There is a module not defined in eth2spec.
197+
# Add it to our list of problems.
198+
if t not in expected_mods:
199+
print('unexpected:', t)
200+
problems.append('unexpected: ' + t)
201+
202+
if problems:
203+
raise Exception('[ERROR] module problems:\n ' + '\n '.join(problems))

tests/generators/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ if __name__ == "__main__":
185185
PHASE0: phase_0_mods,
186186
ALTAIR: altair_mods,
187187
}
188+
check_mods(all_mods, "sanity")
188189

189190
run_state_test_generators(runner_name="sanity", all_mods=all_mods)
190191
```

tests/generators/epoch_processing/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
1+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
22
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
33

44

@@ -63,5 +63,6 @@
6363
DENEB: deneb_mods,
6464
ELECTRA: electra_mods,
6565
}
66+
check_mods(all_mods, "epoch_processing")
6667

6768
run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods)

tests/generators/finality/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
1+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
22
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
33

44

@@ -18,5 +18,6 @@
1818
DENEB: deneb_mods,
1919
ELECTRA: electra_mods,
2020
}
21+
check_mods(all_mods, "finality")
2122

2223
run_state_test_generators(runner_name="finality", all_mods=all_mods)

tests/generators/fork_choice/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
1+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
22
from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
33

44

@@ -37,5 +37,6 @@
3737
DENEB: deneb_mods,
3838
ELECTRA: electra_mods,
3939
}
40+
check_mods(all_mods, "fork_choice")
4041

4142
run_state_test_generators(runner_name="fork_choice", all_mods=all_mods)

tests/generators/genesis/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
1+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
22
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
33

44

@@ -26,5 +26,6 @@
2626
DENEB: deneb_mods,
2727
ELECTRA: electra_mods,
2828
}
29+
check_mods(all_mods, "genesis")
2930

3031
run_state_test_generators(runner_name="genesis", all_mods=all_mods)

tests/generators/light_client/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
2-
from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators
2+
from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators, check_mods
33

44

55
if __name__ == "__main__":
@@ -24,5 +24,6 @@
2424
DENEB: deneb_mods,
2525
ELECTRA: electra_mods,
2626
}
27+
check_mods(all_mods, "light_client")
2728

2829
run_state_test_generators(runner_name="light_client", all_mods=all_mods)

tests/generators/merkle_proof/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from eth2spec.test.helpers.constants import DENEB, ELECTRA, EIP7594
2-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
2+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
33

44

55
if __name__ == "__main__":
@@ -17,5 +17,6 @@
1717
ELECTRA: electra_mods,
1818
EIP7594: eip_7594_mods,
1919
}
20+
check_mods(all_mods, "merkle_proof")
2021

2122
run_state_test_generators(runner_name="merkle_proof", all_mods=all_mods)

tests/generators/networking/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22
from eth2spec.test.helpers.constants import EIP7594
3-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
3+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods
44

55

66
if __name__ == "__main__":
@@ -10,5 +10,6 @@
1010
all_mods = {
1111
EIP7594: eip7594_mods
1212
}
13+
check_mods(all_mods, "networking")
1314

1415
run_state_test_generators(runner_name="networking", all_mods=all_mods)

tests/generators/operations/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods
1+
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods
22
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA
33

44

@@ -61,5 +61,6 @@
6161
DENEB: deneb_mods,
6262
ELECTRA: electra_mods,
6363
}
64+
check_mods(all_mods, "block_processing")
6465

6566
run_state_test_generators(runner_name="operations", all_mods=all_mods)

0 commit comments

Comments
 (0)