Skip to content

Commit 10d62b5

Browse files
carvercarver
authored
Merge pull request #1577 from lithp/lithp/use-blockchain-state-tests
Use the state tests embedded in the blockchain tests
2 parents bc66da9 + 9d28e83 commit 10d62b5

File tree

4 files changed

+162
-467
lines changed

4 files changed

+162
-467
lines changed

‎.circleci/config.yml

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -102,42 +102,42 @@ jobs:
102102
environment:
103103
TOXENV: py36-docs
104104

105-
py35-native-state-byzantium:
105+
py35-native-blockchain-byzantium:
106106
<<: *common
107107
docker:
108108
- image: circleci/python:3.5
109109
environment:
110-
TOXENV: py35-native-state-byzantium
111-
py35-native-state-frontier:
110+
TOXENV: py35-native-blockchain-byzantium
111+
py35-native-blockchain-frontier:
112112
<<: *common
113113
docker:
114114
- image: circleci/python:3.5
115115
environment:
116-
TOXENV: py35-native-state-frontier
117-
py35-native-state-homestead:
116+
TOXENV: py35-native-blockchain-frontier
117+
py35-native-blockchain-homestead:
118118
<<: *common
119119
docker:
120120
- image: circleci/python:3.5
121121
environment:
122-
TOXENV: py35-native-state-homestead
123-
py35-native-state-eip150:
122+
TOXENV: py35-native-blockchain-homestead
123+
py35-native-blockchain-eip150:
124124
<<: *common
125125
docker:
126126
- image: circleci/python:3.5
127127
environment:
128-
TOXENV: py35-native-state-eip150
129-
py35-native-state-eip158:
128+
TOXENV: py35-native-blockchain-eip150
129+
py35-native-blockchain-eip158:
130130
<<: *common
131131
docker:
132132
- image: circleci/python:3.5
133133
environment:
134-
TOXENV: py35-native-state-eip158
135-
py35-native-blockchain:
134+
TOXENV: py35-native-blockchain-eip158
135+
py35-native-blockchain-transition:
136136
<<: *common
137137
docker:
138138
- image: circleci/python:3.5
139139
environment:
140-
TOXENV: py35-native-blockchain
140+
TOXENV: py35-native-blockchain-transition
141141
py35-vm:
142142
<<: *common
143143
docker:
@@ -169,36 +169,42 @@ jobs:
169169
- image: circleci/python:3.6
170170
environment:
171171
TOXENV: py36-benchmark
172-
py36-native-state-byzantium:
172+
py36-native-blockchain-byzantium:
173173
<<: *common
174174
docker:
175175
- image: circleci/python:3.6
176176
environment:
177-
TOXENV: py36-native-state-byzantium
178-
py36-native-state-frontier:
177+
TOXENV: py36-native-blockchain-byzantium
178+
py36-native-blockchain-frontier:
179179
<<: *common
180180
docker:
181181
- image: circleci/python:3.6
182182
environment:
183-
TOXENV: py36-native-state-frontier
184-
py36-native-state-homestead:
183+
TOXENV: py36-native-blockchain-frontier
184+
py36-native-blockchain-homestead:
185185
<<: *common
186186
docker:
187187
- image: circleci/python:3.6
188188
environment:
189-
TOXENV: py36-native-state-homestead
190-
py36-native-state-eip150:
189+
TOXENV: py36-native-blockchain-homestead
190+
py36-native-blockchain-eip150:
191191
<<: *common
192192
docker:
193193
- image: circleci/python:3.6
194194
environment:
195-
TOXENV: py36-native-state-eip150
196-
py36-native-state-eip158:
195+
TOXENV: py36-native-blockchain-eip150
196+
py36-native-blockchain-eip158:
197197
<<: *common
198198
docker:
199199
- image: circleci/python:3.6
200200
environment:
201-
TOXENV: py36-native-state-eip158
201+
TOXENV: py36-native-blockchain-eip158
202+
py36-native-blockchain-transition:
203+
<<: *common
204+
docker:
205+
- image: circleci/python:3.6
206+
environment:
207+
TOXENV: py36-native-blockchain-transition
202208
py36-rpc-state-byzantium:
203209
<<: *common
204210
docker:
@@ -229,12 +235,6 @@ jobs:
229235
- image: circleci/python:3.6
230236
environment:
231237
TOXENV: py36-rpc-state-eip158
232-
py36-native-blockchain:
233-
<<: *common
234-
docker:
235-
- image: circleci/python:3.6
236-
environment:
237-
TOXENV: py36-native-blockchain
238238
py36-rpc-blockchain:
239239
<<: *common
240240
docker:
@@ -339,17 +339,17 @@ workflows:
339339
- py37-trinity-integration
340340
- py37-beacon
341341

342-
- py36-native-state-byzantium
343-
- py36-native-state-frontier
344-
- py36-native-state-homestead
345-
- py36-native-state-eip150
346-
- py36-native-state-eip158
342+
- py36-native-blockchain-byzantium
343+
- py36-native-blockchain-frontier
344+
- py36-native-blockchain-homestead
345+
- py36-native-blockchain-eip150
346+
- py36-native-blockchain-eip158
347+
- py36-native-blockchain-transition
347348
- py36-rpc-state-byzantium
348349
- py36-rpc-state-frontier
349350
- py36-rpc-state-homestead
350351
- py36-rpc-state-eip150
351352
- py36-rpc-state-eip158
352-
- py36-native-blockchain
353353
- py36-rpc-blockchain
354354
- py36-vm
355355
- py36-benchmark
@@ -363,12 +363,12 @@ workflows:
363363
- py36-rpc-state-quadratic
364364
- py36-beacon
365365

366-
- py35-native-state-byzantium
367-
- py35-native-state-frontier
368-
- py35-native-state-homestead
369-
- py35-native-state-eip150
370-
- py35-native-state-eip158
371-
- py35-native-blockchain
366+
- py35-native-blockchain-byzantium
367+
- py35-native-blockchain-frontier
368+
- py35-native-blockchain-homestead
369+
- py35-native-blockchain-eip150
370+
- py35-native-blockchain-eip158
371+
- py35-native-blockchain-transition
372372
- py35-vm
373373
- py35-core
374374
- py35-transactions

‎tests/json-fixtures/test_blockchain.py

Lines changed: 112 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import rlp
44

55
from eth_utils import (
6+
to_tuple,
67
ValidationError,
78
)
89

@@ -24,6 +25,7 @@
2425
genesis_params_from_fixture,
2526
load_fixture,
2627
new_chain_from_fixture,
28+
should_run_slow_tests,
2729
verify_account_db,
2830
)
2931

@@ -34,6 +36,84 @@
3436
BASE_FIXTURE_PATH = os.path.join(ROOT_PROJECT_DIR, 'fixtures', 'BlockchainTests')
3537

3638

39+
# These are the slowest tests from the full blockchain test run. This list
40+
# should be regenerated occasionally using `--durations 100`.
41+
SLOWEST_TESTS = {
42+
('GeneralStateTests/stStaticCall/static_Call50000_sha256_d0g0v0.json', 'static_Call50000_sha256_d0g0v0_Byzantium'), # noqa: E501
43+
('GeneralStateTests/stStaticCall/static_Call50000_sha256_d1g0v0.json', 'static_Call50000_sha256_d1g0v0_Byzantium'), # noqa: E501
44+
('GeneralStateTests/stStaticCall/static_Call50000_rip160_d0g0v0.json', 'static_Call50000_rip160_d0g0v0_Byzantium'), # noqa: E501
45+
('GeneralStateTests/stStaticCall/static_Call50000_d1g0v0.json', 'static_Call50000_d1g0v0_Byzantium'), # noqa: E501
46+
('GeneralStateTests/stStaticCall/static_Call50000_ecrec_d1g0v0.json', 'static_Call50000_ecrec_d1g0v0_Byzantium'), # noqa: E501
47+
('GeneralStateTests/stStaticCall/static_Call50000_rip160_d1g0v0.json', 'static_Call50000_rip160_d1g0v0_Byzantium'), # noqa: E501
48+
('GeneralStateTests/stStaticCall/static_Call50000_identity2_d1g0v0.json', 'static_Call50000_identity2_d1g0v0_Byzantium'), # noqa: E501
49+
('GeneralStateTests/stStaticCall/static_Call50000_identity_d1g0v0.json', 'static_Call50000_identity_d1g0v0_Byzantium'), # noqa: E501
50+
('GeneralStateTests/stStaticCall/static_Return50000_2_d0g0v0.json', 'static_Return50000_2_d0g0v0_Byzantium'), # noqa: E501
51+
('GeneralStateTests/stStaticCall/static_Call50000_d0g0v0.json', 'static_Call50000_d0g0v0_Byzantium'), # noqa: E501
52+
('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d1g0v0.json', 'static_Call50000bytesContract50_1_d1g0v0_Byzantium'), # noqa: E501
53+
('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2_d1g0v0.json', 'static_Call50000bytesContract50_2_d1g0v0_Byzantium'), # noqa: E501
54+
('GeneralStateTests/stChangedEIP150/Call1024PreCalls_d0g0v0.json', 'Call1024PreCalls_d0g0v0_Byzantium'), # noqa: E501
55+
('GeneralStateTests/stChangedEIP150/Call1024PreCalls_d0g0v0.json', 'Call1024PreCalls_d0g0v0_EIP150'), # noqa: E501
56+
('GeneralStateTests/stDelegatecallTestHomestead/Call1024PreCalls_d0g0v0.json', 'Call1024PreCalls_d0g0v0_Byzantium'), # noqa: E501
57+
('GeneralStateTests/stStaticCall/static_Call50000_ecrec_d0g0v0.json', 'static_Call50000_ecrec_d0g0v0_Byzantium'), # noqa: E501
58+
('GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json', 'ContractCreationSpam_d0g0v0_Frontier'), # noqa: E501
59+
('GeneralStateTests/stAttackTest/ContractCreationSpam_d0g0v0.json', 'ContractCreationSpam_d0g0v0_Homestead'), # noqa: E501
60+
('GeneralStateTests/stCallCreateCallCodeTest/Call1024PreCalls_d0g0v0.json', 'Call1024PreCalls_d0g0v0_Byzantium'), # noqa: E501
61+
('GeneralStateTests/stStaticCall/static_Call1024PreCalls2_d0g0v0.json', 'static_Call1024PreCalls2_d0g0v0_Byzantium'), # noqa: E501
62+
('GeneralStateTests/stStaticCall/static_Call50000_identity2_d0g0v0.json', 'static_Call50000_identity2_d0g0v0_Byzantium'), # noqa: E501
63+
('GeneralStateTests/stStaticCall/static_Call50000_identity_d0g0v0.json', 'static_Call50000_identity_d0g0v0_Byzantium'), # noqa: E501
64+
('GeneralStateTests/stRecursiveCreate/recursiveCreateReturnValue_d0g0v0.json', 'recursiveCreateReturnValue_d0g0v0_Homestead'), # noqa: E501
65+
('GeneralStateTests/stStaticCall/static_LoopCallsThenRevert_d0g0v0.json', 'static_LoopCallsThenRevert_d0g0v0_Byzantium'), # noqa: E501
66+
('GeneralStateTests/stCallCreateCallCodeTest/Callcode1024OOG_d0g0v0.json', 'Callcode1024OOG_d0g0v0_Frontier'), # noqa: E501
67+
('GeneralStateTests/stStaticCall/static_LoopCallsThenRevert_d0g1v0.json', 'static_LoopCallsThenRevert_d0g1v0_Byzantium'), # noqa: E501
68+
('GeneralStateTests/stStaticCall/static_Call1024PreCalls3_d1g0v0.json', 'static_Call1024PreCalls3_d1g0v0_Byzantium'), # noqa: E501
69+
('GeneralStateTests/stStaticCall/static_Call1024PreCalls_d1g0v0.json', 'static_Call1024PreCalls_d1g0v0_Byzantium'), # noqa: E501
70+
('GeneralStateTests/stStaticCall/static_Call1024PreCalls2_d1g0v0.json', 'static_Call1024PreCalls2_d1g0v0_Byzantium'), # noqa: E501
71+
('GeneralStateTests/stDelegatecallTestHomestead/Delegatecall1024OOG_d0g0v0.json', 'Delegatecall1024OOG_d0g0v0_Homestead'), # noqa: E501
72+
('GeneralStateTests/stChangedEIP150/Call1024BalanceTooLow_d0g0v0.json', 'Call1024BalanceTooLow_d0g0v0_Byzantium'), # noqa: E501
73+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBombLog_d0g0v0.json', 'CallRecursiveBombLog_d0g0v0_Frontier'), # noqa: E501
74+
('GeneralStateTests/stChangedEIP150/Callcode1024BalanceTooLow_d0g0v0.json', 'Callcode1024BalanceTooLow_d0g0v0_Byzantium'), # noqa: E501
75+
('bcForkStressTest/ForkStressTest.json', 'ForkStressTest_Frontier'), # noqa: E501
76+
('bcForkStressTest/ForkStressTest.json', 'ForkStressTest_Byzantium'), # noqa: E501
77+
('GeneralStateTests/stCallCreateCallCodeTest/Call1024OOG_d0g0v0.json', 'Call1024OOG_d0g0v0_Frontier'), # noqa: E501
78+
('GeneralStateTests/stDelegatecallTestHomestead/Call1024OOG_d0g0v0.json', 'Call1024OOG_d0g0v0_Homestead'), # noqa: E501
79+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0.json', 'CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0_Homestead'), # noqa: E501
80+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBombLog_d0g0v0.json', 'CallRecursiveBombLog_d0g0v0_Homestead'), # noqa: E501
81+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb1_d0g0v0.json', 'CallRecursiveBomb1_d0g0v0_Frontier'), # noqa: E501
82+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBombLog2_d0g0v0.json', 'CallRecursiveBombLog2_d0g0v0_Homestead'), # noqa: E501
83+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0.json', 'CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0_Frontier'), # noqa: E501
84+
('GeneralStateTests/stCallCreateCallCodeTest/Call1024BalanceTooLow_d0g0v0.json', 'Call1024BalanceTooLow_d0g0v0_Byzantium'), # noqa: E501
85+
('GeneralStateTests/stChangedEIP150/Call1024BalanceTooLow_d0g0v0.json', 'Call1024BalanceTooLow_d0g0v0_EIP150'), # noqa: E501
86+
('bcForkStressTest/ForkStressTest.json', 'ForkStressTest_EIP150'), # noqa: E501
87+
('GeneralStateTests/stDelegatecallTestHomestead/CallRecursiveBombPreCall_d0g0v0.json', 'CallRecursiveBombPreCall_d0g0v0_Byzantium'), # noqa: E501
88+
('GeneralStateTests/stChangedEIP150/Callcode1024BalanceTooLow_d0g0v0.json', 'Callcode1024BalanceTooLow_d0g0v0_EIP150'), # noqa: E501
89+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb1_d0g0v0.json', 'CallRecursiveBomb1_d0g0v0_Homestead'), # noqa: E501
90+
('GeneralStateTests/stDelegatecallTestHomestead/CallRecursiveBombPreCall_d0g0v0.json', 'CallRecursiveBombPreCall_d0g0v0_Homestead'), # noqa: E501
91+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb2_d0g0v0.json', 'CallRecursiveBomb2_d0g0v0_Homestead'), # noqa: E501
92+
('GeneralStateTests/stCallCreateCallCodeTest/Callcode1024BalanceTooLow_d0g0v0.json', 'Callcode1024BalanceTooLow_d0g0v0_Byzantium'), # noqa: E501
93+
('GeneralStateTests/stSystemOperationsTest/ABAcalls2_d0g0v0.json', 'ABAcalls2_d0g0v0_Frontier'), # noqa: E501
94+
('GeneralStateTests/stDelegatecallTestHomestead/Delegatecall1024_d0g0v0.json', 'Delegatecall1024_d0g0v0_Byzantium'), # noqa: E501
95+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBombLog2_d0g0v0.json', 'CallRecursiveBombLog2_d0g0v0_Frontier'), # noqa: E501
96+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb2_d0g0v0.json', 'CallRecursiveBomb2_d0g0v0_Frontier'), # noqa: E501
97+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0.json', 'CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0_EIP150'), # noqa: E501
98+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_d0g0v0.json', 'CallRecursiveBomb0_d0g0v0_Frontier'), # noqa: E501
99+
('GeneralStateTests/stDelegatecallTestHomestead/Call1024BalanceTooLow_d0g0v0.json', 'Call1024BalanceTooLow_d0g0v0_Byzantium'), # noqa: E501
100+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_d0g0v0.json', 'CallRecursiveBomb0_d0g0v0_Homestead'), # noqa: E501
101+
('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1_d0g0v0.json', 'static_Call50000bytesContract50_1_d0g0v0_Byzantium'), # noqa: E501
102+
('GeneralStateTests/stDelegatecallTestHomestead/Delegatecall1024_d0g0v0.json', 'Delegatecall1024_d0g0v0_EIP150'), # noqa: E501
103+
('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0.json', 'CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0_Byzantium'), # noqa: E501
104+
('GeneralStateTests/stCallCreateCallCodeTest/CallRecursiveBombPreCall_d0g0v0.json', 'CallRecursiveBombPreCall_d0g0v0_Byzantium'), # noqa: E501
105+
('GeneralStateTests/stSystemOperationsTest/ABAcalls1_d0g0v0.json', 'ABAcalls1_d0g0v0_Frontier'), # noqa: E501
106+
('GeneralStateTests/stCallCreateCallCodeTest/CallRecursiveBombPreCall_d0g0v0.json', 'CallRecursiveBombPreCall_d0g0v0_Frontier'), # noqa: E501
107+
('GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth_d1g0v0.json', 'static_Call1MB1024Calldepth_d1g0v0_Byzantium'), # noqa: E501
108+
('GeneralStateTests/stRevertTest/LoopCallsDepthThenRevert2_d0g0v0.json', 'LoopCallsDepthThenRevert2_d0g0v0_Byzantium'), # noqa: E501
109+
('GeneralStateTests/stSystemOperationsTest/ABAcalls2_d0g0v0.json', 'ABAcalls2_d0g0v0_Homestead'), # noqa: E501
110+
('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2_d0g0v0.json', 'static_Call50000bytesContract50_2_d0g0v0_Byzantium'), # noqa: E501
111+
('GeneralStateTests/stRevertTest/LoopCallsDepthThenRevert2_d0g0v0.json', 'LoopCallsDepthThenRevert2_d0g0v0_EIP158'), # noqa: E501
112+
('GeneralStateTests/stRevertTest/LoopCallsDepthThenRevert3_d0g0v0.json', 'LoopCallsDepthThenRevert3_d0g0v0_EIP158'), # noqa: E501
113+
('GeneralStateTests/stRevertTest/LoopCallsDepthThenRevert3_d0g0v0.json', 'LoopCallsDepthThenRevert3_d0g0v0_Byzantium'), # noqa: E501
114+
}
115+
116+
37117
# These are tests that are thought to be incorrect or buggy upstream,
38118
# at the commit currently checked out in submodule `fixtures`.
39119
# Ideally, this list should be empty.
@@ -48,37 +128,60 @@
48128
}
49129

50130

51-
def blockchain_fixture_mark_fn(fixture_path, fixture_name):
131+
def blockchain_fixture_mark_fn(fixture_path, fixture_name, fixture_fork):
132+
fixture_id = (fixture_path, fixture_name)
133+
52134
if fixture_path.startswith('bcExploitTest'):
53135
return pytest.mark.skip("Exploit tests are slow")
54136
elif fixture_path == 'bcWalletTest/walletReorganizeOwners.json':
55137
return pytest.mark.skip("Wallet owner reorganization tests are slow")
56-
elif (fixture_path, fixture_name) in INCORRECT_UPSTREAM_TESTS:
138+
elif fixture_id in INCORRECT_UPSTREAM_TESTS:
57139
return pytest.mark.xfail(reason="Listed in INCORRECT_UPSTREAM_TESTS.")
140+
elif 'stTransactionTest/zeroSigTransa' in fixture_path:
141+
return pytest.mark.skip("EIP-86 not supported.")
142+
elif fixture_id in SLOWEST_TESTS:
143+
if should_run_slow_tests():
144+
return
145+
else:
146+
return pytest.mark.skip("Skipping slow test")
147+
elif 'stQuadraticComplexityTest' in fixture_path:
148+
return pytest.mark.skip("Skipping slow test")
149+
150+
151+
def generate_ignore_fn_for_fork(passed_fork):
152+
if passed_fork:
153+
passed_fork = passed_fork.lower()
154+
155+
def ignore_fn(fixture_path, fixture_key, fixture_fork):
156+
return fixture_fork.lower() != passed_fork
157+
158+
return ignore_fn
58159

59160

60-
def blockchain_fixture_ignore_fn(fixture_path, fixture_name):
61-
if fixture_path.startswith('GeneralStateTests'):
62-
# General state tests are also exported as blockchain tests. We
63-
# skip them here so we don't run them twice
64-
return True
161+
@to_tuple
162+
def expand_fixtures_forks(all_fixtures):
163+
for fixture_path, fixture_key in all_fixtures:
164+
fixture = load_fixture(fixture_path, fixture_key)
165+
yield fixture_path, fixture_key, fixture['network']
65166

66167

67168
def pytest_generate_tests(metafunc):
169+
fork = metafunc.config.getoption('fork')
68170
generate_fixture_tests(
69171
metafunc=metafunc,
70172
base_fixture_path=BASE_FIXTURE_PATH,
173+
preprocess_fn=expand_fixtures_forks,
71174
filter_fn=filter_fixtures(
72175
fixtures_base_dir=BASE_FIXTURE_PATH,
73176
mark_fn=blockchain_fixture_mark_fn,
74-
ignore_fn=blockchain_fixture_ignore_fn,
177+
ignore_fn=generate_ignore_fn_for_fork(fork)
75178
),
76179
)
77180

78181

79182
@pytest.fixture
80183
def fixture(fixture_data):
81-
fixture_path, fixture_key = fixture_data
184+
fixture_path, fixture_key, fixture_fork = fixture_data
82185
fixture = load_fixture(
83186
fixture_path,
84187
fixture_key,

0 commit comments

Comments
 (0)