Skip to content

Commit 2f2d356

Browse files
marioevzmarioevz
authored
fix(tests): EOF - Remove duplicate container tests, automatically check for duplicates (#800)
* feat(specs): Check EOF test duplicates * fix(specs): Skip dupplicates on EOFStateTest automatically * fix(tests): EOF - remove duplicates * fix(specs): tests
1 parent 589ab4e commit 2f2d356

File tree

16 files changed

+103
-178
lines changed

16 files changed

+103
-178
lines changed

‎src/ethereum_test_specs/base.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from pathlib import Path
1010
from typing import Callable, ClassVar, Dict, Generator, Iterator, List, Optional
1111

12+
import pytest
1213
from pydantic import BaseModel, Field
1314

1415
from ethereum_test_base_types import to_hex
@@ -78,6 +79,7 @@ class BaseTest(BaseModel):
7879
def generate(
7980
self,
8081
*,
82+
request: pytest.FixtureRequest,
8183
t8n: TransitionTool,
8284
fork: Fork,
8385
fixture_format: FixtureFormats,

‎src/ethereum_test_specs/blockchain.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from pprint import pprint
66
from typing import Any, Callable, ClassVar, Dict, Generator, List, Optional, Tuple, Type
77

8+
import pytest
89
from pydantic import ConfigDict, Field, field_validator
910

1011
from ethereum_test_base_types import (
@@ -714,6 +715,7 @@ def make_hive_fixture(
714715

715716
def generate(
716717
self,
718+
request: pytest.FixtureRequest,
717719
t8n: TransitionTool,
718720
fork: Fork,
719721
fixture_format: FixtureFormats,

‎src/ethereum_test_specs/eof.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
from pathlib import Path
88
from shutil import which
99
from subprocess import CompletedProcess
10-
from typing import Any, Callable, ClassVar, Generator, List, Optional, Type
10+
from typing import Any, Callable, ClassVar, Dict, Generator, List, Optional, Type
1111

12+
import pytest
1213
from pydantic import Field, model_validator
1314

1415
from ethereum_test_base_types import Account, Bytes
@@ -24,6 +25,8 @@
2425
from .base import BaseTest
2526
from .state import StateTest
2627

28+
existing_tests: Dict[Bytes, str] = {}
29+
2730

2831
class EOFBaseException(Exception):
2932
"""
@@ -186,12 +189,18 @@ def pytest_parameter_name(cls) -> str:
186189
def make_eof_test_fixture(
187190
self,
188191
*,
192+
request: pytest.FixtureRequest,
189193
fork: Fork,
190194
eips: Optional[List[int]],
191195
) -> Fixture:
192196
"""
193197
Generate the EOF test fixture.
194198
"""
199+
if self.data in existing_tests:
200+
pytest.fail(
201+
f"Duplicate EOF test: {self.data}, existing test: {existing_tests[self.data]}"
202+
)
203+
existing_tests[self.data] = request.node.nodeid
195204
vectors = [
196205
Vector(
197206
code=self.data,
@@ -259,6 +268,7 @@ def verify_result(self, result: CompletedProcess, expected_result: Result, code:
259268
def generate(
260269
self,
261270
*,
271+
request: pytest.FixtureRequest,
262272
t8n: TransitionTool,
263273
fork: Fork,
264274
eips: Optional[List[int]] = None,
@@ -269,7 +279,7 @@ def generate(
269279
Generate the BlockchainTest fixture.
270280
"""
271281
if fixture_format == FixtureFormats.EOF_TEST:
272-
return self.make_eof_test_fixture(fork=fork, eips=eips)
282+
return self.make_eof_test_fixture(request=request, fork=fork, eips=eips)
273283

274284
raise Exception(f"Unknown fixture format: {fixture_format}")
275285

@@ -358,6 +368,7 @@ def generate_state_test(self) -> StateTest:
358368
def generate(
359369
self,
360370
*,
371+
request: pytest.FixtureRequest,
361372
t8n: TransitionTool,
362373
fork: Fork,
363374
eips: Optional[List[int]] = None,
@@ -368,14 +379,18 @@ def generate(
368379
Generate the BlockchainTest fixture.
369380
"""
370381
if fixture_format == FixtureFormats.EOF_TEST:
371-
return self.make_eof_test_fixture(fork=fork, eips=eips)
382+
if self.data in existing_tests:
383+
# Gracefully skip duplicate tests because one EOFStateTest can generate multiple
384+
# state fixtures with the same data.
385+
pytest.skip(f"Duplicate EOF container on EOFStateTest: {request.node.nodeid}")
386+
return self.make_eof_test_fixture(request=request, fork=fork, eips=eips)
372387
elif fixture_format in (
373388
FixtureFormats.STATE_TEST,
374389
FixtureFormats.BLOCKCHAIN_TEST,
375390
FixtureFormats.BLOCKCHAIN_TEST_ENGINE,
376391
):
377392
return self.generate_state_test().generate(
378-
t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
393+
request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
379394
)
380395

381396
raise Exception(f"Unknown fixture format: {fixture_format}")

‎src/ethereum_test_specs/state.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
from typing import Any, Callable, ClassVar, Dict, Generator, List, Optional, Type
66

7+
import pytest
8+
79
from ethereum_test_exceptions import EngineAPIError
810
from ethereum_test_fixtures import BaseFixture, FixtureFormats
911
from ethereum_test_fixtures.state import (
@@ -161,6 +163,7 @@ def make_state_test_fixture(
161163

162164
def generate(
163165
self,
166+
request: pytest.FixtureRequest,
164167
t8n: TransitionTool,
165168
fork: Fork,
166169
fixture_format: FixtureFormats,
@@ -171,7 +174,7 @@ def generate(
171174
"""
172175
if fixture_format in BlockchainTest.supported_fixture_formats:
173176
return self.generate_blockchain_test().generate(
174-
t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
177+
request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
175178
)
176179
elif fixture_format == FixtureFormats.STATE_TEST:
177180
return self.make_state_test_fixture(t8n, fork, eips)

‎src/ethereum_test_specs/tests/test_expect.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,9 @@ def test_post_storage_value_mismatch(
115115
Test post state `Account.storage` exceptions during state test fixture generation.
116116
"""
117117
with pytest.raises(Storage.KeyValueMismatch) as e_info:
118-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
118+
state_test.generate(
119+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
120+
)
119121
assert e_info.value == expected_exception
120122

121123

@@ -141,10 +143,14 @@ def test_post_nonce_value_mismatch(pre: Alloc, post: Alloc, state_test, t8n, for
141143
pre_nonce = pre_account.nonce
142144
post_nonce = post_account.nonce
143145
if "nonce" not in post_account.model_fields_set: # no exception
144-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
146+
state_test.generate(
147+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
148+
)
145149
return
146150
with pytest.raises(Account.NonceMismatch) as e_info:
147-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
151+
state_test.generate(
152+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
153+
)
148154
assert e_info.value == Account.NonceMismatch(
149155
address=ADDRESS_UNDER_TEST, want=post_nonce, got=pre_nonce
150156
)
@@ -172,10 +178,14 @@ def test_post_code_value_mismatch(pre: Alloc, post: Alloc, state_test, t8n, fork
172178
pre_code = pre_account.code
173179
post_code = post_account.code
174180
if "code" not in post_account.model_fields_set: # no exception
175-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
181+
state_test.generate(
182+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
183+
)
176184
return
177185
with pytest.raises(Account.CodeMismatch) as e_info:
178-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
186+
state_test.generate(
187+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
188+
)
179189
assert e_info.value == Account.CodeMismatch(
180190
address=ADDRESS_UNDER_TEST, want=post_code, got=pre_code
181191
)
@@ -203,10 +213,14 @@ def test_post_balance_value_mismatch(pre: Alloc, post: Alloc, state_test, t8n, f
203213
pre_balance = pre_account.balance
204214
post_balance = post_account.balance
205215
if "balance" not in post_account.model_fields_set: # no exception
206-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
216+
state_test.generate(
217+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
218+
)
207219
return
208220
with pytest.raises(Account.BalanceMismatch) as e_info:
209-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
221+
state_test.generate(
222+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
223+
)
210224
assert e_info.value == Account.BalanceMismatch(
211225
address=ADDRESS_UNDER_TEST, want=post_balance, got=pre_balance
212226
)
@@ -245,7 +259,11 @@ def test_post_account_mismatch(state_test, t8n, fork, exception_type: Type[Excep
245259
fixture generation.
246260
"""
247261
if exception_type is None:
248-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
262+
state_test.generate(
263+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
264+
)
249265
return
250266
with pytest.raises(exception_type) as _:
251-
state_test.generate(t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST)
267+
state_test.generate(
268+
request=None, t8n=t8n, fork=fork, fixture_format=FixtureFormats.STATE_TEST
269+
)

‎src/ethereum_test_specs/tests/test_fixtures.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,12 @@ def test_make_genesis(fork: Fork, hash: bytes): # noqa: D103
9191
post={},
9292
blocks=[],
9393
tag="some_state_test",
94-
).generate(t8n, fork, fixture_format=FixtureFormats.BLOCKCHAIN_TEST)
94+
).generate(
95+
request=None, # type: ignore
96+
t8n=t8n,
97+
fork=fork,
98+
fixture_format=FixtureFormats.BLOCKCHAIN_TEST,
99+
)
95100
assert isinstance(fixture, BlockchainFixture)
96101
assert fixture.genesis is not None
97102

@@ -154,6 +159,7 @@ def test_fill_state_test(
154159
tx=tx,
155160
tag="my_chain_id_test",
156161
).generate(
162+
request=None, # type: ignore
157163
t8n=t8n,
158164
fork=fork,
159165
fixture_format=fixture_format,
@@ -486,6 +492,7 @@ def blockchain_test_fixture( # noqa: D102
486492
genesis_environment=genesis_environment,
487493
tag="my_blockchain_test_valid_txs",
488494
).generate(
495+
request=None, # type: ignore
489496
t8n=t8n,
490497
fork=fork,
491498
fixture_format=fixture_format,
@@ -868,6 +875,7 @@ def test_fill_blockchain_invalid_txs(fork: Fork, check_hive: bool, expected_json
868875
blocks=blocks,
869876
genesis_environment=genesis_environment,
870877
).generate(
878+
request=None, # type: ignore
871879
t8n=t8n,
872880
fork=fork,
873881
fixture_format=fixture_format,

‎src/ethereum_test_tools/tests/test_code.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,7 @@ def test_switch(tx_data: bytes, switch_bytecode: bytes, expected_storage: Mappin
636636
post=post,
637637
)
638638
state_test.generate(
639+
request=None, # type: ignore
639640
t8n=GethTransitionTool(),
640641
fork=Cancun,
641642
fixture_format=FixtureFormats.BLOCKCHAIN_TEST,

‎src/pytest_plugins/filler/filler.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,7 @@ def __init__(self, *args, **kwargs):
823823
kwargs["pre"] = pre
824824
super(BaseTestWrapper, self).__init__(*args, **kwargs)
825825
fixture = self.generate(
826+
request=request,
826827
t8n=t8n,
827828
fork=fork,
828829
fixture_format=fixture_format,

‎tests/prague/eip7692_eof_v1/eip3540_eof_v1/test_container_validation.py‎

Lines changed: 11 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,6 @@
3535
@pytest.mark.parametrize(
3636
"container",
3737
[
38-
Container(
39-
name="single_code_section_with_data_section",
40-
sections=[
41-
Section.Code(code=Op.STOP),
42-
Section.Data(data="0x00"),
43-
],
44-
),
4538
Container(
4639
name="single_code_section_max_stack_size",
4740
sections=[
@@ -181,37 +174,30 @@ def test_valid_containers(
181174
),
182175
Container(
183176
name="incomplete_magic",
184-
raw_bytes=bytes([0xEF]),
177+
raw_bytes="ef",
185178
validity_error=EOFException.INVALID_MAGIC,
186179
),
187180
Container(
188181
name="no_version",
189-
raw_bytes=bytes([0xEF, 0x00]),
182+
raw_bytes="ef00",
190183
validity_error=[EOFException.INVALID_VERSION, EOFException.INVALID_MAGIC],
191184
),
192185
Container(
193186
name="no_type_header",
194-
raw_bytes=bytes([0xEF, 0x00, 0x01]),
195-
# TODO the exception must be about missing section types
187+
raw_bytes="ef00 01",
196188
validity_error=EOFException.MISSING_HEADERS_TERMINATOR,
197189
),
198190
Container(
199191
name="no_type_section_size",
200-
raw_bytes=bytes(
201-
[0xEF, 0x00, 0x01, 0x01],
202-
),
203-
# TODO the exception must be about incomplete section in the header
192+
raw_bytes="ef00 01 01",
204193
validity_error=[
205194
EOFException.MISSING_HEADERS_TERMINATOR,
206195
EOFException.INVALID_TYPE_SECTION_SIZE,
207196
],
208197
),
209198
Container(
210199
name="incomplete_type_section_size",
211-
raw_bytes=bytes(
212-
[0xEF, 0x00, 0x01, 0x01, 0x00],
213-
),
214-
# TODO the exception must be about incomplete section in the header
200+
raw_bytes="ef00010100",
215201
validity_error=[
216202
EOFException.INCOMPLETE_SECTION_SIZE,
217203
EOFException.INVALID_TYPE_SECTION_SIZE,
@@ -242,7 +228,7 @@ def test_valid_containers(
242228
),
243229
Container(
244230
name="code_section_count_incomplete",
245-
raw_bytes=bytes([0xEF, 0x00, 0x01, 0x01, 0x00, 0x04, 0x02, 0x00]),
231+
raw_bytes="ef00 01 01 0004 02 00",
246232
validity_error=EOFException.INCOMPLETE_SECTION_NUMBER,
247233
),
248234
Container(
@@ -255,7 +241,7 @@ def test_valid_containers(
255241
),
256242
Container(
257243
name="code_section_size_incomplete",
258-
raw_bytes=bytes([0xEF, 0x00, 0x01, 0x01, 0x00, 0x04, 0x02, 0x00, 0x01, 0x00]),
244+
raw_bytes="ef00 01 01 0004 02 0001 00",
259245
validity_error=[EOFException.INCOMPLETE_SECTION_SIZE, EOFException.ZERO_SECTION_SIZE],
260246
),
261247
Container(
@@ -1059,21 +1045,6 @@ def test_valid_containers(
10591045
],
10601046
validity_error=EOFException.INPUTS_OUTPUTS_NUM_ABOVE_LIMIT,
10611047
),
1062-
Container(
1063-
name="code_section_output_too_large_2",
1064-
sections=[
1065-
Section.Code(
1066-
code=Op.JUMPF[1],
1067-
),
1068-
Section.Code(
1069-
code=(Op.PUSH0 * (MAX_CODE_OUTPUTS + 1)) + Op.RETF,
1070-
code_inputs=0,
1071-
code_outputs=(MAX_CODE_OUTPUTS + 1),
1072-
max_stack_height=(MAX_CODE_OUTPUTS + 1),
1073-
),
1074-
],
1075-
validity_error=EOFException.INVALID_NON_RETURNING_FLAG,
1076-
),
10771048
Container(
10781049
name="single_code_section_max_stack_size_too_large",
10791050
sections=[
@@ -1113,6 +1084,8 @@ def test_magic_validation(
11131084
"""
11141085
Verify EOF container 2-byte magic
11151086
"""
1087+
if magic_0 == 0xEF and magic_1 == 0:
1088+
pytest.skip("Valid magic")
11161089
code = bytearray(bytes(VALID_CONTAINER))
11171090
code[0] = magic_0
11181091
code[1] = magic_1
@@ -1130,6 +1103,8 @@ def test_version_validation(
11301103
"""
11311104
Verify EOF container version
11321105
"""
1106+
if version == 1:
1107+
pytest.skip("Valid version")
11331108
code = bytearray(bytes(VALID_CONTAINER))
11341109
code[2] = version
11351110
eof_test(

0 commit comments

Comments
 (0)