Skip to content

Commit 5325008

Browse files
winsvegamarioevz
andauthored
feat(specs): Parse and fill original Filler.json using static test loader (#1362)
* feat(entry-point): Add filler reader, which parses legacy fillers * tox: lint fixes * parse json fillers by python script WIP * uv,pyproject: Add `eth_abi` * fix(fillerconvert): Parsing fixes to main file * fix(fillerconvert): Expect has default `indexes` * fix(fillerconvert): Common: Parse ABI code, use `Address`, `HexNumber`, `Bytes` * fix(fillerconvert): transaction: Handle data with access list, default gasPrice * support labels in expect section support different transaction types parse labels in code * fill convert compile yul * static test try * fillconvert support all tests in etehreum tests * Some fixes * Parametrize fill function using `valid_at` * Fill works 🎉 * fix check of unset account fields * fix account-should-not-exist * fix(plugins/refiller): warn instead of error for parsing failures * refactor * fix exceptions * fix state_static * add refilled test verification uv run fillerconvert mode=verify folder_path=/home/wins/Ethereum/execution-spec-tests/fixtures/state_tests/legacy_state_tests export refilled test in a single .json file * add original stExample test fillers * fix verification * tests that are not converted to python * exclude legacy_state_tests from parsing * fix t8n prevrandao and base fee calculation * fix transaction creation bug * fix yaml loading integers bug * add blank reference specs for pycheck * allow expected storage to be ANY * Error on failed test parsing * remove solidity tests * fix storage keys in access list parsing * fix default current random value (only for state fillers) * verify filled vectors count * fix linter for __init__ files * fix spell check * fix types * remove old unused dublicated logic code * fix pydantic validation syntax * remove all .json files for code review * pyright: disable type-checking * fix: refiller_enabled * refactor(specs): Move static state files from cli.filler... * change `legacy_state_tests` -> `legacy/state` * refine settings/pyright * fix(specs): Conditional clearing of `difficulty` according to fork * fix(cli,specs): Remove hard-coded directories * chore: Remove unnecessary file * docs: Changelog * docs: fix tox --------- Co-authored-by: Mario Vega <[email protected]>
1 parent ad5c0ad commit 5325008

File tree

25 files changed

+1176
-99
lines changed

25 files changed

+1176
-99
lines changed

.vscode/settings.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,10 @@
5555
"ruff.enable": true,
5656
"ruff.lineLength": 99,
5757
"extensions.ignoreRecommendations": false,
58+
"search.exclude": {
59+
"tests/legacy": true
60+
},
61+
"files.watcherExclude": {
62+
"tests/legacy": true
63+
}
5864
}

docs/CHANGELOG.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,18 @@ Test fixtures for use by clients are available for each release on the [Github r
66

77
## 🔜 [Unreleased]
88

9+
### 💥 Breaking Change
10+
11+
### 🛠️ Framework
12+
13+
- ✨ The `static_filler` plug-in now has support for static state tests (from [GeneralStateTests](https://github.com/ethereum/tests/tree/develop/src/GeneralStateTestsFiller)) ([#1362](https://github.com/ethereum/execution-spec-tests/pull/1362)).
14+
15+
### 📋 Misc
16+
17+
### 🧪 Test Cases
18+
19+
## [v4.2.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v4.2.0) - 2025-04-08
20+
921
**Note**: Although not a breaking change, `consume` users should delete the cache directory (typically located at `~/.cache/ethereum-execution-spec-tests`) used to store downloaded fixture release tarballs. This release adds support for [ethereum/tests](https://github.com/ethereum/tests) and [ethereum/legacytests](https://github.com/ethereum/legacytests) fixture release downloads and the structure of the cache directory has been updated to accommodate this change.
1022

1123
To try this feature:

pyproject.toml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ dependencies = [
5353
"prompt_toolkit>=3.0.48,<4", # ensure we have a new enough version for ipython
5454
"ethereum-rlp>=0.1.3,<0.2",
5555
"pytest-regex>=0.2.0,<0.3",
56+
"eth-abi>=5.2.0",
5657
]
5758

5859
[project.urls]
@@ -104,11 +105,15 @@ order_fixtures = "cli.order_fixtures:order_fixtures"
104105
evm_bytes = "cli.evm_bytes:evm_bytes"
105106
hasher = "cli.hasher:main"
106107
eest = "cli.eest.cli:eest"
108+
fillerconvert = "cli.fillerconvert.fillerconvert:main"
107109

108110
[tool.setuptools.packages.find]
109111
where = ["src"]
110112
exclude = ["*tests*"]
111113

114+
[tool.pyright]
115+
exclude = ["**/legacy/**"]
116+
112117
[tool.setuptools.package-data]
113118
ethereum_test_forks = ["forks/contracts/*.bin"]
114119
"cli.eest.make" = ["templates/*.j2"]

pyrightconfig.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"typeCheckingMode": "off",
3+
}
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""Simple CLI tool that reads filler files in the `ethereum/tests` format."""
2+
3+
import argparse
4+
from glob import glob
5+
from pathlib import Path
6+
7+
from .verify_filled import verify_refilled
8+
9+
10+
def main() -> None:
11+
"""Run the main function."""
12+
parser = argparse.ArgumentParser(description="Filler parser.")
13+
14+
parser.add_argument(
15+
"mode", type=str, help="The type of filler we are trying to parse: blockchain/state."
16+
)
17+
parser.add_argument("folder_path", type=Path, help="The path to the JSON/YML filler directory")
18+
parser.add_argument("legacy_path", type=Path, help="The path to the legacy tests directory")
19+
20+
args = parser.parse_args()
21+
args.folder_path = Path(str(args.folder_path).split("=")[-1])
22+
args.mode = str(args.mode).split("=")[-1]
23+
24+
print("Scanning: " + str(args.folder_path))
25+
files = glob(str(args.folder_path / "**" / "*.json"), recursive=True) + glob(
26+
str(args.folder_path / "**" / "*.yml"), recursive=True
27+
)
28+
29+
if args.mode == "blockchain":
30+
raise NotImplementedError("Blockchain filler not implemented yet.")
31+
32+
if args.mode == "verify":
33+
verified_vectors = 0
34+
for file in files:
35+
print("Verify: " + file)
36+
refilled_file = file
37+
relative_file = file.removeprefix(str(args.folder_path))[1:]
38+
original_file = args.legacy_path / "GeneralStateTests" / relative_file
39+
verified_vectors += verify_refilled(Path(refilled_file), original_file)
40+
print(f"Total vectors verified: {verified_vectors}")
41+
42+
# Solidity skipped tests
43+
# or file.endswith("stExample/solidityExampleFiller.yml")
44+
# or file.endswith("vmPerformance/performanceTesterFiller.yml")
45+
# or file.endswith("vmPerformance/loopExpFiller.yml")
46+
# or file.endswith("vmPerformance/loopMulFiller.yml")
47+
# or file.endswith("stRevertTest/RevertRemoteSubCallStorageOOGFiller.yml")
48+
# or file.endswith("stSolidityTest/SelfDestructFiller.yml")
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
"""Verify refilled test vs original generated test."""
2+
3+
import re
4+
from pathlib import Path
5+
6+
from pydantic import BaseModel, RootModel
7+
8+
9+
# Define only relevant data we need to read from the files
10+
class Indexes(BaseModel):
11+
"""Post Section Indexes."""
12+
13+
data: int
14+
gas: int
15+
value: int
16+
17+
18+
class PostRecord(BaseModel):
19+
"""Post results record."""
20+
21+
hash: str
22+
indexes: Indexes
23+
24+
25+
class StateTest(BaseModel):
26+
"""StateTest in filled file."""
27+
28+
post: dict[str, list[PostRecord]]
29+
30+
31+
class FilledStateTest(RootModel[dict[str, StateTest]]):
32+
"""State Test Wrapper."""
33+
34+
35+
def verify_refilled(refilled: Path, original: Path) -> int:
36+
"""
37+
Verify post hash of the refilled test against original:
38+
Regex the original d,g,v from the refilled test name.
39+
Find the post record for this d,g,v and the fork of refilled test.
40+
Compare the post hash.
41+
"""
42+
verified_vectors = 0
43+
json_str = refilled.read_text(encoding="utf-8")
44+
refilled_test_wrapper = FilledStateTest.model_validate_json(json_str)
45+
46+
json_str = original.read_text(encoding="utf-8")
47+
original_test_wrapper = FilledStateTest.model_validate_json(json_str)
48+
49+
# Each original test has only 1 test with many posts for each fork and many txs
50+
original_test_name, test_original = list(original_test_wrapper.root.items())[0]
51+
52+
for refilled_test_name, refilled_test in refilled_test_wrapper.root.items():
53+
# Each refilled test has only 1 post for 1 fork and 1 transaction
54+
refilled_fork, refilled_result = list(refilled_test.post.items())[0]
55+
pattern = r"v=(\d+)-g=(\d+)-d=(\d+)"
56+
match = re.search(pattern, refilled_test_name)
57+
if match:
58+
v, g, d = match.groups()
59+
v, g, d = int(v), int(g), int(d)
60+
61+
found = False
62+
original_result = test_original.post[refilled_fork]
63+
for res in original_result:
64+
if res.indexes.data == d and res.indexes.gas == g and res.indexes.value == v:
65+
print(f"check: {refilled_fork}, d:{d}, g:{g}, v:{v}")
66+
if res.hash != refilled_result[0].hash:
67+
raise Exception(
68+
"\nRefilled test post hash mismatch: \n"
69+
f"test_name: {refilled_test_name}\n"
70+
f"original_name: {original}\n"
71+
f"refilled_hash: {refilled_result[0].hash}\n"
72+
f"original_hash: {res.hash} f: {refilled_fork}, d: {d}, g: {g}, v: {v}"
73+
)
74+
found = True
75+
verified_vectors += 1
76+
break
77+
78+
if not found:
79+
raise Exception(
80+
"\nRefilled test not found in original: \n"
81+
f"test_name: {refilled_test_name}\n"
82+
f"original_name: {original}\n"
83+
)
84+
else:
85+
raise Exception("Could not regex match d.g.v indexes from refilled test name!")
86+
87+
return verified_vectors

src/ethereum_clis/clis/evmone.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ class EvmoneExceptionMapper(ExceptionMapper):
4747
"""Translate between EEST exceptions and error strings returned by Evmone."""
4848

4949
mapping_substring: ClassVar[Dict[ExceptionBase, str]] = {
50+
TransactionException.SENDER_NOT_EOA: "sender not an eoa:",
51+
TransactionException.GAS_ALLOWANCE_EXCEEDED: "gas limit reached",
52+
TransactionException.PRIORITY_GREATER_THAN_MAX_FEE_PER_GAS: (
53+
"max priority fee per gas higher than max fee per gas"
54+
),
55+
TransactionException.NONCE_IS_MAX: "nonce has max value:",
5056
TransactionException.TYPE_4_TX_CONTRACT_CREATION: "set code transaction must ",
5157
TransactionException.TYPE_4_INVALID_AUTHORITY_SIGNATURE: "invalid authorization signature",
5258
TransactionException.TYPE_4_INVALID_AUTHORITY_SIGNATURE_S_TOO_HIGH: (
@@ -69,6 +75,9 @@ class EvmoneExceptionMapper(ExceptionMapper):
6975
TransactionException.TYPE_3_TX_INVALID_BLOB_VERSIONED_HASH: "invalid blob hash version",
7076
TransactionException.TYPE_3_TX_BLOB_COUNT_EXCEEDED: "blob gas limit exceeded",
7177
TransactionException.TYPE_3_TX_ZERO_BLOBS: "empty blob hashes list",
78+
TransactionException.TYPE_3_TX_CONTRACT_CREATION: (
79+
"blob transaction must not be a create transaction"
80+
),
7281
TransactionException.NONCE_MISMATCH_TOO_LOW: "nonce too low",
7382
TransactionException.NONCE_MISMATCH_TOO_HIGH: "nonce too high",
7483
# TODO EVMONE needs to differentiate when the section is missing in the header or body

src/ethereum_clis/clis/execution_specs.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,9 @@ class ExecutionSpecsExceptionMapper(ExceptionMapper):
155155
TransactionException.PRIORITY_GREATER_THAN_MAX_FEE_PER_GAS: "nsaction: ",
156156
TransactionException.NONCE_MISMATCH_TOO_HIGH: "saction: ",
157157
TransactionException.NONCE_MISMATCH_TOO_LOW: "action: ",
158+
TransactionException.TYPE_3_TX_CONTRACT_CREATION: "ction: ",
159+
TransactionException.NONCE_IS_MAX: "tion: ",
160+
TransactionException.GAS_ALLOWANCE_EXCEEDED: "ion: ",
158161
# TODO EVMONE needs to differentiate when the section is missing in the header or body
159162
EOFException.MISSING_STOP_OPCODE: "err: no_terminating_instruction",
160163
EOFException.MISSING_CODE_HEADER: "err: code_section_missing",

src/ethereum_test_base_types/composite_types.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class Storage(EthereumTestRootModel[Dict[StorageKeyValueType, StorageKeyValueTyp
3030

3131
_current_slot: int = PrivateAttr(0)
3232
_hint_map: Dict[StorageKeyValueType, str] = PrivateAttr(default_factory=dict)
33+
_any_map: Dict[StorageKeyValueType, bool] = PrivateAttr(default_factory=dict)
3334

3435
StorageDictType: ClassVar[TypeAlias] = Dict[
3536
str | int | bytes | SupportsBytes, str | int | bytes | SupportsBytes
@@ -181,6 +182,10 @@ def items(self):
181182
"""Return the items of the storage."""
182183
return self.root.items()
183184

185+
def set_expect_any(self, key: StorageKeyValueTypeConvertible | StorageKeyValueType):
186+
"""Mark key to be able to have any expected value when comparing storages."""
187+
self._any_map[StorageKeyValueTypeAdapter.validate_python(key)] = True
188+
184189
def store_next(
185190
self, value: StorageKeyValueTypeConvertible | StorageKeyValueType | bool, hint: str = ""
186191
) -> StorageKeyValueType:
@@ -265,6 +270,9 @@ def must_be_equal(self, address: Address, other: "Storage | None"):
265270
)
266271

267272
elif other[key] != 0:
273+
# Skip key verification if we allow this key to be ANY
274+
if self._any_map.get(key) is True:
275+
continue
268276
raise Storage.KeyValueMismatchError(
269277
address=address,
270278
key=key,

src/ethereum_test_fixtures/collector.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,6 @@ def get_fixture_basename(self, info: TestInfo) -> Path:
125125
return Path(info.get_single_test_name(mode="module"))
126126
else:
127127
module_relative_output_dir = info.get_module_relative_output_dir(self.filler_path)
128-
129128
if self.single_fixture_per_file:
130129
return module_relative_output_dir / info.get_single_test_name(mode="test")
131130
return module_relative_output_dir / info.get_single_test_name(mode="module")

0 commit comments

Comments
 (0)