diff --git a/.github/configs/eels_resolutions.json b/.github/configs/eels_resolutions.json index e2067f9883d..f33180cdefd 100644 --- a/.github/configs/eels_resolutions.json +++ b/.github/configs/eels_resolutions.json @@ -1,33 +1,33 @@ { "Frontier": { - "path": "../../execution-specs/src/ethereum/frontier" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Homestead": { - "path": "../../execution-specs/src/ethereum/homestead" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Byzantium": { - "path": "../../execution-specs/src/ethereum/byzantium" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "ConstantinopleFix": { - "path": "../../execution-specs/src/ethereum/constantinople" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Istanbul": { - "path": "../../execution-specs/src/ethereum/istanbul" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Berlin": { - "path": "../../execution-specs/src/ethereum/berlin" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "London": { - "path": "../../execution-specs/src/ethereum/london" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Merge": { - "path": "../../execution-specs/src/ethereum/paris" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Shanghai": { - "path": "../../execution-specs/src/ethereum/shanghai" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Cancun": { - "path": "../../execution-specs/src/ethereum/cancun" + "path": "$GITHUB_WORKSPACE/execution-specs/" }, "Prague": { "git_url": "https://github.com/ethereum/execution-specs.git", diff --git a/.github/workflows/tox_verify.yaml b/.github/workflows/tox_verify.yaml index b60e3eedffc..72d0e2df621 100644 --- a/.github/workflows/tox_verify.yaml +++ b/.github/workflows/tox_verify.yaml @@ -118,8 +118,6 @@ jobs: repository: ethereum/execution-specs ref: fa847a0e48309debee8edc510ceddb2fd5db2f2e path: execution-specs - sparse-checkout: | - src/ethereum fetch-depth: 1 - name: Install uv ${{ vars.UV_VERSION }} and python ${{ matrix.python }} uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 @@ -134,6 +132,9 @@ jobs: targets: "evmone-t8n" - name: Build GETH EVM uses: ./.github/actions/build-evm-client/geth + - name: Update eels_resolutions.json + run: | + sed -i -e "s|\$GITHUB_WORKSPACE|${GITHUB_WORKSPACE}|g" .github/configs/eels_resolutions.json - name: Run tox - run framework unit tests with pytest env: EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json @@ -161,8 +162,6 @@ jobs: repository: ethereum/execution-specs ref: fa847a0e48309debee8edc510ceddb2fd5db2f2e path: execution-specs - sparse-checkout: | - src/ethereum fetch-depth: 1 - name: Install uv ${{ vars.UV_VERSION }} and python ${{ matrix.python }} uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 @@ -171,6 +170,9 @@ jobs: cache-dependency-glob: "uv.lock" version: ${{ vars.UV_VERSION }} python-version: ${{ matrix.python }} + - name: Update eels_resolutions.json + run: | + sed -i -e "s|\$GITHUB_WORKSPACE|${GITHUB_WORKSPACE}|g" .github/configs/eels_resolutions.json - name: Run tox - fill tests for deployed forks env: EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json diff --git a/pyproject.toml b/pyproject.toml index c8593a5adcc..396c44568c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ classifiers = [ ] dependencies = [ "click>=8.1.0,<9", - "ethereum-execution==1.17.0rc6.dev1", "hive.py @ git+https://github.com/marioevz/hive.py", "ethereum-spec-evm-resolver", "gitpython>=3.1.31,<4", diff --git a/src/ethereum_test_types/account_types.py b/src/ethereum_test_types/account_types.py index fc4f69e6f4b..589ae931a67 100644 --- a/src/ethereum_test_types/account_types.py +++ b/src/ethereum_test_types/account_types.py @@ -1,12 +1,10 @@ """Account-related types for Ethereum tests.""" -from dataclasses import dataclass -from typing import List, Literal +from dataclasses import dataclass, field +from typing import Dict, List, Literal, Optional, Tuple from coincurve.keys import PrivateKey -from ethereum.frontier.fork_types import Account as FrontierAccount -from ethereum.frontier.fork_types import Address as FrontierAddress -from ethereum.frontier.state import State, set_account, set_storage, state_root +from ethereum_types.bytes import Bytes20 from ethereum_types.numeric import U256, Bytes32, Uint from pydantic import PrivateAttr @@ -26,8 +24,70 @@ ) from ethereum_test_vm import EVMCodeType +from .trie import EMPTY_TRIE_ROOT, FrontierAccount, Trie, root, trie_get, trie_set from .utils import keccak256 +FrontierAddress = Bytes20 + + +@dataclass +class State: + """Contains all information that is preserved between transactions.""" + + _main_trie: Trie[Bytes20, Optional[FrontierAccount]] = field( + default_factory=lambda: Trie(secured=True, default=None) + ) + _storage_tries: Dict[Bytes20, Trie[Bytes32, U256]] = field(default_factory=dict) + _snapshots: List[ + Tuple[ + Trie[Bytes20, Optional[FrontierAccount]], + Dict[Bytes20, Trie[Bytes32, U256]], + ] + ] = field(default_factory=list) + + +def set_account(state: State, address: Bytes20, account: Optional[FrontierAccount]) -> None: + """ + Set the `Account` object at an address. Setting to `None` deletes + the account (but not its storage, see `destroy_account()`). + """ + trie_set(state._main_trie, address, account) + + +def set_storage(state: State, address: Bytes20, key: Bytes32, value: U256) -> None: + """ + Set a value at a storage key on an account. Setting to `U256(0)` deletes + the key. + """ + assert trie_get(state._main_trie, address) is not None + + trie = state._storage_tries.get(address) + if trie is None: + trie = Trie(secured=True, default=U256(0)) + state._storage_tries[address] = trie + trie_set(trie, key, value) + if trie._data == {}: + del state._storage_tries[address] + + +def storage_root(state: State, address: Bytes20) -> Bytes32: + """Calculate the storage root of an account.""" + assert not state._snapshots + if address in state._storage_tries: + return root(state._storage_tries[address]) + else: + return EMPTY_TRIE_ROOT + + +def state_root(state: State) -> Bytes32: + """Calculate the state root.""" + assert not state._snapshots + + def get_storage_root(address: Bytes20) -> Bytes32: + return storage_root(state, address) + + return root(state._main_trie, get_storage_root=get_storage_root) + class EOA(Address): """ diff --git a/src/ethereum_test_types/trie.py b/src/ethereum_test_types/trie.py new file mode 100644 index 00000000000..1caa97621ab --- /dev/null +++ b/src/ethereum_test_types/trie.py @@ -0,0 +1,384 @@ +"""The state trie is the structure responsible for storing.""" + +import copy +from dataclasses import dataclass, field +from typing import ( + Callable, + Dict, + Generic, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + cast, +) + +from Crypto.Hash import keccak +from ethereum_rlp import Extended, rlp +from ethereum_types.bytes import Bytes, Bytes20, Bytes32 +from ethereum_types.frozen import slotted_freezable +from ethereum_types.numeric import U256, Uint +from typing_extensions import assert_type + + +@slotted_freezable +@dataclass +class FrontierAccount: + """State associated with an address.""" + + nonce: Uint + balance: U256 + code: Bytes + + +def keccak256(buffer: Bytes) -> Bytes32: + """Compute the keccak256 hash of the input `buffer`.""" + k = keccak.new(digest_bits=256) + return Bytes32(k.update(buffer).digest()) + + +def encode_account(raw_account_data: FrontierAccount, storage_root: Bytes) -> Bytes: + """ + Encode `Account` dataclass. + + Storage is not stored in the `Account` dataclass, so `Accounts` cannot be + encoded without providing a storage root. + """ + return rlp.encode( + ( + raw_account_data.nonce, + raw_account_data.balance, + storage_root, + keccak256(raw_account_data.code), + ) + ) + + +# note: an empty trie (regardless of whether it is secured) has root: +# +# keccak256(RLP(b'')) +# == +# 56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421 # noqa: E501,SC10 +# +# also: +# +# keccak256(RLP(())) +# == +# 1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347 # noqa: E501,SC10 +# +# which is the sha3Uncles hash in block header with no uncles +EMPTY_TRIE_ROOT = Bytes32( + bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") +) + +Node = Union[FrontierAccount, Bytes, Uint, U256, None] +K = TypeVar("K", bound=Bytes) +V = TypeVar( + "V", + Optional[FrontierAccount], + Bytes, + Uint, + U256, +) + + +@slotted_freezable +@dataclass +class LeafNode: + """Leaf node in the Merkle Trie.""" + + rest_of_key: Bytes + value: Extended + + +@slotted_freezable +@dataclass +class ExtensionNode: + """Extension node in the Merkle Trie.""" + + key_segment: Bytes + subnode: Extended + + +BranchSubnodes = Tuple[ + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, + Extended, +] + + +@slotted_freezable +@dataclass +class BranchNode: + """Branch node in the Merkle Trie.""" + + subnodes: BranchSubnodes + value: Extended + + +InternalNode = Union[LeafNode, ExtensionNode, BranchNode] + + +def encode_internal_node(node: Optional[InternalNode]) -> Extended: + """ + Encode a Merkle Trie node into its RLP form. The RLP will then be + serialized into a `Bytes` and hashed unless it is less that 32 bytes + when serialized. + + This function also accepts `None`, representing the absence of a node, + which is encoded to `b""`. + """ + unencoded: Extended + if node is None: + unencoded = b"" + elif isinstance(node, LeafNode): + unencoded = ( + nibble_list_to_compact(node.rest_of_key, True), + node.value, + ) + elif isinstance(node, ExtensionNode): + unencoded = ( + nibble_list_to_compact(node.key_segment, False), + node.subnode, + ) + elif isinstance(node, BranchNode): + unencoded = list(node.subnodes) + [node.value] + else: + raise AssertionError(f"Invalid internal node type {type(node)}!") + + encoded = rlp.encode(unencoded) + if len(encoded) < 32: + return unencoded + else: + return keccak256(encoded) + + +def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: + """ + Encode a Node for storage in the Merkle Trie. + + Currently mostly an unimplemented stub. + """ + if isinstance(node, FrontierAccount): + assert storage_root is not None + return encode_account(node, storage_root) + elif isinstance(node, U256): + return rlp.encode(node) + elif isinstance(node, Bytes): + return node + else: + raise AssertionError(f"encoding for {type(node)} is not currently implemented") + + +@dataclass +class Trie(Generic[K, V]): + """The Merkle Trie.""" + + secured: bool + default: V + _data: Dict[K, V] = field(default_factory=dict) + + +def copy_trie(trie: Trie[K, V]) -> Trie[K, V]: + """ + Create a copy of `trie`. Since only frozen objects may be stored in tries, + the contents are reused. + """ + return Trie(trie.secured, trie.default, copy.copy(trie._data)) + + +def trie_set(trie: Trie[K, V], key: K, value: V) -> None: + """ + Store an item in a Merkle Trie. + + This method deletes the key if `value == trie.default`, because the Merkle + Trie represents the default value by omitting it from the trie. + + """ + if value == trie.default: + if key in trie._data: + del trie._data[key] + else: + trie._data[key] = value + + +def trie_get(trie: Trie[K, V], key: K) -> V: + """ + Get an item from the Merkle Trie. + + This method returns `trie.default` if the key is missing. + + """ + return trie._data.get(key, trie.default) + + +def common_prefix_length(a: Sequence, b: Sequence) -> int: + """Find the longest common prefix of two sequences.""" + for i in range(len(a)): + if i >= len(b) or a[i] != b[i]: + return i + return len(a) + + +def nibble_list_to_compact(x: Bytes, is_leaf: bool) -> Bytes: + """ + Compresses nibble-list into a standard byte array with a flag. + + A nibble-list is a list of byte values no greater than `15`. The flag is + encoded in high nibble of the highest byte. The flag nibble can be broken + down into two two-bit flags. + + Highest nibble:: + + +---+---+----------+--------+ + | _ | _ | is_leaf | parity | + +---+---+----------+--------+ + 3 2 1 0 + + + The lowest bit of the nibble encodes the parity of the length of the + remaining nibbles -- `0` when even and `1` when odd. The second lowest bit + is used to distinguish leaf and extension nodes. The other two bits are not + used. + """ + compact = bytearray() + + if len(x) % 2 == 0: # ie even length + compact.append(16 * (2 * is_leaf)) + for i in range(0, len(x), 2): + compact.append(16 * x[i] + x[i + 1]) + else: + compact.append(16 * ((2 * is_leaf) + 1) + x[0]) + for i in range(1, len(x), 2): + compact.append(16 * x[i] + x[i + 1]) + + return Bytes(compact) + + +def bytes_to_nibble_list(bytes_: Bytes) -> Bytes: + """Convert a `Bytes` into to a sequence of nibbles (bytes with value < 16).""" + nibble_list = bytearray(2 * len(bytes_)) + for byte_index, byte in enumerate(bytes_): + nibble_list[byte_index * 2] = (byte & 0xF0) >> 4 + nibble_list[byte_index * 2 + 1] = byte & 0x0F + return Bytes(nibble_list) + + +def _prepare_trie( + trie: Trie[K, V], + get_storage_root: Optional[Callable[[Bytes20], Bytes32]] = None, +) -> Mapping[Bytes, Bytes]: + """ + Prepare the trie for root calculation. Removes values that are empty, + hashes the keys (if `secured == True`) and encodes all the nodes. + """ + mapped: MutableMapping[Bytes, Bytes] = {} + + for preimage, value in trie._data.items(): + if isinstance(value, FrontierAccount): + assert get_storage_root is not None + address = Bytes20(preimage) + encoded_value = encode_node(value, get_storage_root(address)) + else: + encoded_value = encode_node(value) + if encoded_value == b"": + raise AssertionError + key: Bytes + if trie.secured: + # "secure" tries hash keys once before construction + key = keccak256(preimage) + else: + key = preimage + mapped[bytes_to_nibble_list(key)] = encoded_value + + return mapped + + +def root( + trie: Trie[K, V], + get_storage_root: Optional[Callable[[Bytes20], Bytes32]] = None, +) -> Bytes32: + """Compute the root of a modified merkle patricia trie (MPT).""" + obj = _prepare_trie(trie, get_storage_root) + + root_node = encode_internal_node(patricialize(obj, Uint(0))) + if len(rlp.encode(root_node)) < 32: + return keccak256(rlp.encode(root_node)) + else: + assert isinstance(root_node, Bytes) + return Bytes32(root_node) + + +def patricialize(obj: Mapping[Bytes, Bytes], level: Uint) -> Optional[InternalNode]: + """ + Structural composition function. + + Used to recursively patricialize and merkleize a dictionary. Includes + memoization of the tree structure and hashes. + """ + if len(obj) == 0: + return None + + arbitrary_key = next(iter(obj)) + + # if leaf node + if len(obj) == 1: + leaf = LeafNode(arbitrary_key[level:], obj[arbitrary_key]) + return leaf + + # prepare for extension node check by finding max j such that all keys in + # obj have the same key[i:j] + substring = arbitrary_key[level:] + prefix_length = len(substring) + for key in obj: + prefix_length = min(prefix_length, common_prefix_length(substring, key[level:])) + + # finished searching, found another key at the current level + if prefix_length == 0: + break + + # if extension node + if prefix_length > 0: + prefix = arbitrary_key[int(level) : int(level) + prefix_length] + return ExtensionNode( + prefix, + encode_internal_node(patricialize(obj, level + Uint(prefix_length))), + ) + + branches: List[MutableMapping[Bytes, Bytes]] = [] + for _ in range(16): + branches.append({}) + value = b"" + for key in obj: + if len(key) == level: + # shouldn't ever have an account or receipt in an internal node + if isinstance(obj[key], (FrontierAccount, Uint)): + raise AssertionError + value = obj[key] + else: + branches[key[level]][key] = obj[key] + + subnodes = tuple( + encode_internal_node(patricialize(branches[k], level + Uint(1))) for k in range(16) + ) + return BranchNode( + cast(BranchSubnodes, assert_type(subnodes, Tuple[Extended, ...])), + value, + ) diff --git a/src/pytest_plugins/execute/rpc/hive.py b/src/pytest_plugins/execute/rpc/hive.py index bfb2ed28b43..0d204acdcf0 100644 --- a/src/pytest_plugins/execute/rpc/hive.py +++ b/src/pytest_plugins/execute/rpc/hive.py @@ -10,7 +10,6 @@ from typing import Any, Dict, Generator, List, Mapping, Tuple, cast import pytest -from ethereum.crypto.hash import keccak256 from filelock import FileLock from hive.client import Client, ClientType from hive.simulation import Simulation @@ -39,6 +38,7 @@ Withdrawal, ) from ethereum_test_types import Requests +from ethereum_test_types.trie import keccak256 from pytest_plugins.consume.hive_simulators.ruleset import ruleset diff --git a/uv.lock b/uv.lock index 485f3bdc880..5b0a9fca79e 100644 --- a/uv.lock +++ b/uv.lock @@ -527,23 +527,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/89/251f118fae703d5504bbe63b72124ef346a8a65c5ee0a106b5b7930c397f/eth_utils-2.3.1-py3-none-any.whl", hash = "sha256:614eedc5ffcaf4e6708ca39e23b12bd69526a312068c1170c773bd1307d13972", size = 77778, upload-time = "2023-11-07T20:54:25.529Z" }, ] -[[package]] -name = "ethereum-execution" -version = "1.17.0rc6.dev1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coincurve" }, - { name = "ethereum-rlp" }, - { name = "ethereum-types" }, - { name = "py-ecc" }, - { name = "pycryptodome" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/a1/f20e2b489a4d84afe97dfe10180fdd06d191ff448d1cbfd8c144c0f6c453/ethereum_execution-1.17.0rc6.dev1.tar.gz", hash = "sha256:70e9585ccaaa3e1fd6f8648f392ef25c0fb619aaadb58fb6a54f13f3b3f7bccc", size = 1120570, upload-time = "2025-02-13T21:27:29.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/6f/722677ffcfc0eebe925d20eeb8715f8c9d0dc3de4b9f04ab98601b5ab185/ethereum_execution-1.17.0rc6.dev1-py3-none-any.whl", hash = "sha256:1dd379e88be14fac040219ed59a7660f491d1ac336b56ae5cdc2a3b230b7f93e", size = 1452337, upload-time = "2025-02-13T21:27:26.52Z" }, -] - [[package]] name = "ethereum-execution-spec-tests" version = "1.0.0" @@ -553,7 +536,6 @@ dependencies = [ { name = "coincurve" }, { name = "colorlog" }, { name = "eth-abi" }, - { name = "ethereum-execution" }, { name = "ethereum-rlp" }, { name = "ethereum-spec-evm-resolver" }, { name = "ethereum-types" }, @@ -619,7 +601,6 @@ requires-dist = [ { name = "coincurve", specifier = ">=20.0.0,<21" }, { name = "colorlog", specifier = ">=6.7.0,<7" }, { name = "eth-abi", specifier = ">=5.2.0" }, - { name = "ethereum-execution", specifier = "==1.17.0rc6.dev1" }, { name = "ethereum-rlp", specifier = ">=0.1.3,<0.2" }, { name = "ethereum-spec-evm-resolver", git = "https://github.com/spencer-tb/ethereum-spec-evm-resolver?rev=ee273e7344e24a739ebfbf0ea1f758530c4d032b" }, { name = "ethereum-types", specifier = ">=0.2.1,<0.3" },