feat(consume): add consume engine hive simulator (#691)

* feat(consume): add consume engine to the framework.

* feat(consume): engine hack for errors.

* fix(spec): Add last_block_hash to hive blockchain fixture

* fix(fw): tests

* fix(fw): Consume/RPC typing

* feat(consume): refactor the consume pytest plugin.

* chore: add tox fixes for framework.

* refactor(plugins): move common fixtures into `consume/hive_simulators/conftest.py`

* feat(consume): small refactor, timing for engine.

* chore(consume): update to latest hive.py

* feat(consume): only use one ini file, hive.py fix.

* docs: add changelog.

---------

Co-authored-by: Mario Vega <[email protected]>

Author: spencer-tb

docs/CHANGELOG.md

@@ -52,6 +52,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🔀 Refactor `gentest` to use `ethereum_test_tools.rpc.rpc` by adding to `get_transaction_by_hash`, `debug_trace_call` to `EthRPC` ([#568](https://github.com/ethereum/execution-spec-tests/pull/568)).
 - ✨ Write a properties file to the output directory and enable direct generation of a fixture tarball from `fill` via `--output=fixtures.tgz`([#627](https://github.com/ethereum/execution-spec-tests/pull/627)).
 - 🔀 `ethereum_test_tools` library has been split into multiple libraries ([#645](https://github.com/ethereum/execution-spec-tests/pull/645)).
+- ✨ Add the consume engine simulator and refactor the consume simulator suite. ([#691](https://github.com/ethereum/execution-spec-tests/pull/691)).
 
 ### 🔧 EVM Tools
 

pytest-consume.ini

@@ -0,0 +1,9 @@
+[pytest]
+console_output_style = count
+minversion = 7.0
+python_files = test_*
+addopts = 
+    -rxXs
+    --tb short
+    -p pytest_plugins.consume.consume
+    -p pytest_plugins.test_help.test_help

pytest-framework.ini

@@ -7,4 +7,7 @@ testpaths =
     src
 addopts = 
     -p pytester
-    --ignore=src/pytest_plugins/test_filler/test_filler.py
+    --ignore=src/pytest_plugins/test_filler/test_filler.py
+    --ignore=src/pytest_plugins/consume/direct/test_direct.py
+    --ignore=src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py
+    --ignore=src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py

setup.cfg

@@ -31,7 +31,7 @@ python_requires = >=3.10
 install_requires =
     click>=8.1.0,<9
     ethereum@git+https://github.com/ethereum/execution-specs
-    hive.py@git+https://github.com/danceratopz/hive.py@chore/setup.cfg/move-mypy-deps-to-lint-extras
+    hive.py@git+https://github.com/marioevz/hive.py
     setuptools
     types-setuptools
     gitpython>=3.1.31,<4

src/cli/gen_index.py

@@ -1,6 +1,7 @@
 """
 Generate an index file of all the json fixtures in the specified directory.
 """
+
 import datetime
 import json
 import os
@@ -179,8 +180,6 @@ def generate_fixtures_index(
         for file in input_path.rglob("*.json"):
             if file.name == "index.json":
                 continue
-            if "blockchain_tests_hive" in file.parts:
-                continue
             if any(fixture in str(file) for fixture in fixtures_to_skip):
                 rich.print(f"Skipping '{file}'")
                 continue

src/cli/gentest.py

@@ -60,7 +60,7 @@
 import click
 
 from ethereum_test_base_types import Account, Address, ZeroPaddedHexNumber
-from ethereum_test_tools.rpc.rpc import BlockNumberType, EthRPC
+from ethereum_test_tools.rpc import BlockNumberType, DebugRPC, EthRPC
 from ethereum_test_types import Transaction
 
 
@@ -289,6 +289,7 @@ def __init__(self, node_config: Config.RemoteNode):
             "CF-Access-Client-Secret": node_config.secret,
         }
         self.rpc = EthRPC(node_config.node_url, extra_headers=headers)
+        self.debug_rpc = DebugRPC(node_config.node_url, extra_headers=headers)
 
     def eth_get_transaction_by_hash(self, transaction_hash: str) -> RemoteTransaction:
         """
@@ -337,7 +338,7 @@ def debug_trace_call(self, tr: RemoteTransaction) -> Dict[Address, Account]:
         """
         Get pre-state required for transaction
         """
-        return self.rpc.debug_trace_call(
+        return self.debug_rpc.trace_call(
             {
                 "from": f"{str(tr.transaction.sender)}",
                 "to": f"{str(tr.transaction.to)}",

src/cli/pytest_commands.py

@@ -37,7 +37,7 @@
 import os
 import sys
 import warnings
-from typing import Any, Callable, List, Literal
+from typing import Any, Callable, List, Literal, get_args
 
 import click
 import pytest
@@ -166,14 +166,25 @@ def get_hive_flags_from_env():
     return pytest_args
 
 
-ConsumeCommands = Literal["dirct", "rlp", "engine", "all"]
+ConsumeCommands = Literal["direct", "rlp", "engine"]
 
 
-def consume_ini_path(consume_command: ConsumeCommands) -> str:
+def consume_test_paths(consume_command: ConsumeCommands) -> str:
     """
-    Get the path to the ini file for the specified consume command.
+    Get the test path for the specified consume command.
     """
-    return f"src/pytest_plugins/consume/ini_files/pytest-consume-{consume_command}.ini"
+    consume_path = "src/pytest_plugins/consume"
+    if consume_command == "direct":
+        return f"{consume_path}/{consume_command}/test_{consume_command}.py"
+    else:  # rlp or engine
+        return f"{consume_path}/hive_simulators/{consume_command}/test_via_{consume_command}.py"
+
+
+def all_consume_test_paths() -> list:
+    """
+    Get all test paths within the consume suite.
+    """
+    return [consume_test_paths(command) for command in get_args(ConsumeCommands)]
 
 
 @click.group()
@@ -191,7 +202,7 @@ def consume_direct(pytest_args, help_flag, pytest_help_flag):
     Clients consume directly via the `blocktest` interface.
     """
     args = handle_help_flags(pytest_args, help_flag, pytest_help_flag)
-    args += ["-c", consume_ini_path("direct"), "--rootdir", "./"]
+    args += ["-c", "pytest-consume.ini", "--rootdir", "./", consume_test_paths("direct")]
     if not sys.stdin.isatty():  # the command is receiving input on stdin
         args.extend(["-s", "--input=stdin"])
     pytest.main(args)
@@ -204,7 +215,15 @@ def consume_via_rlp(pytest_args, help_flag, pytest_help_flag):
     Clients consume RLP-encoded blocks on startup.
     """
     args = handle_help_flags(pytest_args, help_flag, pytest_help_flag)
-    args += ["-c", consume_ini_path("rlp"), "--rootdir", "./"]
+    args += [
+        "-c",
+        "pytest-consume.ini",
+        "--rootdir",
+        "./",
+        consume_test_paths("rlp"),
+        "-p",
+        "pytest_plugins.pytest_hive.pytest_hive",
+    ]
     args += get_hive_flags_from_env()
     if not sys.stdin.isatty():  # the command is receiving input on stdin
         args.extend(["-s", "--input=stdin"])
@@ -218,7 +237,15 @@ def consume_via_engine_api(pytest_args, help_flag, pytest_help_flag):
     Clients consume via the Engine API.
     """
     args = handle_help_flags(pytest_args, help_flag, pytest_help_flag)
-    args += ["-c", consume_ini_path("engine"), "--rootdir", "./"]
+    args += [
+        "-c",
+        "pytest-consume.ini",
+        "--rootdir",
+        "./",
+        consume_test_paths("engine"),
+        "-p",
+        "pytest_plugins.pytest_hive.pytest_hive",
+    ]
     args += get_hive_flags_from_env()
     if not sys.stdin.isatty():  # the command is receiving input on stdin
         args.extend(["-s", "--input=stdin"])
@@ -232,7 +259,14 @@ def consume_all(pytest_args, help_flag, pytest_help_flag):
     Clients consume via all available methods (direct, rlp, engine).
     """
     args = handle_help_flags(pytest_args, help_flag, pytest_help_flag)
-    args += ["-c", consume_ini_path("all"), "--rootdir", "./"]
+    args += [
+        "-c",
+        "pytest-consume.ini",
+        "--rootdir",
+        "./",
+        "-p",
+        "pytest_plugins.pytest_hive.pytest_hive",
+    ] + all_consume_test_paths()
     args += get_hive_flags_from_env()
     if not sys.stdin.isatty():  # the command is receiving input on stdin
         args.extend(["-s", "--input=stdin"])

src/ethereum_test_fixtures/__init__.py

@@ -6,6 +6,7 @@
 
 from .base import BaseFixture
 from .blockchain import Fixture as BlockchainFixture
+from .blockchain import FixtureCommon as BlockchainFixtureCommon
 from .blockchain import HiveFixture as BlockchainHiveFixture
 from .collector import FixtureCollector, TestInfo
 from .eof import Fixture as EOFFixture
@@ -23,6 +24,7 @@
     "FIXTURE_TYPES",
     "BaseFixture",
     "BlockchainFixture",
+    "BlockchainFixtureCommon",
     "BlockchainHiveFixture",
     "EOFFixture",
     "FixtureCollector",

src/ethereum_test_fixtures/blockchain.py

@@ -23,6 +23,7 @@
     Number,
     ZeroPaddedHexNumber,
 )
+from ethereum_test_base_types.json import to_json
 from ethereum_test_exceptions import EngineAPIError, ExceptionInstanceOrList
 from ethereum_test_forks import Fork
 from ethereum_test_types.types import (
@@ -255,6 +256,23 @@ class FixtureEngineNewPayload(CamelModel):
         | None
     ) = None
 
+    def args(self) -> List[Any]:
+        """
+        Returns the arguments to be used when calling the Engine API.
+        """
+        args: List[Any] = [to_json(self.execution_payload)]
+        if self.blob_versioned_hashes is not None:
+            args.append([str(versioned_hash) for versioned_hash in self.blob_versioned_hashes])
+        if self.parent_beacon_block_root is not None:
+            args.append(str(self.parent_beacon_block_root))
+        return args
+
+    def valid(self) -> bool:
+        """
+        Returns whether the payload is valid.
+        """
+        return self.validation_error is None
+
     @classmethod
     def from_fixture_header(
         cls,
@@ -420,6 +438,7 @@ class FixtureCommon(BaseFixture):
     genesis: FixtureHeader = Field(..., alias="genesisBlockHeader")
     pre: Alloc
     post_state: Optional[Alloc] = Field(None)
+    last_block_hash: Hash = Field(..., alias="lastblockhash")  # FIXME: lastBlockHash
 
     def get_fork(self) -> str:
         """
@@ -435,7 +454,6 @@ class Fixture(FixtureCommon):
 
     genesis_rlp: Bytes = Field(..., alias="genesisRLP")
     blocks: List[FixtureBlock | InvalidFixtureBlock]
-    last_block_hash: Hash = Field(..., alias="lastblockhash")
     seal_engine: Literal["NoProof"] = Field("NoProof")
 
     format: ClassVar[FixtureFormats] = FixtureFormats.BLOCKCHAIN_TEST

src/ethereum_test_specs/blockchain.py

@@ -698,6 +698,7 @@ def make_hive_fixture(
             pre=pre,
             post_state=alloc,
             sync_payload=sync_payload,
+            last_block_hash=head_hash,
         )
 
     def generate(