diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cab0cbcbfea..1d3160ca943 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -42,6 +42,7 @@ Users can select any of the artifacts depending on their testing needs for their #### `consume` - 🔀 `consume` now automatically avoids GitHub API calls when using direct release URLs (better for CI environments), while release specifiers like `stable@latest` continue to use the API for version resolution ([#1788](https://github.com/ethereum/execution-spec-tests/pull/1788)). +- 🔀 Refactor consume simulator architecture to use explicit pytest plugin structure with forward-looking architecture ([#1801](https://github.com/ethereum/execution-spec-tests/pull/1801)). #### `execute` diff --git a/pytest-framework.ini b/pytest-framework.ini index 03e001e1beb..2842bb7838b 100644 --- a/pytest-framework.ini +++ b/pytest-framework.ini @@ -13,7 +13,7 @@ addopts = --ignore=src/pytest_plugins/consume/test_cache.py --ignore=src/pytest_plugins/consume/direct/ --ignore=src/pytest_plugins/consume/direct/test_via_direct.py - --ignore=src/pytest_plugins/consume/hive_simulators/ - --ignore=src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py - --ignore=src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py + --ignore=src/pytest_plugins/consume/simulators/ + --ignore=src/pytest_plugins/consume/simulators/engine/test_via_engine.py + --ignore=src/pytest_plugins/consume/simulators/rlp/test_via_rlp.py --ignore=src/pytest_plugins/execute/test_recover.py diff --git a/src/cli/extract_config.py b/src/cli/extract_config.py index c6355f0e5b1..503c352eb1c 100755 --- a/src/cli/extract_config.py +++ b/src/cli/extract_config.py @@ -23,7 +23,7 @@ from ethereum_test_fixtures.file import Fixtures from ethereum_test_fixtures.pre_alloc_groups import PreAllocGroup from ethereum_test_forks import Fork -from pytest_plugins.consume.hive_simulators.ruleset import ruleset +from pytest_plugins.consume.simulators.helpers.ruleset import ruleset def get_docker_containers() -> set[str]: diff --git a/src/cli/pytest_commands/consume.py b/src/cli/pytest_commands/consume.py index 63115ec0d60..29bb8e57615 100644 --- a/src/cli/pytest_commands/consume.py +++ b/src/cli/pytest_commands/consume.py @@ -13,14 +13,14 @@ class ConsumeCommand(PytestCommand): """Pytest command for consume operations.""" - def __init__(self, command_paths: List[Path], is_hive: bool = False): + def __init__(self, command_paths: List[Path], is_hive: bool = False, command_name: str = ""): """Initialize consume command with paths and processors.""" processors: List[ArgumentProcessor] = [HelpFlagsProcessor("consume")] if is_hive: processors.extend( [ - HiveEnvironmentProcessor(), + HiveEnvironmentProcessor(command_name=command_name), ConsumeCommandProcessor(is_hive=True), ] ) @@ -54,13 +54,15 @@ def get_command_paths(command_name: str, is_hive: bool) -> List[Path]: base_path = Path("src/pytest_plugins/consume") if command_name == "hive": commands = ["rlp", "engine"] + command_paths = [ + base_path / "simulators" / "hive_tests" / f"test_via_{cmd}.py" for cmd in commands + ] + elif command_name in ["engine", "rlp"]: + command_paths = [base_path / "simulators" / "hive_tests" / f"test_via_{command_name}.py"] + elif command_name == "direct": + command_paths = [base_path / "direct" / "test_via_direct.py"] else: - commands = [command_name] - - command_paths = [ - base_path / ("hive_simulators" if is_hive else "") / cmd / f"test_via_{cmd}.py" - for cmd in commands - ] + raise ValueError(f"Unexpected command: {command_name}.") return command_paths @@ -86,7 +88,7 @@ def decorator(func: Callable[..., Any]) -> click.Command: @common_pytest_options @functools.wraps(func) def command(pytest_args: List[str], **kwargs) -> None: - consume_cmd = ConsumeCommand(command_paths, is_hive) + consume_cmd = ConsumeCommand(command_paths, is_hive, command_name) consume_cmd.execute(list(pytest_args)) return command diff --git a/src/cli/pytest_commands/processors.py b/src/cli/pytest_commands/processors.py index 82089cc5135..fb493964329 100644 --- a/src/cli/pytest_commands/processors.py +++ b/src/cli/pytest_commands/processors.py @@ -74,6 +74,10 @@ def _is_writing_to_stdout(self, args: List[str]) -> bool: class HiveEnvironmentProcessor(ArgumentProcessor): """Processes Hive environment variables for consume commands.""" + def __init__(self, command_name: str): + """Initialize the processor with command name to determine plugin.""" + self.command_name = command_name + def process_args(self, args: List[str]) -> List[str]: """Convert hive environment variables into pytest flags.""" modified_args = args[:] @@ -92,8 +96,12 @@ def process_args(self, args: List[str]) -> List[str]: if os.getenv("HIVE_LOGLEVEL") is not None: warnings.warn("HIVE_LOG_LEVEL is not yet supported.", stacklevel=2) - modified_args.extend(["-p", "pytest_plugins.pytest_hive.pytest_hive"]) - + if self.command_name == "engine": + modified_args.extend(["-p", "pytest_plugins.consume.simulators.engine.conftest"]) + elif self.command_name == "rlp": + modified_args.extend(["-p", "pytest_plugins.consume.simulators.rlp.conftest"]) + else: + raise ValueError(f"Unknown command name: {self.command_name}") return modified_args def _has_regex_or_sim_limit(self, args: List[str]) -> bool: diff --git a/src/pytest_plugins/consume/hive_simulators/conftest.py b/src/pytest_plugins/consume/hive_simulators/conftest.py deleted file mode 100644 index 4d0c92fc6ca..00000000000 --- a/src/pytest_plugins/consume/hive_simulators/conftest.py +++ /dev/null @@ -1,430 +0,0 @@ -"""Common pytest fixtures for the RLP and Engine simulators.""" - -import io -import json -import logging -import textwrap -import urllib -import warnings -from pathlib import Path -from typing import Dict, Generator, List, Literal, cast - -import pytest -import rich -from hive.client import Client, ClientType -from hive.testing import HiveTest - -from ethereum_test_base_types import Number, to_json -from ethereum_test_exceptions import ExceptionMapper -from ethereum_test_fixtures import ( - BaseFixture, - BlockchainFixtureCommon, -) -from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream -from ethereum_test_fixtures.file import Fixtures -from ethereum_test_rpc import EthRPC -from pytest_plugins.consume.consume import FixturesSource -from pytest_plugins.consume.hive_simulators.ruleset import ruleset # TODO: generate dynamically -from pytest_plugins.pytest_hive.hive_info import ClientFile, HiveInfo - -from .exceptions import EXCEPTION_MAPPERS -from .timing import TimingData - -logger = logging.getLogger(__name__) - - -def pytest_addoption(parser): - """Hive simulator specific consume command line options.""" - consume_group = parser.getgroup( - "consume", "Arguments related to consuming fixtures via a client" - ) - consume_group.addoption( - "--timing-data", - action="store_true", - dest="timing_data", - default=False, - help="Log the timing data for each test case execution.", - ) - consume_group.addoption( - "--disable-strict-exception-matching", - action="store", - dest="disable_strict_exception_matching", - default="", - help=( - "Comma-separated list of client names and/or forks which should NOT use strict " - "exception matching." - ), - ) - - -@pytest.fixture(scope="function") -def eth_rpc(client: Client) -> EthRPC: - """Initialize ethereum RPC client for the execution client under test.""" - return EthRPC(f"http://{client.ip}:8545") - - -@pytest.fixture(scope="function") -def hive_clients_yaml_target_filename() -> str: - """Return the name of the target clients YAML file.""" - return "clients_eest.yaml" - - -@pytest.fixture(scope="function") -def hive_clients_yaml_generator_command( - client_type: ClientType, - client_file: ClientFile, - hive_clients_yaml_target_filename: str, - hive_info: HiveInfo, -) -> str: - """Generate a shell command that creates a clients YAML file for the current client.""" - try: - if not client_file: - raise ValueError("No client information available - try updating hive") - client_config = [c for c in client_file.root if c.client in client_type.name] - if not client_config: - raise ValueError(f"Client '{client_type.name}' not found in client file") - try: - yaml_content = ClientFile(root=[client_config[0]]).yaml().replace(" ", " ") - return f'echo "\\\n{yaml_content}" > {hive_clients_yaml_target_filename}' - except Exception as e: - raise ValueError(f"Failed to generate YAML: {str(e)}") from e - except ValueError as e: - error_message = str(e) - warnings.warn( - f"{error_message}. The Hive clients YAML generator command will not be available.", - stacklevel=2, - ) - - issue_title = f"Client {client_type.name} configuration issue" - issue_body = f"Error: {error_message}\nHive version: {hive_info.commit}\n" - issue_url = f"https://github.com/ethereum/execution-spec-tests/issues/new?title={urllib.parse.quote(issue_title)}&body={urllib.parse.quote(issue_body)}" - - return ( - f"Error: {error_message}\n" - f'Please create an issue to report this problem.' - ) - - -@pytest.fixture(scope="function") -def filtered_hive_options(hive_info: HiveInfo) -> List[str]: - """Filter Hive command options to remove unwanted options.""" - logger.info("Hive info: %s", hive_info.command) - - unwanted_options = [ - "--client", # gets overwritten: we specify a single client; the one from the test case - "--client-file", # gets overwritten: we'll write our own client file - "--results-root", # use default value instead (or you have to pass it to ./hiveview) - "--sim.limit", # gets overwritten: we only run the current test case id - "--sim.parallelism", # skip; we'll only be running a single test - ] - - command_parts = [] - skip_next = False - for part in hive_info.command: - if skip_next: - skip_next = False - continue - - if part in unwanted_options: - skip_next = True - continue - - if any(part.startswith(f"{option}=") for option in unwanted_options): - continue - - command_parts.append(part) - - return command_parts - - -@pytest.fixture(scope="function") -def hive_client_config_file_parameter(hive_clients_yaml_target_filename: str) -> str: - """Return the hive client config file parameter.""" - return f"--client-file {hive_clients_yaml_target_filename}" - - -@pytest.fixture(scope="function") -def hive_consume_command( - test_case: TestCaseIndexFile | TestCaseStream, - hive_client_config_file_parameter: str, - filtered_hive_options: List[str], - client_type: ClientType, -) -> str: - """Command to run the test within hive.""" - command_parts = filtered_hive_options.copy() - command_parts.append(f"{hive_client_config_file_parameter}") - command_parts.append(f"--client={client_type.name}") - command_parts.append(f'--sim.limit="id:{test_case.id}"') - - return " ".join(command_parts) - - -@pytest.fixture(scope="function") -def hive_dev_command( - client_type: ClientType, - hive_client_config_file_parameter: str, -) -> str: - """Return the command used to instantiate hive alongside the `consume` command.""" - return f"./hive --dev {hive_client_config_file_parameter} --client {client_type.name}" - - -@pytest.fixture(scope="function") -def eest_consume_command( - test_suite_name: str, - test_case: TestCaseIndexFile | TestCaseStream, - fixture_source_flags: List[str], -) -> str: - """Commands to run the test within EEST using a hive dev back-end.""" - flags = " ".join(fixture_source_flags) - return ( - f"uv run consume {test_suite_name.split('-')[-1]} " - f'{flags} --sim.limit="id:{test_case.id}" -v -s' - ) - - -@pytest.fixture(scope="function") -def test_case_description( - fixture: BaseFixture, - test_case: TestCaseIndexFile | TestCaseStream, - hive_clients_yaml_generator_command: str, - hive_consume_command: str, - hive_dev_command: str, - eest_consume_command: str, -) -> str: - """Create the description of the current blockchain fixture test case.""" - test_url = fixture.info.get("url", "") - - if "description" not in fixture.info or fixture.info["description"] is None: - test_docstring = "No documentation available." - else: - # this prefix was included in the fixture description field for fixtures <= v4.3.0 - test_docstring = fixture.info["description"].replace("Test function documentation:\n", "") # type: ignore - - description = textwrap.dedent(f""" - Test Details - {test_case.id} - {f'[source]' if test_url else ""} - - {test_docstring} - - Run This Test Locally: - To run this test in hive: - {hive_clients_yaml_generator_command} - {hive_consume_command} - - Advanced: Run the test against a hive developer backend using EEST's consume command - Create the client YAML file, as above, then: - 1. Start hive in dev mode: {hive_dev_command} - 2. In the EEST repository root: {eest_consume_command} - """) # noqa: E501 - - description = description.strip() - description = description.replace("\n", "
") - return description - - -@pytest.fixture(scope="function", autouse=True) -def total_timing_data(request) -> Generator[TimingData, None, None]: - """Record timing data for various stages of executing test case.""" - with TimingData("Total (seconds)") as total_timing_data: - yield total_timing_data - if request.config.getoption("timing_data"): - rich.print(f"\n{total_timing_data.formatted()}") - if hasattr(request.node, "rep_call"): # make available for test reports - request.node.rep_call.timings = total_timing_data - - -@pytest.fixture(scope="function") -def client_genesis(fixture: BlockchainFixtureCommon) -> dict: - """Convert the fixture genesis block header and pre-state to a client genesis state.""" - genesis = to_json(fixture.genesis) - alloc = to_json(fixture.pre) - # NOTE: nethermind requires account keys without '0x' prefix - genesis["alloc"] = {k.replace("0x", ""): v for k, v in alloc.items()} - return genesis - - -@pytest.fixture(scope="function") -def check_live_port(test_suite_name: str) -> Literal[8545, 8551]: - """Port used by hive to check for liveness of the client.""" - if test_suite_name == "eest/consume-rlp": - return 8545 - elif test_suite_name == "eest/consume-engine": - return 8551 - raise ValueError( - f"Unexpected test suite name '{test_suite_name}' while setting HIVE_CHECK_LIVE_PORT." - ) - - -@pytest.fixture(scope="function") -def environment( - fixture: BlockchainFixtureCommon, - check_live_port: Literal[8545, 8551], -) -> dict: - """Define the environment that hive will start the client with.""" - assert fixture.fork in ruleset, f"fork '{fixture.fork}' missing in hive ruleset" - return { - "HIVE_CHAIN_ID": str(Number(fixture.config.chain_id)), - "HIVE_FORK_DAO_VOTE": "1", - "HIVE_NODETYPE": "full", - "HIVE_CHECK_LIVE_PORT": str(check_live_port), - **{k: f"{v:d}" for k, v in ruleset[fixture.fork].items()}, - } - - -@pytest.fixture(scope="function") -def buffered_genesis(client_genesis: dict) -> io.BufferedReader: - """Create a buffered reader for the genesis block header of the current test fixture.""" - genesis_json = json.dumps(client_genesis) - genesis_bytes = genesis_json.encode("utf-8") - return io.BufferedReader(cast(io.RawIOBase, io.BytesIO(genesis_bytes))) - - -@pytest.fixture(scope="session") -def client_exception_mapper_cache(): - """Cache for exception mappers by client type.""" - return {} - - -@pytest.fixture(scope="function") -def client_exception_mapper( - client_type: ClientType, client_exception_mapper_cache -) -> ExceptionMapper | None: - """Return the exception mapper for the client type, with caching.""" - if client_type.name not in client_exception_mapper_cache: - for client in EXCEPTION_MAPPERS: - if client in client_type.name: - client_exception_mapper_cache[client_type.name] = EXCEPTION_MAPPERS[client] - break - else: - client_exception_mapper_cache[client_type.name] = None - - return client_exception_mapper_cache[client_type.name] - - -@pytest.fixture(scope="session") -def disable_strict_exception_matching(request: pytest.FixtureRequest) -> List[str]: - """Return the list of clients or forks that should NOT use strict exception matching.""" - config_string = request.config.getoption("disable_strict_exception_matching") - return config_string.split(",") if config_string else [] - - -@pytest.fixture(scope="function") -def client_strict_exception_matching( - client_type: ClientType, - disable_strict_exception_matching: List[str], -) -> bool: - """Return True if the client type should use strict exception matching.""" - return not any( - client.lower() in client_type.name.lower() for client in disable_strict_exception_matching - ) - - -@pytest.fixture(scope="function") -def fork_strict_exception_matching( - fixture: BlockchainFixtureCommon, - disable_strict_exception_matching: List[str], -) -> bool: - """Return True if the fork should use strict exception matching.""" - # NOTE: `in` makes it easier for transition forks ("Prague" in "CancunToPragueAtTime15k") - return not any( - s.lower() in str(fixture.fork).lower() for s in disable_strict_exception_matching - ) - - -@pytest.fixture(scope="function") -def strict_exception_matching( - client_strict_exception_matching: bool, - fork_strict_exception_matching: bool, -) -> bool: - """Return True if the test should use strict exception matching.""" - return client_strict_exception_matching and fork_strict_exception_matching - - -@pytest.fixture(scope="function") -def client( - hive_test: HiveTest, - client_files: dict, # configured within: rlp/conftest.py & engine/conftest.py - environment: dict, - client_type: ClientType, - total_timing_data: TimingData, -) -> Generator[Client, None, None]: - """Initialize the client with the appropriate files and environment variables.""" - logger.info(f"Starting client ({client_type.name})...") - with total_timing_data.time("Start client"): - client = hive_test.start_client( - client_type=client_type, environment=environment, files=client_files - ) - error_message = ( - f"Unable to connect to the client container ({client_type.name}) via Hive during test " - "setup. Check the client or Hive server logs for more information." - ) - assert client is not None, error_message - logger.info(f"Client ({client_type.name}) ready!") - yield client - logger.info(f"Stopping client ({client_type.name})...") - with total_timing_data.time("Stop client"): - client.stop() - logger.info(f"Client ({client_type.name}) stopped!") - - -@pytest.fixture(scope="function", autouse=True) -def timing_data( - total_timing_data: TimingData, client: Client -) -> Generator[TimingData, None, None]: - """Record timing data for the main execution of the test case.""" - with total_timing_data.time("Test case execution") as timing_data: - yield timing_data - - -class FixturesDict(Dict[Path, Fixtures]): - """ - A dictionary caches loaded fixture files to avoid reloading the same file - multiple times. - """ - - def __init__(self) -> None: - """Initialize the dictionary that caches loaded fixture files.""" - self._fixtures: Dict[Path, Fixtures] = {} - - def __getitem__(self, key: Path) -> Fixtures: - """Return the fixtures from the index file, if not found, load from disk.""" - assert key.is_file(), f"Expected a file path, got '{key}'" - if key not in self._fixtures: - self._fixtures[key] = Fixtures.model_validate_json(key.read_text()) - return self._fixtures[key] - - -@pytest.fixture(scope="session") -def fixture_file_loader() -> Dict[Path, Fixtures]: - """Return a singleton dictionary that caches loaded fixture files used in all tests.""" - return FixturesDict() - - -@pytest.fixture(scope="function") -def fixture( - fixtures_source: FixturesSource, - fixture_file_loader: Dict[Path, Fixtures], - test_case: TestCaseIndexFile | TestCaseStream, -) -> BaseFixture: - """ - Load the fixture from a file or from stream in any of the supported - fixture formats. - - The fixture is either already available within the test case (if consume - is taking input on stdin) or loaded from the fixture json file if taking - input from disk (fixture directory with index file). - """ - fixture: BaseFixture - if fixtures_source.is_stdin: - assert isinstance(test_case, TestCaseStream), "Expected a stream test case" - fixture = test_case.fixture - else: - assert isinstance(test_case, TestCaseIndexFile), "Expected an index file test case" - fixtures_file_path = fixtures_source.path / test_case.json_path - fixtures: Fixtures = fixture_file_loader[fixtures_file_path] - fixture = fixtures[test_case.id] - assert isinstance(fixture, test_case.format), ( - f"Expected a {test_case.format.format_name} test fixture" - ) - return fixture diff --git a/src/pytest_plugins/consume/hive_simulators/__init__.py b/src/pytest_plugins/consume/simulators/__init__.py similarity index 100% rename from src/pytest_plugins/consume/hive_simulators/__init__.py rename to src/pytest_plugins/consume/simulators/__init__.py diff --git a/src/pytest_plugins/consume/simulators/base.py b/src/pytest_plugins/consume/simulators/base.py new file mode 100644 index 00000000000..aa93d5297fa --- /dev/null +++ b/src/pytest_plugins/consume/simulators/base.py @@ -0,0 +1,86 @@ +"""Common pytest fixtures for the Hive simulators.""" + +from pathlib import Path +from typing import Dict, Literal + +import pytest +from hive.client import Client + +from ethereum_test_fixtures import ( + BaseFixture, +) +from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream +from ethereum_test_fixtures.file import Fixtures +from ethereum_test_rpc import EthRPC +from pytest_plugins.consume.consume import FixturesSource + + +@pytest.fixture(scope="function") +def eth_rpc(client: Client) -> EthRPC: + """Initialize ethereum RPC client for the execution client under test.""" + return EthRPC(f"http://{client.ip}:8545") + + +@pytest.fixture(scope="function") +def check_live_port(test_suite_name: str) -> Literal[8545, 8551]: + """Port used by hive to check for liveness of the client.""" + if test_suite_name == "eest/consume-rlp": + return 8545 + elif test_suite_name == "eest/consume-engine": + return 8551 + raise ValueError( + f"Unexpected test suite name '{test_suite_name}' while setting HIVE_CHECK_LIVE_PORT." + ) + + +class FixturesDict(Dict[Path, Fixtures]): + """ + A dictionary caches loaded fixture files to avoid reloading the same file + multiple times. + """ + + def __init__(self) -> None: + """Initialize the dictionary that caches loaded fixture files.""" + self._fixtures: Dict[Path, Fixtures] = {} + + def __getitem__(self, key: Path) -> Fixtures: + """Return the fixtures from the index file, if not found, load from disk.""" + assert key.is_file(), f"Expected a file path, got '{key}'" + if key not in self._fixtures: + self._fixtures[key] = Fixtures.model_validate_json(key.read_text()) + return self._fixtures[key] + + +@pytest.fixture(scope="session") +def fixture_file_loader() -> Dict[Path, Fixtures]: + """Return a singleton dictionary that caches loaded fixture files used in all tests.""" + return FixturesDict() + + +@pytest.fixture(scope="function") +def fixture( + fixtures_source: FixturesSource, + fixture_file_loader: Dict[Path, Fixtures], + test_case: TestCaseIndexFile | TestCaseStream, +) -> BaseFixture: + """ + Load the fixture from a file or from stream in any of the supported + fixture formats. + + The fixture is either already available within the test case (if consume + is taking input on stdin) or loaded from the fixture json file if taking + input from disk (fixture directory with index file). + """ + fixture: BaseFixture + if fixtures_source.is_stdin: + assert isinstance(test_case, TestCaseStream), "Expected a stream test case" + fixture = test_case.fixture + else: + assert isinstance(test_case, TestCaseIndexFile), "Expected an index file test case" + fixtures_file_path = fixtures_source.path / test_case.json_path + fixtures: Fixtures = fixture_file_loader[fixtures_file_path] + fixture = fixtures[test_case.id] + assert isinstance(fixture, test_case.format), ( + f"Expected a {test_case.format.format_name} test fixture" + ) + return fixture diff --git a/src/pytest_plugins/consume/hive_simulators/engine/__init__.py b/src/pytest_plugins/consume/simulators/engine/__init__.py similarity index 100% rename from src/pytest_plugins/consume/hive_simulators/engine/__init__.py rename to src/pytest_plugins/consume/simulators/engine/__init__.py diff --git a/src/pytest_plugins/consume/hive_simulators/engine/conftest.py b/src/pytest_plugins/consume/simulators/engine/conftest.py similarity index 83% rename from src/pytest_plugins/consume/hive_simulators/engine/conftest.py rename to src/pytest_plugins/consume/simulators/engine/conftest.py index fd997bc1eec..e7cda8ffa7d 100644 --- a/src/pytest_plugins/consume/hive_simulators/engine/conftest.py +++ b/src/pytest_plugins/consume/simulators/engine/conftest.py @@ -14,6 +14,15 @@ from ethereum_test_fixtures import BlockchainEngineFixture from ethereum_test_rpc import EngineRPC +pytest_plugins = ( + "pytest_plugins.pytest_hive.pytest_hive", + "pytest_plugins.consume.simulators.base", + "pytest_plugins.consume.simulators.single_test_client", + "pytest_plugins.consume.simulators.test_case_description", + "pytest_plugins.consume.simulators.timing_data", + "pytest_plugins.consume.simulators.exceptions", +) + def pytest_configure(config): """Set the supported fixture formats for the engine simulator.""" diff --git a/src/pytest_plugins/consume/simulators/exceptions.py b/src/pytest_plugins/consume/simulators/exceptions.py new file mode 100644 index 00000000000..0e8d4d63a78 --- /dev/null +++ b/src/pytest_plugins/consume/simulators/exceptions.py @@ -0,0 +1,91 @@ +"""Pytest plugin that defines options and fixtures for client exceptions.""" + +from typing import List + +import pytest +from hive.client import ClientType + +from ethereum_test_exceptions import ExceptionMapper +from ethereum_test_fixtures import ( + BlockchainFixtureCommon, +) + +from .helpers.exceptions import EXCEPTION_MAPPERS + + +def pytest_addoption(parser): + """Hive simulator specific consume command line options.""" + consume_group = parser.getgroup( + "consume", "Arguments related to consuming fixtures via a client" + ) + consume_group.addoption( + "--disable-strict-exception-matching", + action="store", + dest="disable_strict_exception_matching", + default="", + help=( + "Comma-separated list of client names and/or forks which should NOT use strict " + "exception matching." + ), + ) + + +@pytest.fixture(scope="session") +def client_exception_mapper_cache(): + """Cache for exception mappers by client type.""" + return {} + + +@pytest.fixture(scope="function") +def client_exception_mapper( + client_type: ClientType, client_exception_mapper_cache +) -> ExceptionMapper | None: + """Return the exception mapper for the client type, with caching.""" + if client_type.name not in client_exception_mapper_cache: + for client in EXCEPTION_MAPPERS: + if client in client_type.name: + client_exception_mapper_cache[client_type.name] = EXCEPTION_MAPPERS[client] + break + else: + client_exception_mapper_cache[client_type.name] = None + + return client_exception_mapper_cache[client_type.name] + + +@pytest.fixture(scope="session") +def disable_strict_exception_matching(request: pytest.FixtureRequest) -> List[str]: + """Return the list of clients or forks that should NOT use strict exception matching.""" + config_string = request.config.getoption("disable_strict_exception_matching") + return config_string.split(",") if config_string else [] + + +@pytest.fixture(scope="function") +def client_strict_exception_matching( + client_type: ClientType, + disable_strict_exception_matching: List[str], +) -> bool: + """Return True if the client type should use strict exception matching.""" + return not any( + client.lower() in client_type.name.lower() for client in disable_strict_exception_matching + ) + + +@pytest.fixture(scope="function") +def fork_strict_exception_matching( + fixture: BlockchainFixtureCommon, + disable_strict_exception_matching: List[str], +) -> bool: + """Return True if the fork should use strict exception matching.""" + # NOTE: `in` makes it easier for transition forks ("Prague" in "CancunToPragueAtTime15k") + return not any( + s.lower() in str(fixture.fork).lower() for s in disable_strict_exception_matching + ) + + +@pytest.fixture(scope="function") +def strict_exception_matching( + client_strict_exception_matching: bool, + fork_strict_exception_matching: bool, +) -> bool: + """Return True if the test should use strict exception matching.""" + return client_strict_exception_matching and fork_strict_exception_matching diff --git a/src/pytest_plugins/consume/simulators/helpers/__init__.py b/src/pytest_plugins/consume/simulators/helpers/__init__.py new file mode 100644 index 00000000000..4464aa65b7c --- /dev/null +++ b/src/pytest_plugins/consume/simulators/helpers/__init__.py @@ -0,0 +1 @@ +"""Helper classes and functions for consume hive simulators.""" diff --git a/src/pytest_plugins/consume/hive_simulators/exceptions.py b/src/pytest_plugins/consume/simulators/helpers/exceptions.py similarity index 100% rename from src/pytest_plugins/consume/hive_simulators/exceptions.py rename to src/pytest_plugins/consume/simulators/helpers/exceptions.py diff --git a/src/pytest_plugins/consume/hive_simulators/ruleset.py b/src/pytest_plugins/consume/simulators/helpers/ruleset.py similarity index 100% rename from src/pytest_plugins/consume/hive_simulators/ruleset.py rename to src/pytest_plugins/consume/simulators/helpers/ruleset.py diff --git a/src/pytest_plugins/consume/hive_simulators/timing.py b/src/pytest_plugins/consume/simulators/helpers/timing.py similarity index 100% rename from src/pytest_plugins/consume/hive_simulators/timing.py rename to src/pytest_plugins/consume/simulators/helpers/timing.py diff --git a/src/pytest_plugins/consume/simulators/hive_tests/__init__.py b/src/pytest_plugins/consume/simulators/hive_tests/__init__.py new file mode 100644 index 00000000000..e3ef68ee3ad --- /dev/null +++ b/src/pytest_plugins/consume/simulators/hive_tests/__init__.py @@ -0,0 +1 @@ +"""Defines the Pytest test functions used by Hive Consume Simulators.""" diff --git a/src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py b/src/pytest_plugins/consume/simulators/hive_tests/test_via_engine.py similarity index 98% rename from src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py rename to src/pytest_plugins/consume/simulators/hive_tests/test_via_engine.py index 34ba6058b9c..4a37878cf81 100644 --- a/src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py +++ b/src/pytest_plugins/consume/simulators/hive_tests/test_via_engine.py @@ -9,10 +9,10 @@ from ethereum_test_fixtures import BlockchainEngineFixture from ethereum_test_rpc import EngineRPC, EthRPC from ethereum_test_rpc.types import ForkchoiceState, JSONRPCError, PayloadStatusEnum -from pytest_plugins.consume.hive_simulators.exceptions import GenesisBlockMismatchExceptionError +from pytest_plugins.consume.simulators.helpers.exceptions import GenesisBlockMismatchExceptionError from pytest_plugins.logging import get_logger -from ..timing import TimingData +from ..helpers.timing import TimingData logger = get_logger(__name__) diff --git a/src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py b/src/pytest_plugins/consume/simulators/hive_tests/test_via_rlp.py similarity index 91% rename from src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py rename to src/pytest_plugins/consume/simulators/hive_tests/test_via_rlp.py index 7146b74b2ed..cf3102d610d 100644 --- a/src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py +++ b/src/pytest_plugins/consume/simulators/hive_tests/test_via_rlp.py @@ -9,9 +9,9 @@ from ethereum_test_fixtures import BlockchainFixture from ethereum_test_rpc import EthRPC -from pytest_plugins.consume.hive_simulators.exceptions import GenesisBlockMismatchExceptionError +from pytest_plugins.consume.simulators.helpers.exceptions import GenesisBlockMismatchExceptionError -from ..timing import TimingData +from ..helpers.timing import TimingData logger = logging.getLogger(__name__) diff --git a/src/pytest_plugins/consume/hive_simulators/rlp/__init__.py b/src/pytest_plugins/consume/simulators/rlp/__init__.py similarity index 100% rename from src/pytest_plugins/consume/hive_simulators/rlp/__init__.py rename to src/pytest_plugins/consume/simulators/rlp/__init__.py diff --git a/src/pytest_plugins/consume/hive_simulators/rlp/conftest.py b/src/pytest_plugins/consume/simulators/rlp/conftest.py similarity index 86% rename from src/pytest_plugins/consume/hive_simulators/rlp/conftest.py rename to src/pytest_plugins/consume/simulators/rlp/conftest.py index 371f1d09967..5cd63a05ab0 100644 --- a/src/pytest_plugins/consume/hive_simulators/rlp/conftest.py +++ b/src/pytest_plugins/consume/simulators/rlp/conftest.py @@ -11,6 +11,15 @@ TestCase = TestCaseIndexFile | TestCaseStream +pytest_plugins = ( + "pytest_plugins.pytest_hive.pytest_hive", + "pytest_plugins.consume.simulators.base", + "pytest_plugins.consume.simulators.single_test_client", + "pytest_plugins.consume.simulators.test_case_description", + "pytest_plugins.consume.simulators.timing_data", + "pytest_plugins.consume.simulators.exceptions", +) + def pytest_configure(config): """Set the supported fixture formats for the rlp simulator.""" diff --git a/src/pytest_plugins/consume/simulators/single_test_client.py b/src/pytest_plugins/consume/simulators/single_test_client.py new file mode 100644 index 00000000000..a519c09eb10 --- /dev/null +++ b/src/pytest_plugins/consume/simulators/single_test_client.py @@ -0,0 +1,88 @@ +"""Common pytest fixtures for simulators with single-test client architecture.""" + +import io +import json +import logging +from typing import Generator, Literal, cast + +import pytest +from hive.client import Client, ClientType +from hive.testing import HiveTest + +from ethereum_test_base_types import Number, to_json +from ethereum_test_fixtures import BlockchainFixtureCommon +from ethereum_test_fixtures.blockchain import FixtureHeader +from pytest_plugins.consume.simulators.helpers.ruleset import ( + ruleset, # TODO: generate dynamically +) + +from .helpers.timing import TimingData + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="function") +def client_genesis(fixture: BlockchainFixtureCommon) -> dict: + """Convert the fixture genesis block header and pre-state to a client genesis state.""" + genesis = to_json(fixture.genesis) + alloc = to_json(fixture.pre) + # NOTE: nethermind requires account keys without '0x' prefix + genesis["alloc"] = {k.replace("0x", ""): v for k, v in alloc.items()} + return genesis + + +@pytest.fixture(scope="function") +def environment( + fixture: BlockchainFixtureCommon, + check_live_port: Literal[8545, 8551], +) -> dict: + """Define the environment that hive will start the client with.""" + assert fixture.fork in ruleset, f"fork '{fixture.fork}' missing in hive ruleset" + return { + "HIVE_CHAIN_ID": str(Number(fixture.config.chain_id)), + "HIVE_FORK_DAO_VOTE": "1", + "HIVE_NODETYPE": "full", + "HIVE_CHECK_LIVE_PORT": str(check_live_port), + **{k: f"{v:d}" for k, v in ruleset[fixture.fork].items()}, + } + + +@pytest.fixture(scope="function") +def buffered_genesis(client_genesis: dict) -> io.BufferedReader: + """Create a buffered reader for the genesis block header of the current test fixture.""" + genesis_json = json.dumps(client_genesis) + genesis_bytes = genesis_json.encode("utf-8") + return io.BufferedReader(cast(io.RawIOBase, io.BytesIO(genesis_bytes))) + + +@pytest.fixture(scope="function") +def genesis_header(fixture: BlockchainFixtureCommon) -> FixtureHeader: + """Provide the genesis header from the shared pre-state group.""" + return fixture.genesis # type: ignore + + +@pytest.fixture(scope="function") +def client( + hive_test: HiveTest, + client_files: dict, # configured within: rlp/conftest.py & engine/conftest.py + environment: dict, + client_type: ClientType, + total_timing_data: TimingData, +) -> Generator[Client, None, None]: + """Initialize the client with the appropriate files and environment variables.""" + logger.info(f"Starting client ({client_type.name})...") + with total_timing_data.time("Start client"): + client = hive_test.start_client( + client_type=client_type, environment=environment, files=client_files + ) + error_message = ( + f"Unable to connect to the client container ({client_type.name}) via Hive during test " + "setup. Check the client or Hive server logs for more information." + ) + assert client is not None, error_message + logger.info(f"Client ({client_type.name}) ready!") + yield client + logger.info(f"Stopping client ({client_type.name})...") + with total_timing_data.time("Stop client"): + client.stop() + logger.info(f"Client ({client_type.name}) stopped!") diff --git a/src/pytest_plugins/consume/simulators/test_case_description.py b/src/pytest_plugins/consume/simulators/test_case_description.py new file mode 100644 index 00000000000..c56f24d35c5 --- /dev/null +++ b/src/pytest_plugins/consume/simulators/test_case_description.py @@ -0,0 +1,176 @@ +"""Pytest fixtures that help create the test case "Description" displayed in the Hive UI.""" + +import logging +import textwrap +import urllib +import warnings +from typing import List + +import pytest +from hive.client import ClientType + +from ethereum_test_fixtures import BaseFixture +from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream +from pytest_plugins.pytest_hive.hive_info import ClientFile, HiveInfo + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="function") +def hive_clients_yaml_target_filename() -> str: + """Return the name of the target clients YAML file.""" + return "clients_eest.yaml" + + +@pytest.fixture(scope="function") +def hive_clients_yaml_generator_command( + client_type: ClientType, + client_file: ClientFile, + hive_clients_yaml_target_filename: str, + hive_info: HiveInfo, +) -> str: + """Generate a shell command that creates a clients YAML file for the current client.""" + try: + if not client_file: + raise ValueError("No client information available - try updating hive") + client_config = [c for c in client_file.root if c.client in client_type.name] + if not client_config: + raise ValueError(f"Client '{client_type.name}' not found in client file") + try: + yaml_content = ClientFile(root=[client_config[0]]).yaml().replace(" ", " ") + return f'echo "\\\n{yaml_content}" > {hive_clients_yaml_target_filename}' + except Exception as e: + raise ValueError(f"Failed to generate YAML: {str(e)}") from e + except ValueError as e: + error_message = str(e) + warnings.warn( + f"{error_message}. The Hive clients YAML generator command will not be available.", + stacklevel=2, + ) + + issue_title = f"Client {client_type.name} configuration issue" + issue_body = f"Error: {error_message}\nHive version: {hive_info.commit}\n" + issue_url = f"https://github.com/ethereum/execution-spec-tests/issues/new?title={urllib.parse.quote(issue_title)}&body={urllib.parse.quote(issue_body)}" + + return ( + f"Error: {error_message}\n" + f'Please create an issue to report this problem.' + ) + + +@pytest.fixture(scope="function") +def filtered_hive_options(hive_info: HiveInfo) -> List[str]: + """Filter Hive command options to remove unwanted options.""" + logger.info("Hive info: %s", hive_info.command) + + unwanted_options = [ + "--client", # gets overwritten: we specify a single client; the one from the test case + "--client-file", # gets overwritten: we'll write our own client file + "--results-root", # use default value instead (or you have to pass it to ./hiveview) + "--sim.limit", # gets overwritten: we only run the current test case id + "--sim.parallelism", # skip; we'll only be running a single test + ] + + command_parts = [] + skip_next = False + for part in hive_info.command: + if skip_next: + skip_next = False + continue + + if part in unwanted_options: + skip_next = True + continue + + if any(part.startswith(f"{option}=") for option in unwanted_options): + continue + + command_parts.append(part) + + return command_parts + + +@pytest.fixture(scope="function") +def hive_client_config_file_parameter(hive_clients_yaml_target_filename: str) -> str: + """Return the hive client config file parameter.""" + return f"--client-file {hive_clients_yaml_target_filename}" + + +@pytest.fixture(scope="function") +def hive_consume_command( + test_case: TestCaseIndexFile | TestCaseStream, + hive_client_config_file_parameter: str, + filtered_hive_options: List[str], + client_type: ClientType, +) -> str: + """Command to run the test within hive.""" + command_parts = filtered_hive_options.copy() + command_parts.append(f"{hive_client_config_file_parameter}") + command_parts.append(f"--client={client_type.name}") + command_parts.append(f'--sim.limit="id:{test_case.id}"') + + return " ".join(command_parts) + + +@pytest.fixture(scope="function") +def hive_dev_command( + client_type: ClientType, + hive_client_config_file_parameter: str, +) -> str: + """Return the command used to instantiate hive alongside the `consume` command.""" + return f"./hive --dev {hive_client_config_file_parameter} --client {client_type.name}" + + +@pytest.fixture(scope="function") +def eest_consume_command( + test_suite_name: str, + test_case: TestCaseIndexFile | TestCaseStream, + fixture_source_flags: List[str], +) -> str: + """Commands to run the test within EEST using a hive dev back-end.""" + flags = " ".join(fixture_source_flags) + return ( + f"uv run consume {test_suite_name.split('-')[-1]} " + f'{flags} --sim.limit="id:{test_case.id}" -v -s' + ) + + +@pytest.fixture(scope="function") +def test_case_description( + fixture: BaseFixture, + test_case: TestCaseIndexFile | TestCaseStream, + hive_clients_yaml_generator_command: str, + hive_consume_command: str, + hive_dev_command: str, + eest_consume_command: str, +) -> str: + """Create the description of the current blockchain fixture test case.""" + test_url = fixture.info.get("url", "") + + if "description" not in fixture.info or fixture.info["description"] is None: + test_docstring = "No documentation available." + else: + # this prefix was included in the fixture description field for fixtures <= v4.3.0 + test_docstring = fixture.info["description"].replace("Test function documentation:\n", "") # type: ignore + + description = textwrap.dedent(f""" + Test Details + {test_case.id} + {f'[source]' if test_url else ""} + + {test_docstring} + + Run This Test Locally: + To run this test in hive: + {hive_clients_yaml_generator_command} + {hive_consume_command} + + Advanced: Run the test against a hive developer backend using EEST's consume command + Create the client YAML file, as above, then: + 1. Start hive in dev mode: {hive_dev_command} + 2. In the EEST repository root: {eest_consume_command} + """) # noqa: E501 + + description = description.strip() + description = description.replace("\n", "
") + return description diff --git a/src/pytest_plugins/consume/simulators/timing_data.py b/src/pytest_plugins/consume/simulators/timing_data.py new file mode 100644 index 00000000000..e63fedfad0b --- /dev/null +++ b/src/pytest_plugins/consume/simulators/timing_data.py @@ -0,0 +1,43 @@ +"""Pytest plugin that helps measure and log timing data in Hive simulators.""" + +from typing import Generator + +import pytest +import rich +from hive.client import Client + +from .helpers.timing import TimingData + + +def pytest_addoption(parser): + """Hive simulator specific consume command line options.""" + consume_group = parser.getgroup( + "consume", "Arguments related to consuming fixtures via a client" + ) + consume_group.addoption( + "--timing-data", + action="store_true", + dest="timing_data", + default=False, + help="Log the timing data for each test case execution.", + ) + + +@pytest.fixture(scope="function", autouse=True) +def total_timing_data(request) -> Generator[TimingData, None, None]: + """Record timing data for various stages of executing test case.""" + with TimingData("Total (seconds)") as total_timing_data: + yield total_timing_data + if request.config.getoption("timing_data"): + rich.print(f"\n{total_timing_data.formatted()}") + if hasattr(request.node, "rep_call"): # make available for test reports + request.node.rep_call.timings = total_timing_data + + +@pytest.fixture(scope="function", autouse=True) +def timing_data( + total_timing_data: TimingData, client: Client +) -> Generator[TimingData, None, None]: + """Record timing data for the main execution of the test case.""" + with total_timing_data.time("Test case execution") as timing_data: + yield timing_data diff --git a/src/pytest_plugins/execute/rpc/hive.py b/src/pytest_plugins/execute/rpc/hive.py index 0d204acdcf0..80c81b758ce 100644 --- a/src/pytest_plugins/execute/rpc/hive.py +++ b/src/pytest_plugins/execute/rpc/hive.py @@ -39,7 +39,7 @@ ) from ethereum_test_types import Requests from ethereum_test_types.trie import keccak256 -from pytest_plugins.consume.hive_simulators.ruleset import ruleset +from pytest_plugins.consume.simulators.helpers.ruleset import ruleset class HashList(RootModel[List[Hash]]):