feat(specs,tests): Update all benchmark tests to use gas_benchmark_value (#1935)

* feat(specs): Check gas used by benchmark tests matches expectation

* feat(tests): Update all benchmark tests to use `gas_benchmark_value`

* fix: tox

* fix(plugins/benchmarking): Increase GIGA_GAS

* fix(tests): Environment usages

* docs: Changelog

* fix(tox): tests-deployed-benchmark command line

* fix: Remove unused parameter

* fix: `test_worst_storage_access_cold` and add more cases

* fix: `test_worst_bytecode_single_opcode` incorrect env overwrite

* fix(plugins): Bump `BENCHMARKING_MAX_GAS` so all benchmarking tests can run from a single genesis

* feat(plugins): Pass `env` fixture to the test spec builder by default if not specified

* fix(benchmark): Remove `env` from tests where it's unused

* fix(benchmark): tests referencing the `fee_recipient` from the environment

* fix(plugins): Set benchmark environment based on test mark

* fix: minor fix

* fix(plugins): Defaults for `env` and `genesis_environment`

Author: marioevz

docs/CHANGELOG.md

@@ -47,7 +47,11 @@ Additionally, writing debugging information to the EVM "dump directory" by defau
 
 Due to the crossover between `zkevm` and `benchmark` tests, all instances of the former have been replaced with the latter nomenclature. Repository PR labels and titles are additionally updated to reflect this change.
 
-This update renames the `zkevm` feature release to `benchmark_30M` and further expands the latter for 1M, 10M, 60M, 90M, and 120M block gas limits in `fixtures_benchmark_1M.tar.gz`, `fixtures_benchmark_10M.tar.gz`, `fixtures_benchmark_30M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz`, and `fixtures_benchmark_120M.tar.gz` respectively.
+This update renames the `zkevm` feature release to `benchmark` and further expands the latter for 1M, 10M, 30M, 45M, 60M, 90M, and 120M block gas limits in `fixtures_benchmark.tar.gz`.
+
+To select a test for a given gas limit, the IDs of the tests have been expanded to contain `benchmark-gas-value_XM`, where `X` can be any of the aforementioned values.
+
+The benchmark release also now includes BlockchainEngineX format that combines most of the tests into a minimal amount of genesis files. For more info see [Blockchain Engine X Tests](https://eest.ethereum.org/main/running_tests/test_formats/blockchain_test_engine_x/) in the EEST documentation.
 
 Users can select any of the artifacts depending on their testing needs for their provers.
 

src/ethereum_test_specs/base.py

@@ -2,6 +2,7 @@
 
 import hashlib
 from abc import abstractmethod
+from enum import StrEnum, unique
 from functools import reduce
 from os import path
 from pathlib import Path
@@ -49,12 +50,23 @@ def verify_result(result: Result, env: Environment):
         assert result.withdrawals_root == to_hex(Withdrawal.list_root(env.withdrawals))
 
 
+@unique
+class OpMode(StrEnum):
+    """Operation mode for the fill and execute."""
+
+    CONSENSUS = "consensus"
+    BENCHMARKING = "benchmarking"
+
+
 class BaseTest(BaseModel):
     """Represents a base Ethereum test which must return a single test fixture."""
 
     tag: str = ""
 
     _request: pytest.FixtureRequest | None = PrivateAttr(None)
+    _operation_mode: OpMode | None = PrivateAttr(None)
+
+    expected_benchmark_gas_used: int | None = None
 
     spec_types: ClassVar[Dict[str, Type["BaseTest"]]] = {}
 
@@ -98,9 +110,11 @@ def from_test(
         new_instance = cls(
             tag=base_test.tag,
             t8n_dump_dir=base_test.t8n_dump_dir,
+            expected_benchmark_gas_used=base_test.expected_benchmark_gas_used,
             **kwargs,
         )
         new_instance._request = base_test._request
+        new_instance._operation_mode = base_test._operation_mode
         return new_instance
 
     @classmethod

src/ethereum_test_specs/blockchain.py

@@ -53,7 +53,7 @@
 from ethereum_test_forks import Fork
 from ethereum_test_types import Alloc, Environment, Removable, Requests, Transaction, Withdrawal
 
-from .base import BaseTest, verify_result
+from .base import BaseTest, OpMode, verify_result
 from .debugging import print_traces
 from .helpers import verify_block, verify_transactions
 
@@ -489,6 +489,7 @@ def generate_block_data(
         block: Block,
         previous_env: Environment,
         previous_alloc: Alloc,
+        last_block: bool,
     ) -> BuiltBlock:
         """Generate common block data for both make_fixture and make_hive_fixture."""
         env = block.set_environment(previous_env)
@@ -554,6 +555,18 @@ def generate_block_data(
             # Verify the header after transition tool processing.
             block.header_verify.verify(header)
 
+        if last_block and self._operation_mode == OpMode.BENCHMARKING:
+            expected_benchmark_gas_used = self.expected_benchmark_gas_used
+            assert expected_benchmark_gas_used is not None, (
+                "expected_benchmark_gas_used is not set"
+            )
+            gas_used = int(transition_tool_output.result.gas_used)
+            assert gas_used == expected_benchmark_gas_used, (
+                f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
+                f"({expected_benchmark_gas_used})"
+                f", difference: {gas_used - expected_benchmark_gas_used}"
+            )
+
         requests_list: List[Bytes] | None = None
         if fork.header_requests_required(header.number, header.timestamp):
             assert transition_tool_output.result.requests is not None, (
@@ -659,7 +672,7 @@ def make_fixture(
         env = environment_from_parent_header(genesis.header)
         head = genesis.header.block_hash
         invalid_blocks = 0
-        for block in self.blocks:
+        for i, block in enumerate(self.blocks):
             # This is the most common case, the RLP needs to be constructed
             # based on the transactions to be included in the block.
             # Set the environment according to the block to execute.
@@ -669,6 +682,7 @@ def make_fixture(
                 block=block,
                 previous_env=env,
                 previous_alloc=alloc,
+                last_block=i == len(self.blocks) - 1,
             )
             fixture_blocks.append(built_block.get_fixture_block())
             if block.exception is None:
@@ -718,13 +732,14 @@ def make_hive_fixture(
         env = environment_from_parent_header(genesis.header)
         head_hash = genesis.header.block_hash
         invalid_blocks = 0
-        for block in self.blocks:
+        for i, block in enumerate(self.blocks):
             built_block = self.generate_block_data(
                 t8n=t8n,
                 fork=fork,
                 block=block,
                 previous_env=env,
                 previous_alloc=alloc,
+                last_block=i == len(self.blocks) - 1,
             )
             fixture_payloads.append(built_block.get_fixture_engine_new_payload())
             if block.exception is None:
@@ -765,6 +780,7 @@ def make_hive_fixture(
                 block=Block(),
                 previous_env=env,
                 previous_alloc=alloc,
+                last_block=False,
             )
             sync_payload = sync_built_block.get_fixture_engine_new_payload()
 

src/ethereum_test_specs/state.py

@@ -32,7 +32,7 @@
 from ethereum_test_forks import Fork
 from ethereum_test_types import Alloc, Environment, Transaction
 
-from .base import BaseTest
+from .base import BaseTest, OpMode
 from .blockchain import Block, BlockchainTest, Header
 from .debugging import print_traces
 from .helpers import verify_transactions
@@ -219,6 +219,18 @@ def make_state_test_fixture(
             pprint(transition_tool_output.alloc)
             raise e
 
+        if self._operation_mode == OpMode.BENCHMARKING:
+            expected_benchmark_gas_used = self.expected_benchmark_gas_used
+            gas_used = int(transition_tool_output.result.gas_used)
+            assert expected_benchmark_gas_used is not None, (
+                "expected_benchmark_gas_used is not set"
+            )
+            assert gas_used == expected_benchmark_gas_used, (
+                f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
+                f"({expected_benchmark_gas_used})"
+                f", difference: {gas_used - expected_benchmark_gas_used}"
+            )
+
         return StateFixture(
             env=FixtureEnvironment(**env.model_dump(exclude_none=True)),
             pre=pre_alloc,

src/pytest_plugins/execute/execute.py

@@ -13,6 +13,7 @@
 from ethereum_test_tools import BaseTest
 from ethereum_test_types import EnvironmentDefaults, TransactionDefaults
 
+from ..shared.execute_fill import ALL_FIXTURE_PARAMETERS
 from ..shared.helpers import (
     get_spec_format_for_item,
     is_help_or_collectonly_mode,
@@ -252,6 +253,7 @@ def base_test_parametrizer(cls: Type[BaseTest]):
     Implementation detail: All spec fixtures must be scoped on test function level to avoid
     leakage between tests.
     """
+    cls_fixture_parameters = [p for p in ALL_FIXTURE_PARAMETERS if p in cls.model_fields]
 
     @pytest.fixture(
         scope="function",
@@ -289,6 +291,11 @@ def __init__(self, *args, **kwargs):
                     kwargs["pre"] = pre
                 elif kwargs["pre"] != pre:
                     raise ValueError("The pre-alloc object was modified by the test.")
+                kwargs |= {
+                    p: request.getfixturevalue(p)
+                    for p in cls_fixture_parameters
+                    if p not in kwargs
+                }
 
                 request.node.config.sender_address = str(pre._sender)
 

src/pytest_plugins/filler/filler.py

@@ -42,6 +42,7 @@
 )
 from ethereum_test_types import EnvironmentDefaults
 
+from ..shared.execute_fill import ALL_FIXTURE_PARAMETERS
 from ..shared.helpers import (
     get_spec_format_for_item,
     is_help_or_collectonly_mode,
@@ -873,6 +874,7 @@ def base_test_parametrizer(cls: Type[BaseTest]):
     Implementation detail: All spec fixtures must be scoped on test function level to avoid
     leakage between tests.
     """
+    cls_fixture_parameters = [p for p in ALL_FIXTURE_PARAMETERS if p in cls.model_fields]
 
     @pytest.fixture(
         scope="function",
@@ -889,6 +891,7 @@ def base_test_parametrizer_func(
         fixture_collector: FixtureCollector,
         test_case_description: str,
         fixture_source_url: str,
+        gas_benchmark_value: int,
     ):
         """
         Fixture used to instantiate an auto-fillable BaseTest object from within
@@ -914,8 +917,17 @@ def __init__(self, *args, **kwargs):
                 kwargs["t8n_dump_dir"] = dump_dir_parameter_level
                 if "pre" not in kwargs:
                     kwargs["pre"] = pre
+                if "expected_benchmark_gas_used" not in kwargs:
+                    kwargs["expected_benchmark_gas_used"] = gas_benchmark_value
+                kwargs |= {
+                    p: request.getfixturevalue(p)
+                    for p in cls_fixture_parameters
+                    if p not in kwargs
+                }
+
                 super(BaseTestWrapper, self).__init__(*args, **kwargs)
                 self._request = request
+                self._operation_mode = request.config.op_mode
 
                 # Phase 1: Generate pre-allocation groups
                 if fixture_format is BlockchainEngineXFixture and request.config.getoption(

src/pytest_plugins/shared/benchmarking.py

@@ -50,12 +50,20 @@ def gas_benchmark_value(request: pytest.FixtureRequest) -> int:
     return EnvironmentDefaults.gas_limit
 
 
-GIGA_GAS = 1_000_000_000
+BENCHMARKING_MAX_GAS = 500_000_000_000
+
+
+@pytest.fixture
+def genesis_environment(request: pytest.FixtureRequest) -> Environment:  # noqa: D103
+    """Return an Environment instance with appropriate gas limit based on test type."""
+    if request.node.get_closest_marker("benchmark") is not None:
+        return Environment(gas_limit=BENCHMARKING_MAX_GAS)
+    return Environment()
 
 
 @pytest.fixture
 def env(request: pytest.FixtureRequest) -> Environment:  # noqa: D103
-    """Return an Environment instance with appropriate gas limit based on operation mode."""
-    if request.config.op_mode == OpMode.BENCHMARKING:  # type: ignore[attr-defined]
-        return Environment(gas_limit=GIGA_GAS)
+    """Return an Environment instance with appropriate gas limit based on test type."""
+    if request.node.get_closest_marker("benchmark") is not None:
+        return Environment(gas_limit=BENCHMARKING_MAX_GAS)
     return Environment()

src/pytest_plugins/shared/execute_fill.py

@@ -8,7 +8,6 @@
 from ethereum_test_execution import BaseExecute, LabeledExecuteFormat
 from ethereum_test_fixtures import BaseFixture, LabeledFixtureFormat
 from ethereum_test_specs import BaseTest
-from ethereum_test_tools import Environment
 from ethereum_test_types import EOA, Alloc
 
 from ..spec_version_checker.spec_version_checker import EIPSpecTestItem
@@ -22,6 +21,19 @@ class OpMode(StrEnum):
     BENCHMARKING = "benchmarking"
 
 
+ALL_FIXTURE_PARAMETERS = {
+    "genesis_environment",
+    "env",
+}
+"""
+List of test parameters that have a default fixture value which can be retrieved and used
+for the test instance if it was not explicitly specified when calling from the test
+function.
+
+All parameter names included in this list must define a fixture in one of the plugins.
+"""
+
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_configure(config: pytest.Config):
     """
@@ -194,8 +206,3 @@ def pytest_addoption(parser: pytest.Parser):
         default=None,
         help=("Enable reading and filling from static test files."),
     )
-
-
-@pytest.fixture
-def env(request: pytest.FixtureRequest) -> Environment:  # noqa: D103
-    return Environment()

tests/benchmark/test_worst_blocks.py

@@ -14,7 +14,6 @@
     Alloc,
     Block,
     BlockchainTestFiller,
-    Environment,
     StateTestFiller,
     Transaction,
 )
@@ -151,11 +150,11 @@ def test_block_full_of_ether_transfers(
     )
 
     blockchain_test(
-        genesis_environment=Environment(),
         pre=pre,
         post=post_state,
         blocks=[Block(txs=txs)],
         exclude_full_post_state_in_output=True,
+        expected_benchmark_gas_used=iteration_count * intrinsic_cost,
     )
 
 
@@ -174,7 +173,6 @@ def test_block_full_data(
     intrinsic_cost: int,
     total_cost_floor_per_token: int,
     gas_benchmark_value: int,
-    env: Environment,
 ):
     """Test a block with empty payload."""
     # Gas cost calculation based on EIP-7683: (https://eips.ethereum.org/EIPS/eip-7683)
@@ -217,7 +215,6 @@ def test_block_full_data(
     )
 
     state_test(
-        env=env,
         pre=pre,
         post={},
         tx=tx,