feat(fill): add --gas-benchmark-values command to support single genesis file (#1895)

* feat(fill): add benchmark gas valu command to support single genesis file

* refactor(tests): update benchmark test for supported command

* refactor(benchmark): consolidate benchmark configurations into a single entry

* doc(fill): update command description and changelog

* chore(fill): remove legacy gas benchmark values command

* refactor(fill): create gas benchmakr value pytest plugin

* test(fill): add pytest plugin test and update state test

* refactor(fill): add env fixture for benchmarking with gas limit configuration

* refactor: support both fill and execute mode

* fix: update ci flag and test command

Author: LouisTsai-Csie

.github/configs/feature.yaml

@@ -11,34 +11,9 @@ static:
   evm-type: static
   fill-params: --until=Osaka --fill-static-tests ./tests/static
   solc: 0.8.21
-benchmark_1M:
+benchmark_test:
   evm-type: benchmark
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 1000000 -m benchmark ./tests
-  solc: 0.8.21
-  feature_only: true
-benchmark_10M:
-  evm-type: benchmark
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 10000000 -m benchmark ./tests
-  solc: 0.8.21
-  feature_only: true
-benchmark_30M:
-  evm-type: benchmark
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 30000000 -m benchmark ./tests
-  solc: 0.8.21
-  feature_only: true
-benchmark_60M:
-  evm-type: benchmark
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 60000000 -m benchmark ./tests
-  solc: 0.8.21
-  feature_only: true
-benchmark_90M:
-  evm-type: benchmark
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 90000000 -m benchmark ./tests
-  solc: 0.8.21
-  feature_only: true
-benchmark_120M:
-  evm-type: benchmark
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 120000000 -m benchmark ./tests
+  fill-params: --from=Cancun --until=Prague --gas-benchmark-values 1,10,30,60,90,120 -m benchmark --generate-all-formats ./tests
   solc: 0.8.21
   feature_only: true
 eip7692:

docs/CHANGELOG.md

@@ -73,6 +73,7 @@ Users can select any of the artifacts depending on their testing needs for their
 - 🔀 Changed INVALID_DEPOSIT_EVENT_LAYOUT to a BlockException instead of a TransactionException ([#1773](https://github.com/ethereum/execution-spec-tests/pull/1773)).
 - 🔀 Disabled writing debugging information to the EVM "dump directory" to improve performance. To obtain debug output, the `--evm-dump-dir` flag must now be explicitly set. As a consequence, the now redundant `--skip-evm-dump` option was removed ([#1874](https://github.com/ethereum/execution-spec-tests/pull/1874)).
 - ✨ Generate unique addresses with Python for compatible static tests, instead of using hard-coded addresses from legacy static test fillers ([#1781](https://github.com/ethereum/execution-spec-tests/pull/1781)).
+- ✨ Added support for the `--benchmark-gas-values` flag in the `fill` command, allowing a single genesis file to be used across different gas limit settings when generating fixtures. ([#1895](https://github.com/ethereum/execution-spec-tests/pull/1895)).
 
 #### `consume`
 

src/cli/pytest_commands/pytest_ini_files/pytest-execute-hive.ini

@@ -14,6 +14,7 @@ addopts =
     -p pytest_plugins.execute.rpc.hive
     -p pytest_plugins.execute.execute
     -p pytest_plugins.shared.execute_fill
+    -p pytest_plugins.shared.benchmarking
     -p pytest_plugins.shared.transaction_fixtures
     -p pytest_plugins.forks.forks
     -p pytest_plugins.pytest_hive.pytest_hive

src/cli/pytest_commands/pytest_ini_files/pytest-execute.ini

@@ -14,6 +14,7 @@ addopts =
     -p pytest_plugins.execute.pre_alloc
     -p pytest_plugins.execute.execute
     -p pytest_plugins.shared.execute_fill
+    -p pytest_plugins.shared.benchmarking
     -p pytest_plugins.shared.transaction_fixtures
     -p pytest_plugins.execute.rpc.remote_seed_sender
     -p pytest_plugins.execute.rpc.remote

src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini

@@ -12,9 +12,10 @@ addopts =
     -p pytest_plugins.concurrency
     -p pytest_plugins.filler.pre_alloc
     -p pytest_plugins.filler.filler
+    -p pytest_plugins.shared.execute_fill
     -p pytest_plugins.filler.ported_tests
     -p pytest_plugins.filler.static_filler
-    -p pytest_plugins.shared.execute_fill
+    -p pytest_plugins.shared.benchmarking
     -p pytest_plugins.shared.transaction_fixtures
     -p pytest_plugins.forks.forks
     -p pytest_plugins.eels_resolver

src/pytest_plugins/filler/tests/test_benchmarking.py

@@ -0,0 +1,118 @@
+"""Test the benchmarking pytest plugin for gas benchmark values."""
+
+import textwrap
+
+import pytest
+
+test_module_dummy = textwrap.dedent(
+    """\
+    import pytest
+
+    from ethereum_test_tools import Environment
+
+    @pytest.mark.valid_at("Istanbul")
+    def test_dummy_benchmark_test(state_test, gas_benchmark_value):
+        state_test(
+            env=env,pre={},post={},tx=None)
+    """
+)
+
+test_module_without_fixture = textwrap.dedent(
+    """\
+    import pytest
+
+    from ethereum_test_tools import Environment
+
+    @pytest.mark.valid_at("Istanbul")
+    def test_dummy_no_benchmark_test(state_test):
+        state_test(env=env, pre={}, post={}, tx=None)
+    """
+)
+
+
+def setup_test_directory_structure(
+    pytester: pytest.Pytester, test_content: str, test_filename: str
+):
+    """
+    Set up the common test directory structure used across multiple tests.
+
+    Args:
+        pytester: The pytest Pytester fixture
+        test_content: The content to write to the test file
+        test_filename: The name of the test file to create
+
+    Returns:
+        The path to the created test module file
+
+    """
+    tests_dir = pytester.mkdir("tests")
+    istanbul_tests_dir = tests_dir / "istanbul"
+    istanbul_tests_dir.mkdir()
+    dummy_dir = istanbul_tests_dir / "dummy_test_module"
+    dummy_dir.mkdir()
+    test_module = dummy_dir / test_filename
+    test_module.write_text(test_content)
+
+    pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
+
+    return test_module
+
+
+def test_gas_benchmark_option_added(pytester: pytest.Pytester):
+    """Test that the --gas-benchmark-values option is properly added."""
+    pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
+
+    # Command: pytest -p pytest_plugins.filler.benchmarking --help
+    result = pytester.runpytest("-c", "pytest-fill.ini", "--help")
+
+    assert result.ret == 0
+    assert any("--gas-benchmark-values" in line for line in result.outlines)
+    assert any("Specify gas benchmark values for tests" in line for line in result.outlines)
+
+
+def test_benchmarking_mode_configured_with_option(pytester: pytest.Pytester):
+    """Test that fill_mode is set to BENCHMARKING when --gas-benchmark-values is used."""
+    setup_test_directory_structure(pytester, test_module_dummy, "test_dummy_benchmark.py")
+
+    # Test with gas benchmark values
+    result = pytester.runpytest(
+        "-c",
+        "pytest-fill.ini",
+        "--fork",
+        "Istanbul",
+        "--gas-benchmark-values",
+        "10,20,30",
+        "tests/istanbul/dummy_test_module/",
+        "--collect-only",
+        "-q",
+    )
+
+    assert result.ret == 0
+    assert any("9 tests collected" in line for line in result.outlines)
+    # Check that the test names include the benchmark gas values
+    assert any("benchmark-gas-value_10M" in line for line in result.outlines)
+    assert any("benchmark-gas-value_20M" in line for line in result.outlines)
+    assert any("benchmark-gas-value_30M" in line for line in result.outlines)
+
+
+def test_benchmarking_mode_not_configured_without_option(pytester: pytest.Pytester):
+    """Test that fill_mode is not set to BENCHMARKING when --gas-benchmark-values is not used."""
+    setup_test_directory_structure(pytester, test_module_dummy, "test_dummy_benchmark.py")
+
+    # Test without gas benchmark values
+    result = pytester.runpytest(
+        "-c",
+        "pytest-fill.ini",
+        "--fork",
+        "Istanbul",
+        "tests/istanbul/dummy_test_module/",
+        "--collect-only",
+        "-q",
+    )
+
+    assert result.ret == 0
+    # Should generate normal test variants (3) without parametrization
+    assert any("3 tests collected" in line for line in result.outlines)
+    assert not any("benchmark-gas-value_10M" in line for line in result.outlines)
+    assert not any("benchmark-gas-value_20M" in line for line in result.outlines)
+    assert not any("benchmark-gas-value_30M" in line for line in result.outlines)

src/pytest_plugins/shared/benchmarking.py

@@ -0,0 +1,61 @@
+"""The module contains the pytest hooks for the gas benchmark values."""
+
+import pytest
+
+from ethereum_test_tools import Environment
+from ethereum_test_types import EnvironmentDefaults
+
+from .execute_fill import OpMode
+
+
+def pytest_addoption(parser: pytest.Parser):
+    """Add command line options for gas benchmark values."""
+    evm_group = parser.getgroup("evm", "Arguments defining evm executable behavior")
+    evm_group.addoption(
+        "--gas-benchmark-values",
+        action="store",
+        dest="gas_benchmark_value",
+        type=str,
+        default=None,
+        help="Specify gas benchmark values for tests as a comma-separated list.",
+    )
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_configure(config):
+    """Configure the fill and execute mode to benchmarking."""
+    if config.getoption("gas_benchmark_value"):
+        config.op_mode = OpMode.BENCHMARKING
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc):
+    """Generate tests for the gas benchmark values."""
+    if "gas_benchmark_value" in metafunc.fixturenames:
+        gas_benchmark_values = metafunc.config.getoption("gas_benchmark_value")
+        if gas_benchmark_values:
+            gas_values = [int(x.strip()) for x in gas_benchmark_values.split(",")]
+            gas_parameters = [
+                pytest.param(gas_value * 1_000_000, id=f"benchmark-gas-value_{gas_value}M")
+                for gas_value in gas_values
+            ]
+            metafunc.parametrize("gas_benchmark_value", gas_parameters, scope="function")
+
+
+@pytest.fixture(scope="function")
+def gas_benchmark_value(request: pytest.FixtureRequest) -> int:
+    """Return a single gas benchmark value for the current test."""
+    if hasattr(request, "param"):
+        return request.param
+
+    return EnvironmentDefaults.gas_limit
+
+
+GIGA_GAS = 1_000_000_000
+
+
+@pytest.fixture
+def env(request: pytest.FixtureRequest) -> Environment:  # noqa: D103
+    """Return an Environment instance with appropriate gas limit based on operation mode."""
+    if request.config.op_mode == OpMode.BENCHMARKING:  # type: ignore[attr-defined]
+        return Environment(gas_limit=GIGA_GAS)
+    return Environment()

src/pytest_plugins/shared/execute_fill.py

@@ -1,17 +1,27 @@
 """Shared pytest fixtures and hooks for EEST generation modes (fill and execute)."""
 
+from enum import StrEnum, unique
 from typing import List
 
 import pytest
 
 from ethereum_test_execution import BaseExecute, LabeledExecuteFormat
 from ethereum_test_fixtures import BaseFixture, LabeledFixtureFormat
 from ethereum_test_specs import BaseTest
+from ethereum_test_tools import Environment
 from ethereum_test_types import EOA, Alloc
 
 from ..spec_version_checker.spec_version_checker import EIPSpecTestItem
 
 
+@unique
+class OpMode(StrEnum):
+    """Operation mode for the fill and execute."""
+
+    CONSENSUS = "consensus"
+    BENCHMARKING = "benchmarking"
+
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_configure(config: pytest.Config):
     """
@@ -60,6 +70,9 @@ def pytest_configure(config: pytest.Config):
                 (f"{marker}: {description}"),
             )
 
+    if not hasattr(config, "op_mode"):
+        config.op_mode = OpMode.CONSENSUS  # type: ignore[attr-defined]
+
     config.addinivalue_line(
         "markers",
         "yul_test: a test case that compiles Yul code.",
@@ -181,3 +194,8 @@ def pytest_addoption(parser: pytest.Parser):
         default=None,
         help=("Enable reading and filling from static test files."),
     )
+
+
+@pytest.fixture
+def env(request: pytest.FixtureRequest) -> Environment:  # noqa: D103
+    return Environment()

tests/benchmark/test_worst_blocks.py

@@ -21,9 +21,9 @@
 
 
 @pytest.fixture
-def iteration_count(intrinsic_cost: int):
+def iteration_count(intrinsic_cost: int, gas_benchmark_value: int):
     """Calculate the number of iterations based on the gas limit and intrinsic cost."""
-    return Environment().gas_limit // intrinsic_cost
+    return gas_benchmark_value // intrinsic_cost
 
 
 @pytest.fixture
@@ -173,10 +173,10 @@ def test_block_full_data(
     zero_byte: bool,
     intrinsic_cost: int,
     total_cost_floor_per_token: int,
+    gas_benchmark_value: int,
+    env: Environment,
 ):
     """Test a block with empty payload."""
-    attack_gas_limit = Environment().gas_limit
-
     # Gas cost calculation based on EIP-7683: (https://eips.ethereum.org/EIPS/eip-7683)
     #
     #   tx.gasUsed = 21000 + max(
@@ -201,7 +201,7 @@ def test_block_full_data(
     #
     # So we calculate how many bytes we can fit into calldata based on available gas.
 
-    gas_available = attack_gas_limit - intrinsic_cost
+    gas_available = gas_benchmark_value - intrinsic_cost
 
     # Calculate the token_in_calldata
     max_tokens_in_calldata = gas_available // total_cost_floor_per_token
@@ -212,12 +212,12 @@ def test_block_full_data(
     tx = Transaction(
         to=pre.fund_eoa(),
         data=byte_data * num_of_bytes,
-        gas_limit=attack_gas_limit,
+        gas_limit=gas_benchmark_value,
         sender=pre.fund_eoa(),
     )
 
     state_test(
-        env=Environment(),
+        env=env,
         pre=pre,
         post={},
         tx=tx,