refactor(benchmark): update starting fork from prague (#2139)

* fix: update starting fork for benchmark test

* doc: update benchmark fork description

* refactor: update benchmark test generation to use a default benchmark fork

Author: LouisTsai-Csie

docs/writing_tests/benchmarks.md

@@ -4,6 +4,8 @@ Benchmark tests aim to maximize the usage of a specific opcode, precompile, or o
 
 To fill a benchmark test, in addition to the usual test flags, you must include the `-m benchmark` flag. This is necessary because benchmark tests are ignored by default; they must be manually selected via the `benchmark` pytest marker (="tag"). This marker is applied to all tests under `./tests/benchmark/` automatically by the framework.
 
+**Note:** Benchmark tests are now only available starting from the `Prague` fork. Tests targeting earlier forks (`Cancun` or prior) are not supported in benchmark mode.
+
 ## Setting the Gas Limit for Benchmarking
 
 To consume the full benchmark gas limit, use the `gas_benchmark_value` fixture as the gas limit:

tests/benchmark/conftest.py

@@ -4,6 +4,26 @@
 
 import pytest
 
+DEFAULT_BENCHMARK_FORK = "Prague"
+
+
+def pytest_generate_tests(metafunc):
+    """Modify test generation to enforce default benchmark fork for benchmark tests."""
+    benchmark_dir = Path(__file__).parent
+    test_file_path = Path(metafunc.definition.fspath)
+
+    # Check if this test is in the benchmark directory
+    is_in_benchmark_dir = benchmark_dir in test_file_path.parents
+
+    if is_in_benchmark_dir:
+        # Add benchmark marker if no valid_from marker exists
+        existing_markers = list(metafunc.definition.iter_markers())
+        has_valid_from = any(marker.name == "valid_from" for marker in existing_markers)
+
+        if not has_valid_from:
+            benchmark_marker = pytest.mark.valid_from(DEFAULT_BENCHMARK_FORK)
+            metafunc.definition.add_marker(benchmark_marker)
+
 
 def pytest_collection_modifyitems(config, items):
     """Add the `benchmark` marker to all tests under `./tests/benchmark`."""

tests/benchmark/test_worst_blocks.py

@@ -105,7 +105,6 @@ def ether_transfer_case(
     return senders, receivers
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "case_id",
     ["a_to_a", "a_to_b", "diff_acc_to_b", "a_to_diff_acc", "diff_acc_to_diff_acc"],
@@ -174,7 +173,6 @@ def total_cost_standard_per_token():
     return 4
 
 
-@pytest.mark.valid_from("Prague")
 @pytest.mark.parametrize("zero_byte", [True, False])
 def test_block_full_data(
     state_test: StateTestFiller,
@@ -231,7 +229,6 @@ def test_block_full_data(
     )
 
 
-@pytest.mark.valid_from("Prague")
 def test_block_full_access_list_and_data(
     state_test: StateTestFiller,
     pre: Alloc,

tests/benchmark/test_worst_bytecode.py

@@ -47,7 +47,6 @@
         Op.EXTCODECOPY,
     ],
 )
-@pytest.mark.valid_from("Cancun")
 def test_worst_bytecode_single_opcode(
     blockchain_test: BlockchainTestFiller,
     pre: Alloc,
@@ -233,7 +232,6 @@ def test_worst_bytecode_single_opcode(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "pattern",
     [
@@ -324,7 +322,6 @@ def test_worst_initcode_jumpdest_analysis(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "opcode",
     [
@@ -432,7 +429,6 @@ def test_worst_create(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "opcode",
     [

tests/benchmark/test_worst_compute.py

@@ -62,7 +62,6 @@ def make_dup(index: int) -> Opcode:
     return Opcode(0x80 + index, pushed_stack_items=1, min_stack_height=index + 1)
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "opcode",
     [
@@ -112,7 +111,6 @@ def test_worst_zero_param(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("calldata_length", [0, 1_000, 10_000])
 def test_worst_calldatasize(
     state_test: StateTestFiller,
@@ -145,7 +143,6 @@ def test_worst_calldatasize(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("non_zero_value", [True, False])
 @pytest.mark.parametrize("from_origin", [True, False])
 def test_worst_callvalue(
@@ -205,7 +202,6 @@ class ReturnDataStyle(Enum):
     IDENTITY = auto()
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "return_data_style",
     [
@@ -267,7 +263,6 @@ def test_worst_returndatasize_nonzero(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 def test_worst_returndatasize_zero(
     state_test: StateTestFiller,
     pre: Alloc,
@@ -299,7 +294,6 @@ def test_worst_returndatasize_zero(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("mem_size", [0, 1, 1_000, 100_000, 1_000_000])
 def test_worst_msize(
     state_test: StateTestFiller,
@@ -339,7 +333,6 @@ def test_worst_msize(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 def test_worst_keccak(
     state_test: StateTestFiller,
     pre: Alloc,
@@ -414,7 +407,6 @@ def test_worst_keccak(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "address,static_cost,per_word_dynamic_cost,bytes_per_unit_of_work",
     [
@@ -492,7 +484,6 @@ def test_worst_precompile_only_data_input(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     ["mod_exp_input"],
     [
@@ -1299,7 +1290,6 @@ def test_worst_modexp(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "precompile_address,parameters",
     [
@@ -1748,8 +1738,6 @@ def test_worst_precompile_fixed_cost(
     )
 
 
-@pytest.mark.valid_from("Cancun")
-@pytest.mark.slow
 def test_worst_jumps(
     state_test: StateTestFiller,
     pre: Alloc,
@@ -1772,7 +1760,6 @@ def test_worst_jumps(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 def test_worst_jumpi_fallthrough(
     state_test: StateTestFiller,
     pre: Alloc,
@@ -1809,7 +1796,6 @@ def jumpi_seq():
     )
 
 
-@pytest.mark.valid_from("Cancun")
 def test_worst_jumpis(
     state_test: StateTestFiller,
     pre: Alloc,
@@ -1832,7 +1818,6 @@ def test_worst_jumpis(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.slow
 def test_worst_jumpdests(
     state_test: StateTestFiller,
@@ -1868,7 +1853,6 @@ def test_worst_jumpdests(
 )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "opcode,opcode_args",
     [
@@ -2043,7 +2027,6 @@ def test_worst_binop_simple(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("opcode", [Op.ISZERO, Op.NOT])
 def test_worst_unop(
     state_test: StateTestFiller,
@@ -2078,7 +2061,6 @@ def test_worst_unop(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 # `key_mut` indicates the key isn't fixed.
 @pytest.mark.parametrize("key_mut", [True, False])
 # `val_mut` indicates that at the end of each big-loop, the value of the target key changes.
@@ -2134,7 +2116,6 @@ def test_worst_tload(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("key_mut", [True, False])
 @pytest.mark.parametrize("dense_val_mut", [True, False])
 def test_worst_tstore(
@@ -2177,7 +2158,6 @@ def test_worst_tstore(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("shift_right", [Op.SHR, Op.SAR])
 def test_worst_shifts(
     state_test: StateTestFiller,
@@ -2261,7 +2241,6 @@ def select_shift_amount(shift_fn, v):
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "blob_index, blobs_present",
     [
@@ -2325,7 +2304,6 @@ def test_worst_blobhash(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("mod_bits", [255, 191, 127, 63])
 @pytest.mark.parametrize("op", [Op.MOD, Op.SMOD])
 def test_worst_mod(
@@ -2448,7 +2426,6 @@ def test_worst_mod(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("opcode", [Op.MLOAD, Op.MSTORE, Op.MSTORE8])
 @pytest.mark.parametrize("offset", [0, 1, 31])
 @pytest.mark.parametrize("offset_initialized", [True, False])
@@ -2492,7 +2469,6 @@ def test_worst_memory_access(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize("mod_bits", [255, 191, 127, 63])
 @pytest.mark.parametrize("op", [Op.ADDMOD, Op.MULMOD])
 def test_worst_modarith(
@@ -2596,7 +2572,6 @@ def test_worst_modarith(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 def test_empty_block(
     blockchain_test: BlockchainTestFiller,
     pre: Alloc,
@@ -2610,7 +2585,6 @@ def test_empty_block(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 def test_amortized_bn128_pairings(
     state_test: StateTestFiller,
     pre: Alloc,

tests/benchmark/test_worst_memory.py

@@ -30,7 +30,6 @@ class CallDataOrigin:
     CALL = 2
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "origin",
     [
@@ -127,7 +126,6 @@ def test_worst_calldatacopy(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "max_code_size_ratio",
     [
@@ -184,7 +182,6 @@ def test_worst_codecopy(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "size",
     [
@@ -268,7 +265,6 @@ def test_worst_returndatacopy(
     )
 
 
-@pytest.mark.valid_from("Cancun")
 @pytest.mark.parametrize(
     "size",
     [