diff --git a/docs/writing_tests/benchmarks.md b/docs/writing_tests/benchmarks.md index 0884c61c28f..63279f5c4b5 100644 --- a/docs/writing_tests/benchmarks.md +++ b/docs/writing_tests/benchmarks.md @@ -4,6 +4,8 @@ Benchmark tests aim to maximize the usage of a specific opcode, precompile, or o To fill a benchmark test, in addition to the usual test flags, you must include the `-m benchmark` flag. This is necessary because benchmark tests are ignored by default; they must be manually selected via the `benchmark` pytest marker (="tag"). This marker is applied to all tests under `./tests/benchmark/` automatically by the framework. +**Note:** Benchmark tests are now only available starting from the `Prague` fork. Tests targeting earlier forks (`Cancun` or prior) are not supported in benchmark mode. + ## Setting the Gas Limit for Benchmarking To consume the full benchmark gas limit, use the `gas_benchmark_value` fixture as the gas limit: diff --git a/tests/benchmark/conftest.py b/tests/benchmark/conftest.py index 30de75b6dbb..3af1bf9ade7 100644 --- a/tests/benchmark/conftest.py +++ b/tests/benchmark/conftest.py @@ -4,6 +4,26 @@ import pytest +DEFAULT_BENCHMARK_FORK = "Prague" + + +def pytest_generate_tests(metafunc): + """Modify test generation to enforce default benchmark fork for benchmark tests.""" + benchmark_dir = Path(__file__).parent + test_file_path = Path(metafunc.definition.fspath) + + # Check if this test is in the benchmark directory + is_in_benchmark_dir = benchmark_dir in test_file_path.parents + + if is_in_benchmark_dir: + # Add benchmark marker if no valid_from marker exists + existing_markers = list(metafunc.definition.iter_markers()) + has_valid_from = any(marker.name == "valid_from" for marker in existing_markers) + + if not has_valid_from: + benchmark_marker = pytest.mark.valid_from(DEFAULT_BENCHMARK_FORK) + metafunc.definition.add_marker(benchmark_marker) + def pytest_collection_modifyitems(config, items): """Add the `benchmark` marker to all tests under `./tests/benchmark`.""" diff --git a/tests/benchmark/test_worst_blocks.py b/tests/benchmark/test_worst_blocks.py index 327cbae00c9..38e6d5f71e6 100644 --- a/tests/benchmark/test_worst_blocks.py +++ b/tests/benchmark/test_worst_blocks.py @@ -105,7 +105,6 @@ def ether_transfer_case( return senders, receivers -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "case_id", ["a_to_a", "a_to_b", "diff_acc_to_b", "a_to_diff_acc", "diff_acc_to_diff_acc"], @@ -174,7 +173,6 @@ def total_cost_standard_per_token(): return 4 -@pytest.mark.valid_from("Prague") @pytest.mark.parametrize("zero_byte", [True, False]) def test_block_full_data( state_test: StateTestFiller, @@ -231,7 +229,6 @@ def test_block_full_data( ) -@pytest.mark.valid_from("Prague") def test_block_full_access_list_and_data( state_test: StateTestFiller, pre: Alloc, diff --git a/tests/benchmark/test_worst_bytecode.py b/tests/benchmark/test_worst_bytecode.py index a09595dcdcf..67b1eca89a0 100644 --- a/tests/benchmark/test_worst_bytecode.py +++ b/tests/benchmark/test_worst_bytecode.py @@ -47,7 +47,6 @@ Op.EXTCODECOPY, ], ) -@pytest.mark.valid_from("Cancun") def test_worst_bytecode_single_opcode( blockchain_test: BlockchainTestFiller, pre: Alloc, @@ -233,7 +232,6 @@ def test_worst_bytecode_single_opcode( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "pattern", [ @@ -324,7 +322,6 @@ def test_worst_initcode_jumpdest_analysis( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "opcode", [ @@ -432,7 +429,6 @@ def test_worst_create( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "opcode", [ diff --git a/tests/benchmark/test_worst_compute.py b/tests/benchmark/test_worst_compute.py index 74361785ba8..4f42b94db0c 100644 --- a/tests/benchmark/test_worst_compute.py +++ b/tests/benchmark/test_worst_compute.py @@ -62,7 +62,6 @@ def make_dup(index: int) -> Opcode: return Opcode(0x80 + index, pushed_stack_items=1, min_stack_height=index + 1) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "opcode", [ @@ -112,7 +111,6 @@ def test_worst_zero_param( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("calldata_length", [0, 1_000, 10_000]) def test_worst_calldatasize( state_test: StateTestFiller, @@ -145,7 +143,6 @@ def test_worst_calldatasize( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("non_zero_value", [True, False]) @pytest.mark.parametrize("from_origin", [True, False]) def test_worst_callvalue( @@ -205,7 +202,6 @@ class ReturnDataStyle(Enum): IDENTITY = auto() -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "return_data_style", [ @@ -267,7 +263,6 @@ def test_worst_returndatasize_nonzero( ) -@pytest.mark.valid_from("Cancun") def test_worst_returndatasize_zero( state_test: StateTestFiller, pre: Alloc, @@ -299,7 +294,6 @@ def test_worst_returndatasize_zero( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("mem_size", [0, 1, 1_000, 100_000, 1_000_000]) def test_worst_msize( state_test: StateTestFiller, @@ -339,7 +333,6 @@ def test_worst_msize( ) -@pytest.mark.valid_from("Cancun") def test_worst_keccak( state_test: StateTestFiller, pre: Alloc, @@ -414,7 +407,6 @@ def test_worst_keccak( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "address,static_cost,per_word_dynamic_cost,bytes_per_unit_of_work", [ @@ -492,7 +484,6 @@ def test_worst_precompile_only_data_input( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( ["mod_exp_input"], [ @@ -1299,7 +1290,6 @@ def test_worst_modexp( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "precompile_address,parameters", [ @@ -1747,8 +1737,6 @@ def test_worst_precompile_fixed_cost( ) -@pytest.mark.valid_from("Cancun") -@pytest.mark.slow def test_worst_jumps( state_test: StateTestFiller, pre: Alloc, @@ -1771,7 +1759,6 @@ def test_worst_jumps( ) -@pytest.mark.valid_from("Cancun") def test_worst_jumpi_fallthrough( state_test: StateTestFiller, pre: Alloc, @@ -1808,7 +1795,6 @@ def jumpi_seq(): ) -@pytest.mark.valid_from("Cancun") def test_worst_jumpis( state_test: StateTestFiller, pre: Alloc, @@ -1831,7 +1817,6 @@ def test_worst_jumpis( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.slow def test_worst_jumpdests( state_test: StateTestFiller, @@ -1867,7 +1852,6 @@ def test_worst_jumpdests( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "opcode,opcode_args", [ @@ -2042,7 +2026,6 @@ def test_worst_binop_simple( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("opcode", [Op.ISZERO, Op.NOT]) def test_worst_unop( state_test: StateTestFiller, @@ -2077,7 +2060,6 @@ def test_worst_unop( ) -@pytest.mark.valid_from("Cancun") # `key_mut` indicates the key isn't fixed. @pytest.mark.parametrize("key_mut", [True, False]) # `val_mut` indicates that at the end of each big-loop, the value of the target key changes. @@ -2133,7 +2115,6 @@ def test_worst_tload( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("key_mut", [True, False]) @pytest.mark.parametrize("dense_val_mut", [True, False]) def test_worst_tstore( @@ -2176,7 +2157,6 @@ def test_worst_tstore( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("shift_right", [Op.SHR, Op.SAR]) def test_worst_shifts( state_test: StateTestFiller, @@ -2260,7 +2240,6 @@ def select_shift_amount(shift_fn, v): ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "blob_index, blobs_present", [ @@ -2324,7 +2303,6 @@ def test_worst_blobhash( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("mod_bits", [255, 191, 127, 63]) @pytest.mark.parametrize("op", [Op.MOD, Op.SMOD]) def test_worst_mod( @@ -2447,7 +2425,6 @@ def test_worst_mod( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("opcode", [Op.MLOAD, Op.MSTORE, Op.MSTORE8]) @pytest.mark.parametrize("offset", [0, 1, 31]) @pytest.mark.parametrize("offset_initialized", [True, False]) @@ -2491,7 +2468,6 @@ def test_worst_memory_access( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("mod_bits", [255, 191, 127, 63]) @pytest.mark.parametrize("op", [Op.ADDMOD, Op.MULMOD]) def test_worst_modarith( @@ -2595,7 +2571,6 @@ def test_worst_modarith( ) -@pytest.mark.valid_from("Cancun") def test_empty_block( blockchain_test: BlockchainTestFiller, pre: Alloc, @@ -2609,7 +2584,6 @@ def test_empty_block( ) -@pytest.mark.valid_from("Cancun") def test_amortized_bn128_pairings( state_test: StateTestFiller, pre: Alloc, diff --git a/tests/benchmark/test_worst_memory.py b/tests/benchmark/test_worst_memory.py index b6d751fdaa3..4a1797097a3 100644 --- a/tests/benchmark/test_worst_memory.py +++ b/tests/benchmark/test_worst_memory.py @@ -30,7 +30,6 @@ class CallDataOrigin: CALL = 2 -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "origin", [ @@ -127,7 +126,6 @@ def test_worst_calldatacopy( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "max_code_size_ratio", [ @@ -184,7 +182,6 @@ def test_worst_codecopy( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "size", [ @@ -268,7 +265,6 @@ def test_worst_returndatacopy( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "size", [ diff --git a/tests/benchmark/test_worst_stateful_opcodes.py b/tests/benchmark/test_worst_stateful_opcodes.py index 5bc5d89aab7..8146175ae7e 100644 --- a/tests/benchmark/test_worst_stateful_opcodes.py +++ b/tests/benchmark/test_worst_stateful_opcodes.py @@ -33,7 +33,6 @@ REFERENCE_SPEC_VERSION = "TODO" -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "opcode", [ @@ -116,7 +115,6 @@ def test_worst_address_state_cold( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "opcode", [ @@ -198,7 +196,6 @@ class TransactionResult: REVERT = 3 -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "storage_action,tx_result", [ @@ -381,7 +378,6 @@ def test_worst_storage_access_cold( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "storage_action", [ @@ -454,7 +450,6 @@ def test_worst_storage_access_warm( ) -@pytest.mark.valid_from("Cancun") def test_worst_blockhash( blockchain_test: BlockchainTestFiller, pre: Alloc, @@ -483,7 +478,6 @@ def test_worst_blockhash( ) -@pytest.mark.valid_from("Cancun") def test_worst_selfbalance( state_test: StateTestFiller, pre: Alloc, @@ -517,7 +511,6 @@ def test_worst_selfbalance( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize( "copied_size", [ @@ -558,7 +551,6 @@ def test_worst_extcodecopy_warm( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("value_bearing", [True, False]) def test_worst_selfdestruct_existing( blockchain_test: BlockchainTestFiller, @@ -698,7 +690,6 @@ def test_worst_selfdestruct_existing( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("value_bearing", [True, False]) def test_worst_selfdestruct_created( state_test: StateTestFiller, @@ -798,7 +789,6 @@ def test_worst_selfdestruct_created( ) -@pytest.mark.valid_from("Cancun") @pytest.mark.parametrize("value_bearing", [True, False]) def test_worst_selfdestruct_initcode( state_test: StateTestFiller,