Skip to content

Commit 5506d25

Browse files
spencer-tbspencer-tb
committed
enhance(test-benchmark): use config file for fixed opcode count scenarios
1 parent 8f2639b commit 5506d25

File tree

4 files changed

+796
-27
lines changed

4 files changed

+796
-27
lines changed

‎packages/testing/src/execution_testing/cli/pytest_commands/plugins/shared/benchmarking.py‎

Lines changed: 124 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
"""The module contains the pytest hooks for the gas benchmark values."""
22

3+
import re
4+
from pathlib import Path
5+
from typing import Any
6+
37
import pytest
48

59
from execution_testing.test_types import Environment, EnvironmentDefaults
@@ -26,7 +30,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
2630
dest="fixed_opcode_count",
2731
type=str,
2832
default=None,
29-
help="Specify fixed opcode counts (in thousands) for benchmark tests as a comma-separated list.",
33+
nargs="?",
34+
const="",
35+
help=(
36+
"Specify fixed opcode counts (in thousands) for benchmark tests as a comma-separated list. "
37+
"If provided without a value, uses defaults from tests/benchmark/configs/fixed_opcode_counts.py."
38+
),
3039
)
3140

3241

@@ -41,20 +50,75 @@ def pytest_configure(config: pytest.Config) -> None:
4150
config.op_mode = OpMode.BENCHMARKING # type: ignore[attr-defined]
4251

4352

53+
def load_opcode_counts_config(
54+
config: pytest.Config,
55+
) -> dict[str, Any] | None:
56+
"""
57+
Load the opcode counts configuration from `tests/benchmark/configs/`.
58+
59+
Returns dictionary with scenario_configs and default_counts, or None
60+
if not found.
61+
"""
62+
config_file = (
63+
Path(config.rootpath)
64+
/ "tests"
65+
/ "benchmark"
66+
/ "configs"
67+
/ "fixed_opcode_counts.py"
68+
)
69+
70+
if not config_file.exists():
71+
return None
72+
73+
config_globals: dict[str, Any] = {}
74+
exec(config_file.read_text(), config_globals) # noqa: S102
75+
76+
return {
77+
"scenario_configs": config_globals.get("SCENARIO_CONFIGS", {}),
78+
"default_counts": config_globals.get("DEFAULT_OPCODE_COUNTS", [1]),
79+
}
80+
81+
82+
def get_opcode_counts_for_test(
83+
test_name: str,
84+
scenario_configs: dict[str, list[int]],
85+
default_counts: list[int],
86+
) -> list[int]:
87+
"""
88+
Get opcode counts for a test using regex pattern matching.
89+
"""
90+
# Try exact match first (faster)
91+
if test_name in scenario_configs:
92+
return scenario_configs[test_name]
93+
94+
# Try regex patterns
95+
for pattern, counts in scenario_configs.items():
96+
if pattern == test_name:
97+
continue
98+
try:
99+
if re.search(pattern, test_name):
100+
return counts
101+
except re.error:
102+
continue
103+
104+
return default_counts
105+
106+
44107
def pytest_collection_modifyitems(
45108
config: pytest.Config, items: list[pytest.Item]
46109
) -> None:
47-
"""Filter tests based on repricing marker"""
48-
gas_benchmark_value = config.getoption("gas_benchmark_value")
110+
"""Remove non-repricing tests when `--fixed-opcode-count` is specified."""
49111
fixed_opcode_count = config.getoption("fixed_opcode_count")
50112

51-
if not gas_benchmark_value and not fixed_opcode_count:
113+
# Only filter if --fixed-opcode-count flag was provided (with or without value)
114+
if fixed_opcode_count is None:
52115
return
53116

54-
# Check if -m repricing marker filter was specified
55-
markexpr = config.getoption("markexpr", "")
56-
if "repricing" not in markexpr:
57-
return
117+
# Load config data if flag provided without value (empty string)
118+
if fixed_opcode_count == "":
119+
config_data = load_opcode_counts_config(config)
120+
if config_data:
121+
config._opcode_counts_config = config_data # type: ignore[attr-defined]
58122

59123
filtered = []
60124
for item in items:
@@ -85,10 +149,10 @@ def pytest_collection_modifyitems(
85149
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
86150
"""Generate tests for the gas benchmark values and fixed opcode counts."""
87151
gas_benchmark_values = metafunc.config.getoption("gas_benchmark_value")
88-
fixed_opcode_counts = metafunc.config.getoption("fixed_opcode_count")
152+
fixed_opcode_counts_cli = metafunc.config.getoption("fixed_opcode_count")
89153

90154
# Ensure mutual exclusivity
91-
if gas_benchmark_values and fixed_opcode_counts:
155+
if gas_benchmark_values and fixed_opcode_counts_cli:
92156
raise pytest.UsageError(
93157
"--gas-benchmark-values and --fixed-opcode-count are mutually exclusive. "
94158
"Use only one at a time."
@@ -111,22 +175,54 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
111175
)
112176

113177
if "fixed_opcode_count" in metafunc.fixturenames:
114-
if fixed_opcode_counts:
115-
opcode_counts = [
116-
int(x.strip()) for x in fixed_opcode_counts.split(",")
117-
]
118-
opcode_count_parameters = [
119-
pytest.param(
120-
opcode_count,
121-
id=f"opcount_{opcode_count}K",
178+
# Only parametrize if test has repricing marker
179+
has_repricing = (
180+
metafunc.definition.get_closest_marker("repricing") is not None
181+
)
182+
if has_repricing:
183+
opcode_counts_to_use = None
184+
185+
if fixed_opcode_counts_cli and fixed_opcode_counts_cli != "":
186+
# CLI flag with value takes precedence
187+
opcode_counts_to_use = [
188+
int(x.strip()) for x in fixed_opcode_counts_cli.split(",")
189+
]
190+
elif fixed_opcode_counts_cli == "":
191+
# Flag provided without value - load from config file
192+
# Check if config data was already loaded in pytest_collection_modifyitems
193+
config_data = getattr(
194+
metafunc.config, "_opcode_counts_config", None
195+
)
196+
197+
# If not loaded yet (pytest_generate_tests runs first), load it now
198+
if config_data is None:
199+
config_data = load_opcode_counts_config(metafunc.config)
200+
if config_data:
201+
metafunc.config._opcode_counts_config = config_data # type: ignore[attr-defined]
202+
203+
if config_data:
204+
# Look up opcode counts using regex pattern matching
205+
test_name = metafunc.function.__name__
206+
opcode_counts_to_use = get_opcode_counts_for_test(
207+
test_name,
208+
config_data.get("scenario_configs", {}),
209+
config_data.get("default_counts", [1]),
210+
)
211+
212+
# Parametrize if we have counts to use
213+
if opcode_counts_to_use:
214+
opcode_count_parameters = [
215+
pytest.param(
216+
opcode_count,
217+
id=f"opcount_{opcode_count}K",
218+
)
219+
for opcode_count in opcode_counts_to_use
220+
]
221+
metafunc.parametrize(
222+
"fixed_opcode_count",
223+
opcode_count_parameters,
224+
scope="function",
122225
)
123-
for opcode_count in opcode_counts
124-
]
125-
metafunc.parametrize(
126-
"fixed_opcode_count",
127-
opcode_count_parameters,
128-
scope="function",
129-
)
130226

131227

132228
@pytest.fixture(scope="function")
@@ -135,8 +231,9 @@ def gas_benchmark_value(request: pytest.FixtureRequest) -> int:
135231
if hasattr(request, "param"):
136232
return request.param
137233

138-
# If --fixed-opcode-count is specified, use high gas limit to avoid gas constraints
139-
if request.config.getoption("fixed_opcode_count"):
234+
# Only use high gas limit if --fixed-opcode-count flag was provided
235+
fixed_opcode_count = request.config.getoption("fixed_opcode_count")
236+
if fixed_opcode_count is not None:
140237
return HIGH_GAS_LIMIT
141238

142239
return EnvironmentDefaults.gas_limit

‎tests/benchmark/configs/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
"""Fixed opcode count configuration package for benchmark tests."""
2+
3+
from .fixed_opcode_counts import DEFAULT_OPCODE_COUNTS, SCENARIO_CONFIGS
4+
5+
__all__ = [
6+
"DEFAULT_OPCODE_COUNTS",
7+
"SCENARIO_CONFIGS",
8+
]

‎tests/benchmark/configs/fixed_opcode_counts.py‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
"""
2+
Default fixed opcode count configurations for benchmark repricing tests.
3+
4+
This config is only used when `--fixed-opcode-count` values are NOT specified
5+
with the CLI.
6+
7+
Values are in thousands (K). For example, [100, 500, 1000] means
8+
100K, 500K, and 1M opcode executions.
9+
10+
All keys use regex patterns in the format: "test_name.*OPCODE.*"
11+
This matches test IDs like:
12+
test_arithmetic[fork_Prague-blockchain_test-opcode_ADD-]
13+
14+
This file is auto-maintained by `parser.py`. Do not edit manually unless
15+
adding new patterns that the parser cannot detect.
16+
"""
17+
18+
DEFAULT_OPCODE_COUNTS = [1]
19+
20+
# Scenario configurations using test_name.*OPCODE.* patterns
21+
# Keys are regex patterns checked in order; first match wins
22+
SCENARIO_CONFIGS = {
23+
# ACCOUNT QUERY OPERATIONS
24+
"test_selfbalance.*": [1, 10],
25+
"test_codesize.*": [1, 10],
26+
"test_ext_account_query_warm.*BALANCE.*": [1, 10],
27+
"test_ext_account_query_warm.*EXTCODESIZE.*": [1, 10],
28+
"test_ext_account_query_warm.*EXTCODEHASH.*": [1, 10],
29+
"test_ext_account_query_warm.*CALL.*": [1, 10],
30+
"test_ext_account_query_warm.*CALLCODE.*": [1, 10],
31+
"test_ext_account_query_warm.*DELEGATECALL.*": [1, 10],
32+
"test_ext_account_query_warm.*STATICCALL.*": [1, 10],
33+
# ARITHMETIC OPERATIONS
34+
"test_arithmetic.*ADD.*": [1, 10],
35+
"test_arithmetic.*MUL.*": [1, 10],
36+
"test_arithmetic.*SUB.*": [1, 10],
37+
"test_arithmetic.*DIV.*": [1, 10],
38+
"test_arithmetic.*SDIV.*": [1, 10],
39+
"test_arithmetic.*MOD.*": [1, 10],
40+
"test_arithmetic.*SMOD.*": [1, 10],
41+
"test_arithmetic.*EXP.*": [1, 10],
42+
"test_arithmetic.*SIGNEXTEND.*": [1, 10],
43+
"test_mod.*MOD.*": [1, 10],
44+
"test_mod.*SMOD.*": [1, 10],
45+
"test_mod_arithmetic.*ADDMOD.*": [1, 10],
46+
"test_mod_arithmetic.*MULMOD.*": [1, 10],
47+
# BITWISE OPERATIONS
48+
"test_bitwise.*AND.*": [1, 10],
49+
"test_bitwise.*OR.*": [1, 10],
50+
"test_bitwise.*XOR.*": [1, 10],
51+
"test_bitwise.*BYTE.*": [1, 10],
52+
"test_bitwise.*SHL.*": [1, 10],
53+
"test_bitwise.*SHR.*": [1, 10],
54+
"test_bitwise.*SAR.*": [1, 10],
55+
"test_not_op.*": [1, 10],
56+
"test_clz_same.*": [1, 10],
57+
# BLOCK CONTEXT OPERATIONS
58+
"test_block_context_ops.*COINBASE.*": [1, 10],
59+
"test_block_context_ops.*TIMESTAMP.*": [1, 10],
60+
"test_block_context_ops.*NUMBER.*": [1, 10],
61+
"test_block_context_ops.*PREVRANDAO.*": [1, 10],
62+
"test_block_context_ops.*GASLIMIT.*": [1, 10],
63+
"test_block_context_ops.*CHAINID.*": [1, 10],
64+
"test_block_context_ops.*BASEFEE.*": [1, 10],
65+
"test_block_context_ops.*BLOBBASEFEE.*": [1, 10],
66+
"test_blockhash.*": [1, 10],
67+
# CALL CONTEXT OPERATIONS
68+
"test_call_frame_context_ops.*ADDRESS.*": [1, 10],
69+
"test_call_frame_context_ops.*CALLER.*": [1, 10],
70+
"test_call_frame_context_ops.*ORIGIN.*": [1, 10],
71+
"test_call_frame_context_ops.*GASPRICE.*": [1, 10],
72+
"test_calldatasize.*": [1, 10],
73+
"test_callvalue.*": [1, 10],
74+
"test_calldataload.*": [1, 10],
75+
"test_returndatasize_nonzero.*": [1, 10],
76+
"test_returndatasize_zero.*": [1, 10],
77+
"test_returndatacopy.*": [1, 10],
78+
# COMPARISON OPERATIONS
79+
"test_comparison.*LT.*": [1, 10],
80+
"test_comparison.*GT.*": [1, 10],
81+
"test_comparison.*SLT.*": [1, 10],
82+
"test_comparison.*SGT.*": [1, 10],
83+
"test_comparison.*EQ.*": [1, 10],
84+
"test_iszero.*": [1, 10],
85+
# CONTROL FLOW OPERATIONS
86+
"test_gas_op.*": [1, 10],
87+
"test_jumpi_fallthrough.*": [1, 10],
88+
"test_jumpdests.*": [1, 10],
89+
# HASHING OPERATIONS
90+
"test_keccak_max_permutations.*": [1, 10],
91+
# LOGGING OPERATIONS
92+
"test_log.*LOG0.*": [1, 10],
93+
"test_log.*LOG1.*": [1, 10],
94+
"test_log.*LOG2.*": [1, 10],
95+
"test_log.*LOG3.*": [1, 10],
96+
"test_log.*LOG4.*": [1, 10],
97+
# MEMORY OPERATIONS
98+
"test_msize.*": [1, 10],
99+
"test_memory_access.*MLOAD.*": [1, 10],
100+
"test_memory_access.*MSTORE.*": [1, 10],
101+
"test_memory_access.*MSTORE8.*": [1, 10],
102+
"test_mcopy.*": [1, 10],
103+
# STACK OPERATIONS
104+
"test_swap.*SWAP1.*": [1, 10],
105+
"test_swap.*SWAP2.*": [1, 10],
106+
"test_swap.*SWAP3.*": [1, 10],
107+
"test_swap.*SWAP4.*": [1, 10],
108+
"test_swap.*SWAP5.*": [1, 10],
109+
"test_swap.*SWAP6.*": [1, 10],
110+
"test_swap.*SWAP7.*": [1, 10],
111+
"test_swap.*SWAP8.*": [1, 10],
112+
"test_swap.*SWAP9.*": [1, 10],
113+
"test_swap.*SWAP10.*": [1, 10],
114+
"test_swap.*SWAP11.*": [1, 10],
115+
"test_swap.*SWAP12.*": [1, 10],
116+
"test_swap.*SWAP13.*": [1, 10],
117+
"test_swap.*SWAP14.*": [1, 10],
118+
"test_swap.*SWAP15.*": [1, 10],
119+
"test_swap.*SWAP16.*": [1, 10],
120+
"test_dup.*DUP1.*": [1, 10],
121+
"test_dup.*DUP2.*": [1, 10],
122+
"test_dup.*DUP3.*": [1, 10],
123+
"test_dup.*DUP4.*": [1, 10],
124+
"test_dup.*DUP5.*": [1, 10],
125+
"test_dup.*DUP6.*": [1, 10],
126+
"test_dup.*DUP7.*": [1, 10],
127+
"test_dup.*DUP8.*": [1, 10],
128+
"test_dup.*DUP9.*": [1, 10],
129+
"test_dup.*DUP10.*": [1, 10],
130+
"test_dup.*DUP11.*": [1, 10],
131+
"test_dup.*DUP12.*": [1, 10],
132+
"test_dup.*DUP13.*": [1, 10],
133+
"test_dup.*DUP14.*": [1, 10],
134+
"test_dup.*DUP15.*": [1, 10],
135+
"test_dup.*DUP16.*": [1, 10],
136+
"test_push.*PUSH0.*": [1, 10],
137+
"test_push.*PUSH1.*": [1, 10],
138+
"test_push.*PUSH2.*": [1, 10],
139+
"test_push.*PUSH3.*": [1, 10],
140+
"test_push.*PUSH4.*": [1, 10],
141+
"test_push.*PUSH5.*": [1, 10],
142+
"test_push.*PUSH6.*": [1, 10],
143+
"test_push.*PUSH7.*": [1, 10],
144+
"test_push.*PUSH8.*": [1, 10],
145+
"test_push.*PUSH9.*": [1, 10],
146+
"test_push.*PUSH10.*": [1, 10],
147+
"test_push.*PUSH11.*": [1, 10],
148+
"test_push.*PUSH12.*": [1, 10],
149+
"test_push.*PUSH13.*": [1, 10],
150+
"test_push.*PUSH14.*": [1, 10],
151+
"test_push.*PUSH15.*": [1, 10],
152+
"test_push.*PUSH16.*": [1, 10],
153+
"test_push.*PUSH17.*": [1, 10],
154+
"test_push.*PUSH18.*": [1, 10],
155+
"test_push.*PUSH19.*": [1, 10],
156+
"test_push.*PUSH20.*": [1, 10],
157+
"test_push.*PUSH21.*": [1, 10],
158+
"test_push.*PUSH22.*": [1, 10],
159+
"test_push.*PUSH23.*": [1, 10],
160+
"test_push.*PUSH24.*": [1, 10],
161+
"test_push.*PUSH25.*": [1, 10],
162+
"test_push.*PUSH26.*": [1, 10],
163+
"test_push.*PUSH27.*": [1, 10],
164+
"test_push.*PUSH28.*": [1, 10],
165+
"test_push.*PUSH29.*": [1, 10],
166+
"test_push.*PUSH30.*": [1, 10],
167+
"test_push.*PUSH31.*": [1, 10],
168+
"test_push.*PUSH32.*": [1, 10],
169+
# STORAGE OPERATIONS
170+
"test_tload.*": [1, 10],
171+
"test_tstore.*": [1, 10],
172+
# SYSTEM OPERATIONS
173+
"test_return_revert.*RETURN.*": [1, 10],
174+
"test_return_revert.*REVERT.*": [1, 10],
175+
# TRANSACTION CONTEXT OPERATIONS
176+
"test_blobhash.*": [1, 10],
177+
}

0 commit comments

Comments
 (0)