enhance(test-benchmark): use config file for fixed opcode count scenarios

Author: spencer-tb

packages/testing/src/execution_testing/cli/pytest_commands/plugins/shared/benchmarking.py

@@ -1,5 +1,9 @@
 """The module contains the pytest hooks for the gas benchmark values."""
 
+import importlib.util
+import sys
+from pathlib import Path
+
 import pytest
 
 from execution_testing.test_types import Environment, EnvironmentDefaults
@@ -26,7 +30,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         dest="fixed_opcode_count",
         type=str,
         default=None,
-        help="Specify fixed opcode counts (in thousands) for benchmark tests as a comma-separated list.",
+        nargs="?",
+        const="",
+        help=(
+            "Specify fixed opcode counts (in thousands) for benchmark tests as a comma-separated list. "
+            "If provided without a value, uses defaults from tests/benchmark/configs/fixed_opcode_counts.py."
+        ),
     )
 
 
@@ -41,15 +50,48 @@ def pytest_configure(config: pytest.Config) -> None:
         config.op_mode = OpMode.BENCHMARKING  # type: ignore[attr-defined]
 
 
+def _load_opcode_counts_config(config: pytest.Config) -> dict[str, list[int]] | None:
+    """
+    Load the opcode counts configuration from `tests/benchmark/configs/fixed_opcode_counts.py`.
+
+    Returns dictionary mapping scenario IDs to opcode counts, or None if config not found.
+    """
+    config_path = Path(config.rootpath) / "tests" / "benchmark" / "configs" / "fixed_opcode_counts.py"
+
+    if not config_path.exists():
+        return None
+
+    # Load the module dynamically
+    spec = importlib.util.spec_from_file_location("fixed_opcode_counts", config_path)
+    if spec is None or spec.loader is None:
+        return None
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["fixed_opcode_counts"] = module
+    spec.loader.exec_module(module)
+
+    return {
+        "scenario_counts": getattr(module, "SCENARIO_OPCODE_COUNTS", {}),
+        "default_counts": getattr(module, "DEFAULT_OPCODE_COUNTS", [100, 500, 1000]),
+    }
+
+
 def pytest_collection_modifyitems(
     config: pytest.Config, items: list[pytest.Item]
 ) -> None:
-    """Remove non-repricing tests when --fixed-opcode-count is specified."""
+    """Remove non-repricing tests when `--fixed-opcode-count` is specified."""
     fixed_opcode_count = config.getoption("fixed_opcode_count")
-    if not fixed_opcode_count:
-        # If --fixed-opcode-count is not specified, don't filter anything
+
+    # Only filter if --fixed-opcode-count flag was provided (with or without value)
+    if fixed_opcode_count is None:
         return
 
+    # Load config data if flag provided without value (empty string)
+    if fixed_opcode_count == "":
+        config_data = _load_opcode_counts_config(config)
+        if config_data:
+            config._opcode_counts_config = config_data  # type: ignore[attr-defined]
+
     filtered = []
     for item in items:
         if not item.get_closest_marker("benchmark"):
@@ -79,10 +121,10 @@ def pytest_collection_modifyitems(
 def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
     """Generate tests for the gas benchmark values and fixed opcode counts."""
     gas_benchmark_values = metafunc.config.getoption("gas_benchmark_value")
-    fixed_opcode_counts = metafunc.config.getoption("fixed_opcode_count")
+    fixed_opcode_counts_cli = metafunc.config.getoption("fixed_opcode_count")
 
     # Ensure mutual exclusivity
-    if gas_benchmark_values and fixed_opcode_counts:
+    if gas_benchmark_values and fixed_opcode_counts_cli:
         raise pytest.UsageError(
             "--gas-benchmark-values and --fixed-opcode-count are mutually exclusive. "
             "Use only one at a time."
@@ -110,16 +152,32 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
             metafunc.definition.get_closest_marker("repricing") is not None
         )
         if has_repricing:
-            if fixed_opcode_counts:
-                opcode_counts = [
-                    int(x.strip()) for x in fixed_opcode_counts.split(",")
+            opcode_counts_to_use = None
+
+            if fixed_opcode_counts_cli and fixed_opcode_counts_cli != "":
+                # CLI flag with value takes precedence
+                opcode_counts_to_use = [
+                    int(x.strip()) for x in fixed_opcode_counts_cli.split(",")
                 ]
+            elif fixed_opcode_counts_cli == "":
+                # Flag provided without value - load from config file
+                config_data = getattr(
+                    metafunc.config, "_opcode_counts_config", None
+                )
+                if config_data:
+                    default_counts = config_data.get(
+                        "default_counts", [100]  # Default to 100k for all.
+                    )
+                    opcode_counts_to_use = default_counts
+
+            # Parametrize if we have counts to use
+            if opcode_counts_to_use:
                 opcode_count_parameters = [
                     pytest.param(
                         opcode_count,
                         id=f"opcount_{opcode_count}K",
                     )
-                    for opcode_count in opcode_counts
+                    for opcode_count in opcode_counts_to_use
                 ]
                 metafunc.parametrize(
                     "fixed_opcode_count",
@@ -134,8 +192,9 @@ def gas_benchmark_value(request: pytest.FixtureRequest) -> int:
     if hasattr(request, "param"):
         return request.param
 
-    # If --fixed-opcode-count is specified, use high gas limit to avoid gas constraints
-    if request.config.getoption("fixed_opcode_count"):
+    # Only use high gas limit if --fixed-opcode-count flag was provided
+    fixed_opcode_count = request.config.getoption("fixed_opcode_count")
+    if fixed_opcode_count is not None:
         return HIGH_GAS_LIMIT
 
     return EnvironmentDefaults.gas_limit