enhance(test-benchmark): use config file for fixed opcode count scenarios

Author: spencer-tb

packages/testing/src/execution_testing/cli/pytest_commands/plugins/shared/benchmarking.py

@@ -1,5 +1,9 @@
 """The module contains the pytest hooks for the gas benchmark values."""
 
+import importlib.util
+import sys
+from pathlib import Path
+
 import pytest
 
 from execution_testing.test_types import Environment, EnvironmentDefaults
@@ -26,7 +30,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         dest="fixed_opcode_count",
         type=str,
         default=None,
-        help="Specify fixed opcode counts (in thousands) for benchmark tests as a comma-separated list.",
+        nargs="?",
+        const="",
+        help=(
+            "Specify fixed opcode counts (in thousands) for benchmark tests as a comma-separated list. "
+            "If provided without a value, uses defaults from tests/benchmark/configs/fixed_opcode_counts.py."
+        ),
     )
 
 
@@ -41,20 +50,47 @@ def pytest_configure(config: pytest.Config) -> None:
         config.op_mode = OpMode.BENCHMARKING  # type: ignore[attr-defined]
 
 
+def _load_opcode_counts_config(config: pytest.Config) -> dict[str, list[int]] | None:
+    """
+    Load the opcode counts configuration from `tests/benchmark/configs/fixed_opcode_counts.py`.
+
+    Returns dictionary mapping scenario IDs to opcode counts, or None if config not found.
+    """
+    config_path = Path(config.rootpath) / "tests" / "benchmark" / "configs" / "fixed_opcode_counts.py"
+
+    if not config_path.exists():
+        return None
+
+    # Load the module dynamically
+    spec = importlib.util.spec_from_file_location("fixed_opcode_counts", config_path)
+    if spec is None or spec.loader is None:
+        return None
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["fixed_opcode_counts"] = module
+    spec.loader.exec_module(module)
+
+    return {
+        "scenario_counts": getattr(module, "SCENARIO_OPCODE_COUNTS", {}),
+        "default_counts": getattr(module, "DEFAULT_OPCODE_COUNTS", [100, 500, 1000]),
+    }
+
+
 def pytest_collection_modifyitems(
     config: pytest.Config, items: list[pytest.Item]
 ) -> None:
-    """Filter tests based on repricing marker"""
-    gas_benchmark_value = config.getoption("gas_benchmark_value")
+    """Remove non-repricing tests when `--fixed-opcode-count` is specified."""
     fixed_opcode_count = config.getoption("fixed_opcode_count")
 
-    if not gas_benchmark_value and not fixed_opcode_count:
+    # Only filter if --fixed-opcode-count flag was provided (with or without value)
+    if fixed_opcode_count is None:
         return
 
-    # Check if -m repricing marker filter was specified
-    markexpr = config.getoption("markexpr", "")
-    if "repricing" not in markexpr:
-        return
+    # Load config data if flag provided without value (empty string)
+    if fixed_opcode_count == "":
+        config_data = _load_opcode_counts_config(config)
+        if config_data:
+            config._opcode_counts_config = config_data  # type: ignore[attr-defined]
 
     filtered = []
     for item in items:
@@ -85,10 +121,10 @@ def pytest_collection_modifyitems(
 def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
     """Generate tests for the gas benchmark values and fixed opcode counts."""
     gas_benchmark_values = metafunc.config.getoption("gas_benchmark_value")
-    fixed_opcode_counts = metafunc.config.getoption("fixed_opcode_count")
+    fixed_opcode_counts_cli = metafunc.config.getoption("fixed_opcode_count")
 
     # Ensure mutual exclusivity
-    if gas_benchmark_values and fixed_opcode_counts:
+    if gas_benchmark_values and fixed_opcode_counts_cli:
         raise pytest.UsageError(
             "--gas-benchmark-values and --fixed-opcode-count are mutually exclusive. "
             "Use only one at a time."
@@ -111,22 +147,43 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
             )
 
     if "fixed_opcode_count" in metafunc.fixturenames:
-        if fixed_opcode_counts:
-            opcode_counts = [
-                int(x.strip()) for x in fixed_opcode_counts.split(",")
-            ]
-            opcode_count_parameters = [
-                pytest.param(
-                    opcode_count,
-                    id=f"opcount_{opcode_count}K",
+        # Only parametrize if test has repricing marker
+        has_repricing = (
+            metafunc.definition.get_closest_marker("repricing") is not None
+        )
+        if has_repricing:
+            opcode_counts_to_use = None
+
+            if fixed_opcode_counts_cli and fixed_opcode_counts_cli != "":
+                # CLI flag with value takes precedence
+                opcode_counts_to_use = [
+                    int(x.strip()) for x in fixed_opcode_counts_cli.split(",")
+                ]
+            elif fixed_opcode_counts_cli == "":
+                # Flag provided without value - load from config file
+                config_data = getattr(
+                    metafunc.config, "_opcode_counts_config", None
+                )
+                if config_data:
+                    default_counts = config_data.get(
+                        "default_counts", [100]  # Default to 100k for all.
+                    )
+                    opcode_counts_to_use = default_counts
+
+            # Parametrize if we have counts to use
+            if opcode_counts_to_use:
+                opcode_count_parameters = [
+                    pytest.param(
+                        opcode_count,
+                        id=f"opcount_{opcode_count}K",
+                    )
+                    for opcode_count in opcode_counts_to_use
+                ]
+                metafunc.parametrize(
+                    "fixed_opcode_count",
+                    opcode_count_parameters,
+                    scope="function",
                 )
-                for opcode_count in opcode_counts
-            ]
-            metafunc.parametrize(
-                "fixed_opcode_count",
-                opcode_count_parameters,
-                scope="function",
-            )
 
 
 @pytest.fixture(scope="function")
@@ -135,8 +192,9 @@ def gas_benchmark_value(request: pytest.FixtureRequest) -> int:
     if hasattr(request, "param"):
         return request.param
 
-    # If --fixed-opcode-count is specified, use high gas limit to avoid gas constraints
-    if request.config.getoption("fixed_opcode_count"):
+    # Only use high gas limit if --fixed-opcode-count flag was provided
+    fixed_opcode_count = request.config.getoption("fixed_opcode_count")
+    if fixed_opcode_count is not None:
         return HIGH_GAS_LIMIT
 
     return EnvironmentDefaults.gas_limit