chore(benchmark): don't fill benchmark tests by default (#1920)

* chore(benchmark): don't fill benchmark tests by default.

* chore(benchmark): change to collection level only.

* chore: optimizations via review.

* chore(tests): pr comments.

Author: spencer-tb

tests/benchmark/conftest.py

@@ -7,6 +7,28 @@
 
 def pytest_collection_modifyitems(config, items):
     """Add the `benchmark` marker to all tests under `./tests/benchmark`."""
-    for item in items:
-        if Path(__file__).parent in Path(item.fspath).parents:
-            item.add_marker(pytest.mark.benchmark)
+    marker_expr = config.getoption("-m", default="")
+    gas_benchmark_values = config.getoption("--gas-benchmark-values", default=None)
+    run_benchmarks = marker_expr and (
+        "benchmark" in marker_expr and "not benchmark" not in marker_expr
+    )
+    if gas_benchmark_values:
+        run_benchmarks = True
+    items_for_removal = []
+    for i, item in enumerate(items):
+        is_in_benchmark_dir = Path(__file__).parent in Path(item.fspath).parents
+        has_benchmark_marker = item.get_closest_marker("benchmark") is not None
+        is_benchmark_test = is_in_benchmark_dir or has_benchmark_marker
+        if is_benchmark_test:
+            if is_in_benchmark_dir and not has_benchmark_marker:
+                benchmark_marker = pytest.mark.benchmark
+
+                item.add_marker(benchmark_marker)
+            if not run_benchmarks:
+                items_for_removal.append(i)
+
+        elif run_benchmarks:
+            items_for_removal.append(i)
+
+    for i in reversed(items_for_removal):
+        items.pop(i)

tox.ini

@@ -83,7 +83,7 @@ description = Fill test cases in ./tests/ for deployed mainnet forks, except for
 setenv =
     # Use custom EELS_RESOLUTIONS_FILE if it is set via the environment (eg, in CI)
     EELS_RESOLUTIONS_FILE = {env:EELS_RESOLUTIONS_FILE:}
-commands = fill -n auto -m "not slow and not benchmark" --output=/tmp/fixtures-tox --clean
+commands = fill -n auto -m "not slow" --output=/tmp/fixtures-tox --clean
 
 [testenv:tests-deployed-benchmark]
 description = Fill benchmarking test cases in ./tests/ for deployed mainnet forks, using evmone-t8n.
@@ -94,7 +94,7 @@ description = Fill test cases in ./tests/ for deployed and development mainnet f
 setenv =
     # Use custom EELS_RESOLUTIONS_FILE if it is set via the environment (eg, in CI)
     EELS_RESOLUTIONS_FILE = {env:EELS_RESOLUTIONS_FILE:}
-commands = fill -n auto --until={[forks]develop} -k "not slow and not benchmark" --output=/tmp/fixtures-tox --clean
+commands = fill -n auto --until={[forks]develop} -k "not slow" --output=/tmp/fixtures-tox --clean
 
 # ----------------------------------------------------------------------------------------------
 # ALIAS ENVIRONMENTS