feat(benchmarks): add bloatnet marker support (#2169)

Co-authored-by: Mario Vega <[email protected]>

Author: LouisTsai-Csie

docs/templates/base.md.j2

@@ -11,6 +11,8 @@ Documentation for [`{{ pytest_node_id }}@{{ short_git_ref }}`]({{ source_code_ur
     ```console
 {% if is_benchmark %}
     fill -v {{ pytest_node_id }} -m benchmark
+{% elif is_stateful %}
+    fill -v {{ pytest_node_id }} -m stateful
 {% else %}
     fill -v {{ pytest_node_id }} --fork {{ target_or_valid_fork }}
 {% endif %}

src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py

@@ -431,6 +431,7 @@ def create_function_page_props(self, test_functions: Dict["str", List[Item]]) ->
             )
 
             is_benchmark = items[0].get_closest_marker("benchmark") is not None
+            is_stateful = items[0].get_closest_marker("stateful") is not None
 
             self.function_page_props[function_id] = FunctionPageProps(
                 title=get_test_function_name(items[0]),
@@ -447,6 +448,7 @@ def create_function_page_props(self, test_functions: Dict["str", List[Item]]) ->
                 html_static_page_target=f"./{get_test_function_name(items[0])}.html",
                 mkdocs_function_page_target=f"./{get_test_function_name(items[0])}/",
                 is_benchmark=is_benchmark,
+                is_stateful=is_stateful,
             )
 
     def create_module_page_props(self) -> None:
@@ -462,6 +464,7 @@ def create_module_page_props(self) -> None:
                     pytest_node_id=str(module_path),
                     package_name=get_import_path(module_path),
                     is_benchmark=function_page.is_benchmark,
+                    is_stateful=function_page.is_stateful,
                     test_functions=[
                         TestFunction(
                             name=function_page.title,
@@ -475,6 +478,8 @@ def create_module_page_props(self) -> None:
                 existing_module_page = self.module_page_props[str(function_page.path)]
                 if function_page.is_benchmark:
                     existing_module_page.is_benchmark = True
+                if function_page.is_stateful:
+                    existing_module_page.is_stateful = True
                 existing_module_page.test_functions.append(
                     TestFunction(
                         name=function_page.title,
@@ -511,7 +516,12 @@ def add_directory_page_props(self) -> None:
             is_benchmark = any(
                 module_page.is_benchmark
                 for module_page in self.module_page_props.values()
-                if module_page.path.parent == directory
+                if directory in module_page.path.parents or module_page.path.parent == directory
+            )
+            is_stateful = any(
+                module_page.is_stateful
+                for module_page in self.module_page_props.values()
+                if directory in module_page.path.parents or module_page.path.parent == directory
             )
 
             self.page_props[str(directory)] = DirectoryPageProps(
@@ -526,6 +536,7 @@ def add_directory_page_props(self) -> None:
                 # init.py will be used for docstrings
                 package_name=get_import_path(directory),
                 is_benchmark=is_benchmark,
+                is_stateful=is_stateful,
             )
 
     def find_files_within_collection_scope(self, file_pattern: str) -> List[Path]:

src/pytest_plugins/filler/gen_test_doc/page_props.py

@@ -111,6 +111,7 @@ class PagePropsBase:
     pytest_node_id: str
     package_name: str
     is_benchmark: bool = False
+    is_stateful: bool = False
 
     @property
     @abstractmethod

src/pytest_plugins/shared/execute_fill.py

@@ -100,6 +100,10 @@ def pytest_configure(config: pytest.Config):
         "markers",
         "benchmark: Tests relevant to benchmarking EVMs.",
     )
+    config.addinivalue_line(
+        "markers",
+        "stateful: Tests for stateful benchmarking scenarios.",
+    )
     config.addinivalue_line(
         "markers",
         "exception_test: Negative tests that include an invalid block or transaction.",

tests/benchmark/bloatnet/__init__.py

@@ -38,21 +38,29 @@ def pytest_collection_modifyitems(config, items):
 
     if gen_docs:
         for item in items:
-            if benchmark_dir in Path(item.fspath).parents and not item.get_closest_marker(
-                "benchmark"
+            if (
+                benchmark_dir in Path(item.fspath).parents
+                and not item.get_closest_marker("benchmark")
+                and not item.get_closest_marker("stateful")
             ):
                 item.add_marker(benchmark_marker)
         return
 
     marker_expr = config.getoption("-m", default="")
     run_benchmarks = (
         marker_expr and "benchmark" in marker_expr and "not benchmark" not in marker_expr
-    ) or config.getoption("--gas-benchmark-values", default=None)
+    )
+    run_stateful_tests = (
+        marker_expr and "stateful" in marker_expr and "not stateful" not in marker_expr
+    )
 
     items_for_removal = []
     for i, item in enumerate(items):
         is_in_benchmark_dir = benchmark_dir in Path(item.fspath).parents
-        is_benchmark_test = is_in_benchmark_dir or item.get_closest_marker("benchmark")
+        has_stateful_marker = item.get_closest_marker("stateful")
+        is_benchmark_test = (
+            is_in_benchmark_dir and not has_stateful_marker
+        ) or item.get_closest_marker("benchmark")
 
         if is_benchmark_test:
             if is_in_benchmark_dir and not item.get_closest_marker("benchmark"):
@@ -61,6 +69,8 @@ def pytest_collection_modifyitems(config, items):
                 items_for_removal.append(i)
         elif run_benchmarks:
             items_for_removal.append(i)
+        elif is_in_benchmark_dir and has_stateful_marker and not run_stateful_tests:
+            items_for_removal.append(i)
 
     for i in reversed(items_for_removal):
         items.pop(i)

tests/benchmark/conftest.py

@@ -0,0 +1 @@
+"""Benchmark state tests package."""

tests/benchmark/stateful/__init__.py

@@ -0,0 +1 @@
+"""Bloatnet benchmark tests package."""

tests/benchmark/stateful/bloatnet/__init__.py

@@ -0,0 +1,6 @@
+"""
+abstract: Tests benchmark worst-case bloatnet scenarios.
+    Tests benchmark worst-case bloatnet scenarios.
+
+Tests running worst-case bloatnet scenarios for benchmarking purposes.
+"""