feat(plugins): Static test filler (#1336)

* feat(specs): Introduce static spec types

* feat(plugins): Add Refiller plugin

* fix: tox

* feat(specs): Implement fork range descriptor parser

* fix(refiller): Handle `ids` and `marks`

* fix(refiller): Handle duplicate test ids

* fix(specs): Add base static helpers

* feat(plugins/forks): Add `valid_at` marker

* fix(plugins/refiller): warn instead of error for parsing failures

* fix(plugins/refiller): Check filler file name before loading

* fix(fixtures): Fix TestInfo to remove `Filler` suffix

* fix(refiller): Remove try-except to print parse errors

* fix(fixtures/collector): Remove unused function

* fix: Tox

* refactor(refiller): Rename as static-filler

* refactor(forks): Move ForkRangeDescriptor, get_fork_by_name

* docs: Add changelog

Author: marioevz

docs/CHANGELOG.md

@@ -40,6 +40,7 @@ consume cache --help
 
 - 🐞 Fix `--fork/from/until` for transition forks when using `fill` [#1311](https://github.com/ethereum/execution-spec-tests/pull/1311).
 - 🐞 Fix the node id for state tests marked by transition forks ([#1313](https://github.com/ethereum/execution-spec-tests/pull/1313)).
+- ✨ Add `static_filler` plug-in which allows to fill static YAML and JSON tests (from [ethereum/tests](https://github.com/ethereum/tests)) by adding flag `--fill-static-tests` to `uv run fill` ([#1336](https://github.com/ethereum/execution-spec-tests/pull/1336)).
 
 ### 📋 Misc
 

docs/filling_tests/index.md

@@ -28,3 +28,11 @@ The "fill-consume" method follows a differential testing approach: A reference i
     Some tests cases, particularly those without straightforward post-checks, such as certain gas calculations, may allow subtle inconsistencies to slip through during filling.
 
     **Consequently, filling the tests does not ensure the client’s correctness. Clients must consume the tests to be considered correctly tested, even if that client was used to fill the tests.**
+
+## Filling Static Tests from [ethereum/tests](https://github.com/ethereum/tests)
+
+Filling static test fillers in YAML or JSON formats from [ethereum/tests](https://github.com/ethereum/tests/tree/develop/src) is possible by adding the `--fill-static-tests` to the `fill` command.
+
+This functionality is only available for backwards compatibility and copying legacy tests from the [ethereum/tests](https://github.com/ethereum/tests) repository into this one.
+
+Adding new static test fillers is otherwise not allowed.

docs/writing_tests/test_markers.md

@@ -16,6 +16,10 @@ These markers are used to specify the forks for which a test is valid.
 
 :::pytest_plugins.forks.forks.ValidUntil
 
+### `@pytest.mark.valid_at("FORK_NAME_1", "FORK_NAME_2", ...)`
+
+:::pytest_plugins.forks.forks.ValidAt
+
 ### `@pytest.mark.valid_at_transition_to("FORK_NAME")`
 
 :::pytest_plugins.forks.forks.ValidAtTransitionTo

pytest.ini

@@ -11,6 +11,7 @@ addopts =
     -p pytest_plugins.filler.pre_alloc
     -p pytest_plugins.solc.solc
     -p pytest_plugins.filler.filler
+    -p pytest_plugins.filler.static_filler
     -p pytest_plugins.shared.execute_fill
     -p pytest_plugins.forks.forks
     -p pytest_plugins.spec_version_checker.spec_version_checker

src/ethereum_test_fixtures/collector.py

@@ -9,7 +9,7 @@
 import sys
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, Literal, Optional, Tuple
+from typing import ClassVar, Dict, Literal, Optional, Tuple
 
 from ethereum_test_base_types import to_json
 
@@ -18,40 +18,26 @@
 from .file import Fixtures
 
 
-def strip_test_prefix(name: str) -> str:
-    """Remove test prefix from a test case name."""
-    test_prefix = "test_"
-    if name.startswith(test_prefix):
-        return name[len(test_prefix) :]
-    return name
-
-
-def get_module_relative_output_dir(test_module: Path, filler_path: Path) -> Path:
-    """
-    Return a directory name for the provided test_module (relative to the
-    base ./tests directory) that can be used for output (within the
-    configured fixtures output path or the base_dump_dir directory).
-
-    Example:
-    tests/shanghai/eip3855_push0/test_push0.py -> shanghai/eip3855_push0/test_push0
-
-    """
-    basename = test_module.with_suffix("").absolute()
-    basename_relative = basename.relative_to(
-        os.path.commonpath([filler_path.absolute(), basename])
-    )
-    module_path = basename_relative.parent / basename_relative.stem
-    return module_path
-
-
 @dataclass(kw_only=True)
 class TestInfo:
     """Contains test information from the current node."""
 
-    name: str  # pytest: Item.name
-    id: str  # pytest: Item.nodeid
-    original_name: str  # pytest: Item.originalname
-    path: Path  # pytest: Item.path
+    name: str  # pytest: Item.name, e.g. test_paris_one[fork_Paris-state_test]
+    id: str  # pytest: Item.nodeid, e.g. tests/paris/test_module_paris.py::test_paris_one[...]
+    original_name: str  # pytest: Item.originalname, e.g. test_paris_one
+    module_path: Path  # pytest: Item.path, e.g. .../tests/paris/test_module_paris.py
+
+    test_prefix: ClassVar[str] = "test_"  # Python test prefix
+    filler_suffix: ClassVar[str] = "Filler"  # Static test suffix
+
+    @classmethod
+    def strip_test_name(cls, name: str) -> str:
+        """Remove test prefix from a python test case name."""
+        if name.startswith(cls.test_prefix):
+            return name[len(cls.test_prefix) :]
+        if name.endswith(cls.filler_suffix):
+            return name[: -len(cls.filler_suffix)]
+        return name
 
     def get_name_and_parameters(self) -> Tuple[str, str]:
         """
@@ -64,10 +50,16 @@ def get_name_and_parameters(self) -> Tuple[str, str]:
         test_name, parameters = self.name.split("[")
         return test_name, re.sub(r"[\[\-]", "_", parameters).replace("]", "")
 
-    def get_single_test_name(self) -> str:
+    def get_single_test_name(self, mode: Literal["module", "test"] = "module") -> str:
         """Convert test name to a single test name."""
-        test_name, test_parameters = self.get_name_and_parameters()
-        return f"{test_name}__{test_parameters}"
+        if mode == "module":
+            # Use the module name as the test name
+            return self.strip_test_name(self.original_name)
+        elif mode == "test":
+            # Mix the module name and the test name/arguments
+            test_name, test_parameters = self.get_name_and_parameters()
+            test_name = self.strip_test_name(test_name)
+            return f"{test_name}__{test_parameters}"
 
     def get_dump_dir_path(
         self,
@@ -78,7 +70,7 @@ def get_dump_dir_path(
         """Path to dump the debug output as defined by the level to dump at."""
         if not base_dump_dir:
             return None
-        test_module_relative_dir = get_module_relative_output_dir(self.path, filler_path)
+        test_module_relative_dir = self.get_module_relative_output_dir(filler_path)
         if level == "test_module":
             return Path(base_dump_dir) / Path(str(test_module_relative_dir).replace(os.sep, "__"))
         test_name, test_parameter_string = self.get_name_and_parameters()
@@ -89,6 +81,27 @@ def get_dump_dir_path(
             return Path(base_dump_dir) / flat_path / test_parameter_string
         raise Exception("Unexpected level.")
 
+    def get_id(self) -> str:
+        """Return the test id."""
+        return self.id
+
+    def get_module_relative_output_dir(self, filler_path: Path) -> Path:
+        """
+        Return a directory name for the provided test_module (relative to the
+        base ./tests directory) that can be used for output (within the
+        configured fixtures output path or the base_dump_dir directory).
+
+        Example:
+        tests/shanghai/eip3855_push0/test_push0.py -> shanghai/eip3855_push0/test_push0
+
+        """
+        basename = self.module_path.with_suffix("").absolute()
+        basename_relative = basename.relative_to(
+            os.path.commonpath([filler_path.absolute(), basename])
+        )
+        module_path = basename_relative.parent / self.strip_test_name(basename_relative.stem)
+        return module_path
+
 
 @dataclass(kw_only=True)
 class FixtureCollector:
@@ -108,19 +121,14 @@ def get_fixture_basename(self, info: TestInfo) -> Path:
         """Return basename of the fixture file for a given test case."""
         if self.flat_output:
             if self.single_fixture_per_file:
-                return Path(strip_test_prefix(info.get_single_test_name()))
-            return Path(strip_test_prefix(info.original_name))
+                return Path(info.get_single_test_name(mode="test"))
+            return Path(info.get_single_test_name(mode="module"))
         else:
-            relative_fixture_output_dir = Path(info.path).parent / strip_test_prefix(
-                Path(info.path).stem
-            )
-            module_relative_output_dir = get_module_relative_output_dir(
-                relative_fixture_output_dir, self.filler_path
-            )
+            module_relative_output_dir = info.get_module_relative_output_dir(self.filler_path)
 
             if self.single_fixture_per_file:
-                return module_relative_output_dir / strip_test_prefix(info.get_single_test_name())
-            return module_relative_output_dir / strip_test_prefix(info.original_name)
+                return module_relative_output_dir / info.get_single_test_name(mode="test")
+            return module_relative_output_dir / info.get_single_test_name(mode="module")
 
     def add_fixture(self, info: TestInfo, fixture: BaseFixture) -> Path:
         """Add fixture to the list of fixtures of a given test case."""
@@ -135,7 +143,7 @@ def add_fixture(self, info: TestInfo, fixture: BaseFixture) -> Path:
             self.all_fixtures[fixture_path] = Fixtures(root={})
             self.json_path_to_test_item[fixture_path] = info
 
-        self.all_fixtures[fixture_path][info.id] = fixture
+        self.all_fixtures[fixture_path][info.get_id()] = fixture
 
         return fixture_path
 

src/ethereum_test_forks/__init__.py

@@ -27,12 +27,14 @@
 )
 from .gas_costs import GasCosts
 from .helpers import (
+    ForkRangeDescriptor,
     InvalidForkError,
     forks_from,
     forks_from_until,
     get_closest_fork_with_solc_support,
     get_deployed_forks,
     get_development_forks,
+    get_fork_by_name,
     get_forks,
     get_forks_with_no_descendants,
     get_forks_with_no_parents,
@@ -57,6 +59,7 @@
     "Byzantium",
     "Constantinople",
     "ConstantinopleFix",
+    "ForkRangeDescriptor",
     "Frontier",
     "GrayGlacier",
     "Homestead",
@@ -80,6 +83,7 @@
     "get_development_forks",
     "get_transition_fork_predecessor",
     "get_transition_fork_successor",
+    "get_fork_by_name",
     "get_forks_with_no_descendants",
     "get_forks_with_no_parents",
     "get_forks_with_solc_support",

src/ethereum_test_forks/helpers.py

@@ -1,7 +1,9 @@
 """Helper methods to resolve forks during test filling."""
 
-from typing import List, Optional, Set
+import re
+from typing import Any, List, Optional, Set
 
+from pydantic import BaseModel, ConfigDict, ValidatorFunctionWrapHandler, model_validator
 from semver import Version
 
 from .base_fork import BaseFork, Fork
@@ -17,18 +19,20 @@ def __init__(self, message):
         super().__init__(message)
 
 
+all_forks: List[Fork] = []
+for fork_name in forks.__dict__:
+    fork = forks.__dict__[fork_name]
+    if not isinstance(fork, type):
+        continue
+    if issubclass(fork, BaseFork) and fork is not BaseFork:
+        all_forks.append(fork)
+
+
 def get_forks() -> List[Fork]:
     """
     Return list of all the fork classes implemented by
     `ethereum_test_forks` ordered chronologically by deployment.
     """
-    all_forks: List[Fork] = []
-    for fork_name in forks.__dict__:
-        fork = forks.__dict__[fork_name]
-        if not isinstance(fork, type):
-            continue
-        if issubclass(fork, BaseFork) and fork is not BaseFork:
-            all_forks.append(fork)
     return all_forks
 
 
@@ -250,3 +254,61 @@ def get_relative_fork_markers(fork_identifier: Fork | str, strict_mode: bool = T
         return [fork_class.name(), fork_class.transitions_to().name()]
     else:
         return [fork_class.name()]
+
+
+def get_fork_by_name(fork_name: str) -> Fork | None:
+    """Get a fork by name."""
+    for fork in get_forks():
+        if fork.name() == fork_name:
+            return fork
+    return None
+
+
+class ForkRangeDescriptor(BaseModel):
+    """Fork descriptor parsed from string normally contained in ethereum/tests fillers."""
+
+    greater_equal: Fork | None = None
+    less_than: Fork | None = None
+    model_config = ConfigDict(frozen=True)
+
+    def fork_in_range(self, fork: Fork) -> bool:
+        """Return whether the given fork is within range."""
+        if self.greater_equal is not None and fork < self.greater_equal:
+            return False
+        if self.less_than is not None and fork >= self.less_than:
+            return False
+        return True
+
+    @model_validator(mode="wrap")
+    @classmethod
+    def validate_fork_range_descriptor(cls, v: Any, handler: ValidatorFunctionWrapHandler):
+        """
+        Validate the fork range descriptor from a string.
+
+        Examples:
+        - ">=Osaka" validates to {greater_equal=Osaka, less_than=None}
+        - ">=Prague<Osaka" validates to {greater_equal=Prague, less_than=Osaka}
+
+        """
+        if isinstance(v, str):
+            # Decompose the string into its parts
+            descriptor_string = re.sub(r"\s+", "", v.strip())
+            v = {}
+            if m := re.search(r">=(\w+)", descriptor_string):
+                fork: Fork | None = get_fork_by_name(m.group(1))
+                if fork is None:
+                    raise Exception(f"Unable to parse fork name: {m.group(1)}")
+                v["greater_equal"] = fork
+                descriptor_string = re.sub(r">=(\w+)", "", descriptor_string)
+            if m := re.search(r"<(\w+)", descriptor_string):
+                fork = get_fork_by_name(m.group(1))
+                if fork is None:
+                    raise Exception(f"Unable to parse fork name: {m.group(1)}")
+                v["less_than"] = fork
+                descriptor_string = re.sub(r"<(\w+)", "", descriptor_string)
+            if descriptor_string:
+                raise Exception(
+                    "Unable to completely parse fork range descriptor. "
+                    + f'Remaining string: "{descriptor_string}"'
+                )
+        return handler(v)

src/ethereum_test_forks/tests/test_fork_range_descriptor.py

@@ -0,0 +1,36 @@
+"""Test fork range descriptor parsing from string."""
+
+import pytest
+
+from ..forks.forks import Osaka, Prague
+from ..helpers import ForkRangeDescriptor
+
+
+@pytest.mark.parametrize(
+    "fork_range_descriptor_string,expected_fork_range_descriptor",
+    [
+        (
+            ">=Osaka",
+            ForkRangeDescriptor(
+                greater_equal=Osaka,
+                less_than=None,
+            ),
+        ),
+        (
+            ">= Prague < Osaka",
+            ForkRangeDescriptor(
+                greater_equal=Prague,
+                less_than=Osaka,
+            ),
+        ),
+    ],
+)
+def test_parsing_fork_range_descriptor_from_string(
+    fork_range_descriptor_string: str,
+    expected_fork_range_descriptor: ForkRangeDescriptor,
+):
+    """Test multiple strings used as fork range descriptors in ethereum/tests."""
+    assert (
+        ForkRangeDescriptor.model_validate(fork_range_descriptor_string)
+        == expected_fork_range_descriptor
+    )

src/ethereum_test_specs/__init__.py

@@ -3,6 +3,7 @@
 from typing import List, Type
 
 from .base import BaseTest, TestSpec
+from .base_static import BaseStaticTest
 from .blockchain import (
     BlockchainTest,
     BlockchainTestFiller,
@@ -30,6 +31,7 @@
 
 __all__ = (
     "SPEC_TYPES",
+    "BaseStaticTest",
     "BaseTest",
     "BlockchainTest",
     "BlockchainTestEngineFiller",