fix(consume): allow consumption of fixtures with unknown forks (#1318)

Co-authored-by: spencer <[email protected]>

Author: danceratopz

docs/CHANGELOG.md

@@ -13,8 +13,9 @@ Test fixtures for use by clients are available for each release on the [Github r
 #### `consume`
 
 - ✨ Add support for Nethermind's `nethtest` command to `consume direct` ([#1250](https://github.com/ethereum/execution-spec-tests/pull/1250)).
-- ✨ Allow filtering of test cases by fork via pytest marks (e.g., `-m "Cancun or Prague"`) ([#1304](https://github.com/ethereum/execution-spec-tests/pull/1304)).
+- ✨ Allow filtering of test cases by fork via pytest marks (e.g., `-m "Cancun or Prague"`) ([#1304](https://github.com/ethereum/execution-spec-tests/pull/1304), [#1318](https://github.com/ethereum/execution-spec-tests/pull/1318)).
 - ✨ Allow filtering of test cases by fixture format via pytest marks (e.g., `-m blockchain_test`) ([#1314](https://github.com/ethereum/execution-spec-tests/pull/1314)).
+- ✨ Add top-level entries `forks` and `fixture_formats` to the index file that list all the forks and fixture formats used in the indexed fixtures ([#1318](https://github.com/ethereum/execution-spec-tests/pull/1318)).
 - 🐞 Don't parametrize tests for unsupported fixture formats; improve `consume` test collection ([#1315](https://github.com/ethereum/execution-spec-tests/pull/1314)).
 - 🐞 Improve index generation of ethereum/tests fixtures: Allow generation at any directory level and include `generatedTestHash` in the index file for the `fixture_hash` ([#1303](https://github.com/ethereum/execution-spec-tests/pull/1303)).
 

src/cli/gen_index.py

@@ -140,7 +140,8 @@ def generate_fixtures_index(
         disable=quiet_mode,
     ) as progress:  # type: Progress
         task_id = progress.add_task("[cyan]Processing files...", total=total_files, filename="...")
-
+        forks = set()
+        fixture_formats = set()
         test_cases: List[TestCaseIndexFile] = []
         for file in input_path.rglob("*.json"):
             if file.name == "index.json" or ".meta" in file.parts:
@@ -165,6 +166,8 @@ def generate_fixtures_index(
                         format=fixture.__class__,
                     )
                 )
+                forks.add(fixture.get_fork())
+                fixture_formats.add(fixture.format_name)
 
             display_filename = file.name
             if len(display_filename) > filename_display_width:
@@ -185,6 +188,8 @@ def generate_fixtures_index(
         root_hash=root_hash,
         created_at=datetime.datetime.now(),
         test_count=len(test_cases),
+        forks=list(forks),
+        fixture_formats=list(fixture_formats),
     )
 
     with open(output_file, "w") as f:

src/ethereum_test_fixtures/consume.py

@@ -3,7 +3,7 @@
 import datetime
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import List, TextIO
+from typing import List, Optional, TextIO
 
 from pydantic import BaseModel, RootModel
 
@@ -80,6 +80,8 @@ class IndexFile(BaseModel):
     root_hash: HexNumber | None
     created_at: datetime.datetime
     test_count: int
+    forks: Optional[List[str]] = []
+    fixture_formats: Optional[List[str]] = []
     test_cases: List[TestCaseIndexFile]
 
 

src/ethereum_test_forks/helpers.py

@@ -221,12 +221,17 @@ def forks_from(fork: Fork, deployed_only: bool = True) -> List[Fork]:
     return forks_from_until(fork, latest_fork)
 
 
-def get_relative_fork_markers(fork_identifier: Fork | str) -> list[str]:
+def get_relative_fork_markers(fork_identifier: Fork | str, strict_mode: bool = True) -> list[str]:
     """
     Return a list of marker names for a given fork.
+
     For a base fork (e.g. `Shanghai`), return [ `Shanghai` ].
     For a transition fork (e.g. `ShanghaiToCancunAtTime15k` which transitions to `Cancun`),
     return [ `ShanghaiToCancunAtTime15k`, `Cancun` ].
+
+    If `strict_mode` is set to `True`, raise an `InvalidForkError` if the fork is not found,
+    otherwise, simply return the provided (str) `fork_identifier` (this is required to run
+    `consume` with forks that are unknown to EEST).
     """
     all_forks = set(get_forks()) | set(get_transition_forks())
     if isinstance(fork_identifier, str):
@@ -235,8 +240,9 @@ def get_relative_fork_markers(fork_identifier: Fork | str) -> list[str]:
             if candidate.name() == fork_identifier:
                 fork_class = candidate
                 break
-        if fork_class is None:
+        if strict_mode and fork_class is None:
             raise InvalidForkError(f"Unknown fork: {fork_identifier}")
+        return [fork_identifier]
     else:
         fork_class = fork_identifier
 

src/pytest_plugins/consume/consume.py

@@ -16,7 +16,7 @@
 
 from cli.gen_index import generate_fixtures_index
 from ethereum_test_fixtures import BaseFixture
-from ethereum_test_fixtures.consume import TestCases
+from ethereum_test_fixtures.consume import IndexFile, TestCases
 from ethereum_test_forks import get_forks, get_relative_fork_markers, get_transition_forks
 from ethereum_test_tools.utility.versioning import get_current_commit_hash_or_tag
 
@@ -291,17 +291,23 @@ def pytest_configure(config):  # noqa: D103
             force_flag=False,
             disable_infer_format=False,
         )
-    config.test_cases = TestCases.from_index_file(index_file)
 
-    all_forks_with_transitions = {  # type: ignore
-        fork for fork in set(get_forks()) | get_transition_forks() if not fork.ignore()
-    }
+    index = IndexFile.model_validate_json(index_file.read_text())
+    config.test_cases = index.test_cases
+
     for fixture_format in BaseFixture.formats.values():
         config.addinivalue_line(
             "markers",
             f"{fixture_format.format_name}: Tests in `{fixture_format.format_name}` format ",
         )
-    for fork in all_forks_with_transitions:
+
+    # All forked defined within EEST
+    all_forks = {  # type: ignore
+        fork for fork in set(get_forks()) | get_transition_forks() if not fork.ignore()
+    }
+    # Append all forks within the index file (compatibility with `ethereum/tests`)
+    all_forks.update(getattr(index, "forks", []))
+    for fork in all_forks:
         config.addinivalue_line("markers", f"{fork}: Tests for the {fork} fork")
 
     if config.option.sim_limit:
@@ -364,7 +370,7 @@ def pytest_generate_tests(metafunc):
     for test_case in test_cases:
         if test_case.format.format_name not in metafunc.config._supported_fixture_formats:
             continue
-        fork_markers = get_relative_fork_markers(test_case.fork)
+        fork_markers = get_relative_fork_markers(test_case.fork, strict_mode=False)
         param = pytest.param(
             test_case,
             id=test_case.id,