feat(unit-tests): verify transition tool support (#1263)

* unit test to check transition tool support

* feat(clis): Implement `is_installed` method for `EthereumCLI`

* fix(clis): Geth non-optional binary

* fix(clis): Tests: Make t8n support unit tests run on installed t8n tools only

* fix(clis): type fix

* fix(clis): unit test non-sorted forks

* feat(clis): Add unit test to CI-only checks

* build evms for unit tests framework CI

* support evmone action on macOS

* add Prague resolution for eels in CI

* refactor(fw): Generalize `installed_t8n` fixture

---------

Co-authored-by: Mario Vega <[email protected]>

Author: winsvega

.github/actions/build-evm-client/evmone/action.yaml

@@ -27,9 +27,15 @@ runs:
       uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be
       with:
         cmake-version: '3.x'
-    - name: "Install GMP"
+    - name: "Install GMP Linux"
+      if: runner.os == 'Linux'
       shell: bash
       run: sudo apt-get -q update && sudo apt-get -qy install libgmp-dev
+    - name: Install GMP macOS
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        brew update && brew install gmp
     - name: Build evmone binary
       shell: bash
       run: |

.github/configs/eels_resolutions.json

@@ -28,5 +28,9 @@
     },
     "Cancun": {
         "path": "../../execution-specs/src/ethereum/cancun"
+    },
+    "Prague": {
+        "git_url": "https://github.com/ethereum/execution-specs.git",
+        "branch": "devnets/prague/6"
     }
 }

.github/workflows/tox_verify.yaml

@@ -113,6 +113,12 @@ jobs:
           cache-dependency-glob: "uv.lock"
           version: ${{ vars.UV_VERSION }}
           python-version: ${{ matrix.python }}
+      - name: Build EVMONE EVM
+        uses: ./.github/actions/build-evm-client/evmone
+        with:
+          targets: "evmone-t8n"
+      - name: Build GETH EVM
+        uses: ./.github/actions/build-evm-client/geth
       - name: Run tox - run framework unit tests with pytest
         env:
           EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json

src/conftest.py

@@ -1,11 +1,11 @@
-"""Local pytest configuration for framework tests."""
+"""Local pytest configuration used on multiple framework tests."""
 
 import os
-from typing import Generator
+from typing import Dict, Generator
 
 import pytest
 
-from ethereum_clis import ExecutionSpecsTransitionTool, TransitionTool
+from ethereum_clis import BesuTransitionTool, ExecutionSpecsTransitionTool, TransitionTool
 
 
 def pytest_runtest_setup(item):
@@ -16,21 +16,82 @@ def pytest_runtest_setup(item):
             pytest.skip("Skipping test because pytest-xdist is running with more than one worker.")
 
 
-DEFAULT_T8N_FOR_UNIT_TESTS = ExecutionSpecsTransitionTool
+DEFAULT_TRANSITION_TOOL_FOR_UNIT_TESTS = ExecutionSpecsTransitionTool
+
+INSTALLED_TRANSITION_TOOLS = [
+    transition_tool
+    for transition_tool in TransitionTool.registered_tools
+    if (
+        transition_tool.is_installed()
+        # Currently, Besu has the same `default_binary` as Geth, so we can't use `is_installed`.
+        and transition_tool != BesuTransitionTool
+    )
+]
 
 
 @pytest.fixture(scope="session")
-def default_t8n_instance() -> Generator[TransitionTool, None, None]:
-    """Fixture to provide a default t8n instance."""
-    instance = ExecutionSpecsTransitionTool()
-    instance.start_server()
-    yield instance
-    instance.shutdown()
+def installed_transition_tool_instances() -> Generator[
+    Dict[str, TransitionTool | Exception], None, None
+]:
+    """Return all instantiated transition tools."""
+    instances: Dict[str, TransitionTool | Exception] = {}
+    for transition_tool_class in INSTALLED_TRANSITION_TOOLS:
+        try:
+            instances[transition_tool_class.__name__] = transition_tool_class()
+        except Exception as e:
+            # Record the exception in order to provide context when failing the appropriate test
+            instances[transition_tool_class.__name__] = e
+    yield instances
+    for instance in instances.values():
+        if isinstance(instance, TransitionTool):
+            instance.shutdown()
+
+
+@pytest.fixture(
+    params=INSTALLED_TRANSITION_TOOLS,
+    ids=[transition_tool_class.__name__ for transition_tool_class in INSTALLED_TRANSITION_TOOLS],
+)
+def installed_t8n(
+    request: pytest.FixtureRequest,
+    installed_transition_tool_instances: Dict[str, TransitionTool | Exception],
+) -> TransitionTool:
+    """
+    Return an instantiated transition tool.
+
+    Tests using this fixture will be automatically parameterized with all
+    installed transition tools.
+    """
+    transition_tool_class = request.param
+    assert issubclass(transition_tool_class, TransitionTool)
+    assert transition_tool_class.__name__ in installed_transition_tool_instances, (
+        f"{transition_tool_class.__name__} not instantiated"
+    )
+    instance_or_error = installed_transition_tool_instances[transition_tool_class.__name__]
+    if isinstance(instance_or_error, Exception):
+        raise Exception(
+            f"Failed to instantiate {transition_tool_class.__name__}"
+        ) from instance_or_error
+    return instance_or_error
 
 
 @pytest.fixture
 def default_t8n(
-    default_t8n_instance: TransitionTool,
+    installed_transition_tool_instances: Dict[str, TransitionTool | Exception],
 ) -> TransitionTool:
     """Fixture to provide a default t8n instance."""
-    return default_t8n_instance
+    instance = installed_transition_tool_instances.get(
+        DEFAULT_TRANSITION_TOOL_FOR_UNIT_TESTS.__name__
+    )
+    if instance is None:
+        raise Exception(f"Failed to instantiate {DEFAULT_TRANSITION_TOOL_FOR_UNIT_TESTS.__name__}")
+    if isinstance(instance, Exception):
+        raise Exception(
+            f"Failed to instantiate {DEFAULT_TRANSITION_TOOL_FOR_UNIT_TESTS.__name__}"
+        ) from instance
+    return instance
+
+
+@pytest.fixture(scope="session")
+def running_in_ci() -> bool:
+    """Return whether the test is running in a CI environment."""
+    return "CI" in os.environ

src/ethereum_clis/clis/geth.py

@@ -104,14 +104,13 @@ class GethEvm(EthereumCLI):
 
     def __init__(
         self,
-        binary: Path,
+        binary: Optional[Path] = None,
         trace: bool = False,
-        exception_mapper: ExceptionMapper | None = None,
     ):
         """Initialize the GethEvm class."""
-        self.binary = binary
+        self.binary = binary if binary else self.default_binary
         self.trace = trace
-        self.exception_mapper = exception_mapper if exception_mapper else GethExceptionMapper()
+        self._info_metadata: Optional[Dict[str, Any]] = {}
 
     def _run_command(self, command: List[str]) -> subprocess.CompletedProcess:
         try:
@@ -167,12 +166,18 @@ class GethTransitionTool(GethEvm, TransitionTool):
     trace: bool
     t8n_use_stream = True
 
-    def __init__(self, *, binary: Path, trace: bool = False):
+    def __init__(
+        self,
+        *,
+        exception_mapper: Optional[ExceptionMapper] = None,
+        binary: Optional[Path] = None,
+        trace: bool = False,
+    ):
         """Initialize the GethTransitionTool class."""
+        if not exception_mapper:
+            exception_mapper = GethExceptionMapper()
         GethEvm.__init__(self, binary=binary, trace=trace)
-        TransitionTool.__init__(
-            self, exception_mapper=self.exception_mapper, binary=binary, trace=trace
-        )
+        TransitionTool.__init__(self, binary=binary, exception_mapper=exception_mapper)
         help_command = [str(self.binary), str(self.subcommand), "--help"]
         result = self._run_command(help_command)
         self.help_string = result.stdout

src/ethereum_clis/ethereum_cli.py

@@ -126,6 +126,18 @@ def detect_binary(cls, binary_output: str) -> bool:
 
         return cls.detect_binary_pattern.match(binary_output) is not None
 
+    @classmethod
+    def is_installed(cls, binary_path: Optional[Path] = None) -> bool:
+        """Return whether the tool is installed in the current system."""
+        if binary_path is None:
+            binary_path = cls.default_binary
+        else:
+            resolved_path = Path(os.path.expanduser(binary_path)).resolve()
+            if resolved_path.exists():
+                binary_path = resolved_path
+        binary = shutil.which(binary_path)
+        return binary is not None
+
     def version(self) -> str:
         """Return the name and version of the CLI as reported by the CLI's version flag."""
         if self.cached_version is None: