feat,test(consume): use regex for --sim.limit; don't use xdist if --sim.parallelism=1 (#1221)

* chore(deps): add pytest-regex as a dependency

* feat(consume): use `--regex` for hive `--sim.limit` args

* feat(consume): add `--sim.limit` flag to consume

* tests(consume): add tests for `--sim.limit` in collect-only mode

* chore(tests): fix typo in docstring

* docs: update changelog

* fix(consume): add quotes around argument value in help string

* docs: move this pr's entry from v4 to unreleased

* refactor(consume): specify the type of `--sim.limit` to `SimLimitBehavior`

* tests(consume): fix-up tests following rebase on #1237

* tests(consume): try to fix macOS fails

* test(consume): print output upon non-clean fill exit code

* test(consume): fix tox -pytest fails

* test(gentest): add debug output in case of unclean fill exit

* fix(consume): rebase import issue

---------

Co-authored-by: Mario Vega <marioevz@gmail.com>

Author: danceratopz

docs/CHANGELOG.md

@@ -17,6 +17,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Allow `consume direct --collect-only` without specifying a fixture consumer binary on the command-line ([#1237](https://github.com/ethereum/execution-spec-tests/pull/1237)).
 - ✨ Report the (resolved) fixture tarball URL and local fixture cache directory when `consume`'s `--input` flag is a release spec or URL  [#1239](https://github.com/ethereum/execution-spec-tests/pull/1239).
 - ✨ EOF Container validation tests (`eof_test`) now generate container deployment state tests, by wrapping the EOF container in an init-container and sending a deploy transaction ([#783](https://github.com/ethereum/execution-spec-tests/pull/783), [#1233](https://github.com/ethereum/execution-spec-tests/pull/1233)).
+- ✨ Use regexes for Hive's `--sim.limit` argument and don't use xdist if `--sim.parallelism==1` in the `eest/consume-rlp` and `eest/consume-rlp` simulators ([#1220](https://github.com/ethereum/execution-spec-tests/pull/1220)).
 
 ### 📋 Misc
 
@@ -292,8 +293,8 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🔀 Refactor `gentest` to use `ethereum_test_tools.rpc.rpc` by adding to `get_transaction_by_hash`, `debug_trace_call` to `EthRPC` ([#568](https://github.com/ethereum/execution-spec-tests/pull/568)).
 - ✨ Write a properties file to the output directory and enable direct generation of a fixture tarball from `fill` via `--output=fixtures.tgz`([#627](https://github.com/ethereum/execution-spec-tests/pull/627)).
 - 🔀 `ethereum_test_tools` library has been split into multiple libraries ([#645](https://github.com/ethereum/execution-spec-tests/pull/645)).
-- ✨ Add the consume engine simulator and refactor the consume simulator suite. ([#691](https://github.com/ethereum/execution-spec-tests/pull/691)).
-- 🐞 Prevents forcing consume to use stdin as an input when running from hive. ([#701](https://github.com/ethereum/execution-spec-tests/pull/701)).
+- ✨ Add the consume engine simulator and refactor the consume simulator suite ([#691](https://github.com/ethereum/execution-spec-tests/pull/691)).
+- 🐞 Prevents forcing consume to use stdin as an input when running from hive ([#701](https://github.com/ethereum/execution-spec-tests/pull/701)).
 
 ### 📋 Misc
 

pyproject.toml

@@ -51,6 +51,7 @@ dependencies = [
     "questionary @ git+https://github.com/tmbo/questionary@ff22aeae1cd9c1c734f14329934e349bec7873bc",
     "prompt_toolkit>=3.0.48,<4", # ensure we have a new enough version for ipython
     "ethereum-rlp>=0.1.3,<0.2",
+    "pytest-regex>=0.2.0,<0.3",
 ]
 
 [project.urls]

src/cli/gentest/tests/test_cli.py

@@ -98,4 +98,4 @@ def get_mock_context(self: StateTestProvider) -> dict:
 
     ## Fill ##
     fill_result = runner.invoke(fill, ["-c", "pytest.ini", "--skip-evm-dump", output_file])
-    assert fill_result.exit_code == 0
+    assert fill_result.exit_code == 0, f"Fill command failed:\n{fill_result.output}"

src/cli/pytest_commands/consume.py

@@ -14,14 +14,14 @@
 
 def handle_hive_env_flags(args: List[str]) -> List[str]:
     """Convert hive environment variables into pytest flags."""
-    env_var_mappings = {
-        "HIVE_TEST_PATTERN": ["-k"],
-        "HIVE_PARALLELISM": ["-n"],
-    }
-    for env_var, pytest_flag in env_var_mappings.items():
-        value = os.getenv(env_var)
-        if value is not None:
-            args.extend(pytest_flag + [value])
+    # handle hive --sim.limit arg
+    hive_test_pattern = os.getenv("HIVE_TEST_PATTERN")
+    if hive_test_pattern and ("--regex" not in args and "--sim.limit" not in args):
+        args += ["--sim.limit", hive_test_pattern]
+    hive_parallelism = os.getenv("HIVE_PARALLELISM")
+    # handle hive --sim.parallelism arg
+    if hive_parallelism not in [None, "", "1"] and "-n" not in args:
+        args += ["-n", str(hive_parallelism)]
     if os.getenv("HIVE_RANDOM_SEED") is not None:
         warnings.warn("HIVE_RANDOM_SEED is not yet supported.", stacklevel=2)
     if os.getenv("HIVE_LOGLEVEL") is not None:

src/pytest_plugins/consume/consume.py

@@ -1,5 +1,6 @@
 """A pytest plugin providing common functionality for consuming test fixtures."""
 
+import re
 import sys
 import tarfile
 from dataclasses import dataclass
@@ -127,6 +128,56 @@ def download_and_extract(url: str, base_directory: Path) -> Tuple[bool, Path]:
     return already_cached, extract_to / "fixtures"
 
 
+class SimLimitBehavior:
+    """Represents options derived from the `--sim.limit` argument."""
+
+    def __init__(self, pattern: str, collectonly: bool = False):  # noqa: D107
+        self.pattern = pattern
+        self.collectonly = collectonly
+
+    @staticmethod
+    def _escape_id(pattern: str) -> str:
+        """
+        Escape regex char in the pattern; prepend and append '.*' (for `fill` IDs).
+
+        The `pattern` is prefixed and suffixed with a wildcard match to allow `fill`
+        test case IDs to be specified, otherwise the full `consume` test ID must be
+        specified.
+        """
+        return f".*{re.escape(pattern)}.*"
+
+    @classmethod
+    def from_string(cls, pattern: str) -> "SimLimitBehavior":
+        """
+        Parse the `--sim.limit` argument and return a `SimLimitBehavior` instance.
+
+        If `pattern`:
+        - Is "collectonly", enable collection mode without filtering.
+        - Starts with "collectonly:", enable collection mode and use the rest as a regex pattern.
+        - Starts with "id:", treat the rest as a literal test ID and escape special regex chars.
+        - Starts with "collectonly:id:", enable collection mode with a literal test ID.
+        """
+        if pattern == "collectonly":
+            return cls(pattern=".*", collectonly=True)
+
+        if pattern.startswith("collectonly:id:"):
+            literal_id = pattern[len("collectonly:id:") :]
+            if not literal_id:
+                raise ValueError("Empty literal ID provided.")
+            return cls(pattern=cls._escape_id(literal_id), collectonly=True)
+
+        if pattern.startswith("collectonly:"):
+            return cls(pattern=pattern[len("collectonly:") :], collectonly=True)
+
+        if pattern.startswith("id:"):
+            literal_id = pattern[len("id:") :]
+            if not literal_id:
+                raise ValueError("Empty literal ID provided.")
+            return cls(pattern=cls._escape_id(literal_id))
+
+        return cls(pattern=pattern)
+
+
 def pytest_addoption(parser):  # noqa: D103
     consume_group = parser.getgroup(
         "consume", "Arguments related to consuming fixtures via a client"
@@ -173,6 +224,22 @@ def pytest_addoption(parser):  # noqa: D103
             "The --html flag can be used to specify a different path."
         ),
     )
+    consume_group.addoption(
+        "--sim.limit",
+        action="store",
+        dest="sim_limit",
+        type=SimLimitBehavior.from_string,
+        default=SimLimitBehavior(".*"),
+        help=(
+            "Filter tests by either a regex pattern or a literal test case ID. To match a "
+            "test case by its exact ID, prefix the ID with `id:`. The string following `id:` "
+            "will be automatically escaped so that all special regex characters are treated as "
+            "literals. Without the `id:` prefix, the argument is interpreted as a Python regex "
+            "pattern. To see which test cases are matched, without executing them, prefix with "
+            '`collectonly:`, e.g. `--sim.limit "collectonly:.*eip4788.*fork_Prague.*"`. '
+            "To list all available test case IDs, set the value to `collectonly`."
+        ),
+    )
 
 
 @pytest.hookimpl(tryfirst=True)
@@ -231,6 +298,17 @@ def pytest_configure(config):  # noqa: D103
         )
     config.test_cases = TestCases.from_index_file(index_file)
 
+    if config.option.sim_limit:
+        if config.option.dest_regex != ".*":
+            pytest.exit(
+                "Both the --sim.limit (via env var?) and the --regex flags are set. "
+                "Please only set one of them."
+            )
+        config.option.dest_regex = config.option.sim_limit.pattern
+        if config.option.sim_limit.collectonly:
+            config.option.collectonly = True
+            config.option.verbose = -1  # equivalent to -q; only print test ids
+
     if config.option.collectonly:
         return
     if not config.getoption("disable_html") and config.getoption("htmlpath") is None:

src/pytest_plugins/consume/direct/conftest.py

@@ -19,8 +19,7 @@
 from ethereum_test_fixtures import BaseFixture
 from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream
 from ethereum_test_fixtures.file import Fixtures
-
-from ..consume import FixturesSource
+from pytest_plugins.consume.consume import FixturesSource
 
 
 class CollectOnlyCLI(EthereumCLI):

src/pytest_plugins/consume/direct/test_via_direct.py

@@ -7,8 +7,7 @@
 
 from ethereum_test_fixtures import BaseFixture, FixtureConsumer, FixtureFormat
 from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream
-
-from ..decorator import fixture_format
+from pytest_plugins.consume.decorator import fixture_format
 
 
 @fixture_format(*BaseFixture.formats.values())

src/pytest_plugins/consume/tests/test_consume_args.py

@@ -0,0 +1,184 @@
+"""Test the consume plugins with various cli arguments."""
+
+import re
+import shutil
+from pathlib import Path
+from typing import List
+
+import pytest
+from click.testing import CliRunner
+from pytest import Pytester, TempPathFactory
+
+from cli.pytest_commands.fill import fill
+
+
+@pytest.fixture(scope="module")
+def test_paths() -> list[Path]:
+    """Specify the test paths to be filled."""
+    return [
+        Path("tests/istanbul/eip1344_chainid/test_chainid.py"),
+    ]
+
+
+@pytest.fixture(scope="module")
+def consume_test_case_ids() -> list[str]:
+    """Hard-coded expected output of `consume direct --collectonly -q`."""
+    return [
+        "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Cancun-blockchain_test_from_state_test]]",
+        "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Paris-blockchain_test_from_state_test]]",
+        "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Shanghai-blockchain_test_from_state_test]]",
+        "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Cancun-state_test]]",
+        "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Paris-state_test]]",
+        "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Shanghai-state_test]]",
+    ]
+
+
+@pytest.fixture(scope="module")
+def fill_fork_from() -> str:
+    """Specify the value for `fill`'s `--from` argument."""
+    return "Paris"
+
+
+@pytest.fixture(scope="module")
+def fill_fork_until() -> str:
+    """Specify the value for `fill`'s `--until` argument."""
+    return "Cancun"
+
+
+@pytest.fixture(scope="module")
+def fixtures_dir(tmp_path_factory: TempPathFactory) -> Path:
+    """Define the temporary test fixture directory for fill output."""
+    return tmp_path_factory.mktemp("fixtures")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def fill_tests(
+    fixtures_dir: Path, fill_fork_from: str, fill_fork_until: str, test_paths: List[Path]
+) -> None:
+    """Run fill to generate test fixtures for use with testing consume."""
+    fill_result = CliRunner().invoke(
+        fill,
+        [
+            "-c",
+            "pytest.ini",
+            "--skip-evm-dump",
+            "-m",
+            "(not blockchain_test_engine) and (not eip_version_check)",
+            f"--from={fill_fork_from}",
+            f"--until={fill_fork_until}",
+            f"--output={str(fixtures_dir)}",
+            *[str(p) for p in test_paths],
+            # if we fill many tests, it might help to add -n 8/auto
+        ],
+    )
+    assert fill_result.exit_code == 0, f"Fill command failed:\n{fill_result.output}"
+
+
+@pytest.fixture(autouse=True, scope="function")
+def test_fixtures(pytester: Pytester, fixtures_dir: Path, fill_tests: None) -> List[Path]:
+    """
+    Copy test fixtures from the regular temp path to the pytester temporary dir.
+
+    We intentionally copy the `.meta/index.json` file to test its compatibility with consume.
+    """
+    test_fixtures = []
+    for json_file in fixtures_dir.rglob("*.json"):
+        target_dir = Path(pytester.path) / json_file.parent
+        if not target_dir.exists():
+            target_dir.mkdir(parents=True)
+        pytester.copy_example(name=json_file.as_posix())
+        shutil.move(json_file.name, target_dir / json_file.name)
+        if ".meta" not in str(json_file):
+            test_fixtures.append(json_file)
+    return test_fixtures
+
+
+@pytest.fixture(autouse=True)
+def copy_consume_test_paths(pytester: Pytester):
+    """Specify and copy the consume test paths to the testdir."""
+    local_test_paths = [Path("src/pytest_plugins/consume/direct/test_via_direct.py")]
+    for test_path in local_test_paths:
+        target_dir = Path(pytester.path) / test_path.parent
+        target_dir.mkdir(parents=True, exist_ok=True)
+        pytester.copy_example(name=str(test_path))
+        pytester.copy_example(name=str(test_path.parent / "conftest.py"))
+        shutil.move(test_path.name, target_dir / test_path.name)
+        shutil.move("conftest.py", target_dir / "conftest.py")
+
+
+single_test_id = "src/pytest_plugins/consume/direct/test_via_direct.py::test_fixture[CollectOnlyFixtureConsumer-tests/istanbul/eip1344_chainid/test_chainid.py::test_chainid[fork_Shanghai-state_test]]"  # noqa: E501
+
+
+@pytest.mark.parametrize(
+    "extra_args, expected_filter_pattern",
+    [
+        pytest.param(
+            ["--collect-only", "-q"],
+            re.compile(r".*"),
+            id="no_extra_args",
+        ),
+        pytest.param(
+            ["--collect-only", "-q", "--sim.limit", ".*fork_Cancun.*"],
+            re.compile(".*Cancun.*"),
+            id="sim_limit_regex",
+        ),
+        pytest.param(
+            ["--sim.limit", "collectonly:.*fork_Cancun.*"],
+            re.compile(".*Cancun.*"),
+            id="sim_limit_collect_only_regex",
+        ),
+        pytest.param(
+            [
+                "--collect-only",
+                "-q",
+                "--sim.limit",
+                f"id:{single_test_id}",
+            ],
+            re.compile(re.escape(f"{single_test_id}")),
+            id="sim_limit_id",
+        ),
+        pytest.param(
+            [
+                "--sim.limit",
+                f"collectonly:id:{single_test_id}",
+            ],
+            re.compile(
+                re.compile(re.escape(f"{single_test_id}")),
+            ),
+            id="sim_limit_collect_only_id",
+        ),
+    ],
+)
+def test_consume_simlimit_collectonly(
+    pytester: Pytester,
+    fixtures_dir: Path,
+    consume_test_case_ids: List[str],
+    extra_args: List[str],
+    expected_filter_pattern: re.Pattern,
+) -> None:
+    """Test consume's --sim.limit argument in collect-only mode."""
+    ini_file = "pytest-consume.ini"
+    pytester.copy_example(name=ini_file)
+    consume_test_path = "src/pytest_plugins/consume/direct/test_via_direct.py"
+    args = [
+        "-c",
+        ini_file,
+        "--input",
+        str(fixtures_dir),
+        consume_test_path,
+        *extra_args,
+    ]
+    result = pytester.runpytest(*args)
+    assert result.ret == 0
+    stdout_lines = str(result.stdout).splitlines()
+    collected_test_ids = [
+        line
+        for line in stdout_lines
+        if line.strip()
+        and "tests collected in" not in line
+        and "pytest-regex selected" not in line
+    ]
+    expected_collected_test_ids = [
+        line for line in consume_test_case_ids if expected_filter_pattern.search(line)
+    ]
+    assert set(collected_test_ids) == set(expected_collected_test_ids)

src/pytest_plugins/filler/tests/test_filler.py

@@ -1,5 +1,5 @@
 """
-Test the forks plugin.
+Test the filler plugin.
 """
 
 import configparser