feat(consume): update --input and fix --help (#745)

* fix: mypy tox issue.

* feat(consume): update --input & fix --help.

* fix(cli): add one-liner help strings for `consume --help`

Specify the help attribute when registering the command; it's too late if the help/__doc__ attribute is set after registration.

* feat(cli): enable `consume -h` flag

* chore: fix missing sys import.

* chore: small review comment changes.

---------

Co-authored-by: danceratopz <[email protected]>

Author: spencer-tb

.vscode/launch.recommended.json

@@ -72,20 +72,7 @@
         "-v"
       ],
       "cwd": "${workspaceFolder}"
-    },
-    {
-      // To help debugging the --test-help command.
-      "name": "Launch fill --test-help",
-      "type": "python",
-      "request": "launch",
-      "module": "pytest",
-      "args": [
-        "-c",
-        "pytest.ini",
-        "--test-help"
-      ],
-      "cwd": "/home/dtopz/code/github/danceratopz/execution-spec-tests"
-    },
+    }
   ],
   "inputs": [
     {

docs/CHANGELOG.md

@@ -36,6 +36,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Added a new flag `--solc-version` to the `fill` command, which allows the user to specify the version of the Solidity compiler to use when compiling Yul source code; this version will now be automatically downloaded by `fill` via [`solc-select`](https://github.com/crytic/solc-select) ([#772](https://github.com/ethereum/execution-spec-tests/pull/772)).
 - 🐞 Fix usage of multiple `@pytest.mark.with_all*` markers which shared parameters, such as `with_all_call_opcodes` and `with_all_create_opcodes` which shared `evm_code_type`, and now only parametrize compatible values ([#762](https://github.com/ethereum/execution-spec-tests/pull/762)).
 - ✨ Added `selector` and `marks` fields to all `@pytest.mark.with_all*` markers, which allows passing lambda functions to select or mark specific parametrized values (see [documentation](https://ethereum.github.io/execution-spec-tests/main/writing_tests/test_markers/#covariant-marker-keyword-arguments) for more information) ([#762](https://github.com/ethereum/execution-spec-tests/pull/762)).
+- ✨ Improves consume input flags for develop and stable fixture releases, fixes `--help` flag for consume ([#745](https://github.com/ethereum/execution-spec-tests/pull/745)).
 
 ### 🔧 EVM Tools
 

docs/getting_started/executing_tests_command_line.md

@@ -119,9 +119,10 @@ usage: fill [-h] [--strict-alloc] [--ca-start CA_START] [--ca-incr CA_INCR]
             [--solc-version SOLC_VERSION] [--evm-bin EVM_BIN] [--traces]
             [--verify-fixtures] [--verify-fixtures-bin VERIFY_FIXTURES_BIN]
             [--filler-path FILLER_PATH] [--output OUTPUT] [--flat-output]
-            [--single-fixture-per-file] [--no-html] [--build-name BUILD_NAME]
-            [--index] [--evm-dump-dir EVM_DUMP_DIR] [--forks] [--fork FORK]
-            [--from FROM] [--until UNTIL] [--test-help]
+            [--single-fixture-per-file] [--no-html] [--strict-alloc]
+            [--ca-start CA_START] [--ca-incr CA_INCR] [--build-name BUILD_NAME]
+            [--evm-dump-dir EVM_DUMP_DIR] [--forks] [--fork FORK] [--from FROM]
+            [--until UNTIL]
 
 options:
   -h, --help            show this help message and exit
@@ -191,9 +192,5 @@ Specify the fork range to generate fixtures for:
   --from FROM           Fill tests from and including the specified fork.
   --until UNTIL         Fill tests until and including the specified fork.
 
-Arguments related to running execution-spec-tests:
-  --test-help           Only show help options specific to a specific execution-
-                        spec-tests command and exit.
-
 Exit: After displaying help.
 ```

src/cli/pytest_commands/common.py

@@ -38,19 +38,19 @@ def common_click_options(func: Callable[..., Any]) -> Decorator:
     return click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED)(func)
 
 
-def handle_help_flags(
-    pytest_args: List[str], help_flag: bool, pytest_help_flag: bool
-) -> List[str]:
+def handle_help_flags(pytest_args: List[str], pytest_type: str) -> List[str]:
     """
     Modifies the help arguments passed to the click CLI command before forwarding to
     the pytest command.
 
     This is to make `--help` more useful because `pytest --help` is extremely
     verbose and lists all flags from pytest and pytest plugins.
     """
-    if help_flag:
-        return ["--test-help"]
-    elif pytest_help_flag:
+    ctx = click.get_current_context()
+
+    if ctx.params.get("help_flag"):
+        return [f"--{pytest_type}-help"] if pytest_type in {"consume", "fill"} else pytest_args
+    elif ctx.params.get("pytest_help_flag"):
         return ["--help"]
-    else:
-        return list(pytest_args)
+
+    return pytest_args

src/cli/pytest_commands/consume.py

@@ -35,26 +35,18 @@ def handle_hive_env_flags(args: List[str]) -> List[str]:
     return args
 
 
-def handle_consume_command_flags(
-    consume_args: List[str],
-    help_flag: bool,
-    pytest_help_flag: bool,
-    is_hive: bool,
-) -> List[str]:
+def handle_consume_command_flags(consume_args: List[str], is_hive: bool) -> List[str]:
     """
     Handle all consume CLI flag pre-processing.
     """
-    args = handle_help_flags(consume_args, help_flag, pytest_help_flag)
+    args = list(handle_help_flags(consume_args, pytest_type="consume"))
     args += ["-c", "pytest-consume.ini"]
     if is_hive:
-        args = handle_hive_env_flags(args)
+        args += handle_hive_env_flags(args)
         args += ["-p", "pytest_plugins.pytest_hive.pytest_hive"]
-    # Pipe input using stdin if no --input is provided and stdin is not a terminal.
-    if not any(arg.startswith("--input") for arg in args) and not sys.stdin.isatty():
-        args.extend(["-s", "--input=stdin"])
-    # Ensure stdout is captured when timing data is enabled.
-    if "--timing-data" in args and "-s" not in args:
-        args.append("-s")
+        # Ensure stdout is captured when timing data is enabled.
+        if "--timing-data" in args and "-s" not in args:
+            args.append("-s")
     return args
 
 
@@ -81,23 +73,25 @@ def consume_command(is_hive: bool = False) -> Callable[[Callable[..., Any]], cli
     """
 
     def create_command(
-        func: Callable[..., Any], command_name: str, command_paths: List[Path], is_hive: bool
+        func: Callable[..., Any],
+        command_name: str,
+        command_help: str | None,
+        command_paths: List[Path],
+        is_hive: bool,
     ) -> click.Command:
         """
         Create the command function to be decorated.
         """
 
-        @consume.command(name=command_name, context_settings=dict(ignore_unknown_options=True))
+        @consume.command(
+            name=command_name,
+            help=command_help,
+            context_settings=dict(ignore_unknown_options=True),
+        )
         @common_click_options
-        def command(pytest_args: List[str], help_flag: bool, pytest_help_flag: bool) -> None:
-            args = handle_consume_command_flags(
-                pytest_args,
-                help_flag,
-                pytest_help_flag,
-                is_hive,
-            )
+        def command(pytest_args: List[str], **kwargs) -> None:
+            args = handle_consume_command_flags(pytest_args, is_hive)
             args += [str(p) for p in command_paths]
-
             if is_hive and not any(arg.startswith("--hive-session-temp-folder") for arg in args):
                 with TemporaryDirectory() as temp_dir:
                     args.extend(["--hive-session-temp-folder", temp_dir])
@@ -106,21 +100,21 @@ def command(pytest_args: List[str], help_flag: bool, pytest_help_flag: bool) ->
                 result = pytest.main(args)
             sys.exit(result)
 
-        command.__doc__ = func.__doc__
         return command
 
     def decorator(func: Callable[..., Any]) -> click.Command:
         command_name = func.__name__
+        command_help = func.__doc__
         command_paths = get_command_paths(command_name, is_hive)
-        return create_command(func, command_name, command_paths, is_hive)
+        return create_command(func, command_name, command_help, command_paths, is_hive)
 
     return decorator
 
 
-@click.group()
+@click.group(context_settings=dict(help_option_names=["-h", "--help"]))
 def consume() -> None:
     """
-    Entry point group to aid client consumption of test fixtures.
+    Consume command to aid client consumption of test fixtures.
     """
     pass
 

src/cli/pytest_commands/fill.py

@@ -45,33 +45,25 @@ def handle_stdout_flags(args: List[str]) -> List[str]:
     return args
 
 
-def handle_fill_command_flags(
-    fill_args: List[str], help_flag: bool, pytest_help_flag: bool
-) -> List[str]:
+def handle_fill_command_flags(fill_args: List[str]) -> List[str]:
     """
     Handles all fill CLI flag pre-processing.
     """
-    args = handle_help_flags(fill_args, help_flag, pytest_help_flag)
+    args = handle_help_flags(fill_args, pytest_type="fill")
     args = handle_stdout_flags(args)
     return args
 
 
 @click.command(context_settings=dict(ignore_unknown_options=True))
 @common_click_options
-def fill(
-    pytest_args: List[str],
-    help_flag: bool,
-    pytest_help_flag: bool,
-) -> None:
+def fill(pytest_args: List[str], **kwargs) -> None:
     """
     Entry point for the fill command.
     """
     with TemporaryDirectory() as temp_dir:
         result = pytest.main(
             handle_fill_command_flags(
                 [f"--session-temp-folder={temp_dir}", "--index", *pytest_args],
-                help_flag,
-                pytest_help_flag,
             ),
         )
     sys.exit(result)

src/pytest_plugins/consume/consume.py

@@ -83,17 +83,10 @@ def pytest_addoption(parser):  # noqa: D103
         dest="fixture_source",
         default=default_input_directory(),
         help=(
-            "A URL or local directory specifying the JSON test fixtures. Default: "
-            f"'{default_input_directory()}'."
-        ),
-    )
-    consume_group.addoption(
-        "--latest",
-        action="store_true",
-        dest="latest_source",
-        default=False,
-        help=(
-            "The latest EEST development JSON test fixtures. Cannot be used alongside `--input`."
+            "Specify the JSON test fixtures source. Can be a local directory, a URL pointing to a "
+            " fixtures.tar.gz archive, or one of the special keywords: 'stdin', "
+            "'latest-stable', 'latest-develop'. "
+            f"Defaults to the following local directory: '{default_input_directory()}'."
         ),
     )
     consume_group.addoption(
@@ -103,13 +96,6 @@ def pytest_addoption(parser):  # noqa: D103
         default=None,
         help="Only consume tests for the specified fork.",
     )
-    consume_group.addoption(
-        "--timing-data",
-        action="store_true",
-        dest="timing_data",
-        default=False,
-        help="Log the timing data for each test case execution.",
-    )
     consume_group.addoption(
         "--no-html",
         action="store_true",
@@ -133,20 +119,17 @@ def pytest_configure(config):  # noqa: D103
     it uses the modified `htmlpath` option.
     """
     input_flag = any(arg.startswith("--input") for arg in config.invocation_params.args)
-    latest_flag = config.getoption("latest_source")
-
-    if input_flag and latest_flag:
-        pytest.exit("Cannot use both `--input` and `--latest`, please select one input flag.")
-
     input_source = config.getoption("fixture_source")
 
     if input_flag and input_source == "stdin":
         config.test_cases = TestCases.from_stream(sys.stdin)
         return
 
-    if latest_flag:
-        release_base_url = "https://github.com/ethereum/execution-spec-tests/releases"
-        input_source = f"{release_base_url}/latest/download/fixtures_develop.tar.gz"
+    latest_base_url = "https://github.com/ethereum/execution-spec-tests/releases/latest/download"
+    if input_source == "latest-stable-release" or input_source == "latest-stable":
+        input_source = f"{latest_base_url}/fixtures_stable.tar.gz"
+    if input_source == "latest-develop-release" or input_source == "latest-develop":
+        input_source = f"{latest_base_url}/fixtures_develop.tar.gz"
 
     if is_url(input_source):
         cached_downloads_directory.mkdir(parents=True, exist_ok=True)
@@ -162,6 +145,7 @@ def pytest_configure(config):  # noqa: D103
         )
 
     index_file = input_source / ".meta" / "index.json"
+    index_file.parent.mkdir(parents=True, exist_ok=True)
     if not index_file.exists():
         rich.print(f"Generating index file [bold cyan]{index_file}[/]...")
         generate_fixtures_index(

src/pytest_plugins/consume/hive_simulators/conftest.py

@@ -20,6 +20,22 @@
 from .timing import TimingData
 
 
+def pytest_addoption(parser):
+    """
+    Hive simulator specific consume command line options.
+    """
+    consume_group = parser.getgroup(
+        "consume", "Arguments related to consuming fixtures via a client"
+    )
+    consume_group.addoption(
+        "--timing-data",
+        action="store_true",
+        dest="timing_data",
+        default=False,
+        help="Log the timing data for each test case execution.",
+    )
+
+
 @pytest.fixture(scope="function")
 def eth_rpc(client: Client) -> EthRPC:
     """
@@ -55,7 +71,10 @@ def eest_consume_commands(
     Commands to run the test within EEST using a hive dev back-end.
     """
     hive_dev = f"./hive --dev --client-file configs/develop.yaml --client {client_type.name}"
-    consume = f'consume {test_suite_name.split("-")[-1]} -v --latest -k "{test_case.id}"'
+    consume = (
+        f'consume {test_suite_name.split("-")[-1]} -v --input latest-develop-release -k '
+        f'"{test_case.id}"'
+    )
     return [hive_dev, consume]