diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/dumper.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/dumper.py index f8ec386d2b..c68b71cfd0 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/dumper.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/dumper.py @@ -70,6 +70,21 @@ def dump_ssz(self, test_case: TestCase, name: str, data: bytes) -> None: with path.open("wb") as f: f.write(compress(data)) + def dump_manifest(self, test_case: TestCase) -> None: + """Write manifest.yml file containing test case metadata.""" + manifest_data = { + 'config_name': test_case.preset_name, + 'fork_name': test_case.fork_name, + 'runner_name': test_case.runner_name, + 'handler_name': test_case.handler_name, + 'suite_name': test_case.suite_name, + 'case_name': test_case.case_name, + } + # Use cfg_yaml which has block style formatting (default_flow_style=False) + # This ensures each field appears on a separate line, matching data.yaml format + self._dump_yaml(test_case, "manifest", manifest_data, self.cfg_yaml) + + def _dump_yaml(self, test_case: TestCase, name: str, data: any, yaml_encoder: YAML) -> None: """Helper to write YAML files for test case.""" path = test_case.dir / f"{name}.yaml" diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 46512a25b3..a3ec19d4c5 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -109,6 +109,9 @@ def execute_test(test_case: TestCase, dumper: Dumper): if meta: dumper.dump_meta(test_case, meta) + # Always write manifest.yml for every test case + dumper.dump_manifest(test_case) + def run_generator(input_test_cases: Iterable[TestCase], args=None): start_time = time.time() diff --git a/tests/formats/README.md b/tests/formats/README.md index c7a11b19af..7c4f56aeb5 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -22,6 +22,7 @@ testing. - [Common output formats](#common-output-formats) - [Special output parts](#special-output-parts) - [`meta.yaml`](#metayaml) + - [`manifest.yaml`](#manifestyaml) - [`config.yaml`](#configyaml) - [Config sourcing](#config-sourcing) - [Note for implementers](#note-for-implementers) @@ -213,6 +214,28 @@ bls_setting: int -- optional, can have 3 different values: 2: known as "BLS ignored" - if the test validity is strictly dependent on BLS being OFF ``` +##### `manifest.yaml` + +Included in every test case. Contains metadata that identifies the test vector: + +```yaml +config_name: minimal # Configuration preset (mainnet, minimal, general) +fork_name: phase0 # Fork/phase name +runner_name: bls # Test runner category +handler_name: eth_aggregate_pubkeys # Specific handler +suite_name: bls # Test suite name +case_name: eth_aggregate_pubkeys_valid_0 # Individual test case name +``` + +This metadata duplicates what is already encoded in the directory path: + +``` +tests/////// +``` + +Having it in a file means test vectors can be moved or distributed without +losing context. + ##### `config.yaml` The runtime-configurables may be different for specific tests. When present, @@ -247,13 +270,13 @@ The basic pattern for test-suite loading and running is: 1. For a specific config, load it first (and only need to do so once), then continue with the tests defined in the config folder. -2. Select a fork. Repeat for each fork if running tests for multiple forks. -3. Select the category and specialization of interest (e.g. +1. Select a fork. Repeat for each fork if running tests for multiple forks. +1. Select the category and specialization of interest (e.g. `operations > deposits`). Again, repeat for each if running all. -4. Select a test suite. Or repeat for each. -5. Select a test case. Or repeat for each. -6. Load the parts of the case. And `meta.yaml` if present. -7. Run the test, as defined by the test format. +1. Select a test suite. Or repeat for each. +1. Select a test case. Or repeat for each. +1. Load the parts of the case. `manifest.yaml` and `meta.yaml` if present. +1. Run the test, as defined by the test format. Step 1 may be a step with compile time selection of a configuration, if desired for optimization. The base requirement is just to use the same set of constants,