Rename testdata.yaml to test_group.yaml

Also took the time to replace some occurrences of test{case,group,node} with test{ ,_}{case,group,node}.
Verified by grepping `(?<![Cc]heck)testdata` that the phrase "testdata" no longer occurs, except in upgrade.py.

Author: mpsijm

bin/export.py

@@ -213,7 +213,7 @@ def add_testcase(in_file: Path) -> None:
     # substitute constants.
     if problem.settings.constants:
         constants_supported = [
-            "data/**/testdata.yaml",
+            "data/**/test_group.yaml",
             f"{InputValidator.source_dir}/**/*",
             f"{AnswerValidator.source_dir}/**/*",
             f"{OutputValidator.source_dir}/**/*",
@@ -298,7 +298,7 @@ def add_testcase(in_file: Path) -> None:
             ryaml_filter(limits, "time_limit")
         # validator_flags
         validator_flags = " ".join(
-            problem.get_testdata_yaml(
+            problem.get_test_group_yaml(
                 problem.path / "data",
                 OutputValidator.args_key,
                 PrintBar("Getting validator_flags for legacy export"),
@@ -325,13 +325,6 @@ def add_testcase(in_file: Path) -> None:
                     else:
                         util.error(f"{f}: no name set for language {lang}.")
 
-        # rename output_validator dir
-        if (export_dir / OutputValidator.source_dir).exists():
-            (export_dir / "output_validators").mkdir(parents=True)
-            (export_dir / OutputValidator.source_dir).rename(
-                export_dir / "output_validators" / OutputValidator.source_dir
-            )
-
         # rename statement dirs
         if (export_dir / "statement").exists():
             (export_dir / "statement").rename(export_dir / "problem_statement")
@@ -352,6 +345,18 @@ def add_testcase(in_file: Path) -> None:
                         add_file(out, f)
             shutil.rmtree(export_dir / d)
 
+        # rename output_validator dir
+        if (export_dir / OutputValidator.source_dir).exists():
+            (export_dir / "output_validators").mkdir(parents=True)
+            (export_dir / OutputValidator.source_dir).rename(
+                export_dir / "output_validators" / OutputValidator.source_dir
+            )
+
+        # rename test_group.yaml back to testdata.yaml
+        for f in (export_dir / "data").rglob("test_group.yaml"):
+            f.rename(f.with_name("testdata.yaml"))
+            # TODO potentially, some keys also need to be renamed, but we don't use this often enough for this to matter (I hope)
+
     # handle yaml updates
     yaml_path.unlink()
     write_yaml(yaml_data, yaml_path)

bin/generate.py

@@ -56,7 +56,7 @@ def get_validator_command():
                 run.testcase.ans_path.resolve(),
                 run.feedbackdir.resolve(),
             ]
-            + run.testcase.testdata_yaml_args(
+            + run.testcase.test_group_yaml_args(
                 output_validator,
                 bar or PrintBar("Run interactive test case"),
             )

bin/interactive.py

@@ -283,7 +283,9 @@ def __init__(
         self.limits = ProblemLimits(parse_setting(yaml_data, "limits", {}), problem, self)
 
         parse_deprecated_setting(
-            yaml_data, "validator_flags", f"{validate.OutputValidator.args_key}' in 'testdata.yaml"
+            yaml_data,
+            "validator_flags",
+            f"{validate.OutputValidator.args_key}' in 'test_group.yaml",
         )
 
         self.keywords: list[str] = parse_optional_list_setting(yaml_data, "keywords", str)
@@ -362,9 +364,9 @@ def __init__(self, path: Path, tmpdir: Path, label: Optional[str] = None):
         self._programs = dict[Path, "Program"]()
         self._program_callbacks = dict[Path, list[Callable[["Program"], None]]]()
         # Dictionary from path to parsed file contents.
-        # TODO #102: Add type for testdata.yaml (typed Namespace?)
-        self._testdata_yamls = dict[Path, dict[str, Any]]()
-        self._testdata_lock = threading.Lock()
+        # TODO #102: Add type for test_group.yaml (typed Namespace?)
+        self._test_group_yamls = dict[Path, dict[str, Any]]()
+        self._test_group_lock = threading.Lock()
 
         # The label for the problem: A, B, A1, A2, X, ...
         self.label = label
@@ -457,25 +459,25 @@ def _read_settings(self):
         self.multi_pass: bool = self.settings.multi_pass
         self.custom_output: bool = self.settings.custom_output
 
-    # TODO #102 move to TestData class
-    def _parse_testdata_yaml(p, path, bar):
+    # TODO #102 move to a new TestGroup class
+    def _parse_test_group_yaml(p, path, bar):
         assert path.is_relative_to(p.path / "data")
         for dir in [path] + list(path.parents):
             # Do not go above the data directory.
             if dir == p.path:
                 return
 
-            f = dir / "testdata.yaml"
-            if not f.is_file() or f in p._testdata_yamls:
+            f = dir / "test_group.yaml"
+            if not f.is_file() or f in p._test_group_yamls:
                 continue
-            with p._testdata_lock:
-                if f not in p._testdata_yamls:
+            with p._test_group_lock:
+                if f not in p._test_group_yamls:
                     raw = substitute(
                         f.read_text(),
                         p.settings.constants,
                         pattern=config.CONSTANT_SUBSTITUTE_REGEX,
                     )
-                    p._testdata_yamls[f] = flags = parse_yaml(raw, path=f, plain=True)
+                    p._test_group_yamls[f] = flags = parse_yaml(raw, path=f, plain=True)
 
                     parse_deprecated_setting(
                         flags, "output_validator_flags", validate.OutputValidator.args_key
@@ -484,7 +486,7 @@ def _parse_testdata_yaml(p, path, bar):
                         flags, "input_validator_flags", validate.InputValidator.args_key
                     )
 
-                    # Verify testdata.yaml
+                    # Verify test_group.yaml
                     for k in flags:
                         match k:
                             case (
@@ -524,7 +526,7 @@ def _parse_testdata_yaml(p, path, bar):
                                 | "static_validation"
                             ):
                                 bar.warn(
-                                    f"{k} in testdata.yaml not implemented in BAPCtools",
+                                    f"{k} in test_group.yaml not implemented in BAPCtools",
                                     print_item=False,
                                 )
                             case _:
@@ -534,28 +536,28 @@ def _parse_testdata_yaml(p, path, bar):
             if dir == p.path / "data":
                 break
 
-    def get_testdata_yaml(
+    def get_test_group_yaml(
         p,
         path: Path,
         key: str,
         bar: BAR_TYPE,
         name: Optional[str] = None,
     ) -> list[str]:
         """
-        Find the testdata flags applying at the given path for the given key.
-        If necessary, walk up from `path` looking for the first testdata.yaml file that applies,
+        Find the value of the given test_group.yaml key applying at the given path.
+        If necessary, walk up from `path` looking for the first test_group.yaml file that applies.
 
         Side effects: parses and caches the file.
 
         Arguments
         ---------
         path: absolute path (a file or a directory)
-        key: The testdata.yaml key to look for (TODO: 'grading' is not yet implemented)
+        key: The test_group.yaml key to look for (TODO: 'grading' is not yet implemented)
         name: If key == 'input_validator_args', optionally the name of the input validator.
 
         Returns:
         --------
-        A list of string arguments, which is empty if no testdata.yaml is found.
+        A list of string arguments, which is empty if no test_group.yaml is found.
         TODO: when 'grading' is supported, it also can return dict
         """
         known_args_keys = [
@@ -572,19 +574,19 @@ def get_testdata_yaml(
                 f"Only input validators support flags by validator name, got {key} and {name}"
             )
 
-        # parse and cache testdata.yaml
-        p._parse_testdata_yaml(path, bar)
+        # parse and cache test_group.yaml
+        p._parse_test_group_yaml(path, bar)
 
         # extract the flags
         for dir in [path] + list(path.parents):
             # Do not go above the data directory.
             if dir == p.path:
                 return []
 
-            f = dir / "testdata.yaml"
-            if f not in p._testdata_yamls:
+            f = dir / "test_group.yaml"
+            if f not in p._test_group_yamls:
                 continue
-            flags = p._testdata_yamls[f]
+            flags = p._test_group_yamls[f]
             if key in flags:
                 args = flags[key]
                 if key == validate.InputValidator.args_key:
@@ -1331,7 +1333,7 @@ def validate_valid_extra_data(p) -> bool:
         if not p.validators(validate.OutputValidator, strict=True, print_warn=False):
             return True
 
-        args = p.get_testdata_yaml(
+        args = p.get_test_group_yaml(
             p.path / "data" / "valid_output",
             "output_validator_args",
             PrintBar("Generic Output Validation"),
@@ -1492,7 +1494,7 @@ def run_all(select_verdict, select):
                 return None, None, None
 
             def get_slowest(result):
-                slowest_pair = result.slowest_testcase()
+                slowest_pair = result.slowest_test_case()
                 assert slowest_pair is not None
                 return slowest_pair
 

bin/problem.py

@@ -228,7 +228,7 @@ def _validate_output(self, bar: BAR_TYPE) -> Optional[ExecResult]:
         return output_validator.run(
             self.testcase,
             self,
-            args=self.testcase.testdata_yaml_args(output_validator, bar),
+            args=self.testcase.test_group_yaml_args(output_validator, bar),
         )
 
     def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]:
@@ -242,7 +242,7 @@ def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]:
             self.testcase.ans_path.resolve(),
             self.out_path if not self.problem.interactive else None,
             self.feedbackdir,
-            args=self.testcase.testdata_yaml_args(output_visualizer, bar),
+            args=self.testcase.test_group_yaml_args(output_visualizer, bar),
         )
 
 
@@ -501,15 +501,15 @@ def process_run(run: Run):
         else:
             color = Fore.GREEN if self.verdict in self.expected_verdicts else Fore.RED
 
-        (salient_testcase, salient_duration) = verdicts.salient_testcase()
+        (salient_testcase, salient_duration) = verdicts.salient_test_case()
         salient_print_verdict = self.verdict
         salient_duration_style = Style.BRIGHT if salient_duration >= self.limits["timeout"] else ""
 
         # Summary line is the only thing shown.
         message = f"{color}{salient_print_verdict.short():>3}{salient_duration_style}{salient_duration:6.3f}s{Style.RESET_ALL} {Style.DIM}@ {salient_testcase:{max_testcase_len}}{Style.RESET_ALL}"
 
         if verdicts.run_until in [RunUntil.DURATION, RunUntil.ALL]:
-            slowest_pair = verdicts.slowest_testcase()
+            slowest_pair = verdicts.slowest_test_case()
             assert slowest_pair is not None
             (slowest_testcase, slowest_duration) = slowest_pair
             slowest_verdict = verdicts[slowest_testcase]

bin/run.py

@@ -122,7 +122,7 @@ def new_problem() -> None:
         "author": author,
         "type": problem_type,
         OutputValidator.args_key: output_validator_args,
-        "testdata_yaml_comment": "#" if output_validator_args[0] == "#" else "",
+        "test_group_yaml_comment": "#" if output_validator_args[0] == "#" else "",
     }
 
     source_name = ask_variable_string(

bin/skel.py

@@ -22,6 +22,7 @@
     import problem
 
 
+# TODO #102: Consistently separate the compound noun "test case", e.g. "TestCase" or "test_case"
 class Testcase:
     """
     A single test case. It consists of files with matching base names, typically
@@ -59,8 +60,8 @@ class Testcase:
     ans_path: Path
         Like `hamiltonicity/data/secret/cubic/petersen.ans`.
 
-    testdata_yaml: dict
-        The YAML-parsed test data flags that apply to this test case.
+    out_path: Path
+        Like `hamiltonicity/data/secret/cubic/petersen.out`.
 
     """
 
@@ -115,13 +116,13 @@ def __repr__(self) -> str:
     def with_suffix(self, ext: str) -> Path:
         return self.in_path.with_suffix(ext)
 
-    def testdata_yaml_args(
+    def test_group_yaml_args(
         self,
         program: "validate.AnyValidator | visualize.AnyVisualizer",
         bar: BAR_TYPE,
     ) -> list[str]:
         """
-        The flags specified in testdata.yaml for the given validator applying to this testcase.
+        The flags specified in test_group.yaml for the given validator applying to this testcase.
 
         Returns
         -------
@@ -131,7 +132,7 @@ def testdata_yaml_args(
         """
 
         path = self.problem.path / "data" / self.short_path
-        return self.problem.get_testdata_yaml(
+        return self.problem.get_test_group_yaml(
             path,
             type(program).args_key,
             bar,
@@ -156,7 +157,7 @@ def validator_hashes(
         d = dict()
 
         for validator in validators:
-            flags = self.testdata_yaml_args(validator, bar)
+            flags = self.test_group_yaml_args(validator, bar)
             flags_string = " ".join(flags)
             h = combine_hashes_dict(
                 {
@@ -287,7 +288,7 @@ def _run_validators(
             if isinstance(validator, validate.OutputValidator) and mode == validate.Mode.ANSWER:
                 args += ["case_sensitive", "space_change_sensitive"]
                 name = f"{name} (ans)"
-            flags = self.testdata_yaml_args(validator, bar)
+            flags = self.test_group_yaml_args(validator, bar)
             flags = flags + args
 
             ret = validator.run(self, mode=mode, constraints=constraints, args=flags)