From fdb99353fcf00fe9d4c7af64010b4ee70cee096d Mon Sep 17 00:00:00 2001 From: mzuenni Date: Thu, 20 Nov 2025 02:49:24 +0100 Subject: [PATCH 1/7] implemented test group class --- bin/export.py | 3 +- bin/generate.py | 9 +- bin/interactive.py | 7 +- bin/problem.py | 261 ++++++++++++++++++++++++--------------------- bin/run.py | 4 +- bin/testcase.py | 33 ++---- bin/util.py | 40 ++++--- 7 files changed, 183 insertions(+), 174 deletions(-) diff --git a/bin/export.py b/bin/export.py index 3d7bb03d..a13f434c 100644 --- a/bin/export.py +++ b/bin/export.py @@ -330,9 +330,8 @@ def add_testcase(in_file: Path) -> None: validator_flags = " ".join( problem.get_test_case_yaml( problem.path / "data", - OutputValidator.args_key, PrintBar("Zip", item="Getting validator_flags for legacy export"), - ) + ).output_validator_args ) if validator_flags: yaml_data["validator_flags"] = validator_flags diff --git a/bin/generate.py b/bin/generate.py index 39cfda18..80e64ac5 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -391,10 +391,8 @@ def __init__( self.random_salt: str = "" self.retries: int = 1 else: - self.needs_default_solution = parent_config.needs_default_solution - self.solution = parent_config.solution - self.random_salt = parent_config.random_salt - self.retries = parent_config.retries + for key, value in vars(parent_config).items(): + setattr(self, key, value) if yaml is not None: if "solution" in yaml: @@ -1098,10 +1096,12 @@ def use_feedback_image(feedbackdir: Path, source: str) -> None: visualize.InputVisualizer ) output_visualizer = problem.visualizer(visualize.OutputVisualizer) + visualizer_args = testcase.get_test_case_yaml(bar).input_visualizer_args if output_visualizer is not None: if out_path.is_file() or problem.settings.ans_is_output: if visualizer is None or out_path.is_file(): visualizer = output_visualizer + visualizer_args = testcase.get_test_case_yaml(bar).output_visualizer_args if not out_path.is_file(): assert problem.settings.ans_is_output out_path = ans_path @@ -1112,7 +1112,6 @@ def use_feedback_image(feedbackdir: Path, source: str) -> None: use_feedback_image(feedbackdir, "validator") return True - visualizer_args = testcase.test_case_yaml_args(visualizer, bar) visualizer_hash: dict[object, object] = { "visualizer_hash": visualizer.hash, "visualizer_args": visualizer_args, diff --git a/bin/interactive.py b/bin/interactive.py index cd1753c7..58866310 100644 --- a/bin/interactive.py +++ b/bin/interactive.py @@ -67,10 +67,9 @@ def get_validator_command() -> Sequence[str | Path]: run.in_path.absolute(), run.testcase.ans_path.absolute(), run.feedbackdir.absolute(), - *run.testcase.test_case_yaml_args( - output_validator, - bar or PrintBar("Run interactive test case"), - ), + *run.testcase.get_test_case_yaml( + bar or PrintBar("Run interactive test case") + ).output_validator_args, ] assert run.submission.run_command, "Submission must be built" diff --git a/bin/problem.py b/bin/problem.py index 07995c27..8127b9dc 100644 --- a/bin/problem.py +++ b/bin/problem.py @@ -2,7 +2,7 @@ import re import shutil import threading -from collections.abc import Callable, Sequence +from collections.abc import Callable, Mapping, Sequence from colorama import Fore, Style from pathlib import Path from typing import Final, Literal, Optional, overload, TYPE_CHECKING @@ -370,6 +370,104 @@ def type_name(self) -> str: return " ".join(parts) +class TestGroup: + def __init__( + self, + problem: "Problem", + file: Optional[Path], + yaml_data: object, + parent: Optional["TestGroup"], + bar: BAR_TYPE, + ) -> None: + if parent is None: + self.args: Sequence[str] = [] + self.output_visualizer_args: Sequence[str] = [] + self.output_validator_args: Sequence[str] = [] + self.input_visualizer_args: Sequence[str] = [] + self.input_validator_args: Sequence[str] | Mapping[str, Sequence[str]] = [] + + # not implemented: + # self.full_feedback: bool + # self.hint: str + # self.description: str + # self.max_score: int | "unbounded" + # self.score_aggregation: "pass-fail" | "sum" | "min" + # self.static_validation_score: int | "pass-fail" + # self.require_pass: str | Sequence[str] + else: + for key, value in vars(parent).items(): + setattr(self, key, value) + + self.file = file + + if not isinstance(yaml_data, dict): + bar.error(f"could not parse {file}. SKIPPED.") + return + + parser = YamlParser(str(file) if file else "default test_group.yaml", yaml_data, bar=bar) + + # parse deprecated keys + parser.extract_deprecated("output_validator_flags", validate.OutputValidator.args_key) + parser.extract_deprecated("input_validator_flags", validate.InputValidator.args_key) + + # parse args + self.args = parser.extract_optional_list("args", str, allow_value=False) + self.output_visualizer_args = parser.extract_optional_list( + visualize.OutputVisualizer.args_key, str, allow_value=False + ) + self.output_validator_args = parser.extract_optional_list( + validate.OutputValidator.args_key, str, allow_value=False + ) + self.input_visualizer_args = parser.extract_optional_list( + visualize.InputVisualizer.args_key, str, allow_value=False + ) + assert validate.OutputValidator.args_key == validate.AnswerValidator.args_key + + if validate.InputValidator.args_key in parser.yaml: + if isinstance(parser.yaml[validate.InputValidator.args_key], list): + self.input_validator_args = parser.extract_optional_list( + validate.InputValidator.args_key, str, allow_value=False + ) + elif isinstance(value, dict): + # only the hole dict is inherited not individual entries + validator_args_parser = parser.extract_parser(validate.InputValidator.args_key) + self.input_validator_args = {} + for val in problem.validators(validate.InputValidator): + self.input_validator_args[val.name] = parser.extract_optional_list( + val.name, str, allow_value=False + ) + validator_args_parser.check_unknown_keys() + elif value is None: + self.input_validator_args = [] + else: + bar.warn( + f"incompatible value for key `{validate.InputValidator.args_key}` in {parser.source}. SKIPPED." + ) + + # parse keys not currently used + parser.extract_optional("full_feedback", bool) + parser.extract_optional("hint", str) + parser.extract_optional("description", str) + + # check test group only keys + if file is None or not file.with_suffix(".in").is_file(): + for key in ["max_score", "score_aggregation", "static_validation_score"]: + if parser.pop(key) is not None: + bar.error( + f"key `{key}` not supported by BAPCtools in {parser.source}. SKIPPED." + ) + + parser.check_unknown_keys() + + def get_args(self, program: validate.AnyValidator | visualize.AnyVisualizer) -> Sequence[str]: + assert hasattr(self, type(program).args_key) + args = getattr(self, type(program).args_key) + if isinstance(args, dict): + args = args.get(program.name, []) + assert isinstance(args, list) + return args + + # A problem. class Problem: _SHORTNAME_REGEX_STRING: Final[str] = "[a-z0-9]{2,255}" @@ -400,8 +498,8 @@ def __init__(self, path: Path, tmpdir: Path, label: Optional[str] = None): self._programs = dict[Path, "Program"]() self._program_callbacks = dict[Path, list[Callable[["Program"], None]]]() # Dictionary from path to parsed file contents. - # TODO #102: Add type for test_group.yaml (typed Namespace?) - self._test_case_yamls = dict[Path, dict[object, object]]() + self._root_test_case_yaml: Optional[TestGroup] = None + self._test_case_yamls = dict[Path, TestGroup]() self._test_group_lock = threading.Lock() # The label for the problem: A, B, A1, A2, X, ... @@ -498,81 +596,31 @@ def _read_settings(self) -> None: self.multi_pass: bool = self.settings.multi_pass self.custom_output: bool = self.settings.custom_output - # TODO #102 move to a new TestGroup class - def _parse_test_case_and_groups_yaml(p, path: Path, bar: BAR_TYPE) -> None: - assert path.is_relative_to(p.path / "data"), f"{path} is not in data" - for f in [path] + list(path.parents): - # Do not go above the data directory. - if f == p.path: - return + def _parse_test_case_yaml(p, file: Path, parent: TestGroup, bar: BAR_TYPE) -> None: + assert file.is_file() + assert file.is_relative_to(p.path / "data"), f"{file} is not in data" - if f.is_dir(): - f = f / "test_group.yaml" - with p._test_group_lock: - if not f.is_file() or f in p._test_case_yamls: - continue - raw = substitute( - f.read_text(), - p.settings.constants, - pattern=config.CONSTANT_SUBSTITUTE_REGEX, - ) - flags = parse_yaml(raw, path=f, plain=True) - if not isinstance(flags, dict): - bar.error(f"could not parse {f}. SKIPPED.") - continue - p._test_case_yamls[f] = flags + with p._test_group_lock: + # handle race conditions + if file in p._test_case_yamls: + return - parser = YamlParser(str(f), flags) - parser.extract_deprecated( - "output_validator_flags", validate.OutputValidator.args_key - ) - parser.extract_deprecated("input_validator_flags", validate.InputValidator.args_key) - - # Verify test_group.yaml - for k in flags: - match k: - case ( - validate.OutputValidator.args_key - | validate.AnswerValidator.args_key - | visualize.InputVisualizer.args_key - | visualize.OutputVisualizer.args_key - ): - if not isinstance(flags[k], list): - bar.error( - f"{k} must be a list of strings", resume=True, print_item=False - ) - case validate.InputValidator.args_key: - if not isinstance(flags[k], (list, dict)): - bar.error(f"{k} must be list or map", resume=True, print_item=False) - if isinstance(flags[k], dict): - input_validator_names = set( - val.name for val in p.validators(validate.InputValidator) - ) - for name in set(flags[k]) - input_validator_names: - bar.warn( - f"Unknown input validator {name}; expected {input_validator_names}", - print_item=False, - ) - case "description" | "hint": - pass # We don't do anything with hint or description in BAPCtools, but no need to warn about this - case "args" | "full_feedback" | "scoring" | "static_validation": - bar.warn( - f"{k} in test_group.yaml not implemented in BAPCtools", - print_item=False, - ) - case _: - path = f.relative_to(p.path / "data") - bar.warn(f'Unknown key "{k}" in {path}', print_item=False) + # substitute constants + raw = substitute( + file.read_text(), + p.settings.constants, + pattern=config.CONSTANT_SUBSTITUTE_REGEX, + ) + yaml_data = parse_yaml(raw, path=file, plain=True) + p._test_case_yamls[file] = TestGroup(p, file, yaml_data, parent, bar) def get_test_case_yaml( p, path: Path, - key: str, bar: BAR_TYPE, - name: Optional[str] = None, - ) -> list[str]: + ) -> TestGroup: """ - Find the value of the given test_group.yaml key applying at the given path. + Find the test_group.yaml for the given path. If necessary, walk up from `path` looking for the first test_group.yaml file that applies. Side effects: parses and caches the file. @@ -580,68 +628,36 @@ def get_test_case_yaml( Arguments --------- path: absolute path (a .yaml file or a test group directory) - key: The test_group.yaml key to look for (TODO: 'grading' is not yet implemented) - name: If key == 'input_validator_args', optionally the name of the input validator. Returns: -------- - A list of string arguments, which is empty if no test_group.yaml is found. - TODO: when 'grading' is supported, it also can return dict + A TestGroup object """ - known_args_keys = [ - validate.InputValidator.args_key, - validate.OutputValidator.args_key, - validate.AnswerValidator.args_key, - visualize.InputVisualizer.args_key, - visualize.OutputVisualizer.args_key, - ] - if key not in known_args_keys: - raise NotImplementedError(key) - if key != validate.InputValidator.args_key and name is not None: - raise ValueError( - f"Only input validators support flags by validator name, got {key} and {name}" - ) - # parse and cache .yaml and test_group.yaml - p._parse_test_case_and_groups_yaml(path, bar) - - # extract the flags - for f in [path] + list(path.parents): + paths = [] + for f in [path, *path.parents]: # Do not go above the data directory. if f == p.path: - return [] + break + paths.append(f) + # create a root TestGroup object + if p._root_test_case_yaml is None: + with p._test_group_lock: + if p._root_test_case_yaml is None: + p._root_test_case_yaml = TestGroup(p, None, {}, None, bar) + + test_group_yaml = p._root_test_case_yaml + for f in reversed(paths): if f.is_dir(): f = f / "test_group.yaml" - if f not in p._test_case_yamls: + if not f.is_file(): continue - flags = p._test_case_yamls[f] - if key in flags: - args = flags[key] - if key == validate.InputValidator.args_key: - if not isinstance(args, (list, dict)): - bar.error(f"{key} must be list of strings or map of lists") - return [] - if isinstance(args, list): - if any(not isinstance(arg, str) for arg in args): - bar.error(f"{key} must be list of strings or map of lists") - return [] - return args - elif name in args: - args = args[name] - if not isinstance(args, list) or any( - not isinstance(arg, str) for arg in args - ): - bar.error(f"{key} must be list of strings or map of lists") - return [] - return args - elif key in known_args_keys: - if not isinstance(args, list) or any(not isinstance(arg, str) for arg in args): - bar.error(f"{key} must be a list of strings") - return [] - return args - - return [] + if f not in p._test_case_yamls: + p._parse_test_case_yaml(f, test_group_yaml, bar) + assert f in p._test_case_yamls + test_group_yaml = p._test_case_yamls[f] + return test_group_yaml # Because Problem.testcases() may be called multiple times (e.g. validating multiple modes, or with `bt all`), # this cache makes sure that some warnings (like malformed test case names) only appear once. @@ -1434,9 +1450,8 @@ def validate_valid_extra_data(p) -> bool: args = p.get_test_case_yaml( p.path / "data" / "valid_output", - "output_validator_args", PrintBar("Generic Output Validation"), - ) + ).output_validator_args is_space_sensitive = "space_change_sensitive" in args is_case_sensitive = "case_sensitive" in args diff --git a/bin/run.py b/bin/run.py index eb4f2956..f669689b 100644 --- a/bin/run.py +++ b/bin/run.py @@ -244,7 +244,7 @@ def _validate_output(self, bar: ProgressBar) -> Optional[ExecResult]: return output_validator.run( self.testcase, self, - args=self.testcase.test_case_yaml_args(output_validator, bar), + args=self.testcase.get_test_case_yaml(bar).output_validator_args, ) def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]: @@ -258,7 +258,7 @@ def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]: self.testcase.ans_path.absolute(), self.out_path if not self.problem.interactive else None, self.feedbackdir, - args=self.testcase.test_case_yaml_args(output_visualizer, bar), + args=self.testcase.get_test_case_yaml(bar).output_visualizer_args, ) diff --git a/bin/testcase.py b/bin/testcase.py index 294a9f54..4b5fa14e 100644 --- a/bin/testcase.py +++ b/bin/testcase.py @@ -19,7 +19,6 @@ if TYPE_CHECKING: # Prevent circular import: https://stackoverflow.com/a/39757388 import problem - import visualize # TODO #102: Consistently separate the compound noun "test case", e.g. "TestCase" or "test_case" @@ -115,26 +114,11 @@ def __repr__(self) -> str: def with_suffix(self, ext: str) -> Path: return self.in_path.with_suffix(ext) - def test_case_yaml_args( - self, - program: "validate.AnyValidator | visualize.AnyVisualizer", - bar: BAR_TYPE, - ) -> list[str]: - """ - The flags specified in test_group.yaml for the given validator applying to this testcase. - - Returns - ------- - - A nonempty list of strings, such as ["space_change_sensitive", "case_sensitive"] - or ["--max_N", "50"] or even [""]. - """ - + def get_test_case_yaml(self, bar: BAR_TYPE) -> "problem.TestGroup": + assert self.in_path.is_file() return self.problem.get_test_case_yaml( self.problem.path / "data" / self.short_path.with_suffix(".yaml"), - type(program).args_key, bar, - name=program.name if isinstance(program, validate.InputValidator) else None, ) def validator_hashes( @@ -155,7 +139,7 @@ def validator_hashes( d = dict() for validator in validators: - flags = self.test_case_yaml_args(validator, bar) + flags = self.get_test_case_yaml(bar).get_args(validator) flags_string = " ".join(flags) h = combine_hashes_dict( { @@ -280,23 +264,22 @@ def _run_validators( constraints: Optional[validate.ConstraintsDict] = None, warn_instead_of_error: bool = False, ) -> bool: - args = [] results = [] output_validator_crash = False for validator in validators: name = validator.name + args = [] if isinstance(validator, validate.OutputValidator) and mode == validate.Mode.ANSWER: args += ["case_sensitive", "space_change_sensitive"] name = f"{name} (ans)" - flags = self.test_case_yaml_args(validator, bar) - flags = flags + args + args = [*args, *self.get_test_case_yaml(bar).get_args(validator)] - ret = validator.run(self, mode=mode, constraints=constraints, args=flags) + ret = validator.run(self, mode=mode, constraints=constraints, args=args) results.append(ret.status) message = name - if flags: - message += " [" + ", ".join(flags) + "]" + if args: + message += " [" + ", ".join(args) + "]" message += ": " if ret.status: message += "accepted" diff --git a/bin/util.py b/bin/util.py index 2add520c..364e9bf2 100644 --- a/bin/util.py +++ b/bin/util.py @@ -772,13 +772,20 @@ def normalize_yaml_value(value: object, t: type[object]) -> object: class YamlParser: T = TypeVar("T") - def __init__(self, source: str, yaml: dict[object, object], parent_path: Optional[str] = None): + def __init__( + self, + source: str, + yaml: dict[object, object], + parent_path: Optional[str] = None, + bar: BAR_TYPE = PrintBar(), + ): assert isinstance(yaml, dict) self.errors = 0 self.source = source self.yaml = yaml self.parent_path = parent_path self.parent_str = "root" if parent_path is None else f"`{parent_path}`" + self.bar = bar def _key_path(self, key: str) -> str: return key if self.parent_path is None else f"{self.parent_path}.{key}" @@ -786,16 +793,21 @@ def _key_path(self, key: str) -> str: def check_unknown_keys(self) -> None: for key in self.yaml: if not isinstance(key, str): - warn(f"invalid {self.source} key: {key} in {self.parent_str}") + self.bar.warn(f"invalid {self.source} key: {key} in {self.parent_str}") else: - warn(f"found unknown {self.source} key: {key} in {self.parent_str}") + self.bar.warn(f"found unknown {self.source} key: {key} in {self.parent_str}") + + def pop(self, key: str) -> object: + return self.yaml.pop(key, None) def extract_optional(self, key: str, t: type[T]) -> Optional[T]: if key in self.yaml: value = normalize_yaml_value(self.yaml.pop(key), t) if value is None or isinstance(value, t): return value - warn(f"incompatible value for key `{self._key_path(key)}` in {self.source}. SKIPPED.") + self.bar.warn( + f"incompatible value for key `{self._key_path(key)}` in {self.source}. SKIPPED." + ) return None def extract(self, key: str, default: T, constraint: Optional[str] = None) -> T: @@ -805,7 +817,7 @@ def extract(self, key: str, default: T, constraint: Optional[str] = None) -> T: assert isinstance(result, (float, int)) assert eval(f"{default} {constraint}") if not eval(f"{result} {constraint}"): - warn( + self.bar.warn( f"value for `{self._key_path(key)}` in {self.source} should be {constraint} but is {result}. SKIPPED." ) return default @@ -816,37 +828,39 @@ def extract_and_error(self, key: str, t: type[T]) -> T: value = normalize_yaml_value(self.yaml.pop(key), t) if isinstance(value, t): return value - error(f"incompatible value for key '{key}' in {self.source}.") + self.bar.error(f"incompatible value for key '{key}' in {self.source}.") else: - error(f"missing key `{self._key_path(key)}` in {self.source}.") + self.bar.error(f"missing key `{self._key_path(key)}` in {self.source}.") self.errors += 1 return t() def extract_deprecated(self, key: str, new: Optional[str] = None) -> None: if key in self.yaml: use = f", use `{new}` instead" if new else "" - warn(f"key `{self._key_path(key)}` is deprecated{use}. SKIPPED.") + self.bar.warn(f"key `{self._key_path(key)}` is deprecated{use}. SKIPPED.") self.yaml.pop(key) - def extract_optional_list(self, key: str, t: type[T]) -> list[T]: + def extract_optional_list(self, key: str, t: type[T], *, allow_value: bool = True) -> list[T]: if key in self.yaml: value = self.yaml.pop(key) if value is None: return [] - if isinstance(value, t): + if allow_value and isinstance(value, t): return [value] if isinstance(value, list): if not all(isinstance(v, t) for v in value): - warn( + self.bar.warn( f"some values for key `{self._key_path(key)}` in {self.source} do not have type {t.__name__}. SKIPPED." ) return [] if not value: - warn( + self.bar.warn( f"value for `{self._key_path(key)}` in {self.source} should not be an empty list." ) return value - warn(f"incompatible value for key `{self._key_path(key)}` in {self.source}. SKIPPED.") + self.bar.warn( + f"incompatible value for key `{self._key_path(key)}` in {self.source}. SKIPPED." + ) return [] def extract_parser(self, key: str) -> "YamlParser": From 60a79fc59ae7b52892509996d42244330f27353c Mon Sep 17 00:00:00 2001 From: mzuenni Date: Thu, 20 Nov 2025 13:05:37 +0100 Subject: [PATCH 2/7] update tests --- bin/util.py | 2 +- test/test_problem_yaml.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/util.py b/bin/util.py index 364e9bf2..c8a76742 100644 --- a/bin/util.py +++ b/bin/util.py @@ -864,7 +864,7 @@ def extract_optional_list(self, key: str, t: type[T], *, allow_value: bool = Tru return [] def extract_parser(self, key: str) -> "YamlParser": - return YamlParser(self.source, self.extract(key, {}), self._key_path(key)) + return YamlParser(self.source, self.extract(key, {}), self._key_path(key), self.bar) if has_ryaml: diff --git a/test/test_problem_yaml.py b/test/test_problem_yaml.py index 31da03e9..5370336f 100644 --- a/test/test_problem_yaml.py +++ b/test/test_problem_yaml.py @@ -6,6 +6,7 @@ import config import problem +from util import PrintBar RUN_DIR = Path.cwd().absolute() @@ -72,6 +73,10 @@ def test_invalid(self, monkeypatch, test_data): fatal = MagicMock(name="fatal", side_effect=SystemExit(-42)) error = MagicMock(name="error") warn = MagicMock(name="warn") + + monkeypatch.setattr(PrintBar, "fatal", fatal) + monkeypatch.setattr(PrintBar, "error", error) + monkeypatch.setattr(PrintBar, "warn", warn) for module in ["problem", "util"]: monkeypatch.setattr(f"{module}.fatal", fatal) monkeypatch.setattr(f"{module}.error", error) From 5d7af406499daaa05215e74dfc1c29075cb6a78b Mon Sep 17 00:00:00 2001 From: mzuenni Date: Thu, 20 Nov 2025 13:23:48 +0100 Subject: [PATCH 3/7] fix test --- test/problems/identity/generators/hint_desc_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/problems/identity/generators/hint_desc_yaml.py b/test/problems/identity/generators/hint_desc_yaml.py index c842f91b..768d60ca 100644 --- a/test/problems/identity/generators/hint_desc_yaml.py +++ b/test/problems/identity/generators/hint_desc_yaml.py @@ -5,4 +5,4 @@ n = sys.argv[1] Path("testcase.in").write_text(n + "\n") Path("testcase.ans").write_text(n + "\n") -Path("testcase.yaml").write_text("hint: " + n + "\ndescription: " + n + "\n") +Path("testcase.yaml").write_text(f'hint: "{n}"\ndescription: "{n}"\n') From 0d8863563feca03bac0f9a83a1044839bc7843c1 Mon Sep 17 00:00:00 2001 From: mzuenni Date: Thu, 20 Nov 2025 14:23:20 +0100 Subject: [PATCH 4/7] fix test and implementation --- bin/problem.py | 44 +++++++++++++++++++++------------- bin/util.py | 6 +++-- test/yaml/problem/invalid.yaml | 6 ----- test/yaml/problem/valid.yaml | 6 +++++ 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/bin/problem.py b/bin/problem.py index 8127b9dc..0533caf9 100644 --- a/bin/problem.py +++ b/bin/problem.py @@ -314,7 +314,7 @@ def __init__( f"{validate.OutputValidator.args_key}' in 'test_group.yaml", ) - self.keywords: list[str] = parser.extract_optional_list("keywords", str) + self.keywords: list[str] = parser.extract_optional_list("keywords", str, allow_empty=True) # Not implemented in BAPCtools. We always test all languages in languages.yaml. self.languages: list[str] = parser.extract_optional_list("languages", str) # Not implemented in BAPCtools @@ -411,35 +411,47 @@ def __init__( parser.extract_deprecated("input_validator_flags", validate.InputValidator.args_key) # parse args - self.args = parser.extract_optional_list("args", str, allow_value=False) - self.output_visualizer_args = parser.extract_optional_list( - visualize.OutputVisualizer.args_key, str, allow_value=False - ) - self.output_validator_args = parser.extract_optional_list( - validate.OutputValidator.args_key, str, allow_value=False - ) - self.input_visualizer_args = parser.extract_optional_list( - visualize.InputVisualizer.args_key, str, allow_value=False - ) assert validate.OutputValidator.args_key == validate.AnswerValidator.args_key + for key in [ + "args", + visualize.OutputVisualizer.args_key, + validate.OutputValidator.args_key, + visualize.InputVisualizer.args_key, + ]: + if key in parser.yaml: + setattr( + self, + key, + parser.extract_optional_list(key, str, allow_value=False, allow_empty=True), + ) if validate.InputValidator.args_key in parser.yaml: if isinstance(parser.yaml[validate.InputValidator.args_key], list): self.input_validator_args = parser.extract_optional_list( - validate.InputValidator.args_key, str, allow_value=False + validate.InputValidator.args_key, + str, + allow_value=False, + allow_empty=True, ) - elif isinstance(value, dict): + elif isinstance(parser.yaml[validate.InputValidator.args_key], dict): # only the hole dict is inherited not individual entries validator_args_parser = parser.extract_parser(validate.InputValidator.args_key) self.input_validator_args = {} for val in problem.validators(validate.InputValidator): - self.input_validator_args[val.name] = parser.extract_optional_list( - val.name, str, allow_value=False + self.input_validator_args[val.name] = ( + validator_args_parser.extract_optional_list( + val.name, + str, + allow_value=False, + allow_empty=True, + ) ) validator_args_parser.check_unknown_keys() - elif value is None: + elif parser.yaml[validate.InputValidator.args_key] is None: + parser.pop(validate.InputValidator.args_key) self.input_validator_args = [] else: + parser.pop(validate.InputValidator.args_key) bar.warn( f"incompatible value for key `{validate.InputValidator.args_key}` in {parser.source}. SKIPPED." ) diff --git a/bin/util.py b/bin/util.py index c8a76742..29c261d6 100644 --- a/bin/util.py +++ b/bin/util.py @@ -840,7 +840,9 @@ def extract_deprecated(self, key: str, new: Optional[str] = None) -> None: self.bar.warn(f"key `{self._key_path(key)}` is deprecated{use}. SKIPPED.") self.yaml.pop(key) - def extract_optional_list(self, key: str, t: type[T], *, allow_value: bool = True) -> list[T]: + def extract_optional_list( + self, key: str, t: type[T], *, allow_value: bool = True, allow_empty: bool = False + ) -> list[T]: if key in self.yaml: value = self.yaml.pop(key) if value is None: @@ -853,7 +855,7 @@ def extract_optional_list(self, key: str, t: type[T], *, allow_value: bool = Tru f"some values for key `{self._key_path(key)}` in {self.source} do not have type {t.__name__}. SKIPPED." ) return [] - if not value: + if not value and not allow_empty: self.bar.warn( f"value for `{self._key_path(key)}` in {self.source} should not be an empty list." ) diff --git a/test/yaml/problem/invalid.yaml b/test/yaml/problem/invalid.yaml index 5189f1b6..030d0ab5 100644 --- a/test/yaml/problem/invalid.yaml +++ b/test/yaml/problem/invalid.yaml @@ -154,12 +154,6 @@ yaml: name: pass-fail type from empty type type: [] warn: "value for `type` in problem.yaml should not be an empty list." ---- -yaml: - problem_format_version: 2025-09 - name: Empty list - keywords: [] -warn: "value for `keywords` in problem.yaml should not be an empty list." --- # Credits diff --git a/test/yaml/problem/valid.yaml b/test/yaml/problem/valid.yaml index 22e10040..57c10cc3 100644 --- a/test/yaml/problem/valid.yaml +++ b/test/yaml/problem/valid.yaml @@ -245,3 +245,9 @@ yaml: problem_format_version: 2025-09 name: Embargo datetime embargo_until: 2025-12-31T23:59:59 +--- +# keywords +yaml: + problem_format_version: 2025-09 + name: Empty keywords list + keywords: [] From 1fbf60ffa7acae1b6c4d92dced0fbe4fe6899863 Mon Sep 17 00:00:00 2001 From: mzuenni Date: Thu, 20 Nov 2025 22:08:33 +0100 Subject: [PATCH 5/7] better handling of ignored files --- bin/generate.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/bin/generate.py b/bin/generate.py index 80e64ac5..c60ac304 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -487,6 +487,11 @@ def __init__( # used to handle duplicated testcase rules self.copy_of = None + # set during generate + self.generate_success = False + + self.process = generator_config.process_testcase(parent.path / name) + bar = PrintBar("generators.yaml", item=parent.path / name) if name.endswith(".in"): @@ -683,6 +688,8 @@ def write(self) -> None: def link( t, problem: Problem, generator_config: "GeneratorConfig", bar: ProgressBar, dst: Path ) -> None: + assert t.process + src_dir = problem.path / "data" / t.path.parent src = src_dir / (t.name + ".in") @@ -720,6 +727,8 @@ def validate_in( meta_yaml: "TestcaseRule.MetaYaml", bar: ProgressBar, ) -> bool: + assert t.process + infile = problem.tmpdir / "data" / t.hash / "testcase.in" assert infile.is_file() @@ -767,6 +776,8 @@ def validate_ans_and_out( meta_yaml: "TestcaseRule.MetaYaml", bar: ProgressBar, ) -> bool: + assert t.process + infile = problem.tmpdir / "data" / t.hash / "testcase.in" assert infile.is_file() @@ -818,9 +829,9 @@ def validate_ans_and_out( def generate( t, problem: Problem, generator_config: "GeneratorConfig", parent_bar: ProgressBar ) -> None: - bar = parent_bar.start(str(t.path)) + assert t.process - t.generate_success = False + bar = parent_bar.start(str(t.path)) if t.copy_of is not None and not t.intended_copy: bar.warn( @@ -1427,7 +1438,7 @@ def walk( if isinstance(d, Directory): d.walk(testcase_f, dir_f) elif isinstance(d, TestcaseRule): - if testcase_f: + if testcase_f and d.process: testcase_f(d) else: assert False @@ -1484,6 +1495,11 @@ def generate_includes( ansfile = problem.path / "data" / target.parent / (target.name + ".ans") new_infile = problem.path / "data" / d.path / (target.name + ".in") + if not t.process: + bar.warn(f"Included case {target} was not processed.") + bar.done() + continue + if not t.generate_success: bar.error(f"Included case {target} has errors.") bar.done() @@ -1712,10 +1728,6 @@ def parse( if has_count(yaml): name += f"-{count_index + 1:0{len(str(count))}}" - # If a list of testcases was passed and this one is not in it, skip it. - if not self.process_testcase(parent.path / name): - continue - t = TestcaseRule(self.problem, self, key, name, yaml, parent, count_index) if t.path in self.known_cases: PrintBar("generators.yaml", item=t.path).error( From 649370d6cd7b0cd0630dc3ff045ae0df7b495973 Mon Sep 17 00:00:00 2001 From: mzuenni Date: Fri, 21 Nov 2025 12:52:48 +0100 Subject: [PATCH 6/7] skip includes if restricted --- bin/generate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/generate.py b/bin/generate.py index c60ac304..7e4bedc0 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -1490,6 +1490,10 @@ def generate_includes( t = d.includes[key] target = t.path new_case = d.path / target.name + + if not generator_config.process_testcase(new_case): + continue + bar.start(str(new_case)) infile = problem.path / "data" / target.parent / (target.name + ".in") ansfile = problem.path / "data" / target.parent / (target.name + ".ans") From e0fc70c8f6cb4bcb0d447ef2116e086ee5e5013a Mon Sep 17 00:00:00 2001 From: mzuenni Date: Fri, 21 Nov 2025 15:23:15 +0100 Subject: [PATCH 7/7] show more warnings --- bin/generate.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/generate.py b/bin/generate.py index 7e4bedc0..7fd98ba0 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -1407,6 +1407,8 @@ def __init__( def walk( self, testcase_f: Optional[Callable[["TestcaseRule | Directory"], object]], + *, + skip_restricted: bool = True, ) -> None: ... # This overload takes one function for test cases and a separate function for directories. @@ -1415,6 +1417,8 @@ def walk( self, testcase_f: Optional[Callable[[TestcaseRule], object]], dir_f: Optional[Callable[["Directory"], object]], + *, + skip_restricted: bool = True, ) -> None: ... # Map a function over all test cases directory tree. @@ -1428,6 +1432,8 @@ def walk( | Optional[ Callable[["TestcaseRule | Directory"], object] | Callable[["Directory"], object] ] = True, + *, + skip_restricted: bool = True, ) -> None: if dir_f is True: dir_f = cast(Optional[Callable[["TestcaseRule | Directory"], object]], testcase_f) @@ -1438,7 +1444,9 @@ def walk( if isinstance(d, Directory): d.walk(testcase_f, dir_f) elif isinstance(d, TestcaseRule): - if testcase_f and d.process: + if not d.process and skip_restricted: + continue + if testcase_f: testcase_f(d) else: assert False @@ -1872,6 +1880,7 @@ def add_included_case(t: TestcaseRule) -> None: obj.walk( add_included_case, lambda d: list(map(add_included_case, d.includes.values())), + skip_restricted=False, ) pass else: