implement match for generate.yaml

bin/generate.py

@@ -82,8 +82,8 @@ def assert_type(
 UNIQUE_TESTCASE_KEYS: Final[Sequence[str]] = (
     "copy",
     "generate",
-    "retries",
     "count",
+    "match",
     *(e[1:] for e in config.KNOWN_TEXT_DATA_EXTENSIONS),
 )
 
@@ -327,27 +327,22 @@ def default_solution_path(generator_config: "GeneratorConfig") -> Path:
 
 KNOWN_TESTCASE_KEYS: Final[Sequence[str]] = (
     "type",
-    "generate",
-    "copy",
     "solution",
     "random_salt",
     "retries",
-    "count",
-    *(e[1:] for e in config.KNOWN_TEXT_DATA_EXTENSIONS),
+    *UNIQUE_TESTCASE_KEYS,
 )
-RESERVED_TESTCASE_KEYS: Final[Sequence[str]] = ("data", "test_group.yaml", "include")
+UNIQUE_DIRECTORY_KEYS: Final[Sequence[str]] = ("data", "test_group.yaml", "include")
 KNOWN_DIRECTORY_KEYS: Final[Sequence[str]] = (
     "type",
-    "data",
-    "test_group.yaml",
-    "include",
     "solution",
     "random_salt",
     "retries",
+    *UNIQUE_DIRECTORY_KEYS,
 )
 RESERVED_DIRECTORY_KEYS: Final[Sequence[str]] = ("command",)
-KNOWN_ROOT_KEYS: Final[Sequence[str]] = ("generators", "parallel", "version")
-DEPRECATED_ROOT_KEYS: Final[Sequence[str]] = ("gitignore_generated", "visualizer")
+KNOWN_ROOT_KEYS: Final[Sequence[str]] = ("generators", "version")
+DEPRECATED_ROOT_KEYS: Final[Sequence[str]] = ("gitignore_generated", "parallel", "visualizer")
 
 
 # Holds all inheritable configuration options. Currently:
@@ -471,7 +466,9 @@ def __init__(
         #    This variable already includes the .in extension, so `.with_suffix()` works nicely.
         self.copy = None
         # 3. Hardcoded cases where the source is in the yaml file itself.
-        self.hardcoded = {}
+        self.hardcoded = dict[str, str]()
+        # list of patterns used to check the generated testcase.in
+        self.patterns = list[re.Pattern[str]]()
 
         # Hash of testcase for caching.
         self.hash: str
@@ -623,9 +620,23 @@ def __init__(
                 for ext, value in self.hardcoded.items():
                     hashes[ext] = hash_string(value)
 
+                if "match" in yaml:
+                    match_entries = yaml["match"]
+                    assert_type("`match`", match_entries, (list, str))
+                    if isinstance(match_entries, str):
+                        match_entries = [match_entries]
+                    assert isinstance(match_entries, list)
+
+                    for i, match_entry in enumerate(match_entries):
+                        assert_type(f"`match[{i}]`", match_entry, str)
+                        try:
+                            self.patterns.append(re.compile(match_entry, re.MULTILINE | re.DOTALL))
+                        except re.error:
+                            raise ParseException(f"could not parse regex `match[{i}]`.")
+
             # Warn/Error for unknown keys.
             for any_key in yaml:
-                if any_key in RESERVED_TESTCASE_KEYS:
+                if any_key in UNIQUE_DIRECTORY_KEYS:
                     raise ParseException(f"Testcase must not contain reserved key {any_key}.")
                 if any_key not in KNOWN_TESTCASE_KEYS:
                     if config.args.action == "generate":
@@ -670,6 +681,7 @@ def get(key: str, default: T) -> T:
             self.rule_hashes: dict[object, object] = get("rule_hashes", {})
             self.generated_extensions: list[object] = get("generated_extensions", [])
             self.input_validator_hashes: dict[object, object] = get("input_validator_hashes", {})
+            self.matches: dict[object, object] = get("matches", {})
             self.solution_hash: dict[object, object] = get("solution_hash", {})
             self.interactor_hash: dict[object, object] = get("interactor_hash", {})
             self.ans_out_validator_hashes: dict[object, object] = get(
@@ -979,6 +991,31 @@ def generate_from_rule() -> bool:
             assert t._has_required_in(infile), f"Failed to generate in file: {infile.name}"
             return True
 
+        def check_match(testcase: Testcase, bar: ProgressBar) -> None:
+            nonlocal meta_yaml
+
+            def get_pattern_str(pattern: re.Pattern[str]) -> str:
+                return pattern.pattern.encode("unicode_escape").decode()
+
+            if all(meta_yaml.matches.get(get_pattern_str(p)) for p in t.patterns):
+                return
+
+            updated = False
+            text = testcase.in_path.read_text()
+            for pattern in t.patterns:
+                if meta_yaml.matches.get(pattern.pattern):
+                    continue
+                match = pattern.search(text)
+                if match:
+                    match_str = f"[{match.start()}, {match.end()})"
+                    bar.debug(f"Found match for '{get_pattern_str(pattern)}'': {match_str}")
+                    meta_yaml.matches[pattern.pattern] = match_str
+                    updated = True
+                else:
+                    bar.warn(f"Found not match for '{get_pattern_str(pattern)}'")
+            if updated:
+                meta_yaml.write()
+
         def generate_from_solution(testcase: Testcase, bar: ProgressBar) -> bool:
             nonlocal meta_yaml
 
@@ -1278,6 +1315,10 @@ def add_test_case_to_cache() -> None:
             if not t.validate_in(problem, testcase, meta_yaml, bar):
                 return
 
+            # Step 3.1: check patterns
+            # this is not a hard error since the testcase is still valid
+            check_match(testcase, bar)
+
             # Step 4: generate .ans and .interaction if needed
             if not generate_from_solution(testcase, bar):
                 return

bin/util.py

@@ -226,7 +226,7 @@ def __init__(
 
         assert not (items and (max_len or count))
         assert items is not None or max_len
-        if items is not None:
+        if items is not None and max_len is None:
             max_len = max((ProgressBar.item_len(x) for x in items), default=0)
         assert max_len is not None
         self.prefix: str = prefix  # The prefix to always print
@@ -575,7 +575,11 @@ def __init__(
         item: Optional[ITEM_TYPE] = None,
     ) -> None:
         self.prefix = str(prefix) if prefix else None
-        self.item_width = max_len + 1 if max_len is not None else None
+        self.item_width = None
+        if item is not None:
+            self.item_width = ProgressBar.item_len(item) + 1
+        if max_len is not None:
+            self.item_width = max_len + 1
         self.item = item
 
     def start(self, item: Optional[ITEM_TYPE] = None) -> "PrintBar":
@@ -742,6 +746,11 @@ def parse_yaml(
                     fatal(f"Duplicate key in yaml file {path}!\n{error.args[0]}\n{error.args[2]}")
                 else:
                     fatal(f"Duplicate key in yaml object!\n{str(error)}")
+            except Exception as e:
+                if suppress_errors:
+                    return None
+                eprint(f"{Fore.YELLOW}{e}{Style.RESET_ALL}", end="")
+                fatal(f"Failed to parse {path}.")
         return ret
 
     else:

doc/generators.md

@@ -29,10 +29,11 @@ The two main object types are `directory` and `generator`. The root of `generato
 - `test_group.yaml`: Optional yaml configuration that will be copied to `test_group.yaml` in this directory.
 - `solution`: Optional invocation of a solution to be used to generate `.ans` files. Set to empty to disable generating `.ans`. (Useful for e.g. the `data/samples/` directory.) This must be an absolute path relative to the problem root.
 - `random_salt`: Optional string that will be prepended to each command before computing its `{seed}`. May be used to regenerate all random cases and to prevent predictable seeds.
+- `retries`: Optional int that specifies the maximum number of invocation that will be tried if the generator fails. Each invocation will use a different value for `{seed}`.
 - `data`: The test cases / test groups contained in this directory. This may take two forms:
   - A dictionary, each key is the name of a test case/test group, and each value must be a `directory` or `generator` object.
   - A list of dictionaries as above. In this case, testcases will be prefixed with zero padded 1-based integers in the order of the list. Items in the same dictionary will get the same number.
-- `input`: Optional list of Directory object names (as strings) e.g. `- "sample"`. All testcases from those directories are linked for this directory.
+- `include`: Optional list of Directory object names (as strings) e.g. `- "sample"`. All testcases from those directories are linked for this directory.
 
 **Generator objects** have the following forms:
 
@@ -41,7 +42,8 @@ The two main object types are `directory` and `generator`. The root of `generato
 - `<ext>: <content>`: A file with extension `ext` and the `content` will be generated. `<ext>` must be a known file extension.
 - `count: <int>`. To generate multiple Generator objects. If `generate` is used and `{seed}` or `{seed:(0-9)+}` is present all Generator objects will use a different seed. The arguments of `generate` may contain `{count}` to refer to the index of this generator invocation.
 Or as a shorthand:
-- `command` followed by the command as for `generate`.
+- `match`: Optional `str` or list of `str`. Each entry should be a regex pattern. If the generated testcase does not match a pattern a warning will be shown.
+- `solution`, `random_salt`, and `retries`: see **Directory objects**.
 
 The follwoing things should hold:
 - A `.in` file must be specified/generated by this

doc/generators.yaml

@@ -117,6 +117,7 @@ data:
       "04": tree 5 # keys from the global generators: dictionary may also be used.
       "05":
         generate: tree 6 # same as above, but with different argument
+        match: "^1 2$" # check that there is an edge from 1 to 2 (can also be a list with multiple pattern)
 
       # Arguments are split on white space: this will pass two arguments: `"a` and `b"`, so probably not what is intended.
       06-string: tree "a b"

support/schemas/generators.cue

@@ -40,6 +40,7 @@ import "strings"
 		// The "copy" key uses a path relative to "/generators/" ending in a test case name,
 		// such as "manual/samples/3".
 		copy?: #dirpath
+		match?: string | [...string]
 
 		["in" | "in.statement" | "in.download" |
 		 "ans" | "ans.statement" | "ans.download" |

support/schemas/generators_yaml_schema.json

@@ -250,6 +250,21 @@
               "title": "Count",
               "description": "Generate this number of test cases, substituting `{count}` in the `generate:` command with values between 1 and `count`, inclusive."
             },
+            "match": {
+              "oneOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "title": "Match",
+              "description": "Regular expression(s) that are searched in the test case. Each regular expression that is not found results in a warning."
+            },
             "in": {
               "type": "string",
               "title": "Input",

test/problems/alternativeencryption/generators/generators.yaml

@@ -21,8 +21,17 @@ data:
             encrypt
             1
             a
-      - letters: eval.py 0 26 sigma[i % 26]
-      - letters: eval.py 0 1000 sigma[i % 26]
+      - letters:
+          generate: eval.py 0 26 sigma[i % 26]
+          match:
+            - "\\Aencrypt$"
+            - "^a$"
+            - "^c$"
+            - "^z$"
+            - "\\A(?!.*^[A-Z]$).*\\Z"
+      - letters:
+          generate: eval.py 0 1000 sigma[i % 26]
+          match: "^a$.*^z$"
       - random_equal: eval.py {seed} 1000 sigma[i % 26] * randrange(1, 101)
       - max_equal: eval.py 0 1000 sigma[i % 26] * 100
       - random2: eval.py {seed} 1000 randstr(2)

test/test_problems.py

@@ -55,6 +55,9 @@ def setup_alternativeencryption_problem(request):
 
 @pytest.mark.usefixtures("setup_alternativeencryption_problem")
 class TestAlternativeencryptionProblem:
+    def test_generate(self):
+        tools.test(["generate"])
+
     def test_check_testing_tool(self):
         tools.test(["check_testing_tool"])
 

test/yaml/generators/invalid_yaml/invalid.generators.yaml

@@ -381,6 +381,26 @@ data:
        generate: my_generator {count}
        count: 101
 ---
+# match must be a string or list of strings
+data:
+  sample: {}
+  secret:
+    data:
+    - '':
+       in: "1 2"
+       match: 1
+---
+# match must be a string or list of strings
+data:
+  sample: {}
+  secret:
+    data:
+    - '':
+       in: "1 2"
+       match:
+        - "1"
+        - 2
+---
 # No test_group.yaml on testcase level
 # TODO Not picked up by JSON schema
 data:

test/yaml/generators/valid_yaml/rich-generators.yaml

@@ -28,6 +28,15 @@ data:
       'morecurlies':
         generate: my_generator {seed:1} --name {name} --ctr {count} --arg {count}
         count: 5
+      'match':
+        in: "1 2"
+        match: "1"
+      'morematch':
+        in: "1 2 3"
+        match:
+          - "1"
+          - "2"
+          - "3"
       'group_with_test_group_yaml':
         test_group.yaml:
           input_validator_args: [--connected, --max_n, "2000"]