Add .download samples and ans_is_output flag (#436)

* partially implement draft for samples

* only set out path if necessary

* fix missing key

* only check necessary .out files

* use outpath if possible

* some types

* use string type name...

* fix union type?

* fix typing

* drop .out support

* dont warn here

* add .out support

* fix tests

* made ans=out assumption optional

* fix code

* allow ans validators for interactive and multipass problems

* add missing validator

* properly handle samples in export

* properly handle samples in export

* properly handle samples in export

* allow more answer validators

* properly find testcases

* [doc] Improve grammar in documentation

* [validate] Replace import of Union with string type hint

* hide A stat for interactive problems

* dont always create empty ans files

* add comment

* rename

* fix samples

* only drop known suffixes

* simplify code

* add more tests

* allow standalone in.statement

* removed outdated assert

* removed outdated if

* undo namechange

* update files

* remove wip file

* [export] bt samplezip: check for duplicate files from attachments/

* [validate] Skip sanity checks for empty .ans files for interactive problems

* [test] samplezip/zip: assert that the correct samples are in the zip files

* [test] Add samples for constants problem

* [export] build_problem_zip: Make sure that .*.download files also end up in the zip

* improve warning

* [problem] Problem._samples: split warning message for has_raw over multiple lines

* [export] Simplify getting of all samples: .interaction is included in KNOWN_DATA_EXTENSIONS

* [generate] For interactive and/or multi-pass samples, allow .in.download and .interaction when both .in and .in.statement are missing

Also generate empty .ans.statement or .ans.download files if they don't exist yet.

* simplify code

* i hate python tuples

* [generate] generate_empty_interactive_sample_ans: stop when .ans file exists

* [generate] Move generate_empty_interactive_sample_ans to later step

* generators.cue: Add '{in,ans}.{statement,download}' to #testcase

* [test] Fix test_schemata.sh: run from correct directory, replace {%placeholders%}

* [test] test_schemata.sh: Skip empty snippets for now

* [generate] Allow writing empty hardcoded files

Kinda ugly, but should be caught by validators and sanity checks anyway,
so having the check here should™ be redundant.
This does allow writing empty .{in,ans}.{statement,download} files,
which are _not_ sanity-checked.

---------

Co-authored-by: Maarten Sijm <[email protected]>

Author: mzuenni

bin/config.py

@@ -61,13 +61,19 @@
     ".pdf",
 ]
 
+KNOWN_SAMPLE_TESTCASE_EXTENSIONS: Final[Sequence[str]] = [
+    ".in.statement",
+    ".ans.statement",
+    ".in.download",
+    ".ans.download",
+]
+
 KNOWN_TEXT_DATA_EXTENSIONS: Final[Sequence[str]] = [
     *KNOWN_TESTCASE_EXTENSIONS,
+    *KNOWN_SAMPLE_TESTCASE_EXTENSIONS,
     ".interaction",
     ".hint",
     ".desc",
-    ".in.statement",
-    ".ans.statement",
     #'.args',
 ]
 

bin/constraints.py

@@ -23,7 +23,7 @@ def check_validators(problem):
     if not in_constraints:
         warn("No constraint validation of input values found in input validators.")
     problem.validate_data(validate.Mode.ANSWER, constraints=ans_constraints)
-    if not problem.interactive and not problem.multi_pass and not ans_constraints:
+    if not problem.settings.ans_is_output and not ans_constraints:
         log("No constraint validation of answer values found in answer or output validators.")
     print()
 

bin/export.py

@@ -33,7 +33,7 @@ def remove_language_suffix(fname, statement_language):
     return out
 
 
-def build_samples_zip(problems, output, statement_language):
+def build_samples_zip(problems: list[Problem], output: Path, statement_language: str):
     zf = zipfile.ZipFile(output, mode="w", compression=zipfile.ZIP_DEFLATED, allowZip64=False)
 
     # Do not include contest PDF for kattis.
@@ -47,42 +47,51 @@ def build_samples_zip(problems, output, statement_language):
                 )
 
     for problem in problems:
+        if not problem.label:
+            fatal(f"Cannot create samples zip: Problem {problem.name} does not have a label!")
+
         outputdir = Path(problem.label)
 
         attachments_dir = problem.path / "attachments"
         if (problem.interactive or problem.multi_pass) and not attachments_dir.is_dir():
-            interactive = "interactive " if problem.interactive else ""
-            multi_pass = "multi-pass " if problem.multi_pass else ""
             util.error(
-                f"{interactive}{multi_pass}problem {problem.name} does not have an attachments/ directory."
+                f"{problem.settings.type_name()} problem {problem.name} does not have an attachments/ directory."
             )
             continue
 
-        empty = True
+        contents: dict[Path, Path] = {}  # Maps desination to source, to allow checking duplicates.
+
+        # Add samples.
+        samples = problem.download_samples()
+        for i, (in_file, ans_file) in enumerate(samples):
+            base_name = outputdir / str(i + 1)
+            contents[base_name.with_suffix(".in")] = in_file
+            if ans_file.stat().st_size > 0:
+                contents[base_name.with_suffix(".ans")] = ans_file
 
         # Add attachments if they exist.
         if attachments_dir.is_dir():
             for f in attachments_dir.iterdir():
                 if f.is_dir():
                     util.error(f"{f} directory attachments are not yet supported.")
                 elif f.is_file() and f.exists():
-                    zf.write(f, outputdir / f.name)
-                    empty = False
+                    destination = outputdir / f.name
+                    if destination in contents:
+                        util.error(
+                            f"Cannot overwrite {destination} from attachments/"
+                            + f" (sourced from {contents[destination]})."
+                            + "\n\tDo not include samples in attachments/,"
+                            + " use .{in,ans}.statement or .{in,ans}.download instead."
+                        )
+                    else:
+                        contents[destination] = f
                 else:
                     util.error(f"Cannot include broken file {f}.")
 
-        # Add samples for non-interactive and non-multi-pass problems.
-        if not problem.interactive and not problem.multi_pass:
-            samples = problem.testcases(only_samples=True)
-            if samples:
-                for i in range(0, len(samples)):
-                    sample = samples[i]
-                    basename = outputdir / str(i + 1)
-                    zf.write(sample.in_path, basename.with_suffix(".in"))
-                    zf.write(sample.ans_path, basename.with_suffix(".ans"))
-                    empty = False
-
-        if empty:
+        if contents:
+            for destination, source in contents.items():
+                zf.write(source, destination)
+        else:
             util.error(f"No attachments or samples found for problem {problem.name}.")
 
     zf.close()
@@ -107,22 +116,8 @@ def build_problem_zip(problem: Problem, output: Path):
         ("attachments/**/*", problem.interactive or problem.multi_pass),
     ]
 
-    testcases = [
-        ("data/secret/**/*.in", True),
-        ("data/sample/**/*.in", not problem.interactive and not problem.multi_pass),
-    ]
-
-    if problem.interactive or problem.multi_pass:
-        # .interaction files don't need a corresponding .in
-        # therefore we can handle them like all other files
-        files += [("data/sample/**/*.interaction", False)]
-
     if not config.args.kattis:
-        files += [
-            (f"problem.{statement_language}.pdf", True),
-            ("data/sample/**/*.in.statement", False),
-            ("data/sample/**/*.ans.statement", False),
-        ]
+        files.append((f"problem.{statement_language}.pdf", True))
 
     if problem.custom_output:
         files.append(("output_validator/**/*", True))
@@ -141,7 +136,7 @@ def build_problem_zip(problem: Problem, output: Path):
         export_dir /= problem.name
     export_dir.mkdir(parents=True, exist_ok=True)
 
-    def add_file(path, source):
+    def add_file(path: Path, source: Path):
         path = export_dir / path
         path.parent.mkdir(parents=True, exist_ok=True)
         ensure_symlink(path, source)
@@ -158,21 +153,32 @@ def add_file(path, source):
                 out = remove_language_suffix(out, statement_language)
                 add_file(out, f)
 
-    # Include all testcases (specified by a .in file) and copy all related files
-    for pattern, required in testcases:
-        paths = list(util.glob(problem.path, pattern))
-        if required and len(paths) == 0:
-            util.error(f"No matches for required path {pattern}.")
-        for f in paths:
+    def add_testcase(in_file: Path):
+        base_name = util.drop_suffix(in_file, [".in", ".in.statement", ".in.download"])
+        for ext in config.KNOWN_DATA_EXTENSIONS:
+            f = base_name.with_suffix(ext)
             if f.is_file():
-                if not f.with_suffix(".ans").is_file():
-                    util.warn(f"No answer file found for {f}, skipping.")
-                else:
-                    for ext in config.KNOWN_DATA_EXTENSIONS:
-                        f2 = f.with_suffix(ext)
-                        if f2.is_file():
-                            out = f2.relative_to(problem.path)
-                            add_file(out, f2)
+                out = f.relative_to(problem.path)
+                add_file(out, f)
+
+    # Include all sample test cases and copy all related files.
+    samples = problem.download_samples()
+    if len(samples) == 0:
+        util.error("No samples found.")
+    for in_file, _ in samples:
+        add_testcase(in_file)
+
+    # Include all secret test cases and copy all related files.
+    pattern = "data/secret/**/*.in"
+    paths = util.glob(problem.path, pattern)
+    if len(paths) == 0:
+        util.error(f"No secret test cases found in {pattern}.")
+    for f in paths:
+        if f.is_file():
+            if f.with_suffix(".ans").is_file():
+                add_testcase(f)
+            else:
+                util.warn(f"No answer file found for {f}, skipping.")
 
     # DOMjudge and Kattis do not support 2023-07-draft yet.
     # TODO: Remove once they do.