tests(opendata): Cleanup & test judging

Author: 5K1PY

tests/test_opendata.py

@@ -11,7 +11,7 @@
 import shutil
 import unittest
 
-from util import TestFixture
+from util import TestFixture, modify_config
 
 from pisek.user_errors import UserError
 from pisek.task_jobs.data.testcase_info import TestcaseGenerationMode
@@ -24,6 +24,20 @@ class TestFixtureOpendata(TestFixture):
     def expecting_success(self) -> bool:
         return True
 
+    def modify_task(self) -> None:
+        pass
+
+    @property
+    def _testcase_files(self) -> list[str]:
+        files = []
+        for file in [self.input_path, self.output_path, self.contestant_path]:
+            if file is not None:
+                files.append(file)
+        return files
+
+    def created_files(self):
+        return [os.path.basename(file) for file in self._testcase_files]
+
     def runTest(self) -> None:
         if not self.fixture_path:
             return
@@ -33,19 +47,28 @@ def runTest(self) -> None:
 
         self.built_task: BuiltTask
 
+        self.input_path: str | None = None
+        self.output_path: str | None = None
+        self.contestant_path: str | None = None
+
         @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
         @unittest.mock.patch("sys.stderr", new_callable=io.StringIO)
         def run(*args) -> bool:
             try:
+                self.modify_task()
                 self._build_task_dir = tempfile.mkdtemp(prefix="pisek-test_")
                 self.built_task = task.build(self._build_task_dir)
                 self.run_opendata_test()
                 return True
-            except UserError:
+            except UserError as e:
+                print(e)
                 return False
 
         self.assertEqual(run(), self.expecting_success())
 
+        for file in self._testcase_files:
+            self.assertTrue(os.path.isfile(file))
+
         self.check_end_state()
         self.check_files()
 
@@ -72,6 +95,23 @@ class TestSumKasiopeaOpendataBuild(TestFixtureOpendata):
     def fixture_path(self) -> str:
         return "../fixtures/sum_kasiopea/"
 
+    def init_testcase(self, name: str) -> None:
+        self.input_path = os.path.join(self.task_dir, f"{name}.opendata.in")
+        self.output_path = os.path.join(self.task_dir, f"{name}.opendata.out")
+        self.testcase = self.built_task.get_testcase(
+            name, int(name), int("deadbeef", 16), self.input_path, self.output_path
+        )
+
+    def create_contestant_file(self, content: str | bytes) -> None:
+        self.contestant_path = os.path.join(self.task_dir, "02.out")
+
+        if isinstance(content, bytes):
+            with open(self.contestant_path, "xb") as f:
+                f.write(content)
+        else:
+            with open(self.contestant_path, "x") as f:
+                f.write(content)
+
 
 class TestSumKasiopeaOpendataListInputs(TestSumKasiopeaOpendataBuild):
     def run_opendata_test(self):
@@ -98,71 +138,70 @@ def run_opendata_test(self):
 
 
 class TestSumKasiopeaOpendataSequential(TestSumKasiopeaOpendataBuild):
-    def created_files(self):
-        return ["01.in", "01.out"]
-
-    def run_opendata_test(self):
-        input_path = os.path.join(self.task_dir, "01.in")
-        output_path = os.path.join(self.task_dir, "01.out")
-
-        testcase = self.built_task.get_testcase(
-            "01", 1, int("deadbeef", 16), input_path, output_path
-        )
-        testcase.gen_input()
-        self.assertTrue(os.path.exists(input_path))
-        testcase.gen_output()
-        self.assertTrue(os.path.exists(output_path))
+    def run_opendata_test(self) -> None:
+        self.init_testcase("01")
+        assert self.input_path is not None
+        assert self.output_path is not None
+
+        self.testcase.gen_input()
+        self.assertTrue(os.path.exists(self.input_path))
+        self.testcase.gen_output()
+        self.assertTrue(os.path.exists(self.output_path))
         self.assertEqual(
-            testcase.judge(output_path),
+            self.testcase.judge(self.output_path),
             OpendataVerdict(Verdict.ok, None, Decimal(4), None, None),
         )
 
 
-class TestSumKasiopeaOpendataJudgeRightaway(TestSumKasiopeaOpendataBuild):
-    def created_files(self):
-        return ["02.in", "02.out"]
+class TestSumKasiopeaOpendataCheckRightaway(TestSumKasiopeaOpendataBuild):
+    def run_opendata_test(self) -> None:
+        self.init_testcase("02")
+        self.assertEqual(
+            self.testcase.judge(os.path.join(self.task_dir, "sample.out")),
+            OpendataVerdict(Verdict.wrong_answer, None, Decimal(0), None, None),
+        )
+
 
+class TestSumKasiopeaOpendataCheckBinary(TestSumKasiopeaOpendataBuild):
     def run_opendata_test(self):
-        input_path = os.path.join(self.task_dir, "02.in")
-        output_path = os.path.join(self.task_dir, "02.out")
+        self.init_testcase("02")
+        self.create_contestant_file(b"\x07\n")
 
-        testcase = self.built_task.get_testcase(
-            "02", 2, int("deadbeef", 16), input_path, output_path
-        )
         self.assertEqual(
-            testcase.judge(os.path.join(self.task_dir, "sample.out")),
-            OpendataVerdict(Verdict.wrong_answer, None, Decimal(0), None, None),
+            self.testcase.judge(os.path.join(self.task_dir, self.contestant_path)),
+            OpendataVerdict(
+                Verdict.normalization_fail,
+                "File contains non-printable character (code 7 at position 0)",
+                Decimal(0),
+                None,
+                None,
+            ),
         )
-        self.assertTrue(os.path.exists(input_path))
-        self.assertTrue(os.path.exists(output_path))
 
 
-class TestSumKasiopeaOpendataJudgeBinary(TestSumKasiopeaOpendataBuild):
-    def created_files(self):
-        return ["02.in", "02.out", "02.ok"]
+class TestSumKasiopeaOpendataJudge(TestSumKasiopeaOpendataBuild):
+    def modify_task(self) -> None:
+        def modification_fn(raw_config):
+            raw_config["tests"]["out_check"] = "judge"
+            raw_config["tests"]["out_judge"] = "judge"
+            raw_config["tests"]["judge_type"] = "opendata-v2"
+
+        modify_config(self.task_dir, modification_fn)
 
     def run_opendata_test(self):
-        input_path = os.path.join(self.task_dir, "02.in")
-        output_path = os.path.join(self.task_dir, "02.ok")
-        contestant_path = os.path.join(self.task_dir, "02.out")
-        with open(contestant_path, "xb") as f:
-            f.write(b"\x07\n")
-
-        testcase = self.built_task.get_testcase(
-            "02", 2, int("deadbeef", 16), input_path, output_path
-        )
+        self.init_testcase("02")
+        self.create_contestant_file("0\n")
+
         self.assertEqual(
-            testcase.judge(os.path.join(self.task_dir, contestant_path)),
+            self.testcase.judge(os.path.join(self.task_dir, self.contestant_path)),
             OpendataVerdict(
-                Verdict.normalization_fail,
-                "File contains non-printable character (code 7 at position 0)",
+                Verdict.wrong_answer,
+                "Wrong answer",
                 Decimal(0),
                 None,
                 None,
             ),
         )
-        self.assertTrue(os.path.exists(input_path))
-        self.assertTrue(os.path.exists(output_path))
 
 
 if __name__ == "__main__":