improve testcase hashing

Author: mzuenni

bin/generate.py

@@ -556,7 +556,7 @@ def __init__(
                     self.rule["copy"] = str(self.copy)
                     for ext in config.KNOWN_TESTCASE_EXTENSIONS:
                         if self.copy.with_suffix(ext).is_file():
-                            hashes[ext] = hash_file(self.copy.with_suffix(ext))
+                            hashes[ext] = hash_file_content(self.copy.with_suffix(ext))
 
                 # 3. hardcoded
                 for ext in config.KNOWN_TEXT_DATA_EXTENSIONS:
@@ -586,14 +586,8 @@ def __init__(
                             color_type=MessageType.LOG,
                         )
 
-            # build ordered list of hashes we want to consider
-            hs = list(hashes.values())
-
             # combine hashes
-            if len(hs) == 1:
-                self.hash = hs[0]
-            else:
-                self.hash = combine_hashes(hs)
+            self.hash = combine_hashes_dict(hashes)
 
             if self.hash in generator_config.rules_cache:
                 self.copy_of = generator_config.rules_cache[self.hash]
@@ -1170,24 +1164,20 @@ def add_testdata_to_cache():
 
             # consider specific files for the uniqueness of this testcase
             relevant_files = {
+                "invalid_input": [".in"],
                 "invalid_answer": [".in", ".ans"],
                 "invalid_output": [".in", ".ans", ".out"],
                 "valid_output": [".in", ".ans", ".out"],
             }
-            extensions = relevant_files.get(t.root, [".in"])
+            relevant_files_default = [".in"] if problem.settings.ans_is_output else [".in", ".ans"]
+            extensions = relevant_files.get(t.root, relevant_files_default)
 
             for ext in extensions:
                 if target_infile.with_suffix(ext).is_file():
-                    hashes[ext] = hash_file(target_infile.with_suffix(ext))
-
-            # build ordered list of hashes we want to consider
-            hs = list(hashes.values())
+                    hashes[ext] = hash_file_content(target_infile.with_suffix(ext))
 
             # combine hashes
-            if len(hs) == 1:
-                test_hash = hs[0]
-            else:
-                test_hash = combine_hashes(hs)
+            test_hash = combine_hashes_dict(hashes)
 
             # check for duplicates
             if test_hash not in generator_config.generated_testdata:

bin/problem.py

@@ -1175,9 +1175,21 @@ def reset_testcase_hashes(self):
 
     # Returns None for new testcases or the Testcase object it equals.
     def matches_existing_testcase(self, t):
-        if t.root in ["invalid_input", "invalid_answer"]:
-            return None
-        h = hash_file_content(t.in_path)
+        hashes = {}
+        relevant_files = {
+            "invalid_input": ["in"],
+            "invalid_answer": [".in", ".ans"],
+            "invalid_output": [".in", ".ans", ".out"],
+            "valid_output": [".in", ".ans", ".out"],
+        }
+        relevant_files_default = [".in"] if self.settings.ans_is_output else [".in", ".ans"]
+        extensions = relevant_files.get(t.root, relevant_files_default)
+
+        for ext in extensions:
+            if t.with_suffix(ext).is_file():
+                hashes[ext] = hash_file_content(t.with_suffix(ext))
+
+        h = combine_hashes_dict(hashes)
         if h in self._testcase_hashes:
             return self._testcase_hashes[h]
         self._testcase_hashes[h] = t
@@ -1394,14 +1406,7 @@ def process_testcase(testcase: testcase.Testcase):
 
             localbar = bar.start(testcase.name)
 
-            if (
-                mode == validate.Mode.INPUT
-                and not testcase.in_path.is_symlink()
-                and not testcase.root == "invalid_answer"
-                and not testcase.root == "invalid_output"
-                and not testcase.root == "valid_output"
-                and not extra
-            ):
+            if mode == validate.Mode.INPUT and not testcase.in_path.is_symlink() and not extra:
                 t2 = problem.matches_existing_testcase(testcase)
                 if t2 is not None:
                     localbar.warn(

bin/util.py

@@ -1463,7 +1463,7 @@ def combine_hashes(values: Sequence[str]) -> str:
     return hasher.hexdigest()
 
 
-def combine_hashes_dict(d: dict[str, Optional[str]]) -> str:
+def combine_hashes_dict(d: Mapping[str, Optional[str]]) -> str:
     hasher = hashlib.sha512(usedforsecurity=False)
     for key, value in d.items():
         hasher.update(key.encode())