fix(similarity): maximize score instead of minimize

Author: AmitMY

signwriting_evaluation/metrics/base.py

@@ -29,9 +29,10 @@ def corpus_score(self, hypotheses: list[str], references: list[list[str]]) -> fl
         transpose_references = list(zip(*references))
         return sum(self.score_max(h, r) for h, r in zip(hypotheses, transpose_references)) / len(hypotheses)
 
-    def score_all(self, hypotheses: list[str], references: list[str]) -> list[list[float]]:
+    def score_all(self, hypotheses: list[str], references: list[str], progress_bar=True) -> list[list[float]]:
         # Default implementation: call the score function for each hypothesis-reference pair
-        return [[self.score(h, r) for r in references] for h in tqdm(hypotheses, disable=len(hypotheses) == 1)]
+        return [[self.score(h, r) for r in references]
+                for h in tqdm(hypotheses, disable=not progress_bar or len(hypotheses) == 1)]
 
     def __str__(self):
         return self.name

signwriting_evaluation/metrics/similarity.py

@@ -117,8 +117,8 @@ def score(self, hypothesis: str, reference: str) -> float:
             reference_signs += [""] * (max_length - len(reference_signs))
 
         # Match each hypothesis sign with each reference sign
-        cost_matrix = self.score_all(hypothesis_signs, reference_signs)
-        row_ind, col_ind = linear_sum_assignment(cost_matrix)
+        cost_matrix = self.score_all(hypothesis_signs, reference_signs, progress_bar=False)
+        row_ind, col_ind = linear_sum_assignment(1 - np.array(cost_matrix))
         pairs = list(zip(row_ind, col_ind))
         values = [cost_matrix[row][col] for row, col in pairs]
         return sum(values) / len(values)

signwriting_evaluation/metrics/test_similarity.py

@@ -44,6 +44,14 @@ def test_multi_sign_score(self):
         self.assertIsInstance(score, float)
         self.assertAlmostEqual(score, 0.8326259781509948 / 2)
 
+    def test_multi_sign_score_is_order_invariant(self):
+        sign_1 = "M530x538S17600508x462S15a11493x494S20e00488x510S22f03469x517"
+        sign_2 = "M530x538S17600508x462S12a11493x494S20e00488x510S22f13469x517"
+        hypothesis = f"{sign_1} {sign_2}"
+        reference = f"{sign_2} {sign_1}"
+        score = self.metric.score(hypothesis, reference)
+        self.assertAlmostEqual(score, 1)
+
     def test_bad_fsw_equals_0(self):
         bad_fsw = "M<s><s>M<s>p483"
         score = self.metric.corpus_score([bad_fsw], [[bad_fsw]])