Add support for complementary metrics

Author: Marcin Kardas

sota_extractor2/models/linking/bm25_naive.py

@@ -230,33 +230,32 @@ def linked_proposals(proposals):
 
             df = taxonomy_linking(prop.dataset, datasets, desc, topk=topk, debug_info=prop)
             for _, row in df.iterrows():
+                raw_value = prop.raw_value
+                parsed = float(extract_value(raw_value, format))
                 metric = row['metric']
-                # if ("error" in metric or "Error" in metric) and (first_num > 0.5):
-                if (metric.strip().lower() == "error") and (first_num > 0.5):
-                    metric = "Accuracy"
+                if metric != row['true_metric']:
+                    metric = row['true_metric']
+                    parsed = 1 - parsed if parsed < 1 else 100 - parsed
 
                 linked = {
                     'dataset': row['dataset'],
                     'metric': metric,
                     'task': row['task'],
                     'format': format,
-                    'raw_value': prop.raw_value,
+                    'raw_value': raw_value,
                     'model': prop.model_name,
                     'model_type': prop.model_type,
                     'cell_ext_id': prop.cell.cell_ext_id,
                     'confidence': row['confidence'],
                     'struct_model_type': prop.model_type,
-                    'struct_dataset': prop.dataset
+                    'struct_dataset': prop.dataset,
+                    'parsed': parsed
                 }
                 yield linked
 
     # specify columns in case there's no proposal
 
     proposals = pd.DataFrame.from_records(list(linked_proposals(proposals)), columns=proposal_columns)
-
-    if len(proposals):
-        proposals["parsed"]=proposals[["raw_value", "format"]].apply(
-            lambda row: float(extract_value(row.raw_value, row.format)), axis=1)
     return proposals
 
 

sota_extractor2/models/linking/context_search.py

@@ -104,7 +104,7 @@ def _init_structs(self, taxonomy):
         self.all_tasks_trie = EvidenceFinder.make_trie(self.all_tasks)
 
 
-@njit(inline="always")
+@njit
 def axis_logprobs(evidences_for, reverse_probs, found_evidences, noise, pb):
     logprob = 0.0
     empty = typed.Dict.empty(types.unicode_type, types.float64)
@@ -114,6 +114,7 @@ def axis_logprobs(evidences_for, reverse_probs, found_evidences, noise, pb):
     return logprob
 
 
+# compute log-probabilities in a given context and add them to logprobs
 @njit
 def compute_logprobs(taxonomy, reverse_merged_p, reverse_metrics_p, reverse_task_p,
                      dss, mss, tss, noise, ms_noise, ts_noise, ds_pb, ms_pb, ts_pb, logprobs):
@@ -128,7 +129,7 @@ def compute_logprobs(taxonomy, reverse_merged_p, reverse_metrics_p, reverse_task
         if task not in task_cache:
             task_cache[task] = axis_logprobs(task, reverse_task_p, tss, ts_noise, ts_pb)
 
-        logprobs[i] = dataset_cache[dataset] + metric_cache[metric] + task_cache[task]
+        logprobs[i] += dataset_cache[dataset] + metric_cache[metric] + task_cache[task]
 
 
 class ContextSearch:
@@ -262,7 +263,10 @@ def __call__(self, query, datasets, caption, topk=1, debug_info=None):
                 print("[EA] No gold sota record found for the cell")
         # end of error analysis only
         pipeline_logger("linking::taxonomy_linking::topk", ext_id=cellstr, topk=p.head(5))
-        return p.head(topk)
+
+        q = p.head(topk).copy()
+        q["true_metric"] = q.apply(lambda row: self.taxonomy.normalize_metric(row.task, row.dataset, row.metric), axis=1)
+        return q
 
 
 # todo: compare regex approach (old) with find_datasets(.) (current)

sota_extractor2/models/linking/format.py

@@ -29,7 +29,8 @@ def format_to_regexp(format):
     return re.compile('^' + regexp), fn
 
 def extract_value(cell_value, format):
-    cell_value = re.sub(r"\s+%", "%", cell_value).replace(",", "").strip()
+    cell_value = re.sub(r"\s+%", "%", cell_value).replace(",", "")
+    cell_value = cell_value.replace("(", " ").replace(")", " ").strip()
     regexp, fn = format_to_regexp(format)
     match = regexp.match(cell_value)
     if match is None or not len(match.groups()):

sota_extractor2/models/linking/manual_dicts.py

@@ -1,8 +1,9 @@
 metrics = {
+    'Accuracy': ['acc', 'accuracy'],
     'BLEU': ['bleu'],
     'BLEU score': ['bleu'],
     'Character Error Rate': ['cer', 'cers'],
-    'Error': ['error'],
+    'Error': ['error', 'err', 'error rate'],
     'Exact Match Ratio': ['exact match'],
     'F1': ['f1', 'f1 score'],
     'F1 score': ['f1', 'f1 score'],
@@ -11,6 +12,7 @@
                          'phoneme error rate', 'error', 'error rate', 'error rates'],
     'Word Error Rate': ['wer', 'wers', 'word error rate', 'word error rates', 'error', 'error rate', 'error rates'],
     'Word Error Rate (WER)': ['wer', 'wers', 'word error rate', 'word error rates', 'error', 'error rate', 'error rates'],
+    'Word Accuracy': ['accuracy', 'word accuracy', 'acc', 'word acc'],
     'ROUGE-1': ['r1'],
     'ROUGE-2': ['r2'],
     'ROUGE-F': ['rf'],
@@ -31,8 +33,12 @@
     'Category IoU': ['cat iou', 'iou cat'],
     'class iIoU': ['class iiou', 'iiou cla'],
     'Category iIoU': ['cat iiou', 'iiou cat'],
-    'Mean Accuracy': ['mean acc', 'mean', 'acc']
-
+    'Mean Accuracy': ['mean acc', 'mean', 'acc', 'accuracy', 'mean accuracy'],
+    'Mean Error': ['mean err', 'mean', 'err', 'mean error', 'error'],
+    'Top-1 Accuracy': ['top 1 accuracy', 'top 1', 'top 1 acc'],
+    'Top-5 Accuracy': ['top 5 accuracy', 'top 5', 'top 5 acc'],
+    'Top-1 Error Rate': ['top 1 error', 'top 1', 'top 1 err'],
+    'Top-5 Error': ['top 5 error', 'top 5', 'top 5 err']
 }
 
 # datasets[taxonomy name] is a list of normalized evidences for taxonomy name
@@ -153,3 +159,13 @@
 }
 
 tasks = {}
+
+complementary_metrics = {
+    'Accuracy': 'Error',
+    'Error': 'Accuracy',
+    'Percentage Error': 'Accuracy',
+    'Word Error Rate': 'Word Accuracy',
+    'Word Error Rate (WER)': 'Word Accuracy',
+    'Top-1 Accuracy': 'Top-1 Error Rate',
+    'Top-5 Accuracy': 'Top-5 Error',
+}

sota_extractor2/models/linking/taxonomy.py

@@ -1,23 +1,49 @@
 from pathlib import Path
 import json
 from collections import OrderedDict
-
+from sota_extractor2.models.linking.manual_dicts import complementary_metrics
 
 
 class Taxonomy:
     def __init__(self, taxonomy, metrics_info):
-        self.taxonomy = self._read_taxonomy(taxonomy)
+        self.taxonomy = self._get_taxonomy(taxonomy)
         self.metrics_info = self._read_metrics_info(metrics_info)
         self.tasks = self._get_axis('task')
         self.datasets = self._get_axis('dataset')
         self.metrics = self._get_axis('metric')
 
+    def normalize_metric(self, task, dataset, metric):
+        if (task, dataset, metric) in self._complementary:
+            return self._complementary[(task, dataset, metric)][2]
+        return metric
+
     def _read_json(self, path):
         with open(path, "rt") as f:
             return json.load(f)
 
-    def _read_taxonomy(self, path):
-        self._records = self._read_json(path)
+    def _get_complementary_metrics(self, records):
+        complementary = []
+        self._complementary = {}
+        for record in records:
+            metric = record["metric"]
+            if metric in complementary_metrics:
+                task = record["task"]
+                dataset = record["dataset"]
+                comp_metric = complementary_metrics[record["metric"]]
+                complementary.append(
+                    dict(
+                        task=task,
+                        dataset=dataset,
+                        metric=comp_metric
+                    )
+                )
+
+                self._complementary[(task, dataset, comp_metric)] = (task, dataset, metric)
+        return complementary
+
+    def _get_taxonomy(self, path):
+        records = self._read_json(path)
+        self._records = records + self._get_complementary_metrics(records)
         return [(r["task"], r["dataset"], r["metric"]) for r in self._records]
 
     def _get_axis(self, axis):