Use gold_sota_records for error analysis

* change taxonomy format to list of triplets
* change hyphen to spaces when normalizing datasets

Author: Marcin Kardas

sota_extractor2/helpers/explainers.py

@@ -91,10 +91,11 @@ def _repr_html_(self):
 class Explainer:
     _sota_record_columns = ['task', 'dataset', 'metric', 'format', 'model', 'model_type', 'raw_value', 'parsed']
 
-    def __init__(self, pipeline_logger, paper_collection):
+    def __init__(self, pipeline_logger, paper_collection, gold_sota_records=None):
         self.paper_collection = paper_collection
+        self.gold_sota_records = gold_sota_records
         self.spe = StructurePredictionEvaluator(pipeline_logger, paper_collection)
-        self.le = LinkerEvaluator(pipeline_logger, paper_collection)
+        self.le = LinkerEvaluator(pipeline_logger)
         self.fe = FilteringEvaluator(pipeline_logger)
 
     def explain(self, paper, cell_ext_id):
@@ -179,11 +180,20 @@ def linking_metrics(self, experiment_name="unk"):
             print(", ".join(missing))
         papers = [paper for paper in papers.values() if paper is not None]
 
-        if not len(papers):
+        # if not len(papers):
+        #     gold_sota_records = pd.DataFrame(columns=self._sota_record_columns)
+        #     gold_sota_records.index.rename("cell_ext_id", inplace=True)
+        # else:
+        #     gold_sota_records = pd.concat([self._get_sota_records(paper) for paper in papers])
+        if self.gold_sota_records is None:
             gold_sota_records = pd.DataFrame(columns=self._sota_record_columns)
             gold_sota_records.index.rename("cell_ext_id", inplace=True)
         else:
-            gold_sota_records = pd.concat([self._get_sota_records(paper) for paper in papers])
+
+            gold_sota_records = self.gold_sota_records
+            which = gold_sota_records.index.to_series().str.split("/", expand=True)[0]\
+                .isin([paper.paper_id for paper in papers])
+            gold_sota_records = gold_sota_records[which]
 
         df = gold_sota_records.merge(proposals, 'outer', left_index=True, right_index=True, suffixes=['_gold', '_pred'])
         df = df.reindex(sorted(df.columns), axis=1)

sota_extractor2/loggers.py

@@ -144,7 +144,7 @@ def get_table_type_predictions(self, paper_id, table_name):
 
 
 class LinkerEvaluator:
-    def __init__(self, pipeline_logger, pc):
+    def __init__(self, pipeline_logger):
         pipeline_logger.register("linking::call", self.on_before_linking)
         pipeline_logger.register("linking::taxonomy_linking::call", self.on_before_taxonomy)
         pipeline_logger.register("linking::taxonomy_linking::topk", self.on_taxonomy_topk)

sota_extractor2/models/linking/acronym_extractor.py

@@ -1,6 +1,6 @@
 import spacy
 from scispacy.abbreviation import AbbreviationDetector
-from .utils import normalize_cell, normalize_dataset
+from .utils import normalize_cell, normalize_dataset_ws
 
 class AcronymExtractor:
     def __init__(self):
@@ -14,7 +14,7 @@ def __call__(self, text):
         abbrvs = {}
         for abrv in doc._.abbreviations:
             # abbrvs.setdefault(normalize_cell(str(abrv)), Counter())[str(abrv._.long_form)] += 1
-            norm = normalize_cell(normalize_dataset(str(abrv)))
+            norm = normalize_cell(normalize_dataset_ws(str(abrv)))
             if norm != '':
-                abbrvs[norm] = normalize_cell(normalize_dataset(str(abrv._.long_form)))
+                abbrvs[norm] = normalize_cell(normalize_dataset_ws(str(abrv._.long_form)))
         return abbrvs

sota_extractor2/models/linking/context_search.py

@@ -3,7 +3,7 @@
 
 from sota_extractor2.models.linking.acronym_extractor import AcronymExtractor
 from sota_extractor2.models.linking.probs import get_probs, reverse_probs
-from sota_extractor2.models.linking.utils import normalize_dataset, normalize_cell, normalize_cell_ws
+from sota_extractor2.models.linking.utils import normalize_dataset_ws, normalize_cell, normalize_cell_ws
 from scipy.special import softmax
 import re
 import pandas as pd
@@ -201,9 +201,9 @@ def dummy_item(reason):
 
 
 @njit
-def compute_logprobs(dataset_metric, reverse_merged_p, reverse_metrics_p, dss, mss, noise, logprobs):
+def compute_logprobs(taxonomy, reverse_merged_p, reverse_metrics_p, dss, mss, noise, logprobs):
     empty = typed.Dict.empty(types.unicode_type, types.float64)
-    for i, (dataset, metric) in enumerate(dataset_metric):
+    for i, (task, dataset, metric) in enumerate(taxonomy):
         logprob = 0.0
         short_probs = reverse_merged_p.get(dataset, empty)
         met_probs = reverse_metrics_p.get(metric, empty)
@@ -223,16 +223,16 @@ def compute_logprobs(dataset_metric, reverse_merged_p, reverse_metrics_p, dss, m
 class ContextSearch:
     def __init__(self, taxonomy, context_noise=(0.5, 0.2, 0.1), debug_gold_df=None):
         merged_p = \
-        get_probs({k: Counter([normalize_cell(normalize_dataset(x)) for x in v]) for k, v in datasets.items()})[1]
+        get_probs({k: Counter([normalize_cell(normalize_dataset_ws(x)) for x in v]) for k, v in datasets.items()})[1]
         metrics_p = \
-        get_probs({k: Counter([normalize_cell(normalize_dataset(x)) for x in v]) for k, v in metrics.items()})[1]
+        get_probs({k: Counter([normalize_cell(normalize_dataset_ws(x)) for x in v]) for k, v in metrics.items()})[1]
 
 
         self.queries = {}
         self.taxonomy = taxonomy
-        self._dataset_metric = typed.List()
+        self._taxonomy = typed.List()
         for t in self.taxonomy.taxonomy:
-            self._dataset_metric.append(t)
+            self._taxonomy.append(t)
         self.extract_acronyms = AcronymExtractor()
         self.context_noise = context_noise
         self.reverse_merged_p = self._numba_update_nested_dict(reverse_probs(merged_p))
@@ -254,8 +254,9 @@ def _numba_extend_list(self, lst):
         return l
 
     def compute_context_logprobs(self, context, noise, logprobs):
+        context = context or ""
         abbrvs = self.extract_acronyms(context)
-        context = normalize_cell_ws(normalize_dataset(context))
+        context = normalize_cell_ws(normalize_dataset_ws(context))
         dss = set(find_datasets(context)) | set(abbrvs.keys())
         mss = set(find_metrics(context))
         dss -= mss
@@ -265,16 +266,16 @@ def compute_context_logprobs(self, context, noise, logprobs):
         ###print("mss", mss)
         dss = self._numba_extend_list(dss)
         mss = self._numba_extend_list(mss)
-        compute_logprobs(self._dataset_metric, self.reverse_merged_p, self.reverse_metrics_p, dss, mss, noise, logprobs)
+        compute_logprobs(self._taxonomy, self.reverse_merged_p, self.reverse_metrics_p, dss, mss, noise, logprobs)
 
     def match(self, contexts):
         assert len(contexts) == len(self.context_noise)
-        n = len(self._dataset_metric)
-        context_logprobs = np.ones(n)
+        n = len(self._taxonomy)
+        context_logprobs = np.zeros(n)
 
         for context, noise in zip(contexts, self.context_noise):
             self.compute_context_logprobs(context, noise, context_logprobs)
-        keys = self.taxonomy.taxonomy.keys()
+        keys = self.taxonomy.taxonomy
         logprobs = context_logprobs
         #keys, logprobs = zip(*context_logprobs.items())
         probs = softmax(np.array(logprobs))
@@ -290,12 +291,12 @@ def __call__(self, query, datasets, caption, debug_info=None):
         ###print("query:", query, caption)
         if key in self.queries:
             # print(self.queries[key])
-            for context in key:
-                abbrvs = self.extract_acronyms(context)
-                context = normalize_cell_ws(normalize_dataset(context))
-                dss = set(find_datasets(context)) | set(abbrvs.keys())
-                mss = set(find_metrics(context))
-                dss -= mss
+            # for context in key:
+            #     abbrvs = self.extract_acronyms(context)
+            #     context = normalize_cell_ws(normalize_dataset_ws(context))
+            #     dss = set(find_datasets(context)) | set(abbrvs.keys())
+            #     mss = set(find_metrics(context))
+            #     dss -= mss
                 ###print("dss", dss)
                 ###print("mss", mss)
 
@@ -307,7 +308,8 @@ def __call__(self, query, datasets, caption, debug_info=None):
 
             entries = []
             for it, prob in topk:
-                entry = dict(self.taxonomy.taxonomy[it])
+                task, dataset, metric = it
+                entry = dict(task=task, dataset=dataset, metric=metric)
                 entry.update({"evidence": "", "confidence": prob})
                 entries.append(entry)
 
@@ -351,4 +353,4 @@ def from_paper(self, paper):
         return self(text)
 
     def __call__(self, text):
-        return find_datasets(normalize_cell_ws(normalize_dataset(text)))
+        return find_datasets(normalize_cell_ws(normalize_dataset_ws(text)))

sota_extractor2/models/linking/taxonomy.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 import json
+from collections import OrderedDict
 
 
 
@@ -14,7 +15,7 @@ def _read_json(self, path):
 
     def _read_taxonomy(self, path):
         records = self._read_json(path)
-        return {(x['dataset'], x['metric']): x for x in records}
+        return [(r["task"], r["dataset"], r["metric"]) for r in records]
 
     def _read_metrics_info(self, path):
         records = self._read_json(path)

sota_extractor2/models/linking/utils.py

@@ -30,7 +30,7 @@ def clean_cell(cell):
 def remove_references(s):
     return refs_re.sub("", s)
 
-def normalize_dataset2(name):
+def normalize_dataset_ws(name):
     name = remove_references(name)
     name = hyphens_re.sub(" ", name)
     name = year_2k_re.sub(r"\1", name)

sota_extractor2/models/structure/structure_predictor.py

@@ -122,6 +122,8 @@ def to_tables(self, df, transpose=False):
         return X_tables, C_tables, ids
 
     def merge_with_preds(self, df, preds):
+        if not len(df):
+            return []
         ext_id = df.ext_id.str.split("/", expand=True)
         return list(zip(ext_id[0] + "/" + ext_id[1], ext_id[2].astype(int), ext_id[3].astype(int),
                         preds, df.text, df.cell_content, df.cell_layout, df.cell_styles, df.cell_reference, df.label))

sota_extractor2/models/structure/type_predictor.py

@@ -30,8 +30,10 @@ def predict(self, paper, tables):
         if len(tables) == 0:
             predictions = []
         else:
-            df = pd.DataFrame({"caption": [table.caption if table.caption else "" for table in tables]})
-            tl = TextList.from_df(df, cols="caption")
+            column = "caption"
+            df = pd.DataFrame({column: [table.caption if table.caption else "Table" for table in tables]})
+            inputs = df.iloc[:, df_names_to_idx(column, df)]
+            tl = TextList(items=inputs.values[:, 0], path='.', inner_df=df, processor=None)
             self.learner.data.add_test(tl)
             preds, _ = self.learner.get_preds(DatasetType.Test, ordered=True)
             pipeline_logger(f"{TableTypePredictor.step}::multiclass_predicted", paper=paper, tables=tables,