Add pipeline loggers

Author: mkardas

sota_extractor2/config.py

@@ -24,5 +24,7 @@
 datasets_structure = datasets/"structure"
 structure_models = datasets / "structure" / "models"
 
+mocks = datasets / "mocks"
+
 linking_models = datasets / "linking" / "models"
 linking_data = datasets / "linking" / "data"

sota_extractor2/data/paper_collection.py

@@ -7,6 +7,10 @@
 from joblib import Parallel, delayed
 from collections import UserList
 from ..helpers.jupyter import display_table
+import string
+import random
+from extract_tables import extract_tables
+
 
 class Paper:
     def __init__(self, paper_id, text, tables, annotations):
@@ -25,6 +29,27 @@ def __init__(self, paper_id, text, tables, annotations):
             self.gold_tags = ''
 
 
+# todo: make sure multithreading/processing won't cause collisions
+def random_id():
+    return "temp_" + ''.join(random.choice(string.ascii_lowercase) for i in range(10))
+
+
+class TempPaper(Paper):
+    """Similar to Paper, but can be used as context manager, temporarily saving the paper to elastic"""
+    def __init__(self, html):
+        paper_id = random_id()
+        text = PaperText.from_html(html, paper_id)
+        tables = extract_tables(html)
+        super().__init__(paper_id=paper_id, text=text, tables=tables, annotations=None)
+
+    def __enter__(self):
+        self.text.save()
+        return self
+
+    def __exit__(self, exc, value, tb):
+        self.text.delete()
+
+
 arxiv_version_re = re.compile(r"v\d+$")
 def remove_arxiv_version(arxiv_id):
     return arxiv_version_re.sub("", arxiv_id)

sota_extractor2/data/structure.py

@@ -99,6 +99,10 @@ def fix_reference_hightlight(s):
     return partial_highlight_re.sub("xxref-", s)
 
 
+evidence_columns = ["text_sha1", "text_highlited", "text", "header", "cell_type", "cell_content", "cell_reference",
+                    "cell_layout", "cell_styles", "this_paper", "row", "col", "row_context", "col_context", "ext_id"]
+
+
 def create_evidence_records(textfrag, cell, paper, table):
     for text_highlited in textfrag.meta['highlight']['text']:
         text_highlited = fix_reference_hightlight(fix_refs(text_highlited))
@@ -141,15 +145,19 @@ def evidence_for_table(paper, table, paper_limit, corpus_limit):
                                            row=cell.row, col=cell.col, paper_limit=paper_limit, corpus_limit=corpus_limit)
             for record in create_evidence_records(evidence, cell, paper=paper, table=table)
     ]
-    df = pd.DataFrame.from_records(records)
+    df = pd.DataFrame.from_records(records, columns=evidence_columns)
     return df
 
 
 def prepare_data(paper, tables, csv_path, limit_type='interesting'):
-    df = pd.concat([evidence_for_table(paper, table,
+    data = [evidence_for_table(paper, table,
                                        paper_limit=100,
                                        corpus_limit=20,
-                                       limit_type=limit_type) for table in progress_bar(tables)])
+                                       limit_type=limit_type) for table in progress_bar(tables)]
+    if len(data):
+        df = pd.concat(data)
+    else:
+        df = pd.DataFrame(columns=evidence_columns)
     #moved to experiment preprocessing
     #df = df.drop_duplicates(
     #    ["cell_content", "text_highlited", "cell_type", "this_paper"])
@@ -168,4 +176,4 @@ def __call__(self, paper, tables, paper_limit=30, corpus_limit=10):
         dfs = [evidence_for_table(paper, table, paper_limit, corpus_limit) for table in tables]
         if len(dfs):
             return pd.concat(dfs)
-        return pd.DataFrame()
+        return pd.DataFrame(columns=evidence_columns)

sota_extractor2/helpers/__init__.py

@@ -1,5 +1,4 @@
-from .temp_paper import TempPaper
 from .latex_converter import LatexConverter
 from .unpack import Unpack
 
-__all__ = ["TempPaper", "LatexConverter", "Unpack"]
+__all__ = ["LatexConverter", "Unpack"]

sota_extractor2/helpers/temp_paper.py

@@ -4,16 +4,21 @@
 from pathlib import Path
 from shutil import copyfileobj
 from sota_extractor2.errors import UnpackError
+from ..pipeline_logger import pipeline_logger
 
 
 class Unpack:
+    step = "unpack"
+
     def __init__(self):
         self.magic = Magic(mime=True, uncompress=True)
 
     def __call__(self, source, dest):
+        pipeline_logger(f"{Unpack.step}::call", source=source, dest=dest)
         source = Path(source)
         dest = Path(dest)
         mime = self.magic.from_file(str(source))
+        pipeline_logger(f"{Unpack.step}::detect_mime", source=source, mime=mime)
         if mime == 'application/x-tar':
             dest.mkdir(parents=True, exist_ok=True)
             with tarfile.open(source, "r:*") as tar:

sota_extractor2/helpers/unpack.py

@@ -0,0 +1,104 @@
+import sys
+import pandas as pd
+from .models.structure.experiment import Experiment, label_map, Labels
+from .models.structure.type_predictor import TableType
+
+
+class BaseLogger:
+    def __init__(self, pipeline_logger, pattern=".*"):
+        pipeline_logger.register(pattern, self)
+
+    def __call__(self, step, **kwargs):
+        raise NotImplementedError()
+
+
+class StdoutLogger:
+    def __init__(self, pipeline_logger, file=sys.stdout):
+        self.file = file
+        pipeline_logger.register(".*", self)
+
+    def __call__(self, step, **kwargs):
+        print(f"[STEP] {step}: {kwargs}", file=self.file)
+
+
+class StructurePredictionEvaluator:
+    def __init__(self, pipeline_logger, pc):
+        pipeline_logger.register("structure_prediction::tables_labelled", self.on_tables_labelled)
+        pipeline_logger.register("type_prediction::predicted", self.on_type_predicted)
+        self.pc = pc
+        self.results = {}
+        self.type_predictions = {}
+
+    def on_type_predicted(self, step, paper, tables, predictions):
+        self.type_predictions[paper.paper_id] = predictions
+
+    def on_tables_labelled(self, step, paper, tables):
+        golds = [p for p in self.pc if p.text.title == paper.text.title]
+        paper_id = paper.paper_id
+        type_results = []
+        cells_results = []
+        if len(golds) == 1:
+            gold = golds[0]
+            for gold_table, table, table_type in zip(gold.tables, paper.tables, self.type_predictions.get(paper.paper_id, [])):
+                is_important = table_type == TableType.SOTA or table_type == TableType.ABLATION
+                gold_is_important = "sota" in gold_table.gold_tags or "ablation" in gold_table.gold_tags
+                type_results.append({"predicted": is_important, "gold": gold_is_important, "name": table.name})
+                if not is_important:
+                    continue
+                rows, cols = table.df.shape
+                for r in range(rows):
+                    for c in range(cols):
+                        cells_results.append({
+                            "predicted": table.df.iloc[r, c].gold_tags,
+                            "gold": gold_table.df.iloc[r, c].gold_tags,
+                            "ext_id": f"{table.name}/{r}.{c}",
+                            "content": table.df.iloc[r, c].value
+                        })
+
+        self.results[paper_id] = {
+            'type': pd.DataFrame.from_records(type_results),
+            'cells': pd.DataFrame.from_records(cells_results)
+        }
+
+    def map_tags(self, tags):
+        mapping = dict(label_map)
+        mapping[""] = Labels.EMPTY.value
+        return tags.str.strip().apply(lambda x: mapping.get(x, 0))
+
+    def metrics(self, paper_id):
+        if paper_id not in self.results:
+            print(f"No annotations for {paper_id}")
+            return
+        print("Structure prediction:")
+        results = self.results[paper_id]
+        cells_df = results['cells']
+        e = Experiment()
+        e._set_results(paper_id, self.map_tags(results['cells'].predicted), self.map_tags(results['cells'].gold))
+        e.show_results(paper_id, normalize=True)
+
+
+class LinkerEvaluator:
+    def __init__(self, pipeline_logger, pc):
+        pipeline_logger.register("linking::call", self.on_before_linking)
+        pipeline_logger.register("linking::taxonomy_linking::call", self.on_before_taxonomy)
+        pipeline_logger.register("linking::taxonomy_linking::topk", self.on_taxonomy_topk)
+        pipeline_logger.register("linking::linked", self.on_after_linking)
+        self.proposals = {}
+        self.topk = {}
+
+    def on_before_linking(self, step, paper, tables):
+        pass
+
+    def on_after_linking(self, step, paper, tables, proposals):
+        self.proposals[paper.paper_id] = proposals.copy(deep=True)
+
+    def on_before_taxonomy(self, step, ext_id, query, datasets, caption):
+        pass
+
+    def on_taxonomy_topk(self, step, ext_id, topk):
+        paper_id, table_name, rc = ext_id.split('/')
+        row, col = [int(x) for x in rc.split('.')]
+        self.topk[paper_id, table_name, row, col] = topk.copy(deep=True)
+
+    def top_matches(self, paper_id, table_name, row, col):
+        return self.topk[(paper_id, table_name, row, col)]

sota_extractor2/loggers.py

@@ -0,0 +1,7 @@
+class LatexConverterMock:
+    def __init__(self, mock_file):
+        with open(mock_file, "r") as f:
+            self.mock = f.read()
+
+    def to_html(self, source_dir):
+        return self.mock

sota_extractor2/mocks/latex_converter.py

@@ -169,6 +169,11 @@ def handle_pm(value):
                 pass
             # %%
 
+
+proposal_columns = ['dataset', 'metric', 'task', 'format', 'raw_value', 'model', 'model_type', 'cell_ext_id',
+                    'confidence', 'parsed', 'struct_model_type', 'struct_dataset']
+
+
 def generate_proposals_for_table(table_ext_id,  matrix, structure, desc, taxonomy_linking, datasets):
     # %%
     # Proposal generation
@@ -249,9 +254,8 @@ def linked_proposals(proposals):
             yield linked
 
     # specify columns in case there's no proposal
-    columns = ['dataset', 'metric', 'task', 'format', 'raw_value', 'model', 'model_type', 'cell_ext_id', 'confidence', 'parsed',
-               'struct_model_type', 'struct_dataset']
-    proposals = pd.DataFrame.from_records(list(linked_proposals(proposals)), columns=columns)
+
+    proposals = pd.DataFrame.from_records(list(linked_proposals(proposals)), columns=proposal_columns)
 
     if len(proposals):
         proposals["parsed"]=proposals[["raw_value", "format"]].apply(
@@ -274,7 +278,9 @@ def linked_proposals(paper_ext_id, paper, annotated_tables, taxonomy_linking=Mat
 
         if 'sota' in tags and 'no_sota_records' not in tags: # only parse tables that are marked as sota
             proposals.append(generate_proposals_for_table(table_ext_id, matrix, structure, desc, taxonomy_linking, datasets))
-    return pd.concat(proposals)
+    if len(proposals):
+        return pd.concat(proposals)
+    return pd.DataFrame(columns=proposal_columns)
 
 
 def test_link_taxonomy():

sota_extractor2/models/linking/bm25_naive.py

@@ -9,19 +9,21 @@
 import pandas as pd
 import numpy as np
 
+from sota_extractor2.pipeline_logger import pipeline_logger
+
 metrics = {
     'BLEU': ['bleu'],
     'BLEU score': ['bleu'],
-    'Character Error Rate': ['cer'],
+    'Character Error Rate': ['cer', 'cers'],
     'Error': ['error'],
     'Exact Match Ratio': ['exact match'],
     'F1': ['f1', 'f1 score'],
     'F1 score': ['f1', 'f1 score'],
     'MAP': ['map'],
-    'Percentage error': ['wer', 'per', 'word error rate', 'word error rates', 'phoneme error rates',
+    'Percentage error': ['wer', 'per', 'wers', 'pers', 'word error rate', 'word error rates', 'phoneme error rates',
                          'phoneme error rate', 'error', 'error rate', 'error rates'],
-    'Word Error Rate': ['wer', 'word error rate', 'word error rates', 'error', 'error rate', 'error rates'],
-    'Word Error Rate (WER)': ['wer', 'word error rate', 'word error rates', 'error', 'error rate', 'error rates'],
+    'Word Error Rate': ['wer', 'wers', 'word error rate', 'word error rates', 'error', 'error rate', 'error rates'],
+    'Word Error Rate (WER)': ['wer', 'wers', 'word error rate', 'word error rates', 'error', 'error rate', 'error rates'],
     'ROUGE-1': ['r1'],
     'ROUGE-2': ['r2'],
     'ROUGE-F': ['rf'],
@@ -173,10 +175,10 @@ def match(self, contexts):
         return zip(keys, probs)
 
     def __call__(self, query, datasets, caption, debug_info=None):
+        cellstr = debug_info.cell.cell_ext_id
+        pipeline_logger("linking::taxonomy_linking::call", ext_id=cellstr, query=query, datasets=datasets, caption=caption)
         datasets = " ".join(datasets)
-        cell = debug_info.cell
         key = (datasets, caption, query)
-        cellstr = f"{cell.table_ext_id}/{cell.row}.{cell.col}"
         ###print(f"[DEBUG] {cellstr}")
         ###print("[DEBUG]", debug_info)
         ###print("query:", query, caption)
@@ -226,6 +228,7 @@ def __call__(self, query, datasets, caption, debug_info=None):
             else:
                 print("[EA] No gold sota record found for the cell")
         # end of error analysis only
+        pipeline_logger("linking::taxonomy_linking::topk", ext_id=cellstr, topk=p)
         return p.head(1)