Code for initial NBSVM baseline

PiotrCzapla · PiotrCzapla · commit eb2f1bcecbd0 · 2019-06-30T18:12:06.000Z
diff --git a/sota_extractor2/config.py b/sota_extractor2/config.py
@@ -14,3 +14,11 @@
 
 
 elastic = dict(hosts=['localhost'], timeout=20)
+
+
+arxiv = data/'arxiv'
+htmls_raw = arxiv/'htmls'
+htmls_clean = arxiv/'htmls-clean'
+
+datasets = data/"datasets"
+datasets_structure = datasets/"structure"
diff --git a/sota_extractor2/data/structure.py b/sota_extractor2/data/structure.py
@@ -0,0 +1,95 @@
+import re
+import pandas as pd
+from collections import namedtuple
+import hashlib
+from fastai.text import progress_bar
+from .elastic import Fragment
+from .json import *
+
+def get_all_tables(papers):
+    for paper in papers:
+        for table in paper.table_set.all():
+            if 'trash' not in table.gold_tags and table.gold_tags != '':
+                table.paper_id = paper.arxiv_id
+                yield table
+
+def consume_cells(*matrix):
+    Cell = namedtuple('AnnCell', 'row col vals')
+    for row_id, row in enumerate(zip(*matrix)):
+        for col_id, cell_val in enumerate(zip(*row)):
+            yield Cell(row=row_id, col=col_id, vals=cell_val)
+
+
+def fetch_evidence(cell_content, paper_id, paper_limit=10, corpus_limit=10):
+    evidence_query = Fragment.search().highlight(
+        'text', pre_tags="<b>", post_tags="</b>", fragment_size=400)
+    cell_content = cell_content.replace("\xa0", " ")
+    query = {
+        "query": cell_content,
+        "slop": 2
+    }
+    paper_fragments = list(evidence_query
+                           .filter('term', paper_id=paper_id)
+                           .query('match_phrase', text=query)[:paper_limit])
+    other_fagements = list(evidence_query
+                           .exclude('term', paper_id=paper_id)
+                           .query('match_phrase', text=query)[:corpus_limit])
+    return paper_fragments + other_fagements
+
+fix_refs_re = re.compile('\(\?\)|\s[?]+(\s|$)')
+
+
+def fix_refs(text):
+    return fix_refs_re.sub(' xref-unkown ', fix_refs_re.sub(' xref-unkown ', text))
+
+
+highlight_re = re.compile("</?b>")
+
+
+def create_evidence_records(textfrag, cell, table):
+    for text_highlited in textfrag.meta['highlight']['text']:
+        text_highlited = fix_refs(text_highlited)
+        text = highlight_re.sub("", text_highlited)
+        text_sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest()
+
+        cell_ext_id = f"{table.ext_id}/{cell.row}/{cell.col}"
+
+        if len(text.split()) > 50:
+            yield {"text_sha1": text_sha1,
+                   "text_highlited": text_highlited,
+                   "text": text,
+                   "cell_type": cell.vals[1],
+                   "cell_content": fix_refs(cell.vals[0]),
+                   "this_paper": textfrag.paper_id == table.paper_id,
+                   "row": cell.row,
+                   "col": cell.col,
+                   "ext_id": cell_ext_id
+                   #"table_id":table_id
+                   }
+
+
+def filter_cells(cell):
+    return re.search("[a-zA-Z]{2,}", cell.vals[1]) is not None
+
+
+def evidence_for_table(table, paper_limit=10, corpus_limit=1):
+    records = [
+        record
+            for cell in consume_cells(table.matrix, table.matrix_gold_tags) if filter_cells(cell)
+            for evidence in fetch_evidence(cell.vals[0], paper_id=table.paper_id, paper_limit=paper_limit, corpus_limit=corpus_limit)
+            for record in create_evidence_records(evidence, cell, table=table)
+    ]
+    df = pd.DataFrame.from_records(records)
+    return df
+
+
+def evidence_for_tables(tables, paper_limit=100, corpus_limit=20):
+    return pd.concat([evidence_for_table(table,  paper_limit=paper_limit, corpus_limit=corpus_limit) for table in progress_bar(tables)])
+
+def prepare_data(tables, csv_path):
+    df = evidence_for_tables(tables)
+    df = df.drop_duplicates(
+        ["cell_content", "text_highlited", "cell_type", "this_paper"])
+    print("Number of text fragments ", len(df))
+    csv_path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(csv_path, index=None)
diff --git a/sota_extractor2/models/structure/__init__.py b/sota_extractor2/models/structure/__init__.py
@@ -0,0 +1,14 @@
+import numpy as np
+from ...helpers.training import set_seed
+
+
+def split_by_cell_content(df, seed=42, split_column="cell_content"):
+    set_seed(seed, "val_split")
+    contents = np.random.permutation(df[split_column].unique())
+    val_split = int(len(contents)*0.1)
+    val_keys = contents[:val_split]
+    split = df[split_column].isin(val_keys)
+    valid_df = df[split]
+    train_df = df[~split]
+    len(train_df), len(valid_df)
+    return train_df, valid_df
diff --git a/sota_extractor2/models/structure/nbsvm.py b/sota_extractor2/models/structure/nbsvm.py
@@ -0,0 +1,149 @@
+import re
+import string
+from fastai.text import *  # just for utilty functions pd, np, Path etc.
+
+from sklearn.linear_model import LogisticRegression
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+
+from ...helpers.training import set_seed
+
+def transform_df(df):
+    df=df.replace(re.compile(r"(xxref|xxanchor)-[\w\d-]*"), "\\1 ")
+    df=df.replace(re.compile(r"(^|[ ])\d+\.\d+\b"), " xxnum ")
+    df=df.replace(re.compile(r"(^|[ ])\d\b"), " xxnum ")
+    df=df.replace(re.compile(r"\bdata set\b"), " dataset ")
+    df = df.drop_duplicates(["text", "cell_content", "cell_type"]).fillna("")
+    return df
+
+def train_valid_split(df, seed=42, by="cell_content"):
+    set_seed(seed, "val_split")
+    contents = np.random.permutation(df[by].unique())
+    val_split = int(len(contents)*0.1)
+    val_keys = contents[:val_split]
+    split = df[by].isin(val_keys)
+    valid_df = df[split]
+    train_df = df[~split]
+    len(train_df), len(valid_df)
+    return train_df, valid_df
+
+def get_class_column(y, classIdx):
+    if len(y.shape) == 1:
+        return y == classIdx
+    else:
+        return y.iloc[:, classIdx]
+
+def get_number_of_classes(y):
+    if len(y.shape) == 1:
+        return len(np.unique(y))
+    else:
+        return y.shape[1]
+
+class NBSVM:
+    def __init__(self, solver='liblinear', dual=True):
+        self.solver = solver  # 'lbfgs' - large, liblinear for small datasets
+        self.dual = dual
+        pass
+
+    re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
+    
+    def tokenize(self, s): 
+        return self.re_tok.sub(r' \1 ', s).split()
+        
+    def pr(self, y_i, y):
+        p = self.trn_term_doc[y == y_i].sum(0)
+        return (p+1) / ((y == y_i).sum()+1)
+
+    def get_mdl(self, y):
+        y = y.values
+        r = np.log(self.pr(1, y) / self.pr(0, y))
+        m = LogisticRegression(C=4, dual=self.dual, solver=self.solver, max_iter=1000)
+        x_nb = self.trn_term_doc.multiply(r)
+        return m.fit(x_nb, y), r
+
+    def bow(self, X_train):
+        self.n = X_train.shape[0]
+        self.vec = TfidfVectorizer(ngram_range=(1, 2), tokenizer=self.tokenize,
+                                min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1,
+                                smooth_idf=1, sublinear_tf=1)
+        return self.vec.fit_transform(X_train)
+
+    def train_models(self, y_train):
+        self.models = []
+        for i in range(0, self.c):
+            print('fit', i)
+            m, r = self.get_mdl(get_class_column(y_train, i))
+            self.models.append((m, r))
+
+    def fit(self, X_train, y_train):
+        self.trn_term_doc = self.bow(X_train)
+        self.c = get_number_of_classes(y_train)
+        self.train_models(y_train)
+
+    def predict_proba(self, X_test):
+        preds = np.zeros((len(X_test), self.c))
+        test_term_doc = self.vec.transform(X_test)
+        for i in range(0, self.c):
+            m, r = self.models[i]
+            preds[:, i] = m.predict_proba(test_term_doc.multiply(r))[:, 1]
+        return preds
+    
+    def validate(self, X_test, y_test):
+        acc = (np.argmax(self.predict_proba(X_test),  axis=1) == y_test).mean()
+        return acc
+
+def metrics(preds, true_y):
+    y = true_y
+    p = preds
+    acc = (p == y).mean()
+    tp = ((y != 0) & (p == y)).sum()
+    fp = ((p != 0) & (p != y)).sum()
+    prec = tp / (fp + tp)
+    return {
+        "precision": prec,
+        "accuracy": acc,
+        "TP": tp,
+        "FP": fp,
+    }
+
+
+def preds_for_cell_content(test_df, probs, group_by=["cell_content"]):
+    test_df = test_df.copy()
+    test_df["pred"] = np.argmax(probs, axis=1)
+    grouped_preds = test_df.groupby(group_by)["pred"].agg(
+        lambda x: x.value_counts().index[0])
+    grouped_counts = test_df.groupby(group_by)["pred"].count()
+    results = pd.DataFrame({'true': test_df.groupby(group_by)["label"].agg(lambda x: x.value_counts().index[0]),
+                            'pred': grouped_preds,
+                            'counts': grouped_counts})
+    return results
+
+def preds_for_cell_content_multi(test_df, probs, group_by=["cell_content"]):
+    test_df = test_df.copy()
+    probs_df = pd.DataFrame(probs, index=test_df.index)
+    test_df = pd.concat([test_df, probs_df], axis=1)
+    grouped_preds = np.argmax(test_df.groupby(
+        group_by)[probs_df.columns].sum().values, axis=1)
+    grouped_counts = test_df.groupby(group_by)["label"].count()
+    results = pd.DataFrame({'true': test_df.groupby(group_by)["label"].agg(lambda x: x.value_counts().index[0]),
+                            'pred': grouped_preds,
+                            'counts': grouped_counts})
+    return results
+
+def test_model(model, tdf):
+    probs = model(tdf["text"])
+    preds = np.argmax(probs, axis=1)
+    print("Results of categorisation on text fagment level")
+    print(metrics(preds, tdf.label))
+
+    print("Results per cell_content grouped using majority voting")
+    results = preds_for_cell_content(tdf, probs)
+    print(metrics(results["pred"], results["true"]))
+
+    print("Results per cell_content grouped with multi category mean")
+    results = preds_for_cell_content_multi(tdf, probs)
+    print(metrics(results["pred"], results["true"]))
+
+    print("Results per cell_content grouped with multi category mean - only on fragments from the same paper that the coresponding table")
+    results = preds_for_cell_content_multi(
+        tdf[tdf.this_paper], probs[tdf.this_paper])
+    print(metrics(results["pred"], results["true"]))
