Tune searching

Author: Marcin Kardas

sota_extractor2/config.py

@@ -10,7 +10,7 @@
 
 # otherwise use this files
 data = Path("/mnt/efs/pwc/data")
-goldtags_dump = data / "dumps" / "goldtags-2019.06.28_0916.json.gz"
+goldtags_dump = data / "dumps" / "goldtags-2019.07.16_2214.json.gz"
 
 
 elastic = dict(hosts=['localhost'], timeout=20)

sota_extractor2/data/structure.py

@@ -20,7 +20,23 @@ def consume_cells(*matrix):
             yield Cell(row=row_id, col=col_id, vals=cell_val)
 
 
-def fetch_evidence(cell_content, paper_id, paper_limit=10, corpus_limit=10):
+reference_re = re.compile(r"\[[^]]*\]")
+ours_re = re.compile(r"\(ours?\)")
+all_parens_re = re.compile(r"\([^)]*\)")
+
+
+def clear_cell(s):
+    for pat in [reference_re, all_parens_re]:
+        s = pat.sub("", s)
+    s = s.strip()
+    return s
+
+
+def fetch_evidence(cell_content, cell_reference, paper_id, paper_limit=10, corpus_limit=10):
+    cell_content = clear_cell(cell_content)
+    if cell_content == "" and cell_reference == "":
+        return []
+
     evidence_query = Fragment.search().highlight(
         'text', pre_tags="<b>", post_tags="</b>", fragment_size=400)
     cell_content = cell_content.replace("\xa0", " ")
@@ -31,10 +47,21 @@ def fetch_evidence(cell_content, paper_id, paper_limit=10, corpus_limit=10):
     paper_fragments = list(evidence_query
                            .filter('term', paper_id=paper_id)
                            .query('match_phrase', text=query)[:paper_limit])
+    if cell_reference != "":
+        reference_fragments = list(evidence_query
+                                   .filter('term', paper_id=paper_id)
+                                   .query('match_phrase', text={
+                                        "query": cell_reference,
+                                        "slop": 1
+                                    })[:paper_limit])
+    else:
+        reference_fragments = []
     other_fagements = list(evidence_query
                            .exclude('term', paper_id=paper_id)
                            .query('match_phrase', text=query)[:corpus_limit])
-    return paper_fragments + other_fagements
+    if not len(paper_fragments) and not len(reference_fragments) and not len(other_fagements):
+        print(f"No evidences for '{cell_content}' of {paper_id}")
+    return paper_fragments + reference_fragments + other_fagements
 
 fix_refs_re = re.compile('\(\?\)|\s[?]+(\s|$)')
 
@@ -44,29 +71,34 @@ def fix_refs(text):
 
 
 highlight_re = re.compile("</?b>")
+partial_highlight_re = re.compile(r"\<b\>xxref\</b\>-(?!\<b\>)")
+
+
+def fix_reference_hightlight(s):
+    return partial_highlight_re.sub("xxref-", s)
 
 
 def create_evidence_records(textfrag, cell, table):
     for text_highlited in textfrag.meta['highlight']['text']:
-        text_highlited = fix_refs(text_highlited)
+        text_highlited = fix_reference_hightlight(fix_refs(text_highlited))
         text = highlight_re.sub("", text_highlited)
         text_sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest()
 
         cell_ext_id = f"{table.ext_id}/{cell.row}/{cell.col}"
 
-        if len(text.split()) > 50:
-            yield {"text_sha1": text_sha1,
-                   "text_highlited": text_highlited,
-                   "text": text,
-                   "header": textfrag.header,
-                   "cell_type": cell.vals[1],
-                   "cell_content": fix_refs(cell.vals[0]),
-                   "this_paper": textfrag.paper_id == table.paper_id,
-                   "row": cell.row,
-                   "col": cell.col,
-                   "ext_id": cell_ext_id
-                   #"table_id":table_id
-                   }
+        yield {"text_sha1": text_sha1,
+               "text_highlited": text_highlited,
+               "text": text,
+               "header": textfrag.header,
+               "cell_type": cell.vals[1],
+               "cell_content": fix_refs(cell.vals[0]),
+               "cell_reference": cell.vals[2],
+               "this_paper": textfrag.paper_id == table.paper_id,
+               "row": cell.row,
+               "col": cell.col,
+               "ext_id": cell_ext_id
+               #"table_id":table_id
+               }
 
 
 def filter_cells(cell):
@@ -83,8 +115,8 @@ def get_limits(cell_type):
         return dict(paper_limit=paper_limit, corpus_limit=corpus_limit)
     records = [
         record
-            for cell in consume_cells(table.matrix, table.matrix_gold_tags) if filter_cells(cell)
-            for evidence in fetch_evidence(cell.vals[0], paper_id=table.paper_id, **get_limits(cell.vals[1]))
+            for cell in consume_cells(table.matrix, table.matrix_gold_tags, table.matrix_references) if filter_cells(cell)
+            for evidence in fetch_evidence(cell.vals[0], cell.vals[2], paper_id=table.paper_id, **get_limits(cell.vals[1]))
             for record in create_evidence_records(evidence, cell, table=table)
     ]
     df = pd.DataFrame.from_records(records)

sota_extractor2/models/structure/experiment.py

@@ -41,6 +41,8 @@ class Experiment:
     mask: bool = False             # if True and evidence_source = "text_highlited", replace <b>...</b> with xxmask
     evidence_limit: int = None     # maximum number of evidences per cell (grouped by (ext_id, this_paper))
     context_tokens: int = None      # max. number of words before <b> and after </b>
+    analyzer: str = "word"            # "char", "word" or "char_wb"
+    lowercase: bool = True
 
     class_weight: str = None
     multinomial_type: str = "manual"  # "manual", "ovr", "multinomial"
@@ -140,7 +142,7 @@ def _limit_context(self, text):
     def _transform_df(self, df):
         if self.merge_type not in ["concat", "vote_maj", "vote_avg", "vote_max"]:
             raise Exception(f"merge_type must be one of concat, vote_maj, vote_avg, vote_max, but {self.merge_type} was given")
-        df = df[df["cell_type"] != "table-meta"]  # otherwise we get precision 0 on test set
+        #df = df[df["cell_type"] != "table-meta"]  # otherwise we get precision 0 on test set
         if self.evidence_limit is not None:
             df = df.groupby(by=["ext_id", "this_paper"]).head(self.evidence_limit)
         if self.context_tokens is not None:
@@ -181,7 +183,18 @@ def _transform_df(self, df):
         return df
 
     def transform_df(self, *dfs):
-        return [self._transform_df(df) for df in dfs]
+        transformed = [self._transform_df(df) for df in dfs]
+        if len(transformed) == 1:
+            return transformed[0]
+        return transformed
+
+    def _set_results(self, prefix, preds, true_y):
+        m = metrics(preds, true_y)
+        r = {}
+        r[f"{prefix}_accuracy"] = m["accuracy"]
+        r[f"{prefix}_precision"] = m["precision"]
+        r[f"{prefix}_cm"] = confusion_matrix(true_y, preds).tolist()
+        self.update_results(**r)
 
     def evaluate(self, model, train_df, valid_df, test_df):
         for prefix, tdf in zip(["train", "valid", "test"], [train_df, valid_df, test_df]):
@@ -199,13 +212,7 @@ def evaluate(self, model, train_df, valid_df, test_df):
                 true_y = vote_results["true"]
             else:
                 true_y = tdf["label"]
-
-            m = metrics(preds, true_y)
-            r = {}
-            r[f"{prefix}_accuracy"] = m["accuracy"]
-            r[f"{prefix}_precision"] = m["precision"]
-            r[f"{prefix}_cm"] = confusion_matrix(true_y, preds).tolist()
-            self.update_results(**r)
+            self._set_results(prefix, preds, true_y)
 
     def show_results(self, *ds):
         if not len(ds):

sota_extractor2/models/structure/nbsvm.py

@@ -63,7 +63,7 @@ def get_mdl(self, y):
         r = np.log(self.pr(1, y) / self.pr(0, y))
         m = LogisticRegression(C=self.experiment.C, penalty=self.experiment.penalty,
                                dual=self.experiment.dual, solver=self.experiment.solver,
-                               max_iter=self.experiment.max_iter)
+                               max_iter=self.experiment.max_iter, class_weight=self.experiment.class_weight)
         x_nb = self.trn_term_doc.multiply(r)
         return m.fit(x_nb, y), r
 
@@ -74,11 +74,15 @@ def bow(self, X_train):
         if self.experiment.vectorizer == "tfidf":
             self.vec = TfidfVectorizer(ngram_range=self.experiment.ngram_range,
                                        tokenizer=tokenizer,
+                                       lowercase=self.experiment.lowercase,
+                                       analyzer=self.experiment.analyzer,
                                        min_df=self.experiment.min_df, max_df=self.experiment.max_df,
                                        strip_accents='unicode', use_idf=1,
                                        smooth_idf=1, sublinear_tf=1)
         elif self.experiment.vectorizer == "count":
             self.vec = CountVectorizer(ngram_range=self.experiment.ngram_range, tokenizer=tokenizer,
+                                       analyzer=self.experiment.analyzer,
+                                       lowercase=self.experiment.lowercase,
                                        min_df=self.experiment.min_df, max_df=self.experiment.max_df,
                                        strip_accents='unicode')
         else:
@@ -93,11 +97,11 @@ def train_models(self, y_train):
                 #print('fit', i)
                 m, r = self.get_mdl(get_class_column(y_train, i))
                 self.models.append((m, r))
-        elif self.experiment.multinomial_type == "multinomial":
+        elif self.experiment.multinomial_type == "multinomial" or self.experiment.multinomial_type == "ovr":
             m = LogisticRegression(C=self.experiment.C, penalty=self.experiment.penalty,
                                    dual=self.experiment.dual, solver=self.experiment.solver,
                                    max_iter=self.experiment.max_iter,
-                                   multi_class="multinomial", class_weight=self.experiment.class_weight)
+                                   multi_class=self.experiment.multinomial_type, class_weight=self.experiment.class_weight)
             x_nb = self.trn_term_doc
             self.models.append(m.fit(x_nb, y_train))
         else:
@@ -115,7 +119,7 @@ def predict_proba(self, X_test):
             for i in range(0, self.c):
                 m, r = self.models[i]
                 preds[:, i] = m.predict_proba(test_term_doc.multiply(r))[:, 1]
-        elif self.experiment.multinomial_type == "multinomial":
+        elif self.experiment.multinomial_type == "multinomial" or self.experiment.multinomial_type == "ovr":
             preds = self.models[0].predict_proba(test_term_doc)
         else:
             raise Exception(f"Unsupported multinomial_type {self.experiment.multinomial_type}")