Add voting strategies to Experiment

Author: Marcin Kardas

sota_extractor2/models/structure/experiment.py

@@ -25,14 +25,22 @@ class Labels(Enum):
     "model-competing": Labels.COMPETING_MODEL.value
 }
 
+# put here to avoid recompiling, used only in _limit_context
+elastic_tag_split_re = re.compile("(<b>.*?</b>)")
+
 @dataclass
 class Experiment:
     vectorizer: str = "tfidf"
     this_paper: bool = False
     merge_fragments: bool = False
+    merge_type: str = "concat"  # "concat", "vote_maj", "vote_avg", "vote_max"
     evidence_source: str = "text"  # "text" or "text_highlited"
     split_btags: bool = False  # <b>Test</b> -> <b> Test </b>
-    fixed_tokenizer: bool = False  # <b> and </b> are not split
+    fixed_tokenizer: bool = False  # if True, <b> and </b> are not split into < b > and < / b >
+    fixed_this_paper: bool = False # if True and this_paper, filter this_paper before merging fragments
+    mask: bool = False             # if True and evidence_source = "text_highlited", replace <b>...</b> with xxmask
+    evidence_limit: int = None     # maximum number of evidences per cell (grouped by (ext_id, this_paper))
+    context_tokens: int = None      # max. number of words before <b> and after </b>
 
     class_weight: str = None
     multinomial_type: str = "manual"  # "manual", "ovr", "multinomial"
@@ -107,17 +115,61 @@ def get_trained_model(self, train_df):
         self.has_model = True
         return nbsvm
 
+    def _limit_context(self, text):
+        parts = elastic_tag_split_re.split(text)
+        new_parts = []
+        end = len(parts)
+        for i, part in enumerate(parts):
+            if i % 2 == 0:
+                toks = tokenize(part)
+                if i == 0:
+                    toks = toks[-self.context_tokens:]
+                elif i == end:
+                    toks = toks[:self.context_tokens]
+                else:
+                    j = len(toks) - 2 * self.context_tokens
+                    if j > 0:
+                        toks = toks[:self.context_tokens] + toks[-self.context_tokens:]
+                new_parts.append(' '.join(toks))
+            else:
+                new_parts.append(part)
+        return ' '.join(new_parts)
+
+
+
     def _transform_df(self, df):
+        if self.merge_type not in ["concat", "vote_maj", "vote_avg", "vote_max"]:
+            raise Exception(f"merge_type must be one of concat, vote_maj, vote_avg, vote_max, but {self.merge_type} was given")
         df = df[df["cell_type"] != "table-meta"]  # otherwise we get precision 0 on test set
+        if self.evidence_limit is not None:
+            df = df.groupby(by=["ext_id", "this_paper"]).head(self.evidence_limit)
+        if self.context_tokens is not None:
+            df.loc["text_highlited"] = df["text_highlited"].apply(self._limit_context)
+            df.loc["text"] = df["text_highlited"].str.replace("<b>", " ").replace("</b>", " ")
         if self.evidence_source != "text":
             df = df.copy(True)
-            df["text"] = df[self.evidence_source]
-        if self.merge_fragments:
-            df = df.groupby(by=["ext_id", "cell_content", "cell_type", "this_paper"]).text.apply(
-                lambda x: "\n".join(x.values)).reset_index()
-        df = df.drop_duplicates(["text", "cell_content", "cell_type"]).fillna("")
-        if self.this_paper:
-            df = df[df.this_paper]
+            if self.mask:
+                df["text"] = df[self.evidence_source].replace(re.compile("<b>.*?</b>"), " xxmask ")
+            else:
+                df["text"] = df[self.evidence_source]
+
+        elif self.mask:
+            raise Exception("Masking with evidence_source='text' makes no sense")
+        if not self.fixed_this_paper:
+            if self.merge_fragments and self.merge_type == "concat":
+                df = df.groupby(by=["ext_id", "cell_content", "cell_type", "this_paper"]).text.apply(
+                    lambda x: "\n".join(x.values)).reset_index()
+            df = df.drop_duplicates(["text", "cell_content", "cell_type"]).fillna("")
+            if self.this_paper:
+                df = df[df.this_paper]
+        else:
+            if self.this_paper:
+                df = df[df.this_paper]
+            if self.merge_fragments and self.merge_type == "concat":
+                df = df.groupby(by=["ext_id", "cell_content", "cell_type"]).text.apply(
+                    lambda x: "\n".join(x.values)).reset_index()
+            df = df.drop_duplicates(["text", "cell_content", "cell_type"]).fillna("")
+
         if self.split_btags:
             df["text"] = df["text"].replace(re.compile(r"(\</?b\>)"), r" \1 ")
         df = df.replace(re.compile(r"(xxref|xxanchor)-[\w\d-]*"), "\\1 ")
@@ -135,9 +187,20 @@ def evaluate(self, model, train_df, valid_df, test_df):
         for prefix, tdf in zip(["train", "valid", "test"], [train_df, valid_df, test_df]):
             probs = model.predict_proba(tdf["text"])
             preds = np.argmax(probs, axis=1)
-            true_y = tdf["label"]
 
-            m = metrics(preds, tdf.label)
+            if self.merge_fragments and self.merge_type != "concat":
+                if self.merge_type == "vote_maj":
+                    vote_results = preds_for_cell_content(tdf, probs)
+                elif self.merge_type == "vote_avg":
+                    vote_results = preds_for_cell_content_multi(tdf, probs)
+                elif self.merge_type == "vote_max":
+                    vote_results = preds_for_cell_content_max(tdf, probs)
+                preds = vote_results["pred"]
+                true_y = vote_results["true"]
+            else:
+                true_y = tdf["label"]
+
+            m = metrics(preds, true_y)
             r = {}
             r[f"{prefix}_accuracy"] = m["accuracy"]
             r[f"{prefix}_precision"] = m["precision"]

sota_extractor2/models/structure/nbsvm.py

@@ -38,18 +38,21 @@ def get_number_of_classes(y):
     else:
         return y.shape[1]
 
+re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
+re_tok_fixed = re.compile(
+    f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])'.replace('<', '').replace('>', '').replace('/', ''))
+
+def tokenize(s):
+    return re_tok.sub(r' \1 ', s).split()
+
+def tokenize_fixed(s):
+    return re_tok_fixed.sub(r' \1 ', s).split()
+
+
 class NBSVM:
     def __init__(self, experiment):
         self.experiment = experiment
 
-    re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
-    re_tok_fixed = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])'.replace('<', '').replace('>', '').replace('/', ''))
-    
-    def tokenize(self, s):
-        return self.re_tok.sub(r' \1 ', s).split()
-        
-    def tokenize_fixed(self, s):
-        return self.re_tok_fixed.sub(r' \1 ', s).split()
 
     def pr(self, y_i, y):
         p = self.trn_term_doc[y == y_i].sum(0)
@@ -67,14 +70,15 @@ def get_mdl(self, y):
     def bow(self, X_train):
         self.n = X_train.shape[0]
 
+        tokenizer = tokenize_fixed if self.experiment.fixed_tokenizer else tokenize
         if self.experiment.vectorizer == "tfidf":
             self.vec = TfidfVectorizer(ngram_range=self.experiment.ngram_range,
-                                       tokenizer=self.tokenize_fixed if self.experiment.fixed_tokenizer else self.tokenize,
+                                       tokenizer=tokenizer,
                                        min_df=self.experiment.min_df, max_df=self.experiment.max_df,
                                        strip_accents='unicode', use_idf=1,
                                        smooth_idf=1, sublinear_tf=1)
         elif self.experiment.vectorizer == "count":
-            self.vec = CountVectorizer(ngram_range=self.experiment.ngram_range, tokenizer=self.tokenize,
+            self.vec = CountVectorizer(ngram_range=self.experiment.ngram_range, tokenizer=tokenizer,
                                        min_df=self.experiment.min_df, max_df=self.experiment.max_df,
                                        strip_accents='unicode')
         else:
@@ -122,7 +126,7 @@ def sort_features_by_importance(self, label):
         names = np.array(self.vec.get_feature_names())
         if self.experiment.multinomial_type == "manual":
             m, r = self.models[label]
-            f = m.coef_[0] * np.array(r[0])
+            f = m.coef_[0] * np.array(r)[0]
         elif self.experiment.multinomial_type == "multinomial":
             f = self.models[0].coef_[label]
         else:
@@ -133,6 +137,8 @@ def sort_features_by_importance(self, label):
         return names[indices], f[indices]
 
     def get_mismatched(self, df, true_label, predicted_label):
+        if self.experiment.merge_fragments and self.experiment.merge_type != "concat":
+            print("warning: the returned results are before merging")
         true_label = true_label.value
         predicted_label = predicted_label.value
 
@@ -194,12 +200,12 @@ def preds_for_cell_content_multi(test_df, probs, group_by=["cell_content"]):
                             'counts': grouped_counts})
     return results
 
-def preds_for_cell_content_best(test_df, probs, group_by=["cell_content"]):
+def preds_for_cell_content_max(test_df, probs, group_by=["cell_content"]):
     test_df = test_df.copy()
     probs_df = pd.DataFrame(probs, index=test_df.index)
     test_df = pd.concat([test_df, probs_df], axis=1)
     grouped_preds = np.argmax(test_df.groupby(
-        group_by)[probs_df.columns].sum().values, axis=1)
+        group_by)[probs_df.columns].max().values, axis=1)
     grouped_counts = test_df.groupby(group_by)["label"].count()
     results = pd.DataFrame({'true': test_df.groupby(group_by)["label"].agg(lambda x: x.value_counts().index[0]),
                             'pred': grouped_preds,