add features for semantic evaluation: selecting single metrics, postprocessing that removes violations, verbose output

Author: sfluegel

chebai/result/analyse_sem.py

@@ -139,16 +139,118 @@ def load_preds_labels_from_nonwandb(
     return preds, labels
 
 
+def get_label_names(data_module):
+    if os.path.exists(os.path.join(data_module.raw_dir, "classes.txt")):
+        with open(os.path.join(data_module.raw_dir, "classes.txt")) as fin:
+            return [int(line.strip()) for line in fin]
+    return None
+
+
+def get_chebi_graph(data_module, label_names):
+    if os.path.exists(os.path.join(data_module.raw_dir, "chebi.obo")):
+        chebi_graph = data_module.extract_class_hierarchy(
+            os.path.join(data_module.raw_dir, "chebi.obo")
+        )
+        return chebi_graph.subgraph(label_names)
+    return None
+
+
+def get_disjoint_groups():
+    disjoints_owl_file = os.path.join("data", "chebi-disjoints.owl")
+    with open(disjoints_owl_file, "r") as f:
+        plaintext = f.read()
+        segments = plaintext.split("<")
+        disjoint_pairs = []
+        left = None
+        for seg in segments:
+            if seg.startswith("rdf:Description ") or seg.startswith("owl:Class"):
+                left = int(seg.split('rdf:about="&obo;CHEBI_')[1].split('"')[0])
+            elif seg.startswith("owl:disjointWith"):
+                right = int(seg.split('rdf:resource="&obo;CHEBI_')[1].split('"')[0])
+                disjoint_pairs.append([left, right])
+
+        disjoint_groups = []
+        for seg in plaintext.split("<rdf:Description>"):
+            if "owl;AllDisjointClasses" in seg:
+                classes = seg.split('rdf:about="&obo;CHEBI_')[1:]
+                classes = [int(c.split('"')[0]) for c in classes]
+                disjoint_groups.append(classes)
+    disjoint_all = disjoint_pairs + disjoint_groups
+    # one disjointness is commented out in the owl-file
+    # (the correct way would be to parse the owl file and notice the comment symbols, but for this case, it should work)
+    disjoint_all.remove([22729, 51880])
+    print(f"Found {len(disjoint_all)} disjoint groups")
+    return disjoint_all
+
+
+def smooth_preds(preds, label_names, chebi_graph, disjoint_groups):
+    preds_sum_orig = torch.sum(preds)
+    print(f"Preds sum: {preds_sum_orig}")
+    # eliminate implication violations by setting each prediction to maximum of its successors
+    for i, label in enumerate(label_names):
+        succs = [label_names.index(p) for p in chebi_graph.successors(label)] + [i]
+        if len(succs) > 0:
+            preds[:, i] = torch.max(preds[:, succs], dim=1).values
+    print(f"Preds change (step 1): {torch.sum(preds) - preds_sum_orig}")
+    preds_sum_orig = torch.sum(preds)
+    # step 2: eliminate disjointness violations: for group of disjoint classes, set all except max to 0.49 (if it is not already lower)
+    preds_bounded = torch.min(preds, torch.ones_like(preds) * 0.49)
+    for disj_group in disjoint_groups:
+        disj_group = [label_names.index(g) for g in disj_group if g in label_names]
+        if len(disj_group) > 1:
+            old_preds = preds[:, disj_group]
+            disj_max = torch.max(preds[:, disj_group], dim=1)
+            for i, row in enumerate(preds):
+                for l in range(len(preds[i])):
+                    if l in disj_group and l != disj_group[disj_max.indices[i]]:
+                        preds[i, l] = preds_bounded[i, l]
+            samples_changed = 0
+            for i, row in enumerate(preds[:, disj_group]):
+                if any(r != o for r, o in zip(row, old_preds[i])):
+                    samples_changed += 1
+            if samples_changed != 0:
+                print(
+                    f"disjointness group {[label_names[d] for d in disj_group]} changed {samples_changed} samples"
+                )
+    print(
+        f"Preds change after disjointness (step 2): {torch.sum(preds) - preds_sum_orig}"
+    )
+    preds_sum_orig = torch.sum(preds)
+    # step 3: disjointness violation removal may have caused new implication inconsistencies -> set each prediction to min of predecessors
+    for i, label in enumerate(label_names):
+        predecessors = [i] + [
+            label_names.index(p) for p in chebi_graph.predecessors(label)
+        ]
+        lowest_predecessors = torch.min(preds[:, predecessors], dim=1)
+        preds[:, i] = lowest_predecessors.values
+        for idx_idx, idx in enumerate(lowest_predecessors.indices):
+            if idx > 0:
+                print(
+                    f"class {label}: changed prediction of sample {idx_idx} to value of class "
+                    f"{label_names[predecessors[idx]]} ({preds[idx_idx, i].item():.2f})"
+                )
+        if torch.sum(preds) != preds_sum_orig:
+            print(
+                f"Preds change (step 3) for {label}: {torch.sum(preds) - preds_sum_orig}"
+            )
+            preds_sum_orig = torch.sum(preds)
+    return preds
+
+
 def analyse_run(
     preds,
     labels,
     df_hyperparams,  # parameters that are the independent of the semantic loss function used
     labeled_data_cls=ChEBIOver100,  # use labels from this dataset for violations
     chebi_version=231,
     results_path=os.path.join("_semantic", "eval_results.csv"),
+    violation_metrics: [str | list[callable]] = "all",
+    verbose_violation_output=False,
 ):
     """Calculates all semantic metrics for given predictions (and supervised metrics if labels are provided),
     saves results to csv"""
+    if violation_metrics == "all":
+        violation_metrics = [product, lukasiewicz, weak, strict, binary]
     data_module_labeled = labeled_data_cls(chebi_version=chebi_version)
     n_labels = preds.size(1)
     print(f"Found {preds.shape[0]} predictions ({n_labels} classes)")
@@ -173,7 +275,7 @@ def analyse_run(
         del preds_exp
         gc.collect()
 
-        for i, metric in enumerate([product, lukasiewicz, weak, strict, binary]):
+        for i, metric in enumerate(violation_metrics):
             if filter_type == "impl":
                 df_new.append(df_hyperparams.copy())
                 df_new[-1]["metric"] = metric.__name__
@@ -188,6 +290,27 @@ def analyse_run(
             m["fns"] = apply_metric(
                 metric, l_preds, 1 - r_preds if filter_type == "impl" else r_preds
             )
+            if verbose_violation_output:
+                label_names = get_label_names(data_module_labeled)
+                print(f"Found {torch.sum(m['fns'])} {filter_type}-violations")
+                # for k, fn_cls in enumerate(m['fns']):
+                #    if fn_cls > 0:
+                #        print(f"\tThereof, {fn_cls.item()} belong to class {label_names[k]}")
+                if torch.sum(m["fns"]) != 0:
+                    fns = metric(
+                        l_preds, 1 - r_preds if filter_type == "impl" else r_preds
+                    )
+                    print(fns.shape)
+                    for k, row in enumerate(fns):
+                        if torch.sum(row) != 0:
+                            print(f"{torch.sum(row)} violations for entity {k}")
+                            for j, violation in enumerate(row):
+                                if violation > 0:
+                                    print(
+                                        f"\tviolated ({label_names[dl_filter_l[j]]} -> {preds[k, dl_filter_l[j]]:.3f}"
+                                        f", {label_names[dl_filter_r[j]]} -> {preds[k, dl_filter_r[j]]:.3f})"
+                                    )
+
             m_cls = {}
             for key, value in m.items():
                 m_cls[key] = _sort_results_by_label(
@@ -259,14 +382,23 @@ def run_all(
     skip_analyse=False,
     skip_preds=False,
     nonwandb_runs=None,
+    violation_metrics="all",
+    remove_violations=False,
 ):
     # evaluate a list of runs on Hazardous and ChEBIOver100 datasets
     if datasets is None:
         datasets = [(Hazardous, "all"), (ChEBIOver100, "test")]
     timestamp = datetime.now().strftime("%y%m%d-%H%M")
     results_path = os.path.join(
-        "_semloss_eval", f"semloss_results_pc-dis-200k_{timestamp}.csv"
+        "_semloss_eval",
+        f"semloss_results_pc-dis-200k_{timestamp}{'_violations_removed' if remove_violations else ''}.csv",
+    )
+    label_names = get_label_names(ChEBIOver100(chebi_version=chebi_version))
+    chebi_graph = get_chebi_graph(
+        ChEBIOver100(chebi_version=chebi_version), label_names
     )
+    disjoint_groups = get_disjoint_groups()
+
     api = wandb.Api()
     for run_id in run_ids:
         try:
@@ -280,17 +412,50 @@ def run_all(
                     "data_module": test_on.__name__,
                     "chebi_version": chebi_version,
                 }
-                if not skip_preds:
-                    preds, labels = load_preds_labels_from_wandb(
-                        run, epoch, chebi_version, test_on, kind
+                buffer_dir_smoothed = os.path.join(
+                    "results_buffer",
+                    "smoothed3step",
+                    f"{run.name}_ep{epoch}",
+                    f"{test_on.__name__}_{kind}",
+                )
+                if remove_violations and os.path.exists(
+                    os.path.join(buffer_dir_smoothed, "preds000.pt")
+                ):
+                    preds = torch.load(
+                        os.path.join(buffer_dir_smoothed, "preds000.pt"), DEVICE
                     )
+                    labels = None
                 else:
-                    buffer_dir = os.path.join(
-                        "results_buffer",
-                        f"{run.name}_ep{epoch}",
-                        f"{test_on.__name__}_{kind}",
-                    )
-                    preds, labels = load_results_from_buffer(buffer_dir, device=DEVICE)
+                    if not skip_preds:
+                        preds, labels = load_preds_labels_from_wandb(
+                            run, epoch, chebi_version, test_on, kind
+                        )
+                    else:
+                        buffer_dir = os.path.join(
+                            "results_buffer",
+                            f"{run.name}_ep{epoch}",
+                            f"{test_on.__name__}_{kind}",
+                        )
+                        preds, labels = load_results_from_buffer(
+                            buffer_dir, device=DEVICE
+                        )
+                        assert (
+                            preds is not None
+                        ), f"Did not find predictions in dir {buffer_dir}"
+                        if remove_violations:
+                            preds = smooth_preds(
+                                preds, label_names, chebi_graph, disjoint_groups
+                            )
+                            buffer_dir_smoothed = os.path.join(
+                                "results_buffer",
+                                "smoothed3step",
+                                f"{run.name}_ep{epoch}",
+                                f"{test_on.__name__}_{kind}",
+                            )
+                            os.makedirs(buffer_dir_smoothed, exist_ok=True)
+                            torch.save(
+                                preds, os.path.join(buffer_dir_smoothed, "preds000.pt")
+                            )
                 if not skip_analyse:
                     print(
                         f"Calculating metrics for run {run.name} on {test_on.__name__} ({kind})"
@@ -301,6 +466,8 @@ def run_all(
                         df_hyperparams=df,
                         chebi_version=chebi_version,
                         results_path=results_path,
+                        violation_metrics=violation_metrics,
+                        verbose_violation_output=True,
                     )
         except Exception as e:
             print(f"Failed for run {run_id}: {e}")
@@ -330,6 +497,13 @@ def run_all(
                         preds, labels = load_results_from_buffer(
                             buffer_dir, device=DEVICE
                         )
+                        assert (
+                            preds is not None
+                        ), f"Did not find predictions in dir {buffer_dir}"
+                        if remove_violations:
+                            preds = smooth_preds(
+                                preds, label_names, chebi_graph, disjoint_groups
+                            )
                     if not skip_analyse:
                         print(
                             f"Calculating metrics for run {run_name} on {test_on.__name__} ({kind})"
@@ -340,16 +514,15 @@ def run_all(
                             df_hyperparams=df,
                             chebi_version=chebi_version,
                             results_path=results_path,
+                            violation_metrics=violation_metrics,
                         )
             except Exception as e:
                 print(f"Failed for run {run_name}: {e}")
                 print(traceback.format_exc())
 
 
 def run_semloss_eval(mode="eval"):
-    non_wandb_runs = (
-        []
-    )  # ("chebi100_semprodk2_weighted_v231_pc_200k_dis_24042-2000", 195)]
+    non_wandb_runs = []
     if mode == "preds":
         api = wandb.Api()
         runs = api.runs("chebai/chebai", filters={"tags": "eval_semloss_paper"})
@@ -375,8 +548,16 @@ def run_semloss_eval(mode="eval"):
             "tk15yznc",
         ]
         baseline = ["i4wtz1k4", "zd020wkv", "rc1q3t49"]
+        k2 = ["ng3usn0p", "rp0wwzjv", "8fma1q7r"]
         ids = baseline
-        run_all(ids, skip_preds=True, nonwandb_runs=non_wandb_runs)
+        run_all(
+            ids,
+            skip_preds=True,
+            nonwandb_runs=non_wandb_runs,
+            datasets=[(ChEBIOver100, "test")],
+            violation_metrics=[binary],
+            remove_violations=True,
+        )
 
 
 if __name__ == "__main__":