Add SBD to CryoVesNet evaluation

Author: constantinpape

scripts/baselines/cryo_ves_net/evaluate_cooper.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 from elf.evaluation.matching import matching
+from elf.evaluation.dice import symmetric_best_dice_score
 from tqdm import tqdm
 
 INPUT_ROOT = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/cooper/vesicles_processed_v2/testsets"  # noqa
@@ -25,11 +26,11 @@
 ]
 
 
-def evaluate_dataset(ds_name):
+def evaluate_dataset(ds_name, force):
     result_folder = "./results/cooper"
     os.makedirs(result_folder, exist_ok=True)
     result_path = os.path.join(result_folder, f"{ds_name}.csv")
-    if os.path.exists(result_path):
+    if os.path.exists(result_path) and not force:
         results = pd.read_csv(result_path)
         return results
 
@@ -44,6 +45,9 @@ def evaluate_dataset(ds_name):
         mask_key = None
 
     pred_files = sorted(glob(os.path.join(OUTPUT_ROOT, ds_name, "**/*.h5"), recursive=True))
+    if ds_name == "04":
+        pred_names = [os.path.basename(path) for path in pred_files]
+        input_files = [path for path in input_files if os.path.basename(path) in pred_names]
     assert len(input_files) == len(pred_files), f"{len(input_files)}, {len(pred_files)}"
 
     results = {
@@ -52,12 +56,13 @@ def evaluate_dataset(ds_name):
         "precision": [],
         "recall": [],
         "f1-score": [],
+        "sbd-score": [],
     }
     for inf, predf in tqdm(zip(input_files, pred_files), total=len(input_files), desc=f"Evaluate {ds_name}"):
         fname = os.path.basename(inf)
         sub_res_path = os.path.join(result_folder, f"{ds_name}_{fname}.json")
 
-        if os.path.exists(sub_res_path):
+        if os.path.exists(sub_res_path) and not force:
             print("Loading scores from", sub_res_path)
             with open(sub_res_path, "r") as f:
                 scores = json.load(f)
@@ -89,6 +94,8 @@ def evaluate_dataset(ds_name):
                 gt[mask == 0] = 0
 
             scores = matching(seg, gt)
+            sbd_score = symmetric_best_dice_score(seg, gt)
+            scores["sbd"] = sbd_score
 
             with open(sub_res_path, "w") as f:
                 json.dump(scores, f)
@@ -98,16 +105,19 @@ def evaluate_dataset(ds_name):
         results["precision"].append(scores["precision"])
         results["recall"].append(scores["recall"])
         results["f1-score"].append(scores["f1"])
+        results["sbd-score"].append(scores["sbd"])
 
     results = pd.DataFrame(results)
     results.to_csv(result_path, index=False)
     return results
 
 
 def main():
+    force = False
+
     all_results = {}
     for ds in DATASETS:
-        result = evaluate_dataset(ds)
+        result = evaluate_dataset(ds, force=force)
         all_results[ds] = result
 
     groups = {
@@ -123,16 +133,24 @@ def main():
     }
 
     for name, datasets in groups.items():
-        f1_scores = []
+        f1_scores, sbd_scores = [], []
 
         for ds in datasets:
             this_f1_scores = all_results[ds]["f1-score"].values.tolist()
+            this_sbd_scores = all_results[ds]["sbd-score"].values.tolist()
             f1_scores.extend(this_f1_scores)
+            sbd_scores.extend(this_sbd_scores)
 
         mean_f1 = np.mean(f1_scores)
         std_f1 = np.std(f1_scores)
+        print("F1-Score")
         print(name, ":", mean_f1, "+-", std_f1)
 
+        mean_sbd = np.mean(sbd_scores)
+        std_sbd = np.std(sbd_scores)
+        print("SBD-Score")
+        print(name, ":", mean_sbd, "+-", std_sbd)
+
 
 if __name__ == "__main__":
     main()

scripts/baselines/cryo_ves_net/evaluate_cryo.py

@@ -4,17 +4,18 @@
 import h5py
 import pandas as pd
 from elf.evaluation.matching import matching
+from elf.evaluation.dice import symmetric_best_dice_score
 
 
 INPUT_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/fernandez-busnadiego/vesicle_gt/v3"  # noqa
 OUTPUT_FOLDER = "./predictions/cryo"
 
 
-def evaluate_dataset(ds_name="cryo"):
+def evaluate_dataset(ds_name="cryo", force=False):
     result_folder = "./results/cryo"
     os.makedirs(result_folder, exist_ok=True)
     result_path = os.path.join(result_folder, f"{ds_name}.csv")
-    if os.path.exists(result_path):
+    if os.path.exists(result_path) and not force:
         results = pd.read_csv(result_path)
         return results
 
@@ -28,6 +29,7 @@ def evaluate_dataset(ds_name="cryo"):
         "precision": [],
         "recall": [],
         "f1-score": [],
+        "sbd-score": [],
     }
     for inf, predf in zip(input_files, pred_files):
         fname = os.path.basename(inf)
@@ -39,22 +41,29 @@ def evaluate_dataset(ds_name="cryo"):
         assert gt.shape == seg.shape
 
         scores = matching(seg, gt)
+        sbd_score = symmetric_best_dice_score(seg, gt)
+        scores["sbd"] = sbd_score
 
         results["dataset"].append(ds_name)
         results["file"].append(fname)
         results["precision"].append(scores["precision"])
         results["recall"].append(scores["recall"])
         results["f1-score"].append(scores["f1"])
+        results["sbd-score"].append(scores["sbd"])
 
     results = pd.DataFrame(results)
     results.to_csv(result_path, index=False)
     return results
 
 
 def main():
-    result = evaluate_dataset()
+    force = False
+    result = evaluate_dataset(force=force)
     print(result)
+    print("F1-Score")
     print(result["f1-score"].mean())
+    print("SBD-Score")
+    print(result["sbd-score"].mean())
 
 
 if __name__ == "__main__":

scripts/baselines/cryo_ves_net/evaluate_endbulb.py

@@ -4,17 +4,19 @@
 import h5py
 import pandas as pd
 from elf.evaluation.matching import matching
+from elf.evaluation.dice import symmetric_best_dice_score
+from tqdm import tqdm
 
 
 INPUT_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/wichmann/extracted/endbulb_of_held/Automatische_Segmentierung_Dataset_Validierung"  # noqa
 OUTPUT_FOLDER = "./predictions/endbulb"
 
 
-def evaluate_dataset(ds_name="endbulb"):
+def evaluate_dataset(ds_name="endbulb", force=False):
     result_folder = "./results/endbulb"
     os.makedirs(result_folder, exist_ok=True)
     result_path = os.path.join(result_folder, f"{ds_name}.csv")
-    if os.path.exists(result_path):
+    if os.path.exists(result_path) and not force:
         results = pd.read_csv(result_path)
         return results
 
@@ -28,8 +30,9 @@ def evaluate_dataset(ds_name="endbulb"):
         "precision": [],
         "recall": [],
         "f1-score": [],
+        "sbd-score": [],
     }
-    for inf, predf in zip(input_files, pred_files):
+    for inf, predf in tqdm(zip(input_files, pred_files), total=len(input_files), desc="Run evaluation"):
         fname = os.path.basename(inf)
 
         with h5py.File(inf, "r") as f:
@@ -39,24 +42,29 @@ def evaluate_dataset(ds_name="endbulb"):
         assert gt.shape == seg.shape
 
         scores = matching(seg, gt)
+        sbd_score = symmetric_best_dice_score(seg, gt)
 
         results["dataset"].append(ds_name)
         results["file"].append(fname)
         results["precision"].append(scores["precision"])
         results["recall"].append(scores["recall"])
         results["f1-score"].append(scores["f1"])
+        results["sbd-score"].append(sbd_score)
 
     results = pd.DataFrame(results)
     results.to_csv(result_path, index=False)
     return results
 
 
 def main():
-    result = evaluate_dataset()
+    force = False
+    result = evaluate_dataset(force=force)
     print(result)
     print()
-    print(result["f1-score"].mean())
-    print(result["f1-score"].std())
+    print("F1-Score")
+    print(result["f1-score"].mean(), "+-", result["f1-score"].std())
+    print("SBD-Score")
+    print(result["sbd-score"].mean(), "+-", result["sbd-score"].std())
 
 
 if __name__ == "__main__":

scripts/baselines/cryo_ves_net/evaluate_inner_ear.py

@@ -4,17 +4,19 @@
 import h5py
 import pandas as pd
 from elf.evaluation.matching import matching
+from elf.evaluation.dice import symmetric_best_dice_score
+from tqdm import tqdm
 
 
 INPUT_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/moser/vesicle_gt"  # noqa
 OUTPUT_FOLDER = "./predictions/inner_ear"
 
 
-def evaluate_dataset(ds_name="inner_ear"):
+def evaluate_dataset(ds_name="inner_ear", force=False):
     result_folder = "./results/inner_ear"
     os.makedirs(result_folder, exist_ok=True)
     result_path = os.path.join(result_folder, f"{ds_name}.csv")
-    if os.path.exists(result_path):
+    if os.path.exists(result_path) and not force:
         results = pd.read_csv(result_path)
         return results
 
@@ -28,8 +30,9 @@ def evaluate_dataset(ds_name="inner_ear"):
         "precision": [],
         "recall": [],
         "f1-score": [],
+        "sbd-score": [],
     }
-    for inf, predf in zip(input_files, pred_files):
+    for inf, predf in tqdm(zip(input_files, pred_files), total=len(input_files), desc="Run evaluation"):
         fname = os.path.basename(inf)
 
         with h5py.File(inf, "r") as f:
@@ -39,24 +42,29 @@ def evaluate_dataset(ds_name="inner_ear"):
         assert gt.shape == seg.shape
 
         scores = matching(seg, gt)
+        sbd_score = symmetric_best_dice_score(seg, gt)
 
         results["dataset"].append(ds_name)
         results["file"].append(fname)
         results["precision"].append(scores["precision"])
         results["recall"].append(scores["recall"])
         results["f1-score"].append(scores["f1"])
+        results["sbd-score"].append(sbd_score)
 
     results = pd.DataFrame(results)
     results.to_csv(result_path, index=False)
     return results
 
 
 def main():
-    result = evaluate_dataset()
+    force = False
+    result = evaluate_dataset(force=force)
     print(result)
     print()
-    print(result["f1-score"].mean())
-    print(result["f1-score"].std())
+    print("F1-Score")
+    print(result["f1-score"].mean(), "+-", result["f1-score"].std())
+    print("SBD-Score")
+    print(result["sbd-score"].mean(), "+-", result["sbd-score"].std())
 
 
 if __name__ == "__main__":