Skip to content

Commit da7a27a

Browse files
constantinpapeconstantinpape
committed
Finalize SGN evaluation
1 parent 4c0b37c commit da7a27a

File tree

5 files changed

+210
-9
lines changed

5 files changed

+210
-9
lines changed

‎scripts/validation/SGNs/consensus_annotations.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,19 @@
44
from flamingo_tools.validation import create_consensus_annotations
55

66
ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs"
7-
ANNOTATION_FOLDERS = ["AnnotationsAMD", "AnnotationsEK", "AnnotationsLR"]
7+
88
COLOR = ["blue", "yellow", "orange"]
9+
10+
# First iteration of consensus annotations.
911
OUTPUT_FOLDER = os.path.join(ROOT, "for_consensus_annotation")
12+
ANNOTATION_FOLDERS = ["AnnotationsAMD", "AnnotationsEK", "AnnotationsLR"]
13+
14+
# Second iteration of consensus annotations for the two images of rescaled cochlea.
15+
# OUTPUT_FOLDER = os.path.join(ROOT, "for_consensus_annotation2")
16+
# ANNOTATION_FOLDERS = [
17+
# "for_consensus_annotation2/AnnotationsAMD", "for_consensus_annotation2/AnnotationsEK",
18+
# "for_consensus_annotation2/AnnotationsLR"
19+
# ]
1020

1121

1222
def match_annotations(image_path):
@@ -28,8 +38,7 @@ def match_annotations(image_path):
2838
def consensus_annotations(image_path, check):
2939
print("Compute consensus annotations for", image_path)
3040
annotation_paths = match_annotations(image_path)
31-
# TODO change matching distance to be less conservative. Need to try visually.
32-
matching_distance = "" # matching_distance
41+
matching_distance = 8
3342
consensus_annotations, unmatched_annotations = create_consensus_annotations(
3443
annotation_paths, matching_distance=matching_distance, min_matches_for_consensus=2,
3544
)

‎scripts/validation/SGNs/consensus_step2.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66

77
ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs/for_consensus_annotation" # noqa
88

9-
ANNOTATION_FOLDERS = ["AnnotationsAMD", "AnnotationsEK"]
10-
COLOR = ["blue", "yellow"]
9+
ANNOTATION_FOLDERS = ["AnnotationsAMD", "AnnotationsEK", "AnnotationsLR"]
10+
COLOR = ["blue", "yellow", "orange"]
1111
CONSENSUS_ANNOTATIONS = "consensus_annotations"
1212
OUTPUT_FOLDER = os.path.join(ROOT, "final_consensus_annotations")
1313

@@ -20,7 +20,9 @@ def match_annotations(image_path, annotation_folders):
2020
annotations = {}
2121
for annotation_folder in annotation_folders:
2222
all_annotations = glob(os.path.join(ROOT, annotation_folder, "*.csv"))
23-
matches = [ann for ann in all_annotations if os.path.basename(ann).startswith(prefix)]
23+
matches = [
24+
ann for ann in all_annotations if (os.path.basename(ann).startswith(prefix) and "negative" not in ann)
25+
]
2426
if len(matches) != 1:
2527
breakpoint()
2628
assert len(matches) == 1
@@ -34,7 +36,7 @@ def match_annotations(image_path, annotation_folders):
3436
def create_consensus_step2(image_path, check):
3537
print("Compute consensus annotations for", image_path)
3638
annotation_paths = match_annotations(image_path, ANNOTATION_FOLDERS)
37-
matching_distance = 8 # TODO
39+
matching_distance = 8
3840
consensus_annotations, unmatched_annotations = create_consensus_annotations(
3941
annotation_paths, matching_distance=matching_distance, min_matches_for_consensus=2,
4042
)

‎scripts/validation/SGNs/consensus_step3.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import os
2+
from glob import glob
3+
4+
import pandas as pd
5+
from flamingo_tools.validation import create_consensus_annotations
6+
7+
IMAGE_ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs/for_consensus_annotation" # noqa
8+
ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs/final_annotations" # noqa
9+
10+
ANNOTATION_FOLDERS = ["AnnotationsAMD", "AnnotationsEK", "AnnotationsLR"]
11+
COLOR = ["blue", "yellow", "orange"]
12+
CONSENSUS_ANNOTATIONS = "consensus_annotations"
13+
OUTPUT_FOLDER = os.path.join(ROOT, "final_consensus_annotations")
14+
15+
16+
def match_annotations(image_path, annotation_folders):
17+
annotations = {}
18+
prefix = os.path.basename(image_path).split("_")[:3]
19+
prefix = "_".join(prefix)
20+
21+
annotations = {}
22+
for annotation_folder in annotation_folders:
23+
all_annotations = glob(os.path.join(ROOT, annotation_folder, "*.csv"))
24+
matches = [
25+
ann for ann in all_annotations if (os.path.basename(ann).startswith(prefix) and "negative" not in ann)
26+
]
27+
if len(matches) != 1:
28+
breakpoint()
29+
assert len(matches) == 1
30+
annotation_path = matches[0]
31+
annotations[annotation_folder] = annotation_path
32+
33+
assert len(annotations) == len(annotation_folders)
34+
return annotations
35+
36+
37+
def create_consensus_step3(image_path, check):
38+
print("Compute consensus annotations for", image_path)
39+
annotation_paths = match_annotations(image_path, ANNOTATION_FOLDERS)
40+
matching_distance = 8
41+
consensus_annotations, unmatched_annotations = create_consensus_annotations(
42+
annotation_paths, matching_distance=matching_distance, min_matches_for_consensus=2,
43+
)
44+
fname = os.path.basename(image_path)
45+
46+
prev_consensus = match_annotations(image_path, [CONSENSUS_ANNOTATIONS])[CONSENSUS_ANNOTATIONS]
47+
prev_consensus = pd.read_csv(prev_consensus)[["axis-0", "axis-1", "axis-2"]]
48+
49+
if check:
50+
import napari
51+
import tifffile
52+
53+
consensus_annotations = consensus_annotations[["axis-0", "axis-1", "axis-2"]].values
54+
unmatched_annotators = unmatched_annotations.annotator.values
55+
unmatched_annotations = unmatched_annotations[["axis-0", "axis-1", "axis-2"]].values
56+
57+
image = tifffile.imread(image_path)
58+
v = napari.Viewer()
59+
v.add_image(image)
60+
if prev_consensus is not None:
61+
v.add_points(prev_consensus.values, face_color="gray", name="previous-consensus-annotations")
62+
v.add_points(consensus_annotations, face_color="green")
63+
v.add_points(
64+
unmatched_annotations,
65+
properties={"annotator": unmatched_annotators},
66+
face_color="annotator",
67+
face_color_cycle=COLOR, # TODO reorder
68+
)
69+
v.title = os.path.basename(fname)
70+
napari.run()
71+
72+
else:
73+
# Combine consensus and previous annotations.
74+
consensus_annotations = consensus_annotations[["axis-0", "axis-1", "axis-2"]]
75+
n_consensus = len(consensus_annotations)
76+
n_unmatched = len(unmatched_annotations)
77+
if prev_consensus is not None:
78+
n_prev = len(prev_consensus)
79+
print("Number of previous consensus annotations:", n_prev)
80+
81+
print("Number of new consensus annotations:", n_consensus)
82+
print("Number of unmatched annotations:", n_unmatched)
83+
84+
consensus_annotations = pd.concat([consensus_annotations, prev_consensus])
85+
86+
out_name = fname.replace(".tif", ".csv")
87+
out_path = os.path.join(OUTPUT_FOLDER, out_name)
88+
89+
os.makedirs(OUTPUT_FOLDER, exist_ok=True)
90+
consensus_annotations.to_csv(out_path, index=False)
91+
92+
93+
def main():
94+
import argparse
95+
parser = argparse.ArgumentParser()
96+
parser.add_argument("--images", nargs="+")
97+
parser.add_argument("--check", action="store_true")
98+
args = parser.parse_args()
99+
100+
if args.images is None:
101+
image_paths = sorted(glob(os.path.join(IMAGE_ROOT, "*.tif")))
102+
else:
103+
image_paths = args.images
104+
105+
for image_path in image_paths:
106+
create_consensus_step3(image_path, args.check)
107+
108+
109+
if __name__ == "__main__":
110+
main()

‎scripts/validation/SGNs/evaluate_consensus.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import os
2+
from glob import glob
3+
from pathlib import Path
4+
5+
import pandas as pd
6+
from flamingo_tools.validation import match_detections
7+
8+
# The regular root folder.
9+
ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs"
10+
# The root folder for the new annotations for data with scaling issues.
11+
ROOT2 = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs/for_consensus_annotation" # noqa
12+
13+
ANNOTATION_FOLDERS = ["AnnotationsAMD", "AnnotationsEK", "AnnotationsLR"]
14+
CONSENSUS_FOLDER = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs/final_annotations/final_consensus_annotations" # noqa
15+
16+
17+
def match_annotations(consensus_path, sample_name):
18+
annotations = {}
19+
prefix = os.path.basename(consensus_path).split("_")[:3]
20+
prefix = "_".join(prefix)
21+
22+
if sample_name in ("MLR169R_PV_z1913_base_full_rescaled", "MLR169R_PV_z2594_mid_full_rescaled"):
23+
root = ROOT2
24+
else:
25+
root = ROOT
26+
27+
annotations = {}
28+
for annotation_folder in ANNOTATION_FOLDERS:
29+
all_annotations = glob(os.path.join(root, annotation_folder, "*.csv"))
30+
matches = [
31+
ann for ann in all_annotations if (os.path.basename(ann).startswith(prefix) and "negative" not in ann)
32+
]
33+
# TODO continue debugging
34+
if len(matches) != 1:
35+
breakpoint()
36+
assert len(matches) == 1
37+
annotation_path = matches[0]
38+
annotations[annotation_folder] = annotation_path
39+
40+
return annotations
41+
42+
43+
def main():
44+
consensus_files = sorted(glob(os.path.join(CONSENSUS_FOLDER, "*.csv")))
45+
assert len(consensus_files) > 0
46+
47+
results = {
48+
"annotator": [],
49+
"sample": [],
50+
"tps": [],
51+
"fps": [],
52+
"fns": [],
53+
}
54+
for consensus_file in consensus_files:
55+
consensus = pd.read_csv(consensus_file)
56+
consensus = consensus[["axis-0", "axis-1", "axis-2"]]
57+
sample_name = Path(consensus_file).stem
58+
59+
annotations = match_annotations(consensus_file, sample_name)
60+
for name, annotation_path in annotations.items():
61+
annotation = pd.read_csv(annotation_path)[["axis-0", "axis-1", "axis-2"]]
62+
tp, _, fp, fn = match_detections(annotation, consensus, max_dist=8.0)
63+
results["annotator"].append(name)
64+
results["tps"].append(len(tp))
65+
results["fps"].append(len(fp))
66+
results["fns"].append(len(fn))
67+
results["sample"].append(sample_name)
68+
69+
results = pd.DataFrame(results)
70+
print(results)
71+
results.to_csv("results/consensus_evaluation.csv", index=False)
72+
73+
74+
if __name__ == "__main__":
75+
main()

‎scripts/validation/SGNs/run_evaluation.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@
66
fetch_data_for_evaluation, _parse_annotation_path, compute_scores_for_annotated_slice
77
)
88

9-
ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs"
10-
ANNOTATION_FOLDERS = ["AnnotationsEK", "AnnotationsAMD", "AnnotationsLR"]
9+
# OLD EVALUATION FOR INITIAL ANNOTATIONS
10+
# ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs"
11+
# ANNOTATION_FOLDERS = ["AnnotationsEK", "AnnotationsAMD", "AnnotationsLR"]
12+
13+
# NEW EVALUATION FOR FINAL ANNOTATIONS
14+
ROOT = "/mnt/vast-nhr/projects/nim00007/data/moser/cochlea-lightsheet/AnnotatedImageCrops/F1ValidationSGNs/final_annotations" # noqa
15+
ANNOTATION_FOLDERS = ["final_consensus_annotations"]
1116

1217

1318
def run_evaluation(root, annotation_folders, result_file, cache_folder):

0 commit comments

Comments
 (0)