Merge branch 'master' into tracking

Author: constantinpape

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Computational Cell Analytics (Research Group of Prof. Dr. Constantin Pape)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -8,11 +8,15 @@ We implement napari applications for:
 
 **Early beta version**
 
-This is an early beta version. Any feedback is welcome, but please be aware that the functionality is evolving fast and not fully tested.
+This is an early beta version. Any feedback is welcome, but please be aware that the functionality is under active development and that several features are not finalized or thoroughly tested yet.
+Once the functionality has matured we plan to release the interactive annotation applications as [napari plugins](https://napari.org/stable/plugins/index.html).
+
 
 ## Functionality overview
 
 TODO
+- quick explanation and gifs
+
 
 ## Installation
 
@@ -51,15 +55,32 @@ pip install -e .
 
 ## Usage
 
+After the installation the three applications for interactive annotations can be started from the command line or within a python script:
+- **2d segmentation**: via the command `micro_sam.annotator_2d` or with the function `micro_sam.sam_annotator.annotator_2d` from python. Run `micro_sam.annotator_2d -h` or check out [examples/sam_annotator_2d](https://github.com/computational-cell-analytics/micro-sam/blob/master/examples/sam_annotator_2d.py) for details. 
+- **3d segmentation**: via the command `micro_sam.annotator_3d` or with the function `micro_sam.sam_annotator.annotator_3d` from python. Run `micro_sam.annotator_3d -h` or check out [examples/sam_annotator_3d](https://github.com/computational-cell-analytics/micro-sam/blob/master/examples/sam_annotator_3d.py) for details. 
+- **tracking**: via the command `micro_sam.annotator_tracking` or with the function `micro_sam.sam_annotator.annotator_tracking` from python. Run `micro_sam.annotator_tracking -h` or check out [examples/sam_annotator_tracking](https://github.com/computational-cell-analytics/micro-sam/blob/master/examples/sam_annotator_tracking.py) for details. 
+
 TODO
+- show image with annotated user interface (for 3d?!)
+- link to videos that explain the functionality for the 3 plugins
 
 ### Tips & Tricks
 
 TODO
+- speeding things up: precomputing the embeddings with a gpu, making input images smaller
+- correcting existing segmentaitons via `segmentation_results`
+- saving and loading intermediate results via segmentation results
+
+### Limitations
+
+TODO
+- automatic instance segmentation limitations
 
 ## Using the micro_sam library
 
 TODO
+- link to the example image series application
+
 
 ## Contributing
 
@@ -68,8 +89,9 @@ micro_sam <- library with utility functionality for using SAM for microscopy dat
     /sam_annotator <- the napari plugins for annotation
 ```
 
+
 ## Citation
 
 If you are using this repository in your research please cite
 - [SegmentAnything](https://arxiv.org/abs/2304.02643)
-- and our repository on [zenodo](TODO) (we are working on a full publication)
+- and our repository on [zenodo](TODO) (we are working on a publication)

development/.gitignore

@@ -0,0 +1,2 @@
+embeddings/
+*.npy

development/tracking.py

@@ -0,0 +1,46 @@
+from glob import glob
+
+import numpy as np
+from elf.io import open_file
+from micro_sam.sam_annotator import annotator_tracking
+
+
+def debug_tracking(timeseries, embedding_path):
+    import micro_sam.util as util
+    from micro_sam.sam_annotator.annotator_tracking import _track_from_prompts
+
+    predictor = util.get_sam_model()
+    image_embeddings = util.precompute_image_embeddings(predictor, timeseries, embedding_path)
+
+    # seg = np.zeros(timeseries.shape, dtype="uint32")
+    seg = np.load("./seg.npy")
+    assert seg.shape == timeseries.shape
+    slices = np.array([0])
+    stop_upper = False
+
+    _track_from_prompts(seg, predictor, slices, image_embeddings, stop_upper, threshold=0.5, projection="bounding_box")
+
+
+def load_data():
+    pattern = "/home/pape/Work/data/incu_cyte/carmello/videos/MiaPaCa_flat_B3-3_registered/image-*"
+    paths = glob(pattern)
+    paths.sort()
+
+    timeseries = []
+    for p in paths[:45]:
+        with open_file(p, mode="r") as f:
+            timeseries.append(f["phase-contrast"][:])
+    timeseries = np.stack(timeseries)
+    return timeseries
+
+
+def main():
+    timeseries = load_data()
+    embedding_path = "./embeddings/embeddings-tracking.zarr"
+
+    # _check_tracking(timeseries, embedding_path)
+    annotator_tracking(timeseries, embedding_path=embedding_path)
+
+
+if __name__ == "__main__":
+    main()

examples/image_series_annotator_app.py

@@ -0,0 +1,124 @@
+# Example for a small application implemented using napari and the micro_sam library:
+# Iterate over a series of images in a folder and provide annotations with SAM.
+
+import os
+from glob import glob
+
+import imageio
+import micro_sam.util as util
+import napari
+import numpy as np
+
+from magicgui import magicgui
+from micro_sam.segment_from_prompts import segment_from_points
+from micro_sam.sam_annotator.util import create_prompt_menu, prompt_layer_to_points
+from napari import Viewer
+
+
+@magicgui(call_button="Segment Object [S]")
+def segment_wigdet(v: Viewer):
+    points, labels = prompt_layer_to_points(v.layers["prompts"])
+    seg = segment_from_points(PREDICTOR, points, labels)
+    v.layers["segmented_object"].data = seg.squeeze()
+    v.layers["segmented_object"].refresh()
+
+
+def image_series_annotator(image_paths, embedding_save_path, output_folder):
+    global PREDICTOR
+
+    os.makedirs(output_folder, exist_ok=True)
+
+    # get the sam predictor and precompute the image embeddings
+    PREDICTOR = util.get_sam_model()
+    images = np.stack([imageio.imread(p) for p in image_paths])
+    image_embeddings = util.precompute_image_embeddings(PREDICTOR, images, save_path=embedding_save_path)
+    util.set_precomputed(PREDICTOR, image_embeddings, i=0)
+
+    v = napari.Viewer()
+
+    # add the first image
+    next_image_id = 0
+    v.add_image(images[0], name="image")
+
+    # add a layer for the segmented object
+    v.add_labels(data=np.zeros(images.shape[1:], dtype="uint32"), name="segmented_object")
+
+    # create the point layer for the sam prompts and add the widget for toggling the points
+    labels = ["positive", "negative"]
+    prompts = v.add_points(
+        data=[[0.0, 0.0], [0.0, 0.0]],  # FIXME workaround
+        name="prompts",
+        properties={"label": labels},
+        edge_color="label",
+        edge_color_cycle=["green", "red"],
+        symbol="o",
+        face_color="transparent",
+        edge_width=0.5,
+        size=12,
+        ndim=2,
+    )
+    prompts.data = []
+    prompts.edge_color_mode = "cycle"
+    prompt_widget = create_prompt_menu(prompts, labels)
+    v.window.add_dock_widget(prompt_widget)
+
+    # toggle the points between positive / negative
+    @v.bind_key("t")
+    def toggle_label(event=None):
+        # get the currently selected label
+        current_properties = prompts.current_properties
+        current_label = current_properties["label"][0]
+        new_label = "negative" if current_label == "positive" else "positive"
+        current_properties["label"] = np.array([new_label])
+        prompts.current_properties = current_properties
+        prompts.refresh()
+        prompts.refresh_colors()
+
+    # bind the segmentation to a key 's'
+    @v.bind_key("s")
+    def _segmet(v):
+        segment_wigdet(v)
+
+    #
+    # the functionality for saving segmentations and going to the next image
+    #
+
+    def _save_segmentation(seg, output_folder, image_path):
+        fname = os.path.basename(image_path)
+        save_path = os.path.join(output_folder, os.path.splitext(fname)[0] + ".tif")
+        imageio.imwrite(save_path, seg)
+
+    def _next(v):
+        nonlocal next_image_id
+        v.layers["image"].data = images[next_image_id]
+        util.set_precomputed(PREDICTOR, image_embeddings, i=next_image_id)
+
+        v.layers["segmented_object"].data = np.zeros(images[0].shape, dtype="uint32")
+        v.layers["prompts"].data = []
+
+        next_image_id += 1
+        if next_image_id >= images.shape[0]:
+            print("Last image!")
+
+    @v.bind_key("n")
+    def next_image(v):
+        seg = v.layers["segmented_object"].data
+        if seg.max() == 0:
+            print("This image has not been segmented yet, doing nothing!")
+            return
+
+        _save_segmentation(seg, output_folder, image_paths[next_image_id - 1])
+        _next(v)
+
+    napari.run()
+
+
+# this uses data from the cell tracking challenge as example data
+# see 'sam_annotator_tracking' for examples
+def main():
+    image_paths = sorted(glob("./data/DIC-C2DH-HeLa/train/01/*.tif"))[:50]
+    image_series_annotator(image_paths, "./embeddings/embeddings-ctc.zarr", "segmented-series")
+
+
+if __name__ == "__main__":
+    main()

examples/instance_segmentation.py

@@ -0,0 +1,60 @@
+import micro_sam.util as util
+import napari
+
+from elf.io import open_file
+from micro_sam.segment_instances import segment_from_embeddings
+from micro_sam.visualization import compute_pca
+
+
+def mito_segmentation():
+    input_path = "./data/Lucchi++/Test_In"
+    with open_file(input_path) as f:
+        raw = f["*.png"][-1, :768, :768]
+
+    predictor = util.get_sam_model()
+    image_embeddings = util.precompute_image_embeddings(predictor, raw, "./embeddings/embeddings-mito2d.zarr")
+    embedding_pca = compute_pca(image_embeddings["features"])
+
+    seg, initial_seg = segment_from_embeddings(predictor, image_embeddings=image_embeddings, return_initial_seg=True)
+
+    v = napari.Viewer()
+    v.add_image(raw)
+    v.add_image(embedding_pca, scale=(12, 12))
+    v.add_labels(seg)
+    v.add_labels(initial_seg)
+    napari.run()
+
+
+def cell_segmentation():
+    path = "./DIC-C2DH-HeLa/train/01"
+    with open_file(path, mode="r") as f:
+        timeseries = f["*.tif"][:50]
+
+    frame = 11
+
+    predictor = util.get_sam_model()
+    image_embeddings = util.precompute_image_embeddings(predictor, timeseries, "./embeddings/embeddings-ctc.zarr")
+    embedding_pca = compute_pca(image_embeddings["features"][frame])
+
+    seg, initial_seg = segment_from_embeddings(
+        predictor, image_embeddings=image_embeddings, i=frame, return_initial_seg=True
+    )
+
+    v = napari.Viewer()
+    v.add_image(timeseries[frame])
+    v.add_image(embedding_pca, scale=(8, 8))
+    v.add_labels(seg)
+    v.add_labels(initial_seg)
+    napari.run()
+
+
+def main():
+    # automatic segmentation for the data from Lucchi et al. (see 'sam_annotator_3d.py')
+    # mito_segmentation()
+
+    # automatic segmentation for data from the cell tracking challenge (see 'sam_annotator_tracking.py')
+    cell_segmentation()
+
+
+if __name__ == "__main__":
+    main()

examples/sam_annotator_2d.py

@@ -2,12 +2,21 @@
 from micro_sam.sam_annotator import annotator_2d
 
 
-def main():
+# TODO describe how to get the data and don't use hard-coded system path
+def livecell_annotator():
     im = imageio.imread(
         "/home/pape/Work/data/incu_cyte/livecell/images/livecell_test_images/A172_Phase_C7_1_01d04h00m_4.tif"
     )
     embedding_path = "./embeddings/embeddings-livecell_cropped.zarr"
-    annotator_2d(im, embedding_path)
+    annotator_2d(im, embedding_path, show_embeddings=True)
+
+
+def main():
+    # 2d annotator for livecell data
+    # livecell_annotator()
+
+    # 2d annotator for cell tracking challenge hela data
+    hela_2d_annotator()
 
 
 if __name__ == "__main__":

examples/sam_annotator_tracking.py

@@ -1,41 +1,18 @@
-from glob import glob
-
-import h5py
-import numpy as np
+from elf.io import open_file
 from micro_sam.sam_annotator import annotator_tracking
 
 
-# FOR DEBUGGING / DEVELOPMENT
-def _check_tracking(timeseries, embedding_path):
-    import micro_sam.util as util
-    from micro_sam.sam_annotator.annotator_tracking import _track_from_prompts
-
-    predictor = util.get_sam_model()
-    image_embeddings = util.precompute_image_embeddings(predictor, timeseries, embedding_path)
-
-    # seg = np.zeros(timeseries.shape, dtype="uint32")
-    seg = np.load("./seg.npy")
-    assert seg.shape == timeseries.shape
-    slices = np.array([0])
-    stop_upper = False
-
-    _track_from_prompts(seg, predictor, slices, image_embeddings, stop_upper, threshold=0.5, method="bounding_box")
+# TODO describe how to get the data from CTC
+def track_ctc_data():
+    path = "./data/DIC-C2DH-HeLa/train/01"
+    with open_file(path, mode="r") as f:
+        timeseries = f["*.tif"][:50]
+    annotator_tracking(timeseries, embedding_path="./embeddings/embeddings-ctc.zarr")
 
 
 def main():
-    pattern = "/home/pape/Work/data/incu_cyte/carmello/videos/MiaPaCa_flat_B3-3_registered/image-*"
-    paths = glob(pattern)
-    paths.sort()
-
-    timeseries = []
-    for p in paths[:45]:
-        with h5py.File(p) as f:
-            timeseries.append(f["phase-contrast"][:])
-    timeseries = np.stack(timeseries)
-
-    embedding_path = "./embeddings/embeddings-tracking.zarr"
-    # _check_tracking(timeseries, embedding_path)
-    annotator_tracking(timeseries, embedding_path=embedding_path)
+    # run interactive tracking for data from the cell tracking challenge
+    track_ctc_data()
 
 
 if __name__ == "__main__":