Fix non-consecutive ROIs bug.

If ROIs were missing in the ROI mask, the ROI-IDs in the Traces table were no longer matched with the ROI-IDs in the ROI mask and the ROI table.

Author: jonathanoesterle

djimaging/autorois/roi_canvas.py

@@ -1405,7 +1405,8 @@ def split_name(stim_condition):
         return stim, condition
 
     def insert_database(self, roi_mask_tab, field_key):
-        from djimaging.utils.mask_utils import to_igor_format, compare_roi_masks
+        from djimaging.utils.mask_utils import compare_roi_masks
+        from djimaging.utils.mask_format_utils import to_igor_format
 
         pres_and_roi_mask = []
         for pres_key in self.pres_names:

djimaging/tables/core/roi.py

@@ -5,7 +5,7 @@
 
 from djimaging.utils.dj_utils import get_primary_key
 from djimaging.utils.plot_utils import plot_field
-from djimaging.utils.scanm.roi_utils import extract_roi_idxs
+from djimaging.utils.scanm.roi_utils import extract_roi_ids
 
 
 class RoiTemplate(dj.Computed):
@@ -59,7 +59,7 @@ def make(self, key):
         if not np.any(roi_mask):
             return
 
-        roi_idxs = extract_roi_idxs(roi_mask)
+        roi_idxs = extract_roi_ids(roi_mask)
 
         # add every roi to list and the bulk add to roi table
         for roi_idx in roi_idxs:

djimaging/tables/core/roi_mask.py

@@ -12,8 +12,9 @@
 
 from djimaging.utils.filesystem_utils import as_pre_filepath
 from djimaging.utils.dj_utils import get_primary_key, check_unique_one
-from djimaging.utils.mask_utils import to_igor_format, to_python_format, to_roi_mask_file, sort_roi_mask_files, \
+from djimaging.utils.mask_utils import to_roi_mask_file, sort_roi_mask_files, \
     load_preferred_roi_mask_igor, load_preferred_roi_mask_pickle, compare_roi_masks
+from djimaging.utils.mask_format_utils import to_igor_format, to_python_format
 from djimaging.utils.plot_utils import plot_field
 
 

djimaging/tables/misc/sr_index.py

@@ -24,8 +24,8 @@ class SrIndex(misc.SrIndexTemplate):
 import numpy as np
 from matplotlib import pyplot as plt
 
+from djimaging.utils import mask_format_utils
 from djimaging.utils.scanm import read_utils
-from djimaging.utils import mask_utils
 from djimaging.utils.dj_utils import get_primary_key
 
 
@@ -133,7 +133,7 @@ def plot1(self, key=None, sr_threshold=0.5):
 
 def compute_sr_idxs(ch1_stack, roi_ids, roi_mask, npixartifact):
     """Compute SR index for each ROI in roi_ids. SR index is defined as """
-    roi_mask = mask_utils.as_python_format(roi_mask)
+    roi_mask = mask_format_utils.as_python_format(roi_mask)
 
     ch1_avg = np.mean(ch1_stack, axis=2)
 
@@ -154,7 +154,7 @@ def compute_sr_idxs(ch1_stack, roi_ids, roi_mask, npixartifact):
 
 def plot_stack_sr_idxs(ch0_avg, ch1_avg, roi_mask, roi_ids, sr_idxs, sr_threshold=0.5):
     """Plot SR idxs on top of stack averages."""
-    roi_mask = mask_utils.as_python_format(roi_mask)
+    roi_mask = mask_format_utils.as_python_format(roi_mask)
 
     binary_sr_mask = np.isin(roi_mask, roi_ids[sr_idxs >= sr_threshold])
 

djimaging/utils/mask_format_utils.py

@@ -0,0 +1,93 @@
+import numpy as np
+
+
+def assert_igor_format(roi_mask):
+    vmin = np.min(roi_mask)
+    vmax = np.max(roi_mask)
+
+    if roi_mask.ndim != 2:
+        raise ValueError(f'ROI mask must be 2D, but has shape {roi_mask.shape}')
+
+    if np.any(roi_mask != roi_mask.astype(int)):
+        raise ValueError(f'ROI mask must be integer, but has non-integer values')
+
+    if not ((vmax in [0, 1]) and vmin <= 1):
+        raise ValueError(f'ROI mask has unexpected values: vmin={vmin}, vmax={vmax}, value={np.unique(roi_mask)}')
+
+
+def is_igor_format(roi_mask):
+    """This method can fail if the igor format is not used consistently."""
+    vmin = np.min(roi_mask)
+    vmax = np.max(roi_mask)
+
+    if roi_mask.ndim != 2:
+        return False
+
+    if np.any(roi_mask != roi_mask.astype(int)):
+        return False
+
+    if vmax in [0, 1] and vmin <= 1:
+        return True
+    else:
+        return False
+
+
+def as_igor_format(roi_mask):
+    """Convert from python format to igor format"""
+    if is_igor_format(roi_mask):
+        return roi_mask
+    else:
+        return to_igor_format(roi_mask)
+
+
+def to_igor_format(roi_mask):
+    if not is_python_format(roi_mask):
+        raise ValueError(f'ROI mask is not in python format; unique values in mask: {np.unique(roi_mask)}')
+
+    roi_mask = roi_mask.copy()
+    roi_mask[roi_mask == 0] = -1
+    roi_mask = -roi_mask
+
+    return roi_mask
+
+
+def is_python_format(roi_mask):
+    vmin = np.min(roi_mask)
+    vmax = np.max(roi_mask)
+
+    if roi_mask.ndim != 2:
+        return False
+
+    if np.any(roi_mask != roi_mask.astype(int)):
+        return False
+
+    if vmin == 0 and vmax >= 0:
+        return True
+    else:
+        return False
+
+
+def as_python_format(roi_mask):
+    """Convert from igor format to python format if necessary"""
+    if is_python_format(roi_mask):
+        return roi_mask
+    else:
+        return to_python_format(roi_mask)
+
+
+def to_python_format(roi_mask):
+    # Some ROI masks have 11 instead of 1's for some reason
+    rm_vals = np.unique(roi_mask)
+    if set(rm_vals[rm_vals > 0]) == {1, 11}:
+        roi_mask = roi_mask.copy()
+        roi_mask[roi_mask == 11] = 1
+
+    assert_igor_format(roi_mask)
+
+    vmax = np.max(roi_mask)
+
+    roi_mask = roi_mask.copy()
+    roi_mask[roi_mask == vmax] = 0
+    roi_mask = np.abs(roi_mask)
+
+    return roi_mask

djimaging/utils/mask_utils.py

@@ -7,6 +7,7 @@
 from matplotlib import pyplot as plt
 
 from djimaging.utils.alias_utils import check_shared_alias_str
+from djimaging.utils.mask_format_utils import assert_igor_format, to_igor_format
 from djimaging.utils.scanm import read_h5_utils
 from djimaging.utils.cellpose_utils import intersection_over_union
 
@@ -417,92 +418,6 @@ def generate_roi_suggestions(mask_pred, mask_true, n_artifact, threshold=0.1, ve
     return rois_to_add
 
 
-def assert_igor_format(roi_mask):
-    vmin = np.min(roi_mask)
-    vmax = np.max(roi_mask)
-
-    if roi_mask.ndim != 2:
-        raise ValueError(f'ROI mask must be 2D, but has shape {roi_mask.shape}')
-
-    if np.any(roi_mask != roi_mask.astype(int)):
-        raise ValueError(f'ROI mask must be integer, but has non-integer values')
-
-    if not ((vmax in [0, 1]) and vmin <= 1):
-        raise ValueError(f'ROI mask has unexpected values: vmin={vmin}, vmax={vmax}, value={np.unique(roi_mask)}')
-
-
-def is_igor_format(roi_mask):
-    """This method can fail if the igor format is not used consistently."""
-    vmin = np.min(roi_mask)
-    vmax = np.max(roi_mask)
-
-    if roi_mask.ndim != 2:
-        return False
-
-    if np.any(roi_mask != roi_mask.astype(int)):
-        return False
-
-    if vmax in [0, 1] and vmin <= 1:
-        return True
-    else:
-        return False
-
-
-def as_igor_format(roi_mask):
-    """Convert from python format to igor format"""
-    if is_igor_format(roi_mask):
-        return roi_mask
-    else:
-        return to_igor_format(roi_mask)
-
-
-def to_igor_format(roi_mask):
-    if not is_python_format(roi_mask):
-        raise ValueError(f'ROI mask is not in python format; unique values in mask: {np.unique(roi_mask)}')
-
-    roi_mask = roi_mask.copy()
-    roi_mask[roi_mask == 0] = -1
-    roi_mask = -roi_mask
-
-    return roi_mask
-
-
-def is_python_format(roi_mask):
-    vmin = np.min(roi_mask)
-    vmax = np.max(roi_mask)
-
-    if roi_mask.ndim != 2:
-        return False
-
-    if np.any(roi_mask != roi_mask.astype(int)):
-        return False
-
-    if vmin == 0 and vmax >= 0:
-        return True
-    else:
-        return False
-
-
-def as_python_format(roi_mask):
-    """Convert from igor format to python format if necessary"""
-    if is_python_format(roi_mask):
-        return roi_mask
-    else:
-        return to_python_format(roi_mask)
-
-
-def to_python_format(roi_mask):
-    assert_igor_format(roi_mask)
-
-    vmax = np.max(roi_mask)
-
-    roi_mask = roi_mask.copy()
-    roi_mask[roi_mask == vmax] = 0
-    roi_mask = np.abs(roi_mask)
-
-    return roi_mask
-
-
 def to_roi_mask_file(data_file, old_suffix=None, new_suffix='_ROIs.pkl',
                      roi_mask_dir=None, old_prefix=None, new_prefix=None):
     """Get ROI mask file path from data file path"""

djimaging/utils/scanm/read_h5_utils.py

@@ -4,7 +4,7 @@
 import numpy as np
 
 from djimaging.utils.scanm.wparams_utils import check_dims_ch_stack_wparams
-from djimaging.utils.scanm.roi_utils import get_roi2trace
+from djimaging.utils.scanm.roi_utils import get_roi2trace, extract_roi_ids
 
 
 def load_stacks_and_wparams(filepath, ch_names=('wDataCh0', 'wDataCh1')) -> (dict, dict):
@@ -54,9 +54,17 @@ def load_roi2trace(filepath: str, roi_ids: np.ndarray):
     try:
         with h5py.File(filepath, "r", driver="stdio") as h5_file:
             traces, traces_times = extract_traces(h5_file)
+            roi_ids_traces = extract_roi_ids(extract_roi_mask(h5_file, ignore_not_found=False), npixartifact=0)
     except OSError as e:
         raise OSError(f"Error loading file {filepath}: {e}")
-    roi2trace = get_roi2trace(traces=traces, traces_times=traces_times, roi_ids=roi_ids)
+
+    if not set(roi_ids).issubset(set(roi_ids_traces)):
+        raise ValueError(f"roi_ids {roi_ids} do not match roi_ids in traces {roi_ids_traces}")
+    if traces.shape[-1] != len(roi_ids_traces):
+        raise ValueError(f"Number of roi_ids {len(roi_ids_traces)} does not match traces shape {traces.shape[-1]}.")
+
+    roi2trace = get_roi2trace(traces=traces, traces_times=traces_times,
+                              roi_ids_traces=roi_ids_traces, roi_ids_subset=roi_ids)
     frame_dt = np.mean(np.diff(traces_times, axis=0))
     return roi2trace, frame_dt
 

djimaging/utils/scanm/roi_utils.py

@@ -2,16 +2,16 @@
 
 import numpy as np
 
-from djimaging.utils.mask_utils import assert_igor_format
+from djimaging.utils.mask_format_utils import assert_igor_format
 
 
-def extract_roi_idxs(roi_mask, npixartifact=0):
+def extract_roi_ids(roi_mask, npixartifact=0):
     """Return roi idxs as in ROI mask (i.e. negative values)"""
     assert roi_mask.ndim == 2
-    roi_idxs = np.unique(roi_mask[npixartifact:, :])
-    roi_idxs = roi_idxs[roi_idxs < 0]  # remove background indexes (0 or 1)
-    roi_idxs = roi_idxs[np.argsort(np.abs(roi_idxs))]  # Sort by value
-    return roi_idxs.astype(int)
+    roi_ids = np.unique(roi_mask[npixartifact:, :])
+    roi_ids = roi_ids[roi_ids < 0]  # remove background indexes (0 or 1)
+    roi_ids = roi_ids[np.argsort(np.abs(roi_ids))]  # Sort by value
+    return roi_ids.astype(int)
 
 
 def fix_first_or_last_n_nan(trace, n):
@@ -25,22 +25,32 @@ def fix_first_or_last_n_nan(trace, n):
     return trace
 
 
-def get_roi2trace(traces, traces_times, roi_ids):
+def get_roi2trace(traces, traces_times, roi_ids_traces, roi_ids_subset=None):
     """Get dict that holds traces and times accessible by roi_id"""
-    assert np.all(roi_ids >= 1)
+    roi_ids_subset = roi_ids_traces if roi_ids_subset is not None else roi_ids_subset
+
+    assert np.all(roi_ids_traces >= 1)
+    assert np.all(roi_ids_subset >= 1)
+
+    if traces.shape != traces_times.shape:
+        raise ValueError(f"traces shape {traces.shape} does not match traces_times shape {traces_times.shape}.")
+
+    if traces.shape[-1] != len(roi_ids_traces):
+        warnings.warn(f"Number of roi_ids {len(roi_ids_traces)} does not match traces shape {traces.shape[-1]}.")
 
     roi2trace = dict()
 
-    for roi_id in roi_ids:
-        idx = roi_id - 1
+    for i, roi_id in enumerate(roi_ids_traces):
+        if roi_id not in roi_ids_subset:
+            continue
 
-        if traces.ndim == 3 and idx < traces.shape[-1]:
-            trace = traces[:, :, idx]
-            trace_times = traces_times[:, :, idx]
+        if traces.ndim == 3 and i < traces.shape[-1]:
+            trace = traces[:, :, i]
+            trace_times = traces_times[:, :, i]
             trace_valid = 1
-        elif traces.ndim == 2 and idx < traces.shape[-1]:
-            trace = traces[:, idx]
-            trace_times = traces_times[:, idx]
+        elif traces.ndim == 2 and i < traces.shape[-1]:
+            trace = traces[:, i]
+            trace_times = traces_times[:, i]
             trace_valid = 1
         else:
             trace_valid = 0