Merge pull request #953 from fractal-analytics-platform/952_cellvoager_converter

cellvoager converter fixes

Author: jluethi

.github/workflows/manifest_external_packages.yml

@@ -39,13 +39,13 @@ jobs:
             cmd_create_manifest: 'python src/scmultiplex/dev/create_manifest.py'
             custom_dependencies: 'image_registration'
 
-          - package: APx_fractal_task_collection
-            github_repo: Apricot-Therapeutics/APx_fractal_task_collection
-            github_branch: main
-            manifest_path: src/apx_fractal_task_collection/__FRACTAL_MANIFEST__.json
-            cmd_install: 'python -m pip install -e .'
-            cmd_create_manifest: 'python src/apx_fractal_task_collection/dev/update_manifest.py'
-            custom_dependencies: ''
+          # - package: APx_fractal_task_collection
+          #   github_repo: Apricot-Therapeutics/APx_fractal_task_collection
+          #   github_branch: main
+          #   manifest_path: src/apx_fractal_task_collection/__FRACTAL_MANIFEST__.json
+          #   cmd_install: 'python -m pip install -e .'
+          #   cmd_create_manifest: 'python src/apx_fractal_task_collection/dev/update_manifest.py'
+          #   custom_dependencies: ''
 
         exclude:
           - package: skip

CHANGELOG.md

@@ -1,5 +1,12 @@
 **Note**: Numbers like (\#123) point to closed Pull Requests on the fractal-tasks-core repository.
 
+# 1.5.1
+* Tasks:
+    * In `Convert Cellvoyager to OME-Zarr` & `Convert Cellvoyager Multiplexing to OME-Zarr` tasks, handle unset window start & end better (\#953).
+    * In `Convert Cellvoyager to OME-Zarr` & `Convert Cellvoyager Multiplexing to OME-Zarr` tasks, normalize plate names to avoid special characters & spaces (\#953).
+    * In `Convert Cellvoyager Multiplexing to OME-Zarr`, improve error message when no images are found (\#953).
+    * In `Convert Cellvoyager Multiplexing to OME-Zarr`, handle order of acquisitions better (\#953).
+
 # 1.5.0
 
 * Tasks:

fractal_tasks_core/__FRACTAL_MANIFEST__.json

@@ -50,7 +50,7 @@
                   }
                 ],
                 "title": "Window",
-                "description": "Optional `Window` object to set default display settings for napari."
+                "description": "Optional `Window` object to set default display settings. If unset, it will be set to the full bit range of the image (e.g. 0-65535 for 16 bit images)."
               },
               "color": {
                 "title": "Color",
@@ -370,7 +370,7 @@
                   }
                 ],
                 "title": "Window",
-                "description": "Optional `Window` object to set default display settings for napari."
+                "description": "Optional `Window` object to set default display settings. If unset, it will be set to the full bit range of the image (e.g. 0-65535 for 16 bit images)."
               },
               "color": {
                 "title": "Color",

fractal_tasks_core/cellvoyager/metadata.py

@@ -15,6 +15,7 @@
 import fnmatch
 import logging
 import math
+import string
 from pathlib import Path
 from typing import Optional
 from typing import Union
@@ -477,3 +478,30 @@ def check_group_consistency(grouped_df: pd.DataFrame, message: str = ""):
             f"{message}\n"
             f"Difference dataframe: \n{diff_df}"
         )
+
+
+__SPECIAL_CHARACTERS__ = f"{string.punctuation}{string.whitespace}"
+
+
+def sanitize_string(value: str) -> str:
+    """
+    Make string safe to be used in file/folder names.
+
+    Replace any special character with an
+    underscore, where special characters are:
+
+
+        >>> string.punctuation
+        '!"#$%&\'()*+,-./:;<=>?@[\\\\]^_`{|}~'
+        >>> string.whitespace
+        ' \\t\\n\\r\\x0b\\x0c'
+
+    Args:
+        value: Input string
+
+    Returns:
+        Sanitized value
+    """
+    for character in __SPECIAL_CHARACTERS__:
+        new_value = value.replace(character, "_")
+    return new_value

fractal_tasks_core/channels.py

@@ -59,8 +59,9 @@ class OmeroChannel(BaseModel):
         wavelength_id: Unique ID for the channel wavelength, e.g. `A01_C01`.
         index: Do not change. For internal use only.
         label: Name of the channel.
-        window: Optional `Window` object to set default display settings for
-            napari.
+        window: Optional `Window` object to set default display settings. If
+            unset, it will be set to the full bit range of the image
+            (e.g. 0-65535 for 16 bit images).
         color: Optional hex colormap to display the channel in napari (it
             must be of length 6, e.g. `00FFFF`).
         active: Should this channel be shown in the viewer?
@@ -370,6 +371,15 @@ def define_omero_channels(
         if channel.window:
             channel.window.min = 0
             channel.window.max = 2**bit_depth - 1
+        else:
+            # If no channel.window is set, create a new one with full bitrange
+            # min & max
+            channel.window = Window(
+                min=0,
+                max=2**bit_depth - 1,
+                start=0,
+                end=2**bit_depth - 1,
+            )
 
     # Check that channel labels are unique for this image
     labels = [c.label for c in new_channels]

fractal_tasks_core/tasks/cellvoyager_to_ome_zarr_init.py

@@ -27,6 +27,7 @@
 from fractal_tasks_core.cellvoyager.metadata import (
     parse_yokogawa_metadata,
 )
+from fractal_tasks_core.cellvoyager.metadata import sanitize_string
 from fractal_tasks_core.cellvoyager.wells import generate_row_col_split
 from fractal_tasks_core.cellvoyager.wells import get_filename_well_id
 from fractal_tasks_core.channels import check_unique_wavelength_ids
@@ -241,8 +242,9 @@ def cellvoyager_to_ome_zarr_init(
     parallelization_list = []
 
     for plate in plates:
+        plate_name = sanitize_string(plate)
         # Define plate zarr
-        relative_zarrurl = f"{plate}.zarr"
+        relative_zarrurl = f"{plate_name}.zarr"
         in_path = dict_plate_paths[plate]
         logger.info(f"Creating {relative_zarrurl}")
         # Call zarr.open_group wrapper, which handles overwrite=True/False
@@ -337,7 +339,7 @@ def cellvoyager_to_ome_zarr_init(
             well_wavelength_ids = sorted(list(set(well_wavelength_ids)))
             if well_wavelength_ids != actual_wavelength_ids:
                 raise ValueError(
-                    f"ERROR: well {well} in plate {plate} (prefix: "
+                    f"ERROR: well {well} in plate {plate_name} (prefix: "
                     f"{plate_prefix}) has missing channels.\n"
                     f"Expected: {actual_channels}\n"
                     f"Found: {well_wavelength_ids}.\n"
@@ -355,7 +357,7 @@ def cellvoyager_to_ome_zarr_init(
         col_list = sorted(list(set(col_list)))
 
         plate_attrs = {
-            "acquisitions": [{"id": 0, "name": plate}],
+            "acquisitions": [{"id": 0, "name": plate_name}],
             "columns": [{"name": col} for col in col_list],
             "rows": [{"name": row} for row in row_list],
             "version": __OME_NGFF_VERSION__,
@@ -377,7 +379,9 @@ def cellvoyager_to_ome_zarr_init(
         for row, column in well_rows_columns:
             parallelization_list.append(
                 {
-                    "zarr_url": f"{zarr_dir}/{plate}.zarr/{row}/{column}/0",
+                    "zarr_url": (
+                        f"{zarr_dir}/{plate_name}.zarr/{row}/{column}/0"
+                    ),
                     "init_args": InitArgsCellVoyager(
                         image_dir=in_path,
                         plate_prefix=plate_prefix,

fractal_tasks_core/tasks/cellvoyager_to_ome_zarr_init_multiplex.py

@@ -29,6 +29,7 @@
 from fractal_tasks_core.cellvoyager.metadata import (
     parse_yokogawa_metadata,
 )
+from fractal_tasks_core.cellvoyager.metadata import sanitize_string
 from fractal_tasks_core.cellvoyager.wells import generate_row_col_split
 from fractal_tasks_core.cellvoyager.wells import get_filename_well_id
 from fractal_tasks_core.channels import check_unique_wavelength_ids
@@ -151,7 +152,7 @@ def cellvoyager_to_ome_zarr_init_multiplex(
 
     # Identify all plates and all channels, per input folders
     dict_acquisitions: dict = {}
-    acquisitions_sorted = sorted(list(acquisitions.keys()))
+    acquisitions_sorted = sorted(acquisitions.keys(), key=lambda x: int(x))
     for acquisition in acquisitions_sorted:
         acq_input = acquisitions[acquisition]
         dict_acquisitions[acquisition] = {}
@@ -191,6 +192,7 @@ def cellvoyager_to_ome_zarr_init_multiplex(
 
         info = (
             "Listing all plates/channels:\n"
+            f"Folder:   {acq_input.image_dir}\n"
             f"Include patterns: {include_patterns}\n"
             f"Exclude patterns: {exclude_patterns}\n"
             f"Plates:   {plates}\n"
@@ -253,9 +255,12 @@ def cellvoyager_to_ome_zarr_init_multiplex(
     current_plates = [item["plate"] for item in dict_acquisitions.values()]
     if len(set(current_plates)) > 1:
         raise ValueError(f"{current_plates=}")
-    plate = current_plates[0]
+    plate = sanitize_string(current_plates[0])
 
-    zarrurl = dict_acquisitions[acquisitions_sorted[0]]["plate"] + ".zarr"
+    zarrurl = (
+        sanitize_string(dict_acquisitions[acquisitions_sorted[0]]["plate"])
+        + ".zarr"
+    )
     full_zarrurl = str(Path(zarr_dir) / zarrurl)
     logger.info(f"Creating {full_zarrurl=}")
     # Call zarr.open_group wrapper, which handles overwrite=True/False
@@ -416,6 +421,8 @@ def cellvoyager_to_ome_zarr_init_multiplex(
                 Well(**well_attrs)
                 group_well.attrs["well"] = well_attrs
                 zarrurls["well"].append(f"{plate}.zarr/{row}/{column}")
+                print(plate)
+                print(zarrurls["well"])
             except ContainsGroupError:
                 group_well = zarr.open_group(
                     f"{full_zarrurl}/{row}/{column}/", mode="r+"