add_layer_from_images: allow single image folders, retry with bioformats (#829)

* add_layer_from_images: allow folders with single images, retry with bioformats

* add changelog entry, update docs, add one more testcase

* apply PR feedback

* fix for iterables

Author: jstriebel

webknossos/Changelog.md

@@ -17,8 +17,10 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Added
 
 ### Changed
+- `Dataset.from_images` and `dataset.add_layer_from_images` now try to import the images via the [bioformats](https://www.openmicroscopy.org/bio-formats) after all other options as well. [#829](https://github.com/scalableminds/webknossos-libs/pull/829)
 
 ### Fixed
+- `dataset.add_layer_from_images` can now handle paths to folders which only contain a single image. [#829](https://github.com/scalableminds/webknossos-libs/pull/829)
 
 
 ## [0.10.25](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.10.25) - 2022-11-29

webknossos/testdata/single_multipage_tiff_folder/test_C.tif

@@ -0,0 +1 @@
+../various_tiff_formats/test_C.tif

webknossos/tests/dataset/test_add_layer_from_images.py

@@ -62,7 +62,7 @@ def test_compare_tifffile(tmp_path: Path) -> None:
         (64, 64, 2),
     ),
     (
-        "testdata/rgb_tiff/test_rgb.tif",
+        "testdata/rgb_tiff",
         {"mag": 2, "channel": 1, "dtype": "uint32"},
         "uint32",
         1,
@@ -75,6 +75,16 @@ def test_compare_tifffile(tmp_path: Path) -> None:
         1,
         (1024, 1024, 12),
     ),
+    (
+        "testdata/temca2",
+        {"flip_z": True, "batch_size": 2048},
+        "uint8",
+        1,
+        # The topmost folder contains an extra image,
+        # which is included here as well, but not in
+        # the glob pattern above. Therefore z is +1.
+        (1024, 1024, 13),
+    ),
     (
         "testdata/tiff_with_different_dimensions/*",
         {"flip_y": True},
@@ -84,6 +94,8 @@ def test_compare_tifffile(tmp_path: Path) -> None:
     ),
     ("testdata/various_tiff_formats/test_CS.tif", {}, "uint8", 3, (128, 128, 320)),
     ("testdata/various_tiff_formats/test_C.tif", {}, "uint8", 1, (128, 128, 320)),
+    # same as test_C.tif above, but as a single file in a folder:
+    ("testdata/single_multipage_tiff_folder", {}, "uint8", 1, (128, 128, 320)),
     ("testdata/various_tiff_formats/test_I.tif", {}, "uint32", 1, (64, 128, 64)),
     ("testdata/various_tiff_formats/test_S.tif", {}, "uint16", 3, (128, 128, 64)),
 ]

webknossos/webknossos/dataset/_utils/pims_images.py

@@ -32,10 +32,10 @@
 
 try:
     import pims
-except ImportError as e:
+except ImportError as import_error:
     raise RuntimeError(
         "Cannot import pims, please install it e.g. using 'webknossos[all]'"
-    ) from e
+    ) from import_error
 
 
 # Fix ImageIOReader not handling channels correctly. This might get fixed via
@@ -280,6 +280,71 @@ def __init__(
                 self.num_channels = 3
                 self._first_n_channels = 3
 
+    def _normalize_original_images(self) -> Union[str, List[str]]:
+        original_images = self._original_images
+        if isinstance(original_images, (str, Path)):
+            original_images_path = Path(original_images)
+            if original_images_path.is_dir():
+                valid_suffixes = get_valid_pims_suffixes()
+                original_images = [
+                    str(i)
+                    for i in original_images_path.glob("**/*")
+                    if i.is_file() and i.suffix.lstrip(".") in valid_suffixes
+                ]
+                if len(original_images) == 1:
+                    original_images = original_images[0]
+        if isinstance(original_images, str):
+            return original_images
+        elif isinstance(original_images, Iterable):
+            return [str(i) for i in original_images]
+        else:
+            return str(original_images)
+
+    def _open_bioformats_images_raw(
+        self,
+        original_images: Union[str, List[str]],
+        previous_exceptions: List[Exception],
+    ) -> pims.FramesSequence:
+        try:
+            # There is a wrong warning about jpype, supressing it here.
+            # See issue https://github.com/soft-matter/pims/issues/384
+            warnings.filterwarnings(
+                "ignore",
+                "Due to an issue with JPype 0.6.0, reading is slower.*",
+                category=UserWarning,
+                module="pims.bioformats",
+            )
+            try:
+                pims.bioformats._find_jar()
+            except HTTPError:
+                # We cannot use the newest bioformats version,
+                # since it does not include the necessary loci_tools.jar.
+                # Updates to support newer bioformats jars with pims are in PR
+                # https://github.com/soft-matter/pims/pull/403
+                pims.bioformats.download_jar(version="6.7.0")
+
+            if "*" in str(original_images) or isinstance(original_images, list):
+                images_context_manager = pims.ReaderSequence(
+                    original_images, pims.bioformats.BioformatsReader
+                )
+            else:
+                images_context_manager = pims.bioformats.BioformatsReader(
+                    original_images
+                )
+        except Exception as e:
+            if len(previous_exceptions) == 0:
+                raise e
+            else:
+                previous_exceptions.append(e)
+                previous_exceptions_str = "\n".join(
+                    f"{type(e).__name__}: {str(e)}" for e in previous_exceptions
+                )
+                raise ValueError(
+                    f"Tried to open the images {self._original_images} with different methods, "
+                    + f"none succeded. The following errors were raised:\n{previous_exceptions_str}"
+                ) from None
+        return images_context_manager
+
     @contextmanager
     def _open_images(
         self,
@@ -290,48 +355,29 @@ def _open_images(
         For a 2D image this is achieved by wrapping it in a list.
         """
         with warnings.catch_warnings():
-
             if isinstance(self._original_images, pims.FramesSequence):
                 images_context_manager = nullcontext(enter_result=self._original_images)
             else:
-                if self._use_bioformats:
-                    # There is a wrong warning about jpype, supressing it here.
-                    # See issue https://github.com/soft-matter/pims/issues/384
-                    warnings.filterwarnings(
-                        "ignore",
-                        "Due to an issue with JPype 0.6.0, reading is slower.*",
-                        category=UserWarning,
-                        module="pims.bioformats",
-                    )
-                    try:
-                        pims.bioformats._find_jar()
-                    except HTTPError:
-                        # We cannot use the newest bioformats version,
-                        # since it does not include the necessary loci_tools.jar.
-                        # Updates to support newer bioformats jars with pims are in PR
-                        # https://github.com/soft-matter/pims/pull/403
-                        pims.bioformats.download_jar(version="6.7.0")
-                    if "*" in str(self._original_images) or isinstance(
-                        self._original_images, list
-                    ):
-                        images_context_manager = pims.ReaderSequence(
-                            self._original_images, pims.bioformats.BioformatsReader
-                        )
-                    else:
-                        images_context_manager = pims.bioformats.BioformatsReader(
-                            self._original_images
-                        )
-                else:
-                    original_images = self._original_images
-                    if isinstance(original_images, Path):
-                        original_images = str(original_images)
+                pims_open_exceptions = []
+                original_images = self._normalize_original_images()
+                if not self._use_bioformats:
                     try:
                         open_kwargs = {}
                         if self._czi_channel is not None:
                             open_kwargs["czi_channel"] = self._czi_channel
                         images_context_manager = pims.open(original_images)
-                    except Exception:
-                        images_context_manager = pims.ImageSequence(original_images)
+                    except Exception as e1:
+                        pims_open_exceptions.append(e1)
+                        try:
+                            images_context_manager = pims.ImageSequence(original_images)
+                        except Exception as e2:
+                            pims_open_exceptions.append(e2)
+                            self._use_bioformats = True
+
+                if self._use_bioformats:
+                    images_context_manager = self._open_bioformats_images_raw(
+                        original_images, pims_open_exceptions
+                    )
 
             with images_context_manager as images:
                 if isinstance(images, pims.FramesSequenceND):

webknossos/webknossos/dataset/dataset.py

@@ -1020,7 +1020,8 @@ def add_layer_from_images(
         * `swap_xy`: set to `True` to interchange x and y axis before writing to disk
         * `flip_x`, `flip_y`, `flip_z`: set to `True` to reverse the respective axis before writing to disk
         * `dtype`: the read image data will be convertoed to this dtype using `numpy.ndarray.astype`
-        * `use_bioformats`: set to `True` to use the [pims bioformats adapter](https://soft-matter.github.io/pims/v0.6.1/bioformats.html), needs a JVM
+        * `use_bioformats`: set to `True` to only use the
+          [pims bioformats adapter](https://soft-matter.github.io/pims/v0.6.1/bioformats.html) directly, needs a JVM
         * `channel`: may be used to select a single channel, if multiple are available
         * `timepoint`: for timeseries, select a timepoint to use by specifying it as an int, starting from 0
         * `czi_channel`: may be used to select a channel for .czi images, which differs from normal color-channels