from_images: add python-native support for dm3 & dm4 (#842)

* from_images: add python-native support for dm3 & dm4

* add changelog entry

* apply PR feedback

* reduce memory usage by allowing z=1 chunks

* more test logging

* less mem usage in test_compare_tifffile

* formatting

* Revert "more test logging"

This reverts commit 5673b21.

* run gc after every test

Author: jstriebel

webknossos/Changelog.md

@@ -15,6 +15,10 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Breaking Changes
 
 ### Added
+- `Dataset.from_images` and `dataset.add_layer_from_images` have new features: [#842](https://github.com/scalableminds/webknossos-libs/pull/842)
+  * `dm3` and `dm4` datasets can be read without bioformats now.
+  * It's possible to completely disable the bioformats adapter by setting `use_bioformats` to False.
+  * Lists of images can now be handled with other readers, before only images supported by skimage worked in lists.
 
 ### Changed
 

webknossos/README.md

@@ -44,3 +44,10 @@ Please see the [respective documentation page](https://docs.webknossos.org/webkn
 ## License
 [AGPLv3](https://www.gnu.org/licenses/agpl-3.0.html)
 Copyright [scalable minds](https://scalableminds.com)
+
+## Test Data Credits
+Excerpts for testing purposes have been sampled from:
+
+* Dow Jacobo Hossain Siletti Hudspeth (2018). **Connectomics of the zebrafish's lateral-line neuromast reveals wiring and miswiring in a simple microcircuit.** eLife. [DOI:10.7554/eLife.33988](https://elifesciences.org/articles/33988)
+* Zheng Lauritzen Perlman Robinson Nichols Milkie Torrens Price Fisher Sharifi Calle-Schuler Kmecova Ali Karsh Trautman Bogovic Hanslovsky Jefferis Kazhdan Khairy Saalfeld Fetter Bock (2018). **A Complete Electron Microscopy Volume of the Brain of Adult Drosophila melanogaster.** Cell. [DOI:10.1016/j.cell.2018.06.019](https://www.cell.com/cell/fulltext/S0092-8674(18)30787-6). License: [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)
+* Bosch Ackels Pacureanu et al (2022). **Functional and multiscale 3D structural investigation of brain tissue through correlative in vivo physiology, synchrotron microtomography and volume electron microscopy.** Nature Communications. [DOI:10.1038/s41467-022-30199-6](https://www.nature.com/articles/s41467-022-30199-6)

webknossos/test.sh

@@ -10,7 +10,7 @@ export_vars
 # this will ensure that the current directory is added to sys.path
 # (which is standard python behavior). This is necessary so that the imports
 # refer to the checked out (and potentially modified) code.
-PYTEST="poetry run python -m pytest --suppress-no-test-exit-code"
+PYTEST="poetry run python -m pytest --suppress-no-test-exit-code -s -vvv"
 
 
 if [ $# -gt 0 ] && [ "$1" = "--refresh-snapshots" ]; then

webknossos/tests/conftest.py

@@ -1,3 +1,4 @@
+import gc
 import json
 import re
 import warnings
@@ -37,6 +38,11 @@ def pytest_make_parametrize_id(config: Any, val: Any, argname: str) -> Any:
     return None
 
 
+@pytest.fixture(autouse=True)
+def ensure_gc() -> None:
+    gc.collect()
+
+
 ### HYPOTHESIS STRATEGIES (library to test many combinations for data class input)
 
 

webknossos/tests/dataset/test_add_layer_from_images.py

@@ -2,7 +2,7 @@
 from shutil import copy
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from time import gmtime, strftime
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from zipfile import BadZipFile, ZipFile
 
 import httpx
@@ -120,6 +120,7 @@ def test_repo_images(
             layer_name=layer_name,
             compress=True,
             executor=executor,
+            use_bioformats=False,
             **kwargs,
         )
         assert l.dtype_per_channel == np.dtype(dtype)
@@ -131,24 +132,33 @@ def test_repo_images(
     return ds
 
 
-def download_and_unpack(url: str, out_path: Path, filename: str) -> None:
-    with NamedTemporaryFile() as download_file:
-        with httpx.stream("GET", url) as response:
-            total = int(response.headers["Content-Length"])
+def download_and_unpack(
+    url: Union[str, List[str]], out_path: Path, filename: Union[str, List[str]]
+) -> None:
+    if isinstance(url, str):
+        assert isinstance(filename, str)
+        url = [url]
+        filename = [filename]
+    for url_i, filename_i in zip(url, filename):
+        with NamedTemporaryFile() as download_file:
+            with httpx.stream("GET", url_i) as response:
+                total = int(response.headers["Content-Length"])
 
-            with wk.utils.get_rich_progress() as progress:
-                download_task = progress.add_task("Download Image Data", total=total)
-                for chunk in response.iter_bytes():
-                    download_file.write(chunk)
-                    progress.update(
-                        download_task, completed=response.num_bytes_downloaded
+                with wk.utils.get_rich_progress() as progress:
+                    download_task = progress.add_task(
+                        "Download Image Data", total=total
                     )
-        try:
-            with ZipFile(download_file, "r") as zip_file:
-                zip_file.extractall(out_path)
-        except BadZipFile:
-            out_path.mkdir(parents=True, exist_ok=True)
-            copy(download_file.name, out_path / filename)
+                    for chunk in response.iter_bytes():
+                        download_file.write(chunk)
+                        progress.update(
+                            download_task, completed=response.num_bytes_downloaded
+                        )
+            try:
+                with ZipFile(download_file, "r") as zip_file:
+                    zip_file.extractall(out_path)
+            except BadZipFile:
+                out_path.mkdir(parents=True, exist_ok=True)
+                copy(download_file.name, out_path / filename_i)
 
 
 BIOFORMATS_ARGS = [
@@ -241,7 +251,44 @@ def test_bioformats(
     return ds
 
 
+# All scif images used here are published with CC0 license,
+# see https://scif.io/images.
 TEST_IMAGES_ARGS = [
+    (
+        "https://static.webknossos.org/data/webknossos-libs/slice_0420.dm4",
+        "slice_0420.dm4",
+        {"data_format": "zarr"},  # using zarr to allow z=1 chunking
+        "uint16",
+        1,
+        (8192, 8192, 1),
+    ),
+    (
+        "https://static.webknossos.org/data/webknossos-libs/slice_0073.dm3",
+        "slice_0073.dm3",
+        {"data_format": "zarr"},  # using zarr to allow z=1 chunking
+        "uint16",
+        1,
+        (4096, 4096, 1),
+    ),
+    (
+        [
+            "https://static.webknossos.org/data/webknossos-libs/slice_0073.dm3",
+            "https://static.webknossos.org/data/webknossos-libs/slice_0074.dm3",
+        ],
+        ["slice_0073.dm3", "slice_0074.dm3"],
+        {"data_format": "zarr"},  # using zarr to allow smaller chunking
+        "uint16",
+        1,
+        (4096, 4096, 2),
+    ),
+    (
+        "https://samples.scif.io/dnasample1.zip",
+        "dnasample1.dm3",
+        {"data_format": "zarr"},  # using zarr to allow z=1 chunking
+        "int16",
+        1,
+        (4096, 4096, 1),
+    ),
     (
         # published with CC0 license, taken from
         # https://doi.org/10.6084/m9.figshare.c.3727411_D391.v1
@@ -292,41 +339,56 @@ def test_bioformats(
 )
 def test_test_images(
     tmp_path: Path,
-    url: str,
-    filename: str,
+    url: Union[str, List[str]],
+    filename: Union[str, List[str]],
     kwargs: Dict,
     dtype: str,
     num_channels: int,
     size: Tuple[int, int, int],
 ) -> wk.Dataset:
     unzip_path = tmp_path / "unzip"
     download_and_unpack(url, unzip_path, filename)
+    path: Union[Path, List[Path]]
+    if isinstance(filename, list):
+        layer_name = filename[0] + "..."
+        path = [unzip_path / i for i in filename]
+    else:
+        layer_name = filename
+        path = unzip_path / filename
     ds = wk.Dataset(tmp_path / "ds", (1, 1, 1))
     with wk.utils.get_executor_for_args(None) as executor:
-        l_bio = ds.add_layer_from_images(
-            str(unzip_path / filename),
-            layer_name="bioformats_" + filename,
-            compress=True,
-            executor=executor,
-            use_bioformats=True,
-            **kwargs,
-        )
-        assert l_bio.dtype_per_channel == np.dtype(dtype)
-        assert l_bio.num_channels == num_channels
-        assert l_bio.bounding_box == wk.BoundingBox(topleft=(0, 0, 0), size=size)
+        l_bio: Optional[wk.Layer]
+        try:
+            l_bio = ds.add_layer_from_images(
+                path,
+                layer_name="bioformats_" + layer_name,
+                compress=True,
+                executor=executor,
+                use_bioformats=True,
+                **kwargs,
+            )
+        except Exception as e:
+            print(e)
+            l_bio = None
+        else:
+            assert l_bio.dtype_per_channel == np.dtype(dtype)
+            assert l_bio.num_channels == num_channels
+            assert l_bio.bounding_box == wk.BoundingBox(topleft=(0, 0, 0), size=size)
         l_normal = ds.add_layer_from_images(
-            str(unzip_path / filename),
-            layer_name="normal_" + filename,
+            path,
+            layer_name="normal_" + layer_name,
             compress=True,
             executor=executor,
+            use_bioformats=False,
             **kwargs,
         )
         assert l_normal.dtype_per_channel == np.dtype(dtype)
         assert l_normal.num_channels == num_channels
         assert l_normal.bounding_box == wk.BoundingBox(topleft=(0, 0, 0), size=size)
-        assert np.array_equal(
-            l_bio.get_finest_mag().read(), l_normal.get_finest_mag().read()
-        )
+        if l_bio is not None:
+            assert np.array_equal(
+                l_bio.get_finest_mag().read(), l_normal.get_finest_mag().read()
+            )
     return ds
 
 

webknossos/tests/dataset/test_dataset.py

@@ -2525,7 +2525,7 @@ def test_aligned_downsampling(data_format: DataFormat, output_path: Path) -> Non
     ds_path = copy_simple_dataset(data_format, output_path, "aligned_downsampling")
     dataset = Dataset.open(ds_path)
     input_layer = dataset.get_layer("color")
-    input_layer.downsample()
+    input_layer.downsample(coarsest_mag=Mag(2))
     test_layer = dataset.add_layer(
         layer_name="color_2",
         category="color",
@@ -2539,20 +2539,17 @@ def test_aligned_downsampling(data_format: DataFormat, output_path: Path) -> Non
         # assuming the layer has 3 channels:
         data=(np.random.rand(3, 24, 24, 24) * 255).astype(np.uint8),
     )
-    test_layer.downsample()
+    test_layer.downsample(coarsest_mag=Mag(2))
 
     assert (ds_path / "color_2" / "1").exists()
     assert (ds_path / "color_2" / "2").exists()
-    assert (ds_path / "color_2" / "4").exists()
 
     if data_format == DataFormat.Zarr:
         assert (ds_path / "color_2" / "1" / ".zarray").exists()
         assert (ds_path / "color_2" / "2" / ".zarray").exists()
-        assert (ds_path / "color_2" / "4" / ".zarray").exists()
     else:
         assert (ds_path / "color_2" / "1" / "header.wkw").exists()
         assert (ds_path / "color_2" / "2" / "header.wkw").exists()
-        assert (ds_path / "color_2" / "4" / "header.wkw").exists()
 
     assure_exported_properties(dataset)
 
@@ -2565,9 +2562,8 @@ def test_guided_downsampling(data_format: DataFormat, output_path: Path) -> None
     input_layer = input_dataset.get_layer("color")
     # Adding additional mags to the input dataset for testing
     input_layer.get_or_add_mag("2-2-1")
-    input_layer.get_or_add_mag("4-4-2")
     input_layer.redownsample()
-    assert len(input_layer.mags) == 3
+    assert len(input_layer.mags) == 2
     # Use the mag with the best resolution
     finest_input_mag = input_layer.get_finest_mag()
 
@@ -2589,7 +2585,7 @@ def test_guided_downsampling(data_format: DataFormat, output_path: Path) -> None
     # Downsampling the layer to the magnification used in the input dataset
     output_layer.downsample(
         from_mag=output_mag.mag,
-        coarsest_mag=Mag("8-8-4"),
+        coarsest_mag=Mag("4-4-2"),
         align_with_other_layers=input_dataset,
     )
     for mag in input_layer.mags:
@@ -2598,18 +2594,15 @@ def test_guided_downsampling(data_format: DataFormat, output_path: Path) -> None
     assert (output_ds_path / "color" / "1").exists()
     assert (output_ds_path / "color" / "2-2-1").exists()
     assert (output_ds_path / "color" / "4-4-2").exists()
-    assert (output_ds_path / "color" / "8-8-4").exists()
 
     if data_format == DataFormat.Zarr:
         assert (output_ds_path / "color" / "1" / ".zarray").exists()
         assert (output_ds_path / "color" / "2-2-1" / ".zarray").exists()
         assert (output_ds_path / "color" / "4-4-2" / ".zarray").exists()
-        assert (output_ds_path / "color" / "8-8-4" / ".zarray").exists()
     else:
         assert (output_ds_path / "color" / "1" / "header.wkw").exists()
         assert (output_ds_path / "color" / "2-2-1" / "header.wkw").exists()
         assert (output_ds_path / "color" / "4-4-2" / "header.wkw").exists()
-        assert (output_ds_path / "color" / "8-8-4" / "header.wkw").exists()
 
     assure_exported_properties(input_dataset)
 

webknossos/tests/dataset/test_from_images.py

@@ -13,6 +13,7 @@ def test_compare_tifffile(tmp_path: Path) -> None:
         (1, 1, 1),
         compress=True,
         layer_category="segmentation",
+        chunks_per_shard=(8, 8, 8),
         map_filepath_to_layer_name=wk.Dataset.ConversionLayerMapping.ENFORCE_SINGLE_LAYER,
     )
     assert len(ds.layers) == 1

webknossos/webknossos/dataset/_utils/pims_czi_reader.py

@@ -61,20 +61,20 @@ def __init__(self, path: PathLike, czi_channel: int = 0) -> None:
                     # not propagating axes of length one
                     continue
                 self._init_axis(axis, length)
-            czi_pixel_type = czi_file.get_channel_pixel_type(self.czi_channel)
-            if czi_pixel_type.startswith("Bgra"):
+            self._czi_pixel_type = czi_file.get_channel_pixel_type(self.czi_channel)
+            if self._czi_pixel_type.startswith("Bgra"):
                 self._init_axis("c", 4)
-            elif czi_pixel_type.startswith("Bgr"):
+            elif self._czi_pixel_type.startswith("Bgr"):
                 self._init_axis("c", 3)
-            elif czi_pixel_type.startswith("Gray"):
+            elif self._czi_pixel_type.startswith("Gray"):
                 self._init_axis("c", 1)
-            elif czi_pixel_type == "Invalid":
+            elif self._czi_pixel_type == "Invalid":
                 raise ValueError(
                     f"czi_channel {self.czi_channel} does not exist in {self.path}"
                 )
             else:
                 raise ValueError(
-                    f"Got unsupported czi pixel-type {czi_pixel_type} in {self.path}"
+                    f"Got unsupported czi pixel-type {self._czi_pixel_type} in {self.path}"
                 )
 
         self._register_get_frame(self.get_frame_2D, "yxc")
@@ -90,10 +90,7 @@ def available_czi_channels(self) -> List[int]:
 
     @property  # potential @cached_property for py3.8+
     def pixel_type(self) -> np.dtype:
-        with self.czi_file() as czi_file:
-            return np.dtype(
-                PIXEL_TYPE_TO_DTYPE[czi_file.get_channel_pixel_type(self.czi_channel)]
-            )
+        return np.dtype(PIXEL_TYPE_TO_DTYPE[self._czi_pixel_type])
 
     def get_frame_2D(self, **ind: int) -> np.ndarray:
         plane = {k.upper(): v for k, v in ind.items()}