ds.add_layer_from_images: added topleft and dtype kwargs (#818)

* ds.add_layer_from_images: added topleft and enforce_dtype kwargs

* add changelog entries, update docstring

* cleanup dtype typing, make compatible with old numpy versions

* format

* specify ImportError explicitly

* rename enforce_dtype to dtype

Author: jstriebel

webknossos/Changelog.md

@@ -13,8 +13,10 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.10.22...HEAD)
 
 ### Breaking Changes
+- `ds.add_layer_from_images`: Turned some arguments into keyword-only arguments, only affecting positional arguments after the first 8 arguments. [#818](https://github.com/scalableminds/webknossos-libs/pull/818)
 
 ### Added
+- `ds.add_layer_from_images`: added topleft and dtype kw-only arguments. [#818](https://github.com/scalableminds/webknossos-libs/pull/818)
 
 ### Changed
 
@@ -25,11 +27,9 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.10.21...v0.10.22)
 
 ### Fixed
-
 - Fixed a bug where some image sequences could not be read in layer_from_images. [#817](https://github.com/scalableminds/webknossos-libs/pull/817)
 
 
-
 ## [0.10.21](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.10.21) - 2022-10-26
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.10.20...v0.10.21)
 
@@ -44,8 +44,6 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 - `annotation.temporary_volume_layer_copy()` works also with empty volume annotations. [#814](https://github.com/scalableminds/webknossos-libs/pull/814)
 
 
-
-
 ## [0.10.19](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.10.19) - 2022-10-18
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.10.18...v0.10.19)
 

webknossos/tests/dataset/test_add_layer_from_images.py

@@ -22,7 +22,9 @@ def test_compare_tifffile(tmp_path: Path) -> None:
         layer_name="compare_tifffile",
         compress=True,
         category="segmentation",
+        topleft=(100, 100, 55),
     )
+    assert l.bounding_box.topleft == wk.Vec3Int(100, 100, 55)
     data = l.get_finest_mag().read()[0, :, :]
     for z_index in range(0, data.shape[-1]):
         with TiffFile("testdata/tiff/test.0000.tiff") as tif_file:
@@ -60,8 +62,8 @@ def test_compare_tifffile(tmp_path: Path) -> None:
     ),
     (
         "testdata/rgb_tiff/test_rgb.tif",
-        {"mag": 2, "channel": 1},
-        "uint8",
+        {"mag": 2, "channel": 1, "dtype": "uint32"},
+        "uint32",
         1,
         (64, 64, 2),
     ),

webknossos/webknossos/dataset/_utils/pims_images.py

@@ -20,6 +20,7 @@
 
 import numpy as np
 
+from webknossos.dataset.layer import DTypeLike
 from webknossos.dataset.mag_view import MagView
 from webknossos.geometry.vec3_int import Vec3Int
 
@@ -32,6 +33,10 @@
 
 
 class PimsImages:
+    dtype: DTypeLike
+    expected_shape: Vec3Int
+    num_channels: int
+
     def __init__(
         self,
         images: Union[str, Path, "pims.FramesSequence", List[Union[str, PathLike]]],
@@ -294,7 +299,11 @@ def _open_images(
                 yield images
 
     def copy_to_view(
-        self, args: Tuple[int, int], mag_view: MagView, is_segmentation: bool
+        self,
+        args: Tuple[int, int],
+        mag_view: MagView,
+        is_segmentation: bool,
+        dtype: Optional[DTypeLike] = None,
     ) -> Tuple[Tuple[int, int], Optional[int]]:
         """Copies the images according to the passed arguments to the given mag_view.
         args is expected to be the start and end of the z-range, meant for usage with an executor."""
@@ -310,7 +319,7 @@ def copy_to_view(
             if self._flip_z:
                 images = images[::-1]  # pylint: disable=unsubscriptable-object
             with mag_view.get_buffered_slice_writer(
-                absolute_offset=(0, 0, z_start * mag_view.mag.z),
+                relative_offset=(0, 0, z_start * mag_view.mag.z),
                 buffer_size=mag_view.info.chunk_shape.z,
             ) as writer:
                 for image_slice in images[z_start:z_end]:
@@ -339,6 +348,9 @@ def copy_to_view(
                     if self._flip_y:
                         image_slice = np.flip(image_slice, -1)
 
+                    if dtype is not None:
+                        image_slice = image_slice.astype(dtype, order="F")
+
                     if max_id is not None:
                         max_id = max(max_id, image_slice.max())
                     shapes.append(image_slice.shape[-2:])

webknossos/webknossos/dataset/dataset.py

@@ -58,6 +58,7 @@
 from ._utils.from_images import guess_if_segmentation_path
 from ._utils.infer_bounding_box_existing_files import infer_bounding_box_existing_files
 from .layer import (
+    DTypeLike,
     Layer,
     SegmentationLayer,
     _dtype_per_channel_to_element_class,
@@ -738,8 +739,8 @@ def add_layer(
         self,
         layer_name: str,
         category: LayerCategoryType,
-        dtype_per_layer: Optional[Union[str, np.dtype, type]] = None,
-        dtype_per_channel: Optional[Union[str, np.dtype, type]] = None,
+        dtype_per_layer: Optional[DTypeLike] = None,
+        dtype_per_channel: Optional[DTypeLike] = None,
         num_channels: Optional[int] = None,
         data_format: Union[str, DataFormat] = DEFAULT_DATA_FORMAT,
         **kwargs: Any,
@@ -846,8 +847,8 @@ def get_or_add_layer(
         self,
         layer_name: str,
         category: LayerCategoryType,
-        dtype_per_layer: Optional[Union[str, np.dtype, type]] = None,
-        dtype_per_channel: Optional[Union[str, np.dtype, type]] = None,
+        dtype_per_layer: Optional[DTypeLike] = None,
+        dtype_per_channel: Optional[DTypeLike] = None,
         num_channels: Optional[int] = None,
         data_format: Union[str, DataFormat] = DEFAULT_DATA_FORMAT,
         **kwargs: Any,
@@ -975,17 +976,19 @@ def add_layer_from_images(
         chunk_shape: Optional[Union[Vec3IntLike, int]] = None,
         chunks_per_shard: Optional[Union[int, Vec3IntLike]] = None,
         compress: bool = False,
+        *,
         ## other arguments
+        topleft: Vec3IntLike = Vec3Int.zeros(),  # in Mag(1)
         swap_xy: bool = False,
         flip_x: bool = False,
         flip_y: bool = False,
         flip_z: bool = False,
+        dtype: Optional[DTypeLike] = None,
         use_bioformats: bool = False,
         channel: Optional[int] = None,
         timepoint: Optional[int] = None,
         batch_size: Optional[int] = None,  # defaults to shard-size z
         executor: Optional[Executor] = None,
-        *,
         chunk_size: Optional[Union[Vec3IntLike, int]] = None,  # deprecated
     ) -> Layer:
         """
@@ -1005,8 +1008,10 @@ def add_layer_from_images(
         * `data_format`: by default wkw files are written, may be set to "zarr"
         * `mag`: magnification to use for the written data
         * `chunk_shape`, `chunks_per_shard`, `compress`: adjust how the data is stored on disk
+        * `topleft`: set an offset in Mag(1) to start writing the data, only affecting the output
         * `swap_xy`: set to `True` to interchange x and y axis before writing to disk
-        * `flip_x`, `flip_y`, `flip_z`: set to `True` to flip the respective axis before writing to disk
+        * `flip_x`, `flip_y`, `flip_z`: set to `True` to reverse the respective axis before writing to disk
+        * `dtype`: the read image data will be convertoed to this dtype using `numpy.ndarray.astype`
         * `use_bioformats`: set to `True` to use the [pims bioformats adapter](https://soft-matter.github.io/pims/v0.6.1/bioformats.html), needs a JVM
         * `channel`: may be used to select a single channel, if multiple are available,
         * `timepoint`: for timeseries, select a timepoint to use by specifying it as an int, starting from 0
@@ -1041,7 +1046,7 @@ def add_layer_from_images(
             layer_name=layer_name,
             category=category,
             data_format=data_format,
-            dtype_per_channel=pims_images.dtype,
+            dtype_per_channel=pims_images.dtype if dtype is None else dtype,
             num_channels=pims_images.num_channels,
             **add_layer_kwargs,  # type: ignore[arg-type]
         )
@@ -1052,9 +1057,11 @@ def add_layer_from_images(
             compress=compress,
         )
         mag = mag_view.mag
-        layer.bounding_box = BoundingBox(
-            (0, 0, 0), pims_images.expected_shape
-        ).from_mag_to_mag1(mag)
+        layer.bounding_box = (
+            BoundingBox((0, 0, 0), pims_images.expected_shape)
+            .from_mag_to_mag1(mag)
+            .offset(topleft)
+        )
 
         if batch_size is None:
             if compress:
@@ -1074,6 +1081,7 @@ def add_layer_from_images(
             pims_images.copy_to_view,
             mag_view=mag_view,
             is_segmentation=category == "segmentation",
+            dtype=dtype,
         )
 
         args = []
@@ -1109,8 +1117,10 @@ def add_layer_from_images(
                 max_id = max(max_ids)
                 cast(SegmentationLayer, layer).largest_segment_id = max_id
             actual_size = Vec3Int(dimwise_max(shapes) + (pims_images.expected_shape.z,))
-            layer.bounding_box = BoundingBox((0, 0, 0), actual_size).from_mag_to_mag1(
-                mag
+            layer.bounding_box = (
+                BoundingBox((0, 0, 0), actual_size)
+                .from_mag_to_mag1(mag)
+                .offset(topleft)
             )
         if pims_images.expected_shape != actual_size:
             warnings.warn(

webknossos/webknossos/dataset/layer.py

@@ -54,6 +54,11 @@
 )
 from .mag_view import MagView, _find_mag_path_on_disk
 
+try:
+    from numpy.typing import DTypeLike
+except ImportError:
+    DTypeLike = Union[str, np.dtype, type]  # type: ignore[misc]
+
 
 def _is_int(s: str) -> bool:
     try:
@@ -64,7 +69,7 @@ def _is_int(s: str) -> bool:
 
 
 def _convert_dtypes(
-    dtype: Union[str, np.dtype],
+    dtype: DTypeLike,
     num_channels: int,
     dtype_per_layer_to_dtype_per_channel: bool,
 ) -> str:
@@ -81,9 +86,7 @@ def _convert_dtypes(
     return "".join(converted_dtype_parts)
 
 
-def _normalize_dtype_per_channel(
-    dtype_per_channel: Union[str, np.dtype, type]
-) -> np.dtype:
+def _normalize_dtype_per_channel(dtype_per_channel: DTypeLike) -> np.dtype:
     try:
         return np.dtype(dtype_per_channel)
     except TypeError as e:
@@ -92,9 +95,7 @@ def _normalize_dtype_per_channel(
         ) from e
 
 
-def _normalize_dtype_per_layer(
-    dtype_per_layer: Union[str, np.dtype, type]
-) -> Union[str, np.dtype]:
+def _normalize_dtype_per_layer(dtype_per_layer: DTypeLike) -> DTypeLike:
     try:
         dtype_per_layer = str(np.dtype(dtype_per_layer))
     except Exception:
@@ -103,7 +104,7 @@ def _normalize_dtype_per_layer(
 
 
 def _dtype_per_layer_to_dtype_per_channel(
-    dtype_per_layer: Union[str, np.dtype], num_channels: int
+    dtype_per_layer: DTypeLike, num_channels: int
 ) -> np.dtype:
     try:
         return np.dtype(
@@ -118,7 +119,7 @@ def _dtype_per_layer_to_dtype_per_channel(
 
 
 def _dtype_per_channel_to_dtype_per_layer(
-    dtype_per_channel: Union[str, np.dtype], num_channels: int
+    dtype_per_channel: DTypeLike, num_channels: int
 ) -> str:
     return _convert_dtypes(
         np.dtype(dtype_per_channel),
@@ -128,7 +129,7 @@ def _dtype_per_channel_to_dtype_per_layer(
 
 
 def _dtype_per_channel_to_element_class(
-    dtype_per_channel: Union[str, np.dtype], num_channels: int
+    dtype_per_channel: DTypeLike, num_channels: int
 ) -> str:
     dtype_per_layer = _dtype_per_channel_to_dtype_per_layer(
         dtype_per_channel, num_channels