Use Dataset API inside cubing (#418)

rschwanhold · Youri K · web-flow · commit 4176bf53c9f3 · 2021-10-14T16:18:18.000+02:00
* WIP: use dataset API in cubing

* use Dataset API for cubing

* remove unused imports

* cleanup

* use scale 1 for cubing

* update changelog

* format code

* fix comment

* fix typing error

* fix linting

* support segmentation layers for cubing

* format code

* compute largest_segment_id for segmentation layers

* fix a bug that was introduced through the merge

* use new properties of Mag and Vec3Int

* use to_wkw_dict

* use correct mag for comression and downsampling

* change type of cli parameter to Mag

* fix formatting

* fix the behaviour of start_z

* update changelog

* fix formatting

* fix lint

* implement PR feedback

* use better import statement

* format code

* add scale parameter for cubing

* format code

* extend existing bbox

* add scale parameter to test scripts

* fix interpretation of parameter 'start_z' and add parameter 'skip_first_z_slices'

* fix formatting/linting

* add missing parapeter to test script

* add scale parameter to docker smoke test

Co-authored-by: Youri K &lt;youri.kaminsky@student.hpi.de&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -202,6 +202,7 @@ jobs:
             --batch_size 8 \
             --layer_name color \
             --wkw_file_len 32 \
+            --scale 1 \
             testdata/tiff testoutput/tiff
 
     - name: Login to docker
diff --git a/webknossos/webknossos/dataset/downsampling_utils.py b/webknossos/webknossos/dataset/downsampling_utils.py
@@ -99,12 +99,14 @@ def calculate_default_max_mag(dataset_size: Vec3IntLike) -> Mag:
 
 
 def parse_interpolation_mode(
-    interpolation_mode: str, layer_name: str
+    interpolation_mode: str, layer_category: str
 ) -> InterpolationModes:
     if interpolation_mode.upper() == "DEFAULT":
+        from webknossos.dataset.layer import LayerCategories
+
         return (
             InterpolationModes.MODE
-            if layer_name == "segmentation"
+            if layer_category == LayerCategories.SEGMENTATION_TYPE
             else InterpolationModes.MEDIAN
         )
     else:
diff --git a/webknossos/webknossos/utils.py b/webknossos/webknossos/utils.py
@@ -84,11 +84,13 @@ def named_partial(func: F, *args: Any, **kwargs: Any) -> F:
     return partial_func
 
 
-def wait_and_ensure_success(futures: List[Future]) -> None:
+def wait_and_ensure_success(futures: List[Future]) -> List[Any]:
     """Waits for all futures to complete and raises an exception
     as soon as a future resolves with an error."""
+    results = []
     for fut in as_completed(futures):
-        fut.result()
+        results.append(fut.result())
+    return results
 
 
 def snake_to_camel_case(snake_case_name: str) -> str:
diff --git a/wkcuber/Changelog.md b/wkcuber/Changelog.md
@@ -10,9 +10,11 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.8.16...HEAD)
 
 ### Breaking Changes in Config & CLI
+- Use Dataset API inside `cubing` to automatically write metadata. Cubing does now require a scale. [#418](https://github.com/scalableminds/webknossos-libs/pull/418)
 ### Added
 ### Changed
 - Updated scikit-image dependency to 0.18.3. [#435](https://github.com/scalableminds/webknossos-libs/pull/435)
+- Improved the `TIFF` and `CZI` reader to work with a wider variety of formats. The module `convert_image_stack_to_wkw` is now capable of making the result webKnossos compatible. [#335](https://github.com/scalableminds/webknossos-libs/pull/335)
 ### Fixed
 
 
@@ -28,7 +30,6 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 - Adjust downsampling scheme to always try to minimize the scaled difference between the different dimensions of a mag and renamed the sampling mode `auto` to `anisotropic`. [#391](https://github.com/scalableminds/webknossos-libs/pull/391)
 - Make parameter `executor` optional for `View.for_each_chunk` and `View.for_zipped_chunks`. [#404](https://github.com/scalableminds/webknossos-libs/pull/404)
 - Add option to rename foreign layer with add_{symlink,copy}_layer. [#419](https://github.com/scalableminds/webknossos-libs/pull/419)
-- Improved the `TIFF` and `CZI` reader to work with a wider variety of formats. The module `convert_image_stack_to_wkw` is now capable of making the result webKnossos compatible. [#335](https://github.com/scalableminds/webknossos-libs/pull/335)
 
 ### Fixed
 - Reverted that `dataset.add_symlink_layer` and `dataset.add_copy_layer` resolved the layer path if it was a symlink. [#408](https://github.com/scalableminds/webknossos-libs/pull/408)
diff --git a/wkcuber/tests/scripts/in_memory_downsampled_cubing.sh b/wkcuber/tests/scripts/in_memory_downsampled_cubing.sh
@@ -6,6 +6,7 @@ python -m wkcuber.cubing \
   --layer_name color \
   --target_mag 2-2-1 \
   --interpolation_mode default \
+  --scale 1 \
   testdata/tiff testoutput/in_memory_downsampled_tiff
 [ -d testoutput/in_memory_downsampled_tiff/color ]
 [ -d testoutput/in_memory_downsampled_tiff/color/2-2-1 ]
diff --git a/wkcuber/tests/scripts/tiff_cubing.sh b/wkcuber/tests/scripts/tiff_cubing.sh
@@ -5,6 +5,7 @@ python -m wkcuber.cubing \
   --batch_size 8 \
   --layer_name color \
   --wkw_file_len 32 \
+  --scale 1 \
   testdata/tiff testoutput/tiff
 [ -d testoutput/tiff/color ]
 [ -d testoutput/tiff/color/1 ]
diff --git a/wkcuber/tests/scripts/tiff_formats_cubing.sh b/wkcuber/tests/scripts/tiff_formats_cubing.sh
@@ -8,7 +8,7 @@ python -m wkcuber.convert_image_stack_to_wkw \
   --batch_size 8 \
   --layer_name color \
   --max_mag 4 \
-  --target_mag 2\
+  --target_mag 2 \
   --scale 1,1,1 \
   --name awesome_data \
   --no_compress \
diff --git a/wkcuber/tests/scripts/tile_cubing.sh b/wkcuber/tests/scripts/tile_cubing.sh
@@ -4,6 +4,7 @@ python -m wkcuber.tile_cubing \
   --jobs 2 \
   --batch_size 8 \
   --layer_name color \
+  --scale 1 \
   testdata/temca2 testoutput/temca2
 [ -d testoutput/temca2/color ]
 [ -d testoutput/temca2/color/1 ]
diff --git a/wkcuber/tests/test_metadata.py b/wkcuber/tests/test_metadata.py
@@ -2,8 +2,7 @@
 import numpy as np
 import wkw
 
-from wkcuber.cubing import ensure_wkw
-from wkcuber.utils import WkwDatasetInfo, open_wkw
+from wkcuber.utils import WkwDatasetInfo, open_wkw, ensure_wkw
 from wkcuber.metadata import (
     write_webknossos_metadata,
     refresh_metadata,
diff --git a/wkcuber/wkcuber/convert_image_stack_to_wkw.py b/wkcuber/wkcuber/convert_image_stack_to_wkw.py
@@ -2,19 +2,15 @@
 from argparse import Namespace, ArgumentParser
 from typing import Sequence
 
-from wkcuber import downsample_mags
-from .compress import compress_mag_inplace
 from .cubing import (
     get_channel_and_sample_count_and_dtype,
     cubing,
     create_parser as create_cubing_parser,
 )
 from .mag import Mag
-from .metadata import write_webknossos_metadata, refresh_metadata
 from .utils import (
     add_isotropic_flag,
     setup_logging,
-    add_scale_flag,
     add_sampling_mode_flag,
     get_executor_args,
     is_wk_compatible_layer_format,
@@ -66,7 +62,6 @@ def create_parser() -> ArgumentParser:
     )
 
     parser.add_argument("--name", "-n", help="Name of the dataset", default=None)
-    add_scale_flag(parser)
     add_isotropic_flag(parser)
     add_sampling_mode_flag(parser)
 
@@ -151,58 +146,44 @@ def main(args: Namespace) -> None:
             sample_iter = range(sample_count)
 
     layer_count = 0
-    bounding_box = None
+    layers = []
     for channel_index in channel_iter:
         for sample_index in sample_iter:
-            bounding_box = cubing(
-                args.source_path,
-                args.target_path,
-                f"{args.layer_name}_{layer_count}"
-                if len(channel_iter) * len(sample_iter) > 1
-                else args.layer_name,
-                arg_dict.get("batch_size"),
-                channel_index,
-                sample_index,
-                arg_dict.get("dtype"),
-                args.target_mag,
-                args.wkw_file_len,
-                args.interpolation_mode,
-                args.start_z,
-                args.pad,
-                args,
+            layers.append(
+                cubing(
+                    args.source_path,
+                    args.target_path,
+                    f"{args.layer_name}_{layer_count}"
+                    if len(channel_iter) * len(sample_iter) > 1
+                    else args.layer_name,
+                    arg_dict.get("batch_size"),
+                    channel_index,
+                    sample_index,
+                    arg_dict.get("dtype"),
+                    args.target_mag,
+                    args.wkw_file_len,
+                    args.interpolation_mode,
+                    args.start_z,
+                    args.skip_first_z_slices,
+                    args.pad,
+                    args.scale,
+                    args,
+                )
             )
             layer_count += 1
 
-    write_webknossos_metadata(
-        args.target_path,
-        args.name,
-        args.scale,
-        compute_max_id=False,
-        exact_bounding_box=bounding_box,
-    )
-
-    for i in range(layer_count):
+    for layer in layers:
         if not args.no_compress:
-            compress_mag_inplace(
-                args.target_path,
-                f"{args.layer_name}_{i}" if layer_count > 1 else args.layer_name,
-                args.target_mag,
-                args,
-            )
+            layer.get_mag(args.target_mag).compress(args=args)
 
-        downsample_mags(
-            path=args.target_path,
-            layer_name=f"{args.layer_name}_{i}" if layer_count > 1 else args.layer_name,
+        layer.downsample(
             from_mag=args.target_mag,
             max_mag=None if args.max_mag is None else Mag(args.max_mag),
-            interpolation_mode="default",
             compress=not args.no_compress,
             sampling_mode=args.sampling_mode,
             args=get_executor_args(args),
         )
 
-    refresh_metadata(args.target_path)
-
 
 if __name__ == "__main__":
     args = create_parser().parse_args()
diff --git a/wkcuber/wkcuber/converter.py b/wkcuber/wkcuber/converter.py
@@ -3,6 +3,7 @@
 from pathlib import Path
 from typing import Iterable, List, Any, Tuple, Dict, Set, Callable, cast, Optional
 
+from webknossos.dataset.properties import LayerViewConfiguration
 from .convert_knossos import (
     main as convert_knossos,
     create_parser as create_knossos_parser,
@@ -318,16 +319,16 @@ def __init__(self) -> None:
         self.dataset_names: Set[str] = set()
 
     @staticmethod
-    def get_view_configuration(index: int) -> Optional[Dict[str, List[int]]]:
+    def get_color_of_view_configuration(index: int) -> Optional[Tuple[int, int, int]]:
         color = None
         if index == 0:
-            color = [255, 0, 0]
+            color = (255, 0, 0)
         elif index == 1:
-            color = [0, 255, 0]
+            color = (0, 255, 0)
         elif index == 2:
-            color = [0, 0, 255]
+            color = (0, 0, 255)
         if color is not None:
-            return {"color": color}
+            return color
         else:
             return None
 
@@ -358,6 +359,9 @@ def convert_input(self, args: Namespace) -> bool:
             args, image_stack_parser, "interpolation_mode"
         )
         put_default_from_argparser_if_not_present(args, image_stack_parser, "start_z")
+        put_default_from_argparser_if_not_present(
+            args, image_stack_parser, "skip_first_z_slices"
+        )
         put_default_from_argparser_if_not_present(args, image_stack_parser, "jobs")
         put_default_from_argparser_if_not_present(
             args, image_stack_parser, "distribution_strategy"
@@ -377,8 +381,6 @@ def convert_input(self, args: Namespace) -> bool:
         ) = self.detect_dataset_name_and_layer_path_to_layer_name()
         put_default_if_not_present(args, "name", dataset_name)
 
-        bounding_box = None
-        view_configuration = dict()
         converted_layers = 0
         for layer_path, layer_name in layer_path_to_name.items():
             if not all_files_of_same_type(
@@ -410,7 +412,7 @@ def convert_input(self, args: Namespace) -> bool:
                         curr_layer_name = layer_name
 
                     arg_dict = vars(args)
-                    bounding_box = cube_image_stack(
+                    layer = cube_image_stack(
                         Path(layer_path),
                         args.target_path,
                         curr_layer_name,
@@ -422,27 +424,25 @@ def convert_input(self, args: Namespace) -> bool:
                         args.wkw_file_len,
                         args.interpolation_mode,
                         args.start_z,
+                        args.skip_first_z_slices,
                         args.pad,
+                        args.scale,
                         executor_args,
                     )
 
                     if not is_wk_compatible_layer_format(sample_count, dtype):
-                        # this means that every sample has to be converted into its own layer, so we want to set a view configuration since first three layers are probably RGB
-                        view_configuration[
-                            curr_layer_name
-                        ] = ImageStackConverter.get_view_configuration(layer_count)
+                        # This means that every sample has to be converted into its own layer, so we want to set a view configuration since first three layers are probably RGB
+                        color = ImageStackConverter.get_color_of_view_configuration(
+                            layer_count
+                        )
+                        if color is not None:
+                            layer.default_view_configuration = LayerViewConfiguration(
+                                color=color
+                            )
                     layer_count += 1
 
         assert converted_layers > 0, "No layer could be converted!"
 
-        write_webknossos_metadata(
-            args.target_path,
-            args.name,
-            args.scale,
-            exact_bounding_box=bounding_box,
-            view_configuration=view_configuration,
-        )
-
         return False
 
     def detect_dataset_name_and_layer_path_to_layer_name(
diff --git a/wkcuber/wkcuber/cubing.py b/wkcuber/wkcuber/cubing.py
diff --git a/wkcuber/wkcuber/upsampling.py b/wkcuber/wkcuber/upsampling.py
