Add only_setup_mag(s) parameter to downsample methods (#610)

* add only_setup_mag(s) parameter to downsample methods to allow parallel downsampling without writing outdated datasource-properties.json

* update changelog

* add test

Author: philippotto

webknossos/Changelog.md

@@ -12,6 +12,7 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Breaking Changes
 
 ### Added
+- Added `only_setup_mag` parameter to downsample-related methods in `Layer`. This parameter allows creating output magnifications before parallelizing downsampling invocations to avoid outdated writes to datasource-properties.json. [#610](https://github.com/scalableminds/webknossos-libs/pull/610)
 - Added `Task.create()` method to create tasks by prodiving a dataset name, location, and rotation. [#605](https://github.com/scalableminds/webknossos-libs/pull/605)
 
 ### Changed

webknossos/webknossos/dataset/layer.py

@@ -515,6 +515,7 @@ def downsample(
         buffer_edge_len: Optional[int] = None,
         force_sampling_scheme: bool = False,
         args: Optional[Namespace] = None,
+        only_setup_mags: bool = False,
     ) -> None:
         """
         Downsamples the data starting from `from_mag` until a magnification is `>= max(max_mag)`.
@@ -602,6 +603,7 @@ def downsample(
                 compress=compress,
                 buffer_edge_len=buffer_edge_len,
                 args=args,
+                only_setup_mag=only_setup_mags,
             )
 
     def downsample_mag(
@@ -613,6 +615,7 @@ def downsample_mag(
         buffer_edge_len: Optional[int] = None,
         args: Optional[Namespace] = None,
         allow_overwrite: bool = False,
+        only_setup_mag: bool = False,
     ) -> None:
         """
         Performs a single downsampling step from `from_mag` to `target_mag`.
@@ -626,6 +629,11 @@ def downsample_mag(
 
         The `args` can contain information to distribute the computation.
         If allow_overwrite is True, an existing Mag may be overwritten.
+
+        If only_setup_mag is True, the magnification is created, but left
+        empty. This parameter can be used to prepare for parallel downsampling
+        of multiple layers while avoiding parallel writes with outdated updates
+        to the datasource-properties.json file.
         """
         assert (
             from_mag in self.mags.keys()
@@ -652,6 +660,9 @@ def downsample_mag(
                 target_mag, prev_mag_view, compress
             )
 
+        if only_setup_mag:
+            return
+
         bb_mag1 = self.bounding_box
 
         aligned_source_bb = bb_mag1.align_with_mag(target_mag, ceil=True).in_mag(
@@ -731,6 +742,7 @@ def downsample_mag_list(
         buffer_edge_len: Optional[int] = None,
         args: Optional[Namespace] = None,
         allow_overwrite: bool = False,
+        only_setup_mags: bool = False,
     ) -> None:
         """
         Downsamples the data starting at `from_mag` to each magnification in `target_mags` iteratively.
@@ -762,6 +774,7 @@ def downsample_mag_list(
                 buffer_edge_len=buffer_edge_len,
                 args=args,
                 allow_overwrite=allow_overwrite,
+                only_setup_mag=only_setup_mags,
             )
             source_mag = target_mag
 

wkcuber/tests/test_downsampling.py

@@ -347,6 +347,24 @@ def test_downsample_mag_list() -> None:
         assert m in layer.mags
 
 
+def test_downsample_mag_list_with_only_setup_mags() -> None:
+    ds = Dataset(TESTOUTPUT_DIR / "downsample_mag_list", scale=(1, 1, 2))
+    layer = ds.add_layer("color", COLOR_CATEGORY)
+    mag = layer.add_mag(1)
+    mag.write(data=(np.random.rand(10, 20, 30) * 255).astype(np.uint8))
+
+    target_mags = [Mag([4, 4, 8]), Mag(2), Mag([32, 32, 8]), Mag(32)]  # unsorted list
+
+    layer.downsample_mag_list(
+        from_mag=Mag(1), target_mags=target_mags, only_setup_mags=True
+    )
+
+    for m in target_mags:
+        assert np.all(
+            layer.get_mag(m).read((0, 0, 0), (10, 20, 30)) == 0
+        ), "The mags should be empty."
+
+
 def test_downsample_with_invalid_mag_list() -> None:
     ds = Dataset(TESTOUTPUT_DIR / "downsample_mag_list", scale=(1, 1, 2))
     layer = ds.add_layer("color", COLOR_CATEGORY)