Fix downsampling memory consumption (#1325)

* Improve memory signature of mode downsampling

* Reintroduce smaller buffer shape for downsampling to avoid excessive RAM consumption

* Revert "Improve memory signature of mode downsampling"

This reverts commit d5d82c5.

* Update changelog

Author: daniel-wer

webknossos/Changelog.md

@@ -19,6 +19,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 ### Changed
 
 ### Fixed
+- Fixed excessive RAM consumption during downsampling by chunking the computation. [#1325](https://github.com/scalableminds/webknossos-libs/pull/1325)
 
 
 ## [2.3.10](https://github.com/scalableminds/webknossos-libs/releases/tag/v2.3.10) - 2025-06-17

webknossos/webknossos/dataset/_downsampling_utils.py

@@ -28,7 +28,16 @@ class InterpolationModes(Enum):
     MIN = 6
 
 
-def determine_buffer_shape(array_info: ArrayInfo) -> Vec3Int:
+def determine_downsample_buffer_shape(array_info: ArrayInfo) -> Vec3Int:
+    # This is the shape of the data in the downsampling target magnification, so the
+    # data that is read is up to 512³ vx in the source magnification. Using larger
+    # shapes uses a lot of RAM, especially for segmentation layers which use the mode filter.
+    # See https://scm.slack.com/archives/CMBMU5684/p1749771929954699 for more context.
+    return Vec3Int.full(256).pairmin(array_info.shard_shape)
+
+
+def determine_upsample_buffer_shape(array_info: ArrayInfo) -> Vec3Int:
+    # This is the shape of the data in the upsampling target magnification.
     return array_info.shard_shape
 
 

webknossos/webknossos/dataset/layer.py

@@ -19,7 +19,8 @@
     calculate_default_coarsest_mag,
     calculate_mags_to_downsample,
     calculate_mags_to_upsample,
-    determine_buffer_shape,
+    determine_downsample_buffer_shape,
+    determine_upsample_buffer_shape,
     downsample_cube_job,
     parse_interpolation_mode,
 )
@@ -1301,7 +1302,7 @@ def downsample_mag(
         # perform downsampling
         with get_executor_for_args(None, executor) as executor:
             if buffer_shape is None:
-                buffer_shape = determine_buffer_shape(prev_mag_view.info)
+                buffer_shape = determine_downsample_buffer_shape(prev_mag_view.info)
             func = named_partial(
                 downsample_cube_job,
                 mag_factors=mag_factors,
@@ -1506,7 +1507,7 @@ def upsample(
             # perform upsampling
             with get_executor_for_args(None, executor) as actual_executor:
                 if buffer_shape is None:
-                    buffer_shape = determine_buffer_shape(prev_mag_view.info)
+                    buffer_shape = determine_upsample_buffer_shape(prev_mag_view.info)
                 else:
                     buffer_shape = Vec3Int.from_vec_or_int(buffer_shape)
                 func = named_partial(