Compressed_write_without_json_update (#942)

* Add test for compressed write bug and implement possible fix.

* Update changelog.

* Update tests.

* Implement requested changes.

* Update test test_repo_images.

* Run formatter.

* Revert changes from dataset.py.

Author: markbader

webknossos/Changelog.md

@@ -19,6 +19,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 ### Changed
 
 ### Fixed
+- Fixed a bug in writing compressed data. [#942](https://github.com/scalableminds/webknossos-libs/pull/942)
 
 
 ## [0.13.6](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.13.6) - 2023-08-17

webknossos/testdata/tiff_with_different_shapes/1.tif

@@ -96,11 +96,11 @@ def test_compare_tifffile(tmp_path: Path) -> None:
         (1024, 1024, 13),
     ),
     (
-        "testdata/tiff_with_different_dimensions/*",
+        "testdata/tiff_with_different_shapes/*",
         {"flip_y": True},
         "uint8",
         1,
-        (2970, 2521, 3),
+        (2970, 2521, 4),
     ),
     ("testdata/various_tiff_formats/test_CS.tif", {}, "uint8", 3, (128, 128, 320)),
     ("testdata/various_tiff_formats/test_C.tif", {}, "uint8", 1, (128, 128, 320)),

webknossos/testdata/tiff_with_different_dimensions/3.tif renamed to webknossos/testdata/tiff_with_different_shapes/3.tif

@@ -63,7 +63,7 @@ def test_multiple_multitiffs(tmp_path: Path) -> None:
 def test_no_slashes_in_layername(tmp_path: Path) -> None:
     (input_path := tmp_path / "tiff" / "subfolder" / "tifffiles").mkdir(parents=True)
     copytree(
-        TESTDATA_DIR / "tiff_with_different_dimensions",
+        TESTDATA_DIR / "tiff_with_different_shapes",
         input_path,
         dirs_exist_ok=True,
     )

webknossos/testdata/tiff_with_different_dimensions/4.tif renamed to webknossos/testdata/tiff_with_different_shapes/4.tif

@@ -202,12 +202,15 @@ def test_convert() -> None:
         assert (wkw_path / PROPERTIES_FILE_NAME).exists()
 
 
-def test_convert_with_all_params() -> None:
+@pytest.mark.parametrize(
+    "origin_path",
+    [TESTDATA_DIR / "tiff", TESTDATA_DIR / "tiff_with_different_shapes"],
+)
+def test_convert_with_all_params(origin_path: Path) -> None:
     """Tests the functionality of convert subcommand."""
 
     with tmp_cwd():
-        origin_path = TESTDATA_DIR / "tiff"
-        wkw_path = Path("wkw_from_tiff_extended")
+        wkw_path = Path(f"wkw_from_{origin_path.name}")
         with pytest.warns(UserWarning):
             result = runner.invoke(
                 app,

webknossos/testdata/tiff_with_different_dimensions/5.tif renamed to webknossos/testdata/tiff_with_different_shapes/5.tif

@@ -277,14 +277,17 @@ def write(
 
         if self._is_compressed():
             for current_mag_bbox, chunked_data in self._prepare_compressed_write(
-                current_mag_bbox, data
+                current_mag_bbox, data, json_update_allowed
             ):
                 self._array.write(current_mag_bbox.topleft, chunked_data)
         else:
             self._array.write(current_mag_bbox.topleft, data)
 
     def _prepare_compressed_write(
-        self, current_mag_bbox: BoundingBox, data: np.ndarray
+        self,
+        current_mag_bbox: BoundingBox,
+        data: np.ndarray,
+        json_update_allowed: bool = True,
     ) -> Iterator[Tuple[BoundingBox, np.ndarray]]:
         """This method takes an arbitrary sized chunk of data with an accompanying bbox,
         divides these into chunks of shard_shape size and delegates
@@ -305,10 +308,15 @@ def _prepare_compressed_write(
                     -current_mag_bbox.topleft
                 ).to_slices()
 
-            yield self._prepare_compressed_write_chunk(chunked_bbox, data[source_slice])
+            yield self._prepare_compressed_write_chunk(
+                chunked_bbox, data[source_slice], json_update_allowed
+            )
 
     def _prepare_compressed_write_chunk(
-        self, current_mag_bbox: BoundingBox, data: np.ndarray
+        self,
+        current_mag_bbox: BoundingBox,
+        data: np.ndarray,
+        json_update_allowed: bool = True,
     ) -> Tuple[BoundingBox, np.ndarray]:
         """This method takes an arbitrary sized chunk of data with an accompanying bbox
         (ideally not larger than a shard) and enlarges that chunk to fit the shard it
@@ -321,7 +329,11 @@ def _prepare_compressed_write_chunk(
         if current_mag_bbox != aligned_bbox:
             # The data bbox should either be aligned or match the dataset's bounding box:
             current_mag_view_bbox = self.bounding_box.in_mag(self._mag)
-            if current_mag_bbox != current_mag_view_bbox.intersected_with(aligned_bbox):
+            if (
+                json_update_allowed
+                and current_mag_bbox
+                != current_mag_view_bbox.intersected_with(aligned_bbox)
+            ):
                 warnings.warn(
                     _BLOCK_ALIGNMENT_WARNING,
                 )
@@ -334,8 +346,8 @@ def _prepare_compressed_write_chunk(
             # overwrite the specified data
             aligned_data[index_slice] = data
             return aligned_bbox, aligned_data
-        else:
-            return current_mag_bbox, data
+
+        return current_mag_bbox, data
 
     def read(
         self,