Skip to content

Commit 9601235

Browse files
authored
Compressed_write_without_json_update (#942)
* Add test for compressed write bug and implement possible fix. * Update changelog. * Update tests. * Implement requested changes. * Update test test_repo_images. * Run formatter. * Revert changes from dataset.py.
1 parent c78a7a8 commit 9601235

File tree

9 files changed

+29
-13
lines changed

9 files changed

+29
-13
lines changed

webknossos/Changelog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
1919
### Changed
2020

2121
### Fixed
22+
- Fixed a bug in writing compressed data. [#942](https://github.com/scalableminds/webknossos-libs/pull/942)
2223

2324

2425
## [0.13.6](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.13.6) - 2023-08-17
61.3 KB
Binary file not shown.

webknossos/tests/dataset/test_add_layer_from_images.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,11 @@ def test_compare_tifffile(tmp_path: Path) -> None:
9696
(1024, 1024, 13),
9797
),
9898
(
99-
"testdata/tiff_with_different_dimensions/*",
99+
"testdata/tiff_with_different_shapes/*",
100100
{"flip_y": True},
101101
"uint8",
102102
1,
103-
(2970, 2521, 3),
103+
(2970, 2521, 4),
104104
),
105105
("testdata/various_tiff_formats/test_CS.tif", {}, "uint8", 3, (128, 128, 320)),
106106
("testdata/various_tiff_formats/test_C.tif", {}, "uint8", 1, (128, 128, 320)),

webknossos/tests/dataset/test_from_images.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ def test_multiple_multitiffs(tmp_path: Path) -> None:
6363
def test_no_slashes_in_layername(tmp_path: Path) -> None:
6464
(input_path := tmp_path / "tiff" / "subfolder" / "tifffiles").mkdir(parents=True)
6565
copytree(
66-
TESTDATA_DIR / "tiff_with_different_dimensions",
66+
TESTDATA_DIR / "tiff_with_different_shapes",
6767
input_path,
6868
dirs_exist_ok=True,
6969
)

webknossos/tests/test_cli.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,12 +202,15 @@ def test_convert() -> None:
202202
assert (wkw_path / PROPERTIES_FILE_NAME).exists()
203203

204204

205-
def test_convert_with_all_params() -> None:
205+
@pytest.mark.parametrize(
206+
"origin_path",
207+
[TESTDATA_DIR / "tiff", TESTDATA_DIR / "tiff_with_different_shapes"],
208+
)
209+
def test_convert_with_all_params(origin_path: Path) -> None:
206210
"""Tests the functionality of convert subcommand."""
207211

208212
with tmp_cwd():
209-
origin_path = TESTDATA_DIR / "tiff"
210-
wkw_path = Path("wkw_from_tiff_extended")
213+
wkw_path = Path(f"wkw_from_{origin_path.name}")
211214
with pytest.warns(UserWarning):
212215
result = runner.invoke(
213216
app,

webknossos/webknossos/dataset/view.py

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -277,14 +277,17 @@ def write(
277277

278278
if self._is_compressed():
279279
for current_mag_bbox, chunked_data in self._prepare_compressed_write(
280-
current_mag_bbox, data
280+
current_mag_bbox, data, json_update_allowed
281281
):
282282
self._array.write(current_mag_bbox.topleft, chunked_data)
283283
else:
284284
self._array.write(current_mag_bbox.topleft, data)
285285

286286
def _prepare_compressed_write(
287-
self, current_mag_bbox: BoundingBox, data: np.ndarray
287+
self,
288+
current_mag_bbox: BoundingBox,
289+
data: np.ndarray,
290+
json_update_allowed: bool = True,
288291
) -> Iterator[Tuple[BoundingBox, np.ndarray]]:
289292
"""This method takes an arbitrary sized chunk of data with an accompanying bbox,
290293
divides these into chunks of shard_shape size and delegates
@@ -305,10 +308,15 @@ def _prepare_compressed_write(
305308
-current_mag_bbox.topleft
306309
).to_slices()
307310

308-
yield self._prepare_compressed_write_chunk(chunked_bbox, data[source_slice])
311+
yield self._prepare_compressed_write_chunk(
312+
chunked_bbox, data[source_slice], json_update_allowed
313+
)
309314

310315
def _prepare_compressed_write_chunk(
311-
self, current_mag_bbox: BoundingBox, data: np.ndarray
316+
self,
317+
current_mag_bbox: BoundingBox,
318+
data: np.ndarray,
319+
json_update_allowed: bool = True,
312320
) -> Tuple[BoundingBox, np.ndarray]:
313321
"""This method takes an arbitrary sized chunk of data with an accompanying bbox
314322
(ideally not larger than a shard) and enlarges that chunk to fit the shard it
@@ -321,7 +329,11 @@ def _prepare_compressed_write_chunk(
321329
if current_mag_bbox != aligned_bbox:
322330
# The data bbox should either be aligned or match the dataset's bounding box:
323331
current_mag_view_bbox = self.bounding_box.in_mag(self._mag)
324-
if current_mag_bbox != current_mag_view_bbox.intersected_with(aligned_bbox):
332+
if (
333+
json_update_allowed
334+
and current_mag_bbox
335+
!= current_mag_view_bbox.intersected_with(aligned_bbox)
336+
):
325337
warnings.warn(
326338
_BLOCK_ALIGNMENT_WARNING,
327339
)
@@ -334,8 +346,8 @@ def _prepare_compressed_write_chunk(
334346
# overwrite the specified data
335347
aligned_data[index_slice] = data
336348
return aligned_bbox, aligned_data
337-
else:
338-
return current_mag_bbox, data
349+
350+
return current_mag_bbox, data
339351

340352
def read(
341353
self,

0 commit comments

Comments
 (0)