adds example for downloading a tiff stack (#862)

* adds example for downloading a tiff stack

* lint

* doh

* url

* try no:faulthandler

* revert previous, update test durations, slightly smaller test-case

* CI: use splitting-algorithm least_duration

* even smaller testcase

---------

Co-authored-by: Jonathan Striebel <[email protected]>
Co-authored-by: Jonathan Striebel <[email protected]>

Author: normanrz

.github/workflows/ci.yml

@@ -172,7 +172,7 @@ jobs:
     - name: Python tests
       env:
         WK_TOKEN: ${{ secrets.WK_TOKEN }}
-      run: ./test.sh --splits 3 --group ${{ matrix.group }}
+      run: ./test.sh --splits 3 --group ${{ matrix.group }}  --splitting-algorithm least_duration
 
     - name: Check if git is dirty
       run: |

docs/mkdocs.yml

@@ -81,6 +81,7 @@ nav:
         - webknossos-py/examples/upload_tiff_stack.md
         - webknossos-py/examples/upload_image_data.md
         - webknossos-py/examples/download_image_data.md
+        - webknossos-py/examples/download_tiff_stack.md
         - webknossos-py/examples/remote_datasets.md
         - webknossos-py/examples/zarr_and_dask.md
       - Annotation Examples:

docs/src/webknossos-py/examples/download_tiff_stack.md

@@ -0,0 +1,9 @@
+# Download datasets as TIFF sequences
+
+This example [accesses a dataset on webKnososs](../../api/webknossos/dataset/dataset.md#Dataset.open_remote) and creates a TIFF sequences from the downloaded data. The [BufferedSliceReader](../../api/webknossos/dataset/view.md#View.get_buffered_slice_reader) is used to efficiently request sections from the remote data.
+
+```python
+--8<--
+webknossos/examples/download_tiff_stack.py
+--8<--
+```

webknossos/.test_durations

@@ -0,0 +1,34 @@
+from tifffile import imwrite
+
+import webknossos as wk
+
+DATASET_NAME = "l4_sample"
+LAYER_NAME = "color"
+MAG = wk.Mag("4-4-1")
+
+
+def main() -> None:
+    dataset = wk.Dataset.open_remote(
+        DATASET_NAME,
+        organization_id="scalable_minds",
+        webknossos_url="https://webknossos.org",
+    )
+    mag_view = dataset.get_layer(LAYER_NAME).get_mag(MAG)
+
+    z = mag_view.layer.bounding_box.topleft.z
+    with mag_view.get_buffered_slice_reader() as reader:
+        for slice_data in reader:
+            slice_data = slice_data[0]  # First channel only
+            slice_data = slice_data.T  # Tiff likes the data transposed
+
+            imwrite(
+                f"l4_sample_tiff/mag{MAG}_z{z:04d}.tiff",
+                slice_data,
+            )
+
+            print(f"Downloaded z={z:04d}")
+            z += MAG.z
+
+
+if __name__ == "__main__":
+    main()

webknossos/examples/download_tiff_stack.py

@@ -19,16 +19,18 @@
 def test_compare_tifffile(tmp_path: Path) -> None:
     ds = wk.Dataset(tmp_path, (1, 1, 1))
     l = ds.add_layer_from_images(
-        "testdata/tiff/test.*.tiff",
+        "testdata/tiff/test.02*.tiff",
         layer_name="compare_tifffile",
         compress=True,
         category="segmentation",
         topleft=(100, 100, 55),
+        chunk_shape=(8, 8, 8),
+        chunks_per_shard=(8, 8, 8),
     )
     assert l.bounding_box.topleft == wk.Vec3Int(100, 100, 55)
     data = l.get_finest_mag().read()[0, :, :]
     for z_index in range(0, data.shape[-1]):
-        with TiffFile("testdata/tiff/test.0000.tiff") as tif_file:
+        with TiffFile("testdata/tiff/test.0200.tiff") as tif_file:
             comparison_slice = tif_file.asarray().T
         assert np.array_equal(data[:, :, z_index], comparison_slice)
 

webknossos/tests/dataset/test_add_layer_from_images.py

@@ -261,3 +261,20 @@ def test_download_segments() -> None:
             len(list(output_path.iterdir()))
             == 2 * mag_view.layer.bounding_box.size.z / mag_view.mag.z
         )
+
+
+@pytest.mark.block_network(allowed_hosts=[".*"])
+@pytest.mark.vcr(ignore_hosts=["webknossos.org", "data-humerus.webknossos.org"])
+def test_download_tiff_stack() -> None:
+    import examples.download_tiff_stack as example
+
+    with tmp_cwd():
+        output_path = Path("l4_sample_tiff")
+        output_path.mkdir()
+
+        (mag_view,) = exec_main_and_get_vars(example, "mag_view")
+
+        assert (
+            len(list(output_path.iterdir()))
+            == mag_view.layer.bounding_box.size.z / mag_view.mag.z
+        )