From de7c200dca292671bb05ca37672ce2a1eb8338de Mon Sep 17 00:00:00 2001 From: jluethi Date: Tue, 28 Jan 2025 15:37:59 +0100 Subject: [PATCH 01/10] Add output_types filter to 2d to 3d task. Closes #8 --- src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json | 3 +++ src/fractal_helper_tasks/dev/task_list.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json index 60cd415..e57f108 100644 --- a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json +++ b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json @@ -49,6 +49,9 @@ "input_types": { "is_3D": false }, + "output_types": { + "is_3D": true + }, "tags": [ "Mixed modality", "2D to 3D workflows" diff --git a/src/fractal_helper_tasks/dev/task_list.py b/src/fractal_helper_tasks/dev/task_list.py index 8289643..6d0c762 100644 --- a/src/fractal_helper_tasks/dev/task_list.py +++ b/src/fractal_helper_tasks/dev/task_list.py @@ -13,6 +13,7 @@ ), ParallelTask( input_types=dict(is_3D=False), + output_types=dict(is_3D=True), name="Convert 2D segmentation to 3D", executable="convert_2D_segmentation_to_3D.py", meta={"cpus_per_task": 2, "mem": 8000}, From 9ab60cd1f80267325f84564ecf7e71922f74add2 Mon Sep 17 00:00:00 2001 From: jluethi Date: Tue, 28 Jan 2025 15:40:04 +0100 Subject: [PATCH 02/10] Expose z chunking options & improve defaults. Closes #21 --- src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json | 5 +++++ .../convert_2D_segmentation_to_3D.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json index e57f108..b4780c0 100644 --- a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json +++ b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json @@ -116,6 +116,11 @@ "type": "string", "description": "If the image name between 2D & 3D don't match, this is the suffix that should be added to the 3D image. If the 2D image is in \"/path/to/my_plate_mip.zarr/B/03/0\" and the 3D image is in \"/path/to/my_plate.zarr/B/03/0_illum_corr\", the value should be \"_illum_corr\"." }, + "z_chunks": { + "title": "Z Chunks", + "type": "integer", + "description": "Chunking for the Z dimension" + }, "overwrite": { "default": false, "title": "Overwrite", diff --git a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py index 91d3d10..3c6a4c9 100644 --- a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py +++ b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py @@ -99,6 +99,7 @@ def convert_2D_segmentation_to_3D( plate_suffix: str = "_mip", image_suffix_2D_to_remove: Optional[str] = None, image_suffix_3D_to_add: Optional[str] = None, + z_chunks: Optional[int] = None, overwrite: bool = False, ) -> None: """Convert 2D segmentation to 3D segmentation. @@ -141,6 +142,7 @@ def convert_2D_segmentation_to_3D( If the 2D image is in "/path/to/my_plate_mip.zarr/B/03/0" and the 3D image is in "/path/to/my_plate.zarr/B/03/0_illum_corr", the value should be "_illum_corr". + z_chunks: Chunking for the Z dimension overwrite: If `True`, overwrite existing label and ROI tables in the 3D OME-Zarr """ @@ -179,6 +181,13 @@ def convert_2D_segmentation_to_3D( with zarr.open(zarr_3D_url, mode="rw+") as zarr_img: zarr_3D = da.from_zarr(zarr_img[0]) new_z_planes = zarr_3D.shape[-3] + z_chunk_3d = zarr_3D.chunksize[-3] + + # TODO: Improve axis detection in ngio refactor? + if z_chunks: + chunks[-3] = z_chunks + else: + chunks[-3] = z_chunk_3d image_meta = load_NgffImageMeta(zarr_3D_url) z_pixel_size = image_meta.get_pixel_sizes_zyx(level=0)[0] From 9cef7b3d8b2019e69b3a799b7388d472a0f1ad86 Mon Sep 17 00:00:00 2001 From: jluethi Date: Tue, 28 Jan 2025 19:16:49 +0100 Subject: [PATCH 03/10] Fix tuple vs list type for chunks --- src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py index 3c6a4c9..419b205 100644 --- a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py +++ b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py @@ -175,7 +175,7 @@ def convert_2D_segmentation_to_3D( # 1a) Load a 2D label image label_img = da.from_zarr(f"{zarr_url}/labels/{label_name}/{level}") - chunks = label_img.chunksize + chunks = list(label_img.chunksize) # 1b) Get number z planes & Z spacing from 3D OME-Zarr file with zarr.open(zarr_3D_url, mode="rw+") as zarr_img: @@ -188,6 +188,7 @@ def convert_2D_segmentation_to_3D( chunks[-3] = z_chunks else: chunks[-3] = z_chunk_3d + chunks = tuple(chunks) image_meta = load_NgffImageMeta(zarr_3D_url) z_pixel_size = image_meta.get_pixel_sizes_zyx(level=0)[0] From 25c9f0868a892927773652015377b35c48a6b898 Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 13:28:15 +0100 Subject: [PATCH 04/10] Remove output type filter for rechunk task --- src/fractal_helper_tasks/rechunk_zarr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fractal_helper_tasks/rechunk_zarr.py b/src/fractal_helper_tasks/rechunk_zarr.py index 5c48f1f..6d81470 100644 --- a/src/fractal_helper_tasks/rechunk_zarr.py +++ b/src/fractal_helper_tasks/rechunk_zarr.py @@ -131,7 +131,6 @@ def rechunk_zarr( types=dict(rechunked=True), ) ], - filters=dict(types=dict(rechunked=True)), ) return output From 841019f5a59c82f38623bbc08f1d9dc6a197467f Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 16:49:48 +0100 Subject: [PATCH 05/10] Change json output of convert 2d to 3d task --- .../convert_2D_segmentation_to_3D.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py index 419b205..95e846f 100644 --- a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py +++ b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py @@ -259,12 +259,27 @@ def convert_2D_segmentation_to_3D( logger.info("Finished 2D to 3D conversion") - output_dict = dict( - filters=dict( - types=dict(is_3D=True), - ) + # FIXME: Remove that, but ensure that the output zarr_url is set for the + # 3D one? Would that work? + # If I don't, but the output types are set => changes the type of 2D + # images to 3D? + # If I don't set an output type, then tasks after it run on 2D images, + # which isn't the goal + # output_dict = dict( + # filters=dict( + # types=dict(is_3D=True), + # ) + # ) + # return output_dict + # New idea, to be tested: + image_list_updates = dict( + image_list_updates=[ + dict( + zarr_url=zarr_3D_url, + ) + ] ) - return output_dict + return image_list_updates if __name__ == "__main__": From 6c907b56c8a8d2152a7047cec779e3b1046679cf Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 17:33:24 +0100 Subject: [PATCH 06/10] Cleanup task function --- .../convert_2D_segmentation_to_3D.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py index 95e846f..cc2eae2 100644 --- a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py +++ b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py @@ -259,19 +259,7 @@ def convert_2D_segmentation_to_3D( logger.info("Finished 2D to 3D conversion") - # FIXME: Remove that, but ensure that the output zarr_url is set for the - # 3D one? Would that work? - # If I don't, but the output types are set => changes the type of 2D - # images to 3D? - # If I don't set an output type, then tasks after it run on 2D images, - # which isn't the goal - # output_dict = dict( - # filters=dict( - # types=dict(is_3D=True), - # ) - # ) - # return output_dict - # New idea, to be tested: + # Give the 3D image as an output so that filters are applied correctly image_list_updates = dict( image_list_updates=[ dict( From e1701d48edfc3b5a95fe42085cf76738fbea47fd Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 17:35:02 +0100 Subject: [PATCH 07/10] Update 2D to 3D docstring --- src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json | 2 +- src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json index b4780c0..8e88495 100644 --- a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json +++ b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json @@ -119,7 +119,7 @@ "z_chunks": { "title": "Z Chunks", "type": "integer", - "description": "Chunking for the Z dimension" + "description": "Chunking for the Z dimension. Set this parameter if you want the label image to be chunked differently from the 3D image in the Z dimension." }, "overwrite": { "default": false, diff --git a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py index cc2eae2..797b0c1 100644 --- a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py +++ b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py @@ -142,7 +142,9 @@ def convert_2D_segmentation_to_3D( If the 2D image is in "/path/to/my_plate_mip.zarr/B/03/0" and the 3D image is in "/path/to/my_plate.zarr/B/03/0_illum_corr", the value should be "_illum_corr". - z_chunks: Chunking for the Z dimension + z_chunks: Chunking for the Z dimension. Set this parameter if you want + the label image to be chunked differently from the 3D image in + the Z dimension. overwrite: If `True`, overwrite existing label and ROI tables in the 3D OME-Zarr """ From 2dce4b896c6b0ac3f5cb76b2b9123156b9f6ffb9 Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 17:39:46 +0100 Subject: [PATCH 08/10] Update tests for rechunking to remove type filters --- tests/test_rechunk_zarr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_rechunk_zarr.py b/tests/test_rechunk_zarr.py index 2e7deb3..6fc19f8 100644 --- a/tests/test_rechunk_zarr.py +++ b/tests/test_rechunk_zarr.py @@ -119,7 +119,6 @@ def test_rechunk_no_overwrite_input(tmp_zenodo_zarr: list[str]): types=dict(rechunked=True), ) ], - filters=dict(types=dict(rechunked=True)), ) assert expected_output == output From 9be7b01f2885643a9e7e40d2e6738752c09ba604 Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 17:42:19 +0100 Subject: [PATCH 09/10] update fractal-tasks-core version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c9d308a..384faee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ authors = [ # Required Python version and dependencies requires-python = ">=3.10" dependencies = [ - "fractal-tasks-core==1.4.0","ngio==0.1.4", + "fractal-tasks-core==1.4.2","ngio==0.1.4", ] # Optional dependencies (e.g. for `pip install -e ".[dev]"`, see From 188b7fb650b0d6480c3d634485b386819dbdc2d5 Mon Sep 17 00:00:00 2001 From: jluethi Date: Mon, 3 Feb 2025 17:47:55 +0100 Subject: [PATCH 10/10] Update ngio version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 384faee..62e13ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ authors = [ # Required Python version and dependencies requires-python = ">=3.10" dependencies = [ - "fractal-tasks-core==1.4.2","ngio==0.1.4", + "fractal-tasks-core==1.4.2","ngio==0.1.6", ] # Optional dependencies (e.g. for `pip install -e ".[dev]"`, see