|
| 1 | +# Copyright 2025 (C) BioVisionCenter, University of Zurich |
| 2 | +# |
| 3 | +# Original authors: |
| 4 | + |
| 5 | +"""Rechunk an existing Zarr.""" |
| 6 | + |
| 7 | +import logging |
| 8 | +import os |
| 9 | +import shutil |
| 10 | +from typing import Any, Optional |
| 11 | + |
| 12 | +import ngio |
| 13 | +from pydantic import validate_call |
| 14 | + |
| 15 | +from fractal_helper_tasks.utils import normalize_chunk_size_dict, rechunk_label |
| 16 | + |
| 17 | +logger = logging.getLogger(__name__) |
| 18 | + |
| 19 | + |
| 20 | +@validate_call |
| 21 | +def rechunk_zarr( |
| 22 | + *, |
| 23 | + zarr_url: str, |
| 24 | + chunk_sizes: Optional[dict[str, Optional[int]]] = None, |
| 25 | + suffix: str = "rechunked", |
| 26 | + rechunk_labels: bool = True, |
| 27 | + rebuild_pyramids: bool = True, |
| 28 | + overwrite_input: bool = True, |
| 29 | + overwrite: bool = False, |
| 30 | +) -> dict[str, Any]: |
| 31 | + """Drops singleton t dimension. |
| 32 | +
|
| 33 | + Args: |
| 34 | + zarr_url: Path or url to the individual OME-Zarr image to be processed. |
| 35 | + (standard argument for Fractal tasks, managed by Fractal server). |
| 36 | + chunk_sizes: Dictionary of chunk sizes to adapt. One can set any of |
| 37 | + the t, c, z, y, x axes that exist in the input image to be resized |
| 38 | + to a different chunk size. For example, {"y": 4000, "x": 4000} |
| 39 | + will set a new x & y chunking while maintaining the other chunk |
| 40 | + sizes. {"z": 10} will just change the Z chunking while keeping |
| 41 | + all other chunk sizes the same as the input. |
| 42 | + suffix: Suffix of the rechunked image. |
| 43 | + rechunk_labels: Whether to apply the same rechunking to all label |
| 44 | + images of the OME-Zarr as well. |
| 45 | + rebuild_pyramids: Whether pyramids are built fresh in the rechunked |
| 46 | + image. This has a small performance overhead, but ensures that |
| 47 | + this task is save against off-by-one issues when pyramid levels |
| 48 | + aren't easily downsampled by 2. |
| 49 | + overwrite_input: Whether the old image without rechunking should be |
| 50 | + overwritten (to avoid duplicating the data needed). |
| 51 | + overwrite: Whether to overwrite potential pre-existing output with the |
| 52 | + name zarr_url_suffix. |
| 53 | + """ |
| 54 | + logger.info(f"Running `rechunk_zarr` on {zarr_url=} with {chunk_sizes=}.") |
| 55 | + |
| 56 | + chunk_sizes = normalize_chunk_size_dict(chunk_sizes) |
| 57 | + |
| 58 | + rechunked_zarr_url = zarr_url + f"_{suffix}" |
| 59 | + ngff_image = ngio.NgffImage(zarr_url) |
| 60 | + pyramid_paths = ngff_image.levels_paths |
| 61 | + highest_res_img = ngff_image.get_image() |
| 62 | + axes_names = highest_res_img.dataset.on_disk_axes_names |
| 63 | + chunks = highest_res_img.on_disk_dask_array.chunks |
| 64 | + |
| 65 | + # Compute the chunksize tuple |
| 66 | + new_chunksize = [c[0] for c in chunks] |
| 67 | + logger.info(f"Initial chunk sizes were: {chunks}") |
| 68 | + # Overwrite chunk_size with user-set chunksize |
| 69 | + for i, axis in enumerate(axes_names): |
| 70 | + if axis in chunk_sizes: |
| 71 | + if chunk_sizes[axis] is not None: |
| 72 | + new_chunksize[i] = chunk_sizes[axis] |
| 73 | + |
| 74 | + for axis in chunk_sizes: |
| 75 | + if axis not in axes_names: |
| 76 | + raise NotImplementedError( |
| 77 | + f"Rechunking with {axis=} is specified, but the OME-Zarr only " |
| 78 | + f"has the following axes: {axes_names}" |
| 79 | + ) |
| 80 | + |
| 81 | + logger.info(f"Chunk sizes after rechunking will be: {new_chunksize=}") |
| 82 | + |
| 83 | + new_ngff_image = ngff_image.derive_new_image( |
| 84 | + store=rechunked_zarr_url, |
| 85 | + name=ngff_image.image_meta.name, |
| 86 | + overwrite=overwrite, |
| 87 | + copy_labels=not rechunk_labels, |
| 88 | + copy_tables=True, |
| 89 | + chunks=new_chunksize, |
| 90 | + ) |
| 91 | + |
| 92 | + ngff_image = ngio.NgffImage(zarr_url) |
| 93 | + |
| 94 | + if rebuild_pyramids: |
| 95 | + # Set the highest resolution, then consolidate to build a new pyramid |
| 96 | + new_ngff_image.get_image(highest_resolution=True).set_array( |
| 97 | + ngff_image.get_image(highest_resolution=True).on_disk_dask_array |
| 98 | + ) |
| 99 | + new_ngff_image.get_image(highest_resolution=True).consolidate() |
| 100 | + else: |
| 101 | + for path in pyramid_paths: |
| 102 | + new_ngff_image.get_image(path=path).set_array( |
| 103 | + ngff_image.get_image(path=path).on_disk_dask_array |
| 104 | + ) |
| 105 | + |
| 106 | + # Copy labels |
| 107 | + if rechunk_labels: |
| 108 | + chunk_sizes["c"] = None |
| 109 | + label_names = ngff_image.labels.list() |
| 110 | + for label in label_names: |
| 111 | + rechunk_label( |
| 112 | + orig_ngff_image=ngff_image, |
| 113 | + new_ngff_image=new_ngff_image, |
| 114 | + label=label, |
| 115 | + chunk_sizes=chunk_sizes, |
| 116 | + overwrite=overwrite, |
| 117 | + rebuild_pyramids=rebuild_pyramids, |
| 118 | + ) |
| 119 | + |
| 120 | + if overwrite_input: |
| 121 | + os.rename(zarr_url, f"{zarr_url}_tmp") |
| 122 | + os.rename(rechunked_zarr_url, zarr_url) |
| 123 | + shutil.rmtree(f"{zarr_url}_tmp") |
| 124 | + return |
| 125 | + else: |
| 126 | + output = dict( |
| 127 | + image_list_updates=[ |
| 128 | + dict( |
| 129 | + zarr_url=rechunked_zarr_url, |
| 130 | + origin=zarr_url, |
| 131 | + types=dict(rechunked=True), |
| 132 | + ) |
| 133 | + ], |
| 134 | + filters=dict(types=dict(rechunked=True)), |
| 135 | + ) |
| 136 | + return output |
| 137 | + |
| 138 | + |
| 139 | +if __name__ == "__main__": |
| 140 | + from fractal_tasks_core.tasks._utils import run_fractal_task |
| 141 | + |
| 142 | + run_fractal_task( |
| 143 | + task_function=rechunk_zarr, |
| 144 | + logger_name=logger.name, |
| 145 | + ) |
0 commit comments