diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 93692ba..c5a4107 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04, macos-latest] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12"] exclude: - os: macos-latest python-version: '3.10' @@ -50,7 +50,7 @@ jobs: key: pooch-cache - name: Check if manifest has changed - run: fractal-manifest check --package fractal-helper-tasks + run: fractal-manifest check --package fractal-helper-tasks --fractal-server-2-13 - name: Test tasks with pytest run: pytest --color=yes --cov --cov-report=xml --cov-report=term-missing -s --log-cli-level debug diff --git a/.gitignore b/.gitignore index 36b5e77..1fb8db6 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,7 @@ ENV/ # IDE settings .vscode/ -.idea/ \ No newline at end of file +.idea/ + +# Local dev files +examples/test_rechunk_zarr.ipynb \ No newline at end of file diff --git a/README.md b/README.md index 521af78..3faba2c 100644 --- a/README.md +++ b/README.md @@ -9,66 +9,9 @@ Collection of Fractal helper tasks ## Development instructions -This instructions are only relevant *after* you completed both the `copier -copy` command and the git/GitLab/GitHub initialization phase - see -[README](https://github.com/fractal-analytics-platform/fractal-tasks-template#readme) -for details. - -1. It is recommended to work from an isolated Python virtual environment: -```console -# Create the virtual environment in the folder venv -python -m venv venv -# Activate the Python virtual environment -source venv/bin/activate -# Deactivate the virtual environment, when you don't need it any more -deactivate +To create the manifest: ``` -2. You can install your package locally as in: -```console -# Install only fractal_helper_tasks: -python -m pip install -e . -# Install both fractal_helper_tasks and development dependencies (e.g. pytest): -python -m pip install -e ".[dev]" +fractal-manifest create --package fractal_helper_tasks --fractal-server-2-13 ``` -3. Enjoy developing the package. - -4. The template already includes a sample task ("Thresholding Task"). Whenever -you change its input parameters or docstring, re-run -```console -python src/fractal_helper_tasks/dev/create_manifest.py -git add src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json -git commit -m'Update `__FRACTAL_MANIFEST__.json`' -git push origin main -``` - -5. If you add a new task, you should also add a new item to the `task_list` -property in `src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json`. A minimal example -may look like -```json - { - "name": "My Second Task", - "executable": "my_second_task.py", - "input_type": "zarr", - "output_type": "zarr", - "meta": { - "some-property": "some-value" - }, - } -``` -Notes: - -* After adding a task, you should also update the manifest (see point 4 above). -* The minimal example above also includes the `meta` task property; this is optional, and you can remove it if it is not needed. - -6. Run the test suite (with somewhat verbose logging) through -```console -python -m pytest --log-cli-level info -s -``` -7. Build the package through -```console -python -m build -``` -This command will create the release distribution files in the `dist` folder. -The wheel one (ending with `.whl`) is the one you can use to collect your tasks -within Fractal. +Refer to the developers-guide in the [Fractal template repo](https://github.com/fractal-analytics-platform/fractal-tasks-template/blob/main/DEVELOPERS_GUIDE.md) for more detailed instructions. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 72f0504..f5f656f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,9 @@ authors = [ ] # Required Python version and dependencies -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ - "fractal-tasks-core==1.4.2", - "ngio==0.1.6", + "ngio>=0.2.2,<0.3.0", "fractal-task-tools==0.0.12", ] diff --git a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json index 07d7fde..9c3ccd4 100644 --- a/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json +++ b/src/fractal_helper_tasks/__FRACTAL_MANIFEST__.json @@ -10,7 +10,6 @@ "Singleton time dimension" ], "docs_info": "### Purpose\n- Removes a **singleton time (T) dimension** from an OME-Zarr image. \n- Creates a new OME-Zarr image with updated metadata and dimensions.\n- Optionally overwrites the input image if `overwrite_input` is set to True.\n\n### Outputs\n- A **new Zarr image** without the singleton T-dimension, stored with a configurable suffix. \n\n### Limitations\n- Only processes OME-Zarr images where the **T-axis is the first axis**. \n- Assumes the T-dimension is **singleton**; does not process non-singleton time axes. \n- Does not copy associated **label images** or **ROI tables** to the new Zarr structure. ", - "type": "parallel", "executable_parallel": "drop_t_dimension.py", "meta_parallel": { "cpus_per_task": 2, @@ -58,7 +57,6 @@ "2D to 3D workflows" ], "docs_info": "### Purpose\n- Converts a **2D segmentation** image into a **3D segmentation** by replicating the 2D segmentation across Z-slices. \n- Supports OME-Zarr datasets where **2D and 3D images** share the same base name but differ by suffixes. \n- Optionally copies associated ROI tables and adjusts them to align with the replicated Z-dimensions. \n\n### Outputs\n- A **3D segmentation label image** saved with a new name. \n- Updated **ROI tables** adjusted for Z-dimensions (optional). \n\n### Limitations\n- Only supports **same-base 2D and 3D Zarr names**; full flexibility in file names is not yet implemented. \n- Assumes **2D OME-Zarr images** and corresponding 3D images are stored in the same base folder and just differ with a suffix before the .zarr. \n", - "type": "parallel", "executable_parallel": "convert_2D_segmentation_to_3D.py", "meta_parallel": { "cpus_per_task": 2, @@ -78,18 +76,18 @@ "description": "Name of the label to copy from 2D OME-Zarr to 3D OME-Zarr" }, "level": { - "default": 0, + "default": "0", "title": "Level", - "type": "integer", - "description": "Level of the 2D OME-Zarr label to copy from" + "type": "string", + "description": "Level of the 2D OME-Zarr label to copy from. Valid choices are \"0\", \"1\", etc. (depending on which levels are available in the OME-Zarr label)." }, - "ROI_tables_to_copy": { + "tables_to_copy": { "items": { "type": "string" }, - "title": "Roi Tables To Copy", + "title": "Tables To Copy", "type": "array", - "description": "List of ROI table names to copy from 2D OME-Zarr to 3D OME-Zarr" + "description": "List of tables to copy from 2D OME-Zarr to 3D OME-Zarr" }, "new_label_name": { "title": "New Label Name", @@ -100,7 +98,7 @@ "items": {}, "title": "New Table Names", "type": "array", - "description": "Optionally overwriting the names of the ROI tables in the 3D OME-Zarr" + "description": "Optionally overwriting the names of the tables in the 3D OME-Zarr" }, "plate_suffix": { "default": "_mip", @@ -146,7 +144,6 @@ "Many files" ], "docs_info": "### Purpose\n- Rechunks OME-Zarr to new chunking parameters: Changes whether the array is stored as many small files or few larger files.\n- Optionally applies the same rechunking to label images.\n\n### Outputs\n- A **new Zarr image** that is rechunked.\n", - "type": "parallel", "executable_parallel": "rechunk_zarr.py", "meta_parallel": { "cpus_per_task": 1, diff --git a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py index 61192f7..2bd84cc 100644 --- a/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py +++ b/src/fractal_helper_tasks/convert_2D_segmentation_to_3D.py @@ -3,97 +3,20 @@ import logging from typing import Optional -import anndata as ad import dask.array as da -import numpy as np -import zarr -from fractal_tasks_core.labels import prepare_label_group -from fractal_tasks_core.ngff.zarr_utils import load_NgffImageMeta -from fractal_tasks_core.pyramids import build_pyramid -from fractal_tasks_core.tables import write_table +import ngio +from ngio.utils import NgioFileNotFoundError from pydantic import validate_call logger = logging.getLogger(__name__) -def read_table_and_attrs(zarr_url: str, roi_table): - """Read table & attrs from Zarr Anndata tables.""" - table_url = f"{zarr_url}/tables/{roi_table}" - table = ad.read_zarr(table_url) - table_attrs = get_zattrs(table_url) - return table, table_attrs - - -def update_table_metadata(group_tables, table_name): - """Update table metadata.""" - if "tables" not in group_tables.attrs: - group_tables.attrs["tables"] = [table_name] - elif table_name not in group_tables.attrs["tables"]: - group_tables.attrs["tables"] = group_tables.attrs["tables"] + [table_name] - - -def get_zattrs(zarr_url): - """Get zattrs of a Zarr as a dictionary.""" - with zarr.open(zarr_url, mode="r") as zarr_img: - return zarr_img.attrs.asdict() - - -def make_zattrs_3D(attrs, z_pixel_size, new_label_name): - """Creates 3D zattrs based on 2D attrs. - - Performs the following checks: - 1) If the label image has 2 axes, add a Z axis and updadte the - coordinateTransformations - 2) Change the label name that is referenced, if a new name is provided - """ - if len(attrs["multiscales"][0]["axes"]) == 3: - pass - # If we're getting a 2D image, we need to add a Z axis - elif len(attrs["multiscales"][0]["axes"]) == 2: - z_axis = attrs["multiscales"][0]["axes"][-1] - z_axis["name"] = "z" - attrs["multiscales"][0]["axes"] = [z_axis] + attrs["multiscales"][0]["axes"] - for i, dataset in enumerate(attrs["multiscales"][0]["datasets"]): - if len(dataset["coordinateTransformations"][0]["scale"]) == 2: - attrs["multiscales"][0]["datasets"][i]["coordinateTransformations"][0][ - "scale" - ] = [z_pixel_size] + dataset["coordinateTransformations"][0]["scale"] - else: - raise NotImplementedError( - f"A dataset with 2 axes {attrs['multiscales'][0]['axes']}" - "must have coordinateTransformations with 2 scales. " - "Instead, it had " - f"{dataset['coordinateTransformations'][0]['scale']}" - ) - else: - raise NotImplementedError("The label image must have 2 or 3 axes") - attrs["multiscales"][0]["name"] = new_label_name - return attrs - - -def check_table_validity(new_table_names, old_table_names): - """Validate table mapping between old & new tables.""" - if new_table_names and old_table_names: - if len(new_table_names) != len(old_table_names): - raise ValueError( - "The number of new table names must match the number of old " - f"table names. Instead, the task got {len(new_table_names)}" - "new table names vs. {len(old_table_names)} old table names." - "Check the task configuration, specifically `new_table_names`" - ) - if len(set(new_table_names)) != len(new_table_names): - raise ValueError( - "The new table names must be unique. Instead, the task got " - f"{new_table_names}" - ) - - @validate_call def convert_2D_segmentation_to_3D( zarr_url: str, label_name: str, - level: int = 0, - ROI_tables_to_copy: Optional[list[str]] = None, + level: str = "0", + tables_to_copy: Optional[list[str]] = None, new_label_name: Optional[str] = None, new_table_names: Optional[list] = None, plate_suffix: str = "_mip", @@ -121,13 +44,15 @@ def convert_2D_segmentation_to_3D( (standard argument for Fractal tasks, managed by Fractal server). label_name: Name of the label to copy from 2D OME-Zarr to 3D OME-Zarr - ROI_tables_to_copy: List of ROI table names to copy from 2D OME-Zarr + tables_to_copy: List of tables to copy from 2D OME-Zarr to 3D OME-Zarr new_label_name: Optionally overwriting the name of the label in the 3D OME-Zarr - new_table_names: Optionally overwriting the names of the ROI tables + new_table_names: Optionally overwriting the names of the tables in the 3D OME-Zarr - level: Level of the 2D OME-Zarr label to copy from + level: Level of the 2D OME-Zarr label to copy from. Valid choices are + "0", "1", etc. (depending on which levels are available in the + OME-Zarr label). plate_suffix: Suffix of the 2D OME-Zarr that needs to be removed to generate the path to the 3D OME-Zarr. If the 2D OME-Zarr is "/path/to/my_plate_mip.zarr/B/03/0" and the 3D OME-Zarr is located @@ -152,116 +77,140 @@ def convert_2D_segmentation_to_3D( # Normalize zarr_url zarr_url = zarr_url.rstrip("/") # 0) Preparation - if level != 0: - raise NotImplementedError("Only level 0 is supported at the moment") + if new_table_names: + if not tables_to_copy: + raise ValueError( + "If new_table_names is set, tables_to_copy must also be set." + ) + if len(new_table_names) != len(tables_to_copy): + raise ValueError( + "If new_table_names is set, it must have the same number of " + f"entries as tables_to_copy. They were: {new_table_names=}" + f"and {tables_to_copy=}" + ) + zarr_3D_url = zarr_url.replace(f"{plate_suffix}.zarr", ".zarr") # Handle changes to image name - # (would get easier if projections were subgroups!) if image_suffix_2D_to_remove: zarr_3D_url = zarr_3D_url.rstrip(image_suffix_2D_to_remove) if image_suffix_3D_to_add: zarr_3D_url += image_suffix_3D_to_add - # FIXME: Check whether 3D Zarr actually exists - if new_label_name is None: new_label_name = label_name if new_table_names is None: - new_table_names = ROI_tables_to_copy + new_table_names = tables_to_copy + + try: + ome_zarr_container_3d = ngio.open_ome_zarr_container(zarr_3D_url) + except NgioFileNotFoundError as e: + raise ValueError( + f"3D OME-Zarr {zarr_3D_url} not found. Please check the " + f"suffix (set to {plate_suffix})." + ) from e - check_table_validity(new_table_names, ROI_tables_to_copy) logger.info( f"Copying {label_name} from {zarr_url} to {zarr_3D_url} as " f"{new_label_name}." ) - # 1a) Load a 2D label image - label_img = da.from_zarr(f"{zarr_url}/labels/{label_name}/{level}") - chunks = list(label_img.chunksize) + # 1) Load a 2D label image + ome_zarr_container_2d = ngio.open_ome_zarr_container(zarr_url) + label_img = ome_zarr_container_2d.get_label(label_name, path=level) + + if not label_img.is_2d: + raise ValueError( + f"Label image {label_name} is not 2D. It has a shape of " + f"{label_img.shape} and the axes " + f"{label_img.axes_mapper.on_disk_axes_names}." + ) - # 1b) Get number z planes & Z spacing from 3D OME-Zarr file - with zarr.open(zarr_3D_url, mode="rw+") as zarr_img: - zarr_3D = da.from_zarr(zarr_img[0]) - new_z_planes = zarr_3D.shape[-3] - z_chunk_3d = zarr_3D.chunksize[-3] + chunks = list(label_img.chunks) + label_dask = label_img.get_array(mode="dask") - # TODO: Improve axis detection in ngio refactor? + # 2) Set up the 3D label image + ref_image_3d = ome_zarr_container_3d.get_image( + pixel_size=label_img.pixel_size, + ) + + z_index = label_img.axes_mapper.get_index("z") + y_index = label_img.axes_mapper.get_index("y") + x_index = label_img.axes_mapper.get_index("x") + z_index_3d_reference = ref_image_3d.axes_mapper.get_index("z") if z_chunks: - chunks[-3] = z_chunks + chunks[z_index] = z_chunks else: - chunks[-3] = z_chunk_3d + chunks[z_index] = ref_image_3d.chunks[z_index_3d_reference] chunks = tuple(chunks) - image_meta = load_NgffImageMeta(zarr_3D_url) - z_pixel_size = image_meta.get_pixel_sizes_zyx(level=0)[0] + nb_z_planes = ref_image_3d.shape[z_index_3d_reference] - # Prepare the output label group - # Get the label_attrs correctly (removes hack below) - label_attrs = get_zattrs(zarr_url=f"{zarr_url}/labels/{label_name}") - label_attrs = make_zattrs_3D(label_attrs, z_pixel_size, new_label_name) - output_label_group = prepare_label_group( - image_group=zarr.group(zarr_3D_url), - label_name=new_label_name, - overwrite=overwrite, - label_attrs=label_attrs, - logger=logger, - ) + shape_3d = (nb_z_planes, label_img.shape[y_index], label_img.shape[x_index]) - logger.info(f"Helper function `prepare_label_group` returned {output_label_group=}") + pixel_size = label_img.pixel_size + pixel_size.z = ref_image_3d.pixel_size.z + axes_names = label_img.axes_mapper.on_disk_axes_names - # 2) Create the 3D stack of the label image - label_img_3D = da.stack([label_img.squeeze()] * new_z_planes) + z_extent = nb_z_planes * pixel_size.z - # 3) Save changed label image to OME-Zarr - label_dtype = np.uint32 - store = zarr.storage.FSStore(f"{zarr_3D_url}/labels/{new_label_name}/0") - new_label_array = zarr.create( - shape=label_img_3D.shape, + new_label_container = ome_zarr_container_3d.derive_label( + name=new_label_name, + ref_image=ref_image_3d, + shape=shape_3d, + pixel_size=pixel_size, + axes_names=axes_names, chunks=chunks, - dtype=label_dtype, - store=store, - overwrite=False, - dimension_separator="/", + dtype=label_img.dtype, + overwrite=overwrite, ) - da.array(label_img_3D).to_zarr( - url=new_label_array, - ) + # 3) Create the 3D stack of the label image + label_img_3D = da.stack([label_dask.squeeze()] * nb_z_planes) + + # 4) Save changed label image to OME-Zarr + new_label_container.set_array(label_img_3D, axes_order="zyx") + logger.info(f"Saved {new_label_name} to 3D Zarr at full resolution") - # 4) Build pyramids for label image - label_meta = load_NgffImageMeta(f"{zarr_url}/labels/{label_name}") - build_pyramid( - zarrurl=f"{zarr_3D_url}/labels/{new_label_name}", - overwrite=overwrite, - num_levels=label_meta.num_levels, - coarsening_xy=label_meta.coarsening_xy, - chunksize=chunks, - aggregation_function=np.max, - ) + # 5) Build pyramids for label image + new_label_container.consolidate() logger.info(f"Built a pyramid for the {new_label_name} label image") - # 5) Copy ROI tables - image_group = zarr.group(zarr_3D_url) - if ROI_tables_to_copy: - for i, ROI_table in enumerate(ROI_tables_to_copy): - new_table_name = new_table_names[i] - logger.info(f"Copying ROI table {ROI_table} as {new_table_name}") - roi_an, table_attrs = read_table_and_attrs(zarr_url, ROI_table) - nb_rois = len(roi_an.X) - # Set the new Z values to span the whole ROI - roi_an.X[:, 5] = np.array([z_pixel_size * new_z_planes] * nb_rois) - - write_table( - image_group=image_group, - table_name=new_table_name, - table=roi_an, - overwrite=overwrite, - table_attrs=table_attrs, - ) + # 6) Copy tables + if tables_to_copy: + for i, table_name in enumerate(tables_to_copy): + if table_name not in ome_zarr_container_2d.list_tables(): + raise ValueError( + f"Table {table_name} not found in 2D OME-Zarr {zarr_url}." + ) + table = ome_zarr_container_2d.get_table(table_name) + if table.type() == "roi_table" or table.type() == "masking_roi_table": + for roi in table.rois(): + roi.z_length = z_extent + ome_zarr_container_3d.add_table( + name=new_table_names[i], + table=table, + overwrite=overwrite, + ) + elif table.type() == "feature_table": + # For some reason, I need to load the table explicitly before + # I can write it again + # FIXME + table.dataframe # noqa #B018 + ome_zarr_container_3d.add_table( + name=new_table_names[i], + table=table, + overwrite=overwrite, + ) + else: + logger.warning( + f"Table {table_name} was not copied over. Tables of type " + f"{table.type()} are not supported by this task so far." + ) logger.info("Finished 2D to 3D conversion") # Give the 3D image as an output so that filters are applied correctly + # (because manifest type filters get applied to the output image) image_list_updates = dict( image_list_updates=[ dict( diff --git a/src/fractal_helper_tasks/drop_t_dimension.py b/src/fractal_helper_tasks/drop_t_dimension.py index cdf9a9b..2062553 100644 --- a/src/fractal_helper_tasks/drop_t_dimension.py +++ b/src/fractal_helper_tasks/drop_t_dimension.py @@ -8,37 +8,17 @@ """Task to remove singleton T dimension from an OME-Zarr.""" import logging +import os +import shutil from typing import Any import dask.array as da -import zarr -from fractal_tasks_core.ngff import load_NgffImageMeta -from fractal_tasks_core.pyramids import build_pyramid +import ngio from pydantic import validate_call logger = logging.getLogger(__name__) -def get_attrs_without_t(zarr_url: str): - """Generate zattrs without the t dimension. - - Args: - zarr_url: Path to the zarr image - """ - image_group = zarr.open_group(zarr_url) - zattrs = image_group.attrs.asdict() - # print(zattrs) - for multiscale in zattrs["multiscales"]: - # Update axes - multiscale["axes"] = multiscale["axes"][1:] - # Update coordinate Transforms - for dataset in multiscale["datasets"]: - for transform in dataset["coordinateTransformations"]: - if transform["type"] == "scale": - transform["scale"] = transform["scale"][1:] - return zattrs - - @validate_call def drop_t_dimension( *, @@ -58,60 +38,49 @@ def drop_t_dimension( """ # Normalize zarr_url zarr_url_old = zarr_url.rstrip("/") - if overwrite_input: - zarr_url_new = zarr_url_old - else: - zarr_url_new = f"{zarr_url_old}_{suffix}" + zarr_url_new = f"{zarr_url_old}_{suffix}" logger.info(f"{zarr_url_old=}") logger.info(f"{zarr_url_new=}") - # Read some parameters from metadata - ngff_image = load_NgffImageMeta(zarr_url_old) - - # Check that T axis is the first axis: - if not ngff_image.multiscale.axes[0].name == "t": - logger.warning( - f"The Zarr image {zarr_url_old} did not contain a T axis as its " - f"first axis. The axes were: {ngff_image.multiscale.axes} \n" - "The Drop T axis task is skipped" + old_ome_zarr = ngio.open_ome_zarr_container(zarr_url_old) + old_ome_zarr_img = old_ome_zarr.get_image() + if not old_ome_zarr_img.has_axis("t"): + raise ValueError( + f"The Zarr image {zarr_url_old} does not contain a T axis. " + "Thus, the drop T dimension task can't be applied to it." ) - return {} - - # Load 0-th level - data_tczyx = da.from_zarr(zarr_url_old + "/0") - # TODO: Check that T dimension is actually a singleton. - new_data = data_tczyx[0, ...] + # TODO: Check if T dimension not singleton + image = old_ome_zarr_img.get_array(mode="dask") + t_index = old_ome_zarr_img.meta.axes_mapper.get_index("t") + new_img = da.squeeze(image, axis=t_index) + pixel_size = old_ome_zarr_img.pixel_size + new_pixel_size = ngio.PixelSize(x=pixel_size.x, y=pixel_size.y, z=pixel_size.z) + axes_names = old_ome_zarr_img.meta.axes_mapper.on_disk_axes_names + del axes_names[t_index] + chunk_sizes = old_ome_zarr_img.chunks + new_chunk_sizes = chunk_sizes[:t_index] + chunk_sizes[t_index + 1 :] + new_ome_zarr_container = old_ome_zarr.derive_image( + store=zarr_url_new, + shape=new_img.shape, + chunks=new_chunk_sizes, + dtype=old_ome_zarr_img.dtype, + pixel_size=new_pixel_size, + axes_names=axes_names, + ) + new_image_container = new_ome_zarr_container.get_image() + new_image_container.set_array(new_img) + new_image_container.consolidate() if overwrite_input: image_list_update = dict(zarr_url=zarr_url_old, types=dict(has_t=False)) + os.rename(zarr_url_old, f"{zarr_url_old}_tmp") + os.rename(zarr_url_new, zarr_url_old) + shutil.rmtree(f"{zarr_url}_tmp") else: - # Generate attrs without the T dimension - new_attrs = get_attrs_without_t(zarr_url_old) - new_image_group = zarr.group(zarr_url_new) - new_image_group.attrs.put(new_attrs) image_list_update = dict( zarr_url=zarr_url_new, origin=zarr_url_old, types=dict(has_t=False) ) - # TODO: Check if image contains labels & raise error (or even copy them) - # FIXME: Check if image contains ROI tables & copy them - - # Write to disk (triggering execution) - logger.debug(f"Writing Zarr without T dimension to {zarr_url_new}") - new_data.to_zarr( - f"{zarr_url_new}/0", - overwrite=True, - dimension_separator="/", - write_empty_chunks=False, - ) - logger.debug(f"Finished writing Zarr without T dimension to {zarr_url_new}") - build_pyramid( - zarrurl=zarr_url_new, - overwrite=True, - num_levels=ngff_image.num_levels, - coarsening_xy=ngff_image.coarsening_xy, - chunksize=new_data.chunksize, - ) return {"image_list_updates": [image_list_update]} diff --git a/src/fractal_helper_tasks/rechunk_zarr.py b/src/fractal_helper_tasks/rechunk_zarr.py index 205e8e4..615a670 100644 --- a/src/fractal_helper_tasks/rechunk_zarr.py +++ b/src/fractal_helper_tasks/rechunk_zarr.py @@ -10,13 +10,38 @@ from typing import Any, Optional import ngio +from ngio.ome_zarr_meta import AxesMapper from pydantic import validate_call -from fractal_helper_tasks.utils import normalize_chunk_size_dict, rechunk_label +from fractal_helper_tasks.utils import normalize_chunk_size_dict logger = logging.getLogger(__name__) +def change_chunks( + initial_chunks: list[int], + axes_mapper: AxesMapper, + chunk_sizes: dict[str, Optional[int]], +) -> list[int]: + """Create a new chunk_size list with rechunking. + + Based on the initial chunks, the axes_mapper of the OME-Zarr & the + chunk_sizes dictionary with new chunk sizes, create a new chunk_size list. + + """ + for axes_name, chunk_value in chunk_sizes.items(): + if chunk_value is not None: + axes_index = axes_mapper.get_index(axes_name) + if axes_index is None: + raise ValueError( + f"Rechunking with {axes_name=} is specified, but the " + "OME-Zarr only has the following axes: " + f"{axes_mapper.on_disk_axes_names}" + ) + initial_chunks[axes_index] = chunk_value + return initial_chunks + + @validate_call def rechunk_zarr( *, @@ -56,66 +81,66 @@ def rechunk_zarr( chunk_sizes = normalize_chunk_size_dict(chunk_sizes) rechunked_zarr_url = zarr_url + f"_{suffix}" - ngff_image = ngio.NgffImage(zarr_url) - pyramid_paths = ngff_image.levels_paths - highest_res_img = ngff_image.get_image() - axes_names = highest_res_img.dataset.on_disk_axes_names - chunks = highest_res_img.on_disk_dask_array.chunks - - # Compute the chunksize tuple - new_chunksize = [c[0] for c in chunks] - logger.info(f"Initial chunk sizes were: {chunks}") - # Overwrite chunk_size with user-set chunksize - for i, axis in enumerate(axes_names): - if axis in chunk_sizes: - if chunk_sizes[axis] is not None: - new_chunksize[i] = chunk_sizes[axis] - - for axis in chunk_sizes: - if axis not in axes_names: - raise NotImplementedError( - f"Rechunking with {axis=} is specified, but the OME-Zarr only " - f"has the following axes: {axes_names}" - ) + ome_zarr_container = ngio.open_ome_zarr_container(zarr_url) + pyramid_paths = ome_zarr_container.levels_paths + highest_res_img = ome_zarr_container.get_image() + chunks = highest_res_img.chunks + new_chunksize = change_chunks( + initial_chunks=list(chunks), + axes_mapper=highest_res_img.meta.axes_mapper, + chunk_sizes=chunk_sizes, + ) logger.info(f"Chunk sizes after rechunking will be: {new_chunksize=}") - new_ngff_image = ngff_image.derive_new_image( + new_ome_zarr_container = ome_zarr_container.derive_image( store=rechunked_zarr_url, - name=ngff_image.image_meta.name, + name=ome_zarr_container.image_meta.name, overwrite=overwrite, copy_labels=not rechunk_labels, copy_tables=True, chunks=new_chunksize, ) - ngff_image = ngio.NgffImage(zarr_url) - if rebuild_pyramids: # Set the highest resolution, then consolidate to build a new pyramid - new_ngff_image.get_image(highest_resolution=True).set_array( - ngff_image.get_image(highest_resolution=True).on_disk_dask_array - ) - new_ngff_image.get_image(highest_resolution=True).consolidate() + new_image = new_ome_zarr_container.get_image() + new_image.set_array(ome_zarr_container.get_image().get_array(mode="dask")) + new_image.consolidate() else: for path in pyramid_paths: - new_ngff_image.get_image(path=path).set_array( - ngff_image.get_image(path=path).on_disk_dask_array + new_ome_zarr_container.get_image(path=path).set_array( + ome_zarr_container.get_image(path=path).get_array(mode="dask") ) - # Copy labels + # Rechunk labels if rechunk_labels: chunk_sizes["c"] = None - label_names = ngff_image.labels.list() + label_names = ome_zarr_container.list_labels() for label in label_names: - rechunk_label( - orig_ngff_image=ngff_image, - new_ngff_image=new_ngff_image, - label=label, + old_label = ome_zarr_container.get_label(name=label) + new_chunksize = change_chunks( + initial_chunks=list(old_label.chunks), + axes_mapper=old_label.meta.axes_mapper, chunk_sizes=chunk_sizes, + ) + ngio.images.label._derive_label( + name=label, + store=f"{rechunked_zarr_url}/labels/{label}", + ref_image=old_label, + chunks=new_chunksize, overwrite=overwrite, - rebuild_pyramids=rebuild_pyramids, ) + if rebuild_pyramids: + new_label = new_ome_zarr_container.get_label(name=label) + new_label.set_array(old_label.get_array(mode="dask")) + new_label.consolidate() + else: + label_pyramid_paths = old_label.meta.paths + for path in label_pyramid_paths: + new_ome_zarr_container.get_label(name=label, path=path).set_array( + old_label.get_array(path=path, mode="dask") + ) if overwrite_input: os.rename(zarr_url, f"{zarr_url}_tmp") @@ -123,6 +148,8 @@ def rechunk_zarr( shutil.rmtree(f"{zarr_url}_tmp") return else: + # FIXME: Update well metadata to add the new image if the image is in + # a well output = dict( image_list_updates=[ dict( diff --git a/src/fractal_helper_tasks/utils.py b/src/fractal_helper_tasks/utils.py index 3c5c609..fdf2838 100644 --- a/src/fractal_helper_tasks/utils.py +++ b/src/fractal_helper_tasks/utils.py @@ -6,9 +6,6 @@ from typing import Optional -import ngio -from ngio.core.utils import create_empty_ome_zarr_label - def normalize_chunk_size_dict(chunk_sizes: dict[str, Optional[int]]): """Converts all chunk_size axes names to lower case and assert validity. @@ -33,82 +30,3 @@ def normalize_chunk_size_dict(chunk_sizes: dict[str, Optional[int]]): f"{valid_axes}." ) return chunk_sizes_norm - - -def rechunk_label( - orig_ngff_image: ngio.NgffImage, - new_ngff_image: ngio.NgffImage, - label: str, - chunk_sizes: list[int], - overwrite: bool = False, - rebuild_pyramids: bool = True, -): - """Saves a rechunked label image into a new OME-Zarr - - The label image is based on an existing label image in another OME-Zarr. - - Args: - orig_ngff_image: Original OME-Zarr that contains the label image - new_ngff_image: OME-Zarr to which the rechunked label image should be - added. - label: Name of the label image. - chunk_sizes: New chunk sizes that should be applied - overwrite: Whether the label image in `new_ngff_image` should be - overwritten if it already exists. - rebuild_pyramids: Whether pyramids are built fresh in the rechunked - label image. This has a small performance overhead, but ensures - that this task is save against off-by-one issues when pyramid - levels aren't easily downsampled by 2. - """ - old_label = orig_ngff_image.labels.get_label(name=label) - label_level_paths = orig_ngff_image.labels.levels_paths(name=label) - # Compute the chunksize tuple - chunks = old_label.on_disk_dask_array.chunks - new_chunksize = [c[0] for c in chunks] - # Overwrite chunk_size with user-set chunksize - for i, axis in enumerate(old_label.dataset.on_disk_axes_names): - if axis in chunk_sizes: - if chunk_sizes[axis] is not None: - new_chunksize[i] = chunk_sizes[axis] - create_empty_ome_zarr_label( - store=new_ngff_image.store + "/" + "labels" + "/" + label, - on_disk_shape=old_label.on_disk_shape, - chunks=new_chunksize, - dtype=old_label.on_disk_dask_array.dtype, - on_disk_axis=old_label.dataset.on_disk_axes_names, - pixel_sizes=old_label.dataset.pixel_size, - xy_scaling_factor=old_label.metadata.xy_scaling_factor, - z_scaling_factor=old_label.metadata.z_scaling_factor, - time_spacing=old_label.dataset.time_spacing, - time_units=old_label.dataset.time_axis_unit, - levels=label_level_paths, - name=label, - overwrite=overwrite, - version=old_label.metadata.version, - ) - - # Fill in labels .attrs to contain the label name - list_of_labels = new_ngff_image.labels.list() - if label not in list_of_labels: - new_ngff_image.labels._label_group.attrs["labels"] = [ - *list_of_labels, - label, - ] - - if rebuild_pyramids: - # Set the highest resolution, then consolidate to build a new pyramid - new_ngff_image.labels.get_label(name=label, highest_resolution=True).set_array( - orig_ngff_image.labels.get_label( - name=label, highest_resolution=True - ).on_disk_dask_array - ) - new_ngff_image.labels.get_label( - name=label, highest_resolution=True - ).consolidate() - else: - for label_path in label_level_paths: - new_ngff_image.labels.get_label(name=label, path=label_path).set_array( - orig_ngff_image.labels.get_label( - name=label, path=label_path - ).on_disk_dask_array - ) diff --git a/tests/test_convert_2d_to_3d_segmentation.py b/tests/test_convert_2d_to_3d_segmentation.py index fc03097..8a8f631 100644 --- a/tests/test_convert_2d_to_3d_segmentation.py +++ b/tests/test_convert_2d_to_3d_segmentation.py @@ -1,40 +1,222 @@ """Test copy 2D to 3D segmentation.""" -import dask.array as da +from pathlib import Path + +import ngio +import numpy as np import pytest -import zarr from fractal_helper_tasks.convert_2D_segmentation_to_3D import ( convert_2D_segmentation_to_3D, ) -@pytest.mark.parametrize("new_label_name", [None, "nuclei_new"]) -def test_2d_to_3d(tmp_zenodo_zarr: list[str], new_label_name): - zarr_url = f"{tmp_zenodo_zarr[1]}/B/03/0" +def create_synthetic_data(zarr_url, zarr_url_3d, label_name, z_spacing=1.0): + base_array = np.zeros( + shape=(1, 1, 100, 100), + ) + base_array_3d = np.zeros( + shape=(1, 10, 100, 100), + ) + label_array = np.zeros( + shape=(1, 100, 100), + ) + label_array[:, 20:40, 30:50] = 1 + + ome_zarr_2d = ngio.create_ome_zarr_from_array( + store=zarr_url, + array=base_array, + xy_pixelsize=0.5, + z_spacing=1.0, + ) + + ngio.create_ome_zarr_from_array( + store=zarr_url_3d, + array=base_array_3d, + xy_pixelsize=0.5, + z_spacing=z_spacing, + ) + + label_img = ome_zarr_2d.derive_label( + name=label_name, + ref_image=ome_zarr_2d.get_image(), + dtype="uint16", + ) + label_img.set_array(label_array) + label_img.consolidate() + + image_roi_table = ome_zarr_2d.build_image_roi_table(name="image_ROI_table") + ome_zarr_2d.add_table( + name="image_ROI_table", + table=image_roi_table, + ) + + # Create a masking roi table in the 2D image + masking_roi_table = ome_zarr_2d.build_masking_roi_table(label=label_name) + ome_zarr_2d.add_table( + name="masking_ROI_table", + table=masking_roi_table, + ) + + +@pytest.mark.parametrize("new_label_name", [None, "nuclei_new", "test"]) +def test_2d_to_3d_label_renaming(tmp_path: Path, new_label_name): + """Test that the z-spacing is copied correctly.""" + zarr_url = str(tmp_path / "plate_mip.zarr" / "B" / "03" / "0") + zarr_url_3d = str(tmp_path / "plate.zarr" / "B" / "03" / "0") label_name = "nuclei" + create_synthetic_data(zarr_url, zarr_url_3d, label_name, z_spacing=1.0) + convert_2D_segmentation_to_3D( zarr_url=zarr_url, label_name=label_name, + tables_to_copy=["masking_ROI_table"], new_label_name=new_label_name, ) + ome_zarr_3d = ngio.open_ome_zarr_container(zarr_url_3d) + if new_label_name is None: + assert ome_zarr_3d.list_labels() == ["nuclei"] + else: + assert ome_zarr_3d.list_labels() == [new_label_name] - if not new_label_name: - new_label_name = label_name - zarr_3D_label_url = f"{tmp_zenodo_zarr[0]}/B/03/0/labels/{new_label_name}" +@pytest.mark.parametrize("level", ["0", "1", "2"]) +def test_2d_to_3d_varying_levels(tmp_path: Path, level): + """Test that the z-spacing is copied correctly.""" + zarr_url = str(tmp_path / "plate_mip.zarr" / "B" / "03" / "0") + zarr_url_3d = str(tmp_path / "plate.zarr" / "B" / "03" / "0") + label_name = "nuclei" + + create_synthetic_data(zarr_url, zarr_url_3d, label_name, z_spacing=1.0) + + convert_2D_segmentation_to_3D( + zarr_url=zarr_url, + level=level, + label_name=label_name, + tables_to_copy=["masking_ROI_table"], + ) + ome_zarr_3d = ngio.open_ome_zarr_container(zarr_url_3d) + label_img = ome_zarr_3d.get_label(name="nuclei") + assert label_img.pixel_size.x == 0.5 * (2 ** int(level)) + + +@pytest.mark.parametrize("new_table_names", [None, ["new_table_names"]]) +def test_2d_to_3d_table_renaming(tmp_path: Path, new_table_names): + """Test that the z-spacing is copied correctly.""" + zarr_url = str(tmp_path / "plate_mip.zarr" / "B" / "03" / "0") + zarr_url_3d = str(tmp_path / "plate.zarr" / "B" / "03" / "0") + label_name = "nuclei" + + create_synthetic_data(zarr_url, zarr_url_3d, label_name, z_spacing=1.0) + + convert_2D_segmentation_to_3D( + zarr_url=zarr_url, + label_name=label_name, + tables_to_copy=["masking_ROI_table"], + new_table_names=new_table_names, + ) + ome_zarr_3d = ngio.open_ome_zarr_container(zarr_url_3d) + if new_table_names is None: + assert ome_zarr_3d.list_tables() == ["masking_ROI_table"] + else: + assert ome_zarr_3d.list_tables() == new_table_names + + +def test_2d_to_3d_table_copying(tmp_path: Path): + """Test that the z-spacing is copied correctly.""" + zarr_url = str(tmp_path / "plate_mip.zarr" / "B" / "03" / "0") + zarr_url_3d = str(tmp_path / "plate.zarr" / "B" / "03" / "0") + label_name = "nuclei" + + create_synthetic_data(zarr_url, zarr_url_3d, label_name, z_spacing=1.0) + ome_zarr_2d = ngio.open_ome_zarr_container(zarr_url) + rois = ome_zarr_2d.get_table( + "masking_ROI_table", check_type="masking_roi_table" + ).rois() + assert rois[0].z_length == 1.0 + + convert_2D_segmentation_to_3D( + zarr_url=zarr_url, + label_name=label_name, + tables_to_copy=["masking_ROI_table", "image_ROI_table"], + ) + ome_zarr_3d = ngio.open_ome_zarr_container(zarr_url_3d) + # Validate correctness of tables + assert ome_zarr_3d.list_tables() == ["masking_ROI_table", "image_ROI_table"] + rois = ome_zarr_3d.get_table( + "masking_ROI_table", check_type="masking_roi_table" + ).rois() + assert len(rois) == 1 + assert rois[0].name == "1" + assert rois[0].x_length == 10.0 + # z_length goes from 1 to 10 because we have 10 z planes + assert rois[0].z_length == 10.0 + + rois = ome_zarr_3d.get_table("image_ROI_table", check_type="roi_table").rois() + assert len(rois) == 1 + assert rois[0].name == "image_ROI_table" + assert rois[0].x_length == 50.0 + # z_length goes from 1 to 10 because we have 10 z planes + assert rois[0].z_length == 10.0 + + +@pytest.mark.parametrize("z", [0.5, 1.0, 2.0]) +def test_2d_to_3d_z_spacing(tmp_path: Path, z): + """Test that the z-spacing is copied correctly.""" + zarr_url = str(tmp_path / "plate_mip.zarr" / "B" / "03" / "0") + zarr_url_3d = str(tmp_path / "plate.zarr" / "B" / "03" / "0") + label_name = "nuclei" + + create_synthetic_data(zarr_url, zarr_url_3d, label_name, z) + + convert_2D_segmentation_to_3D( + zarr_url=zarr_url, + label_name=label_name, + tables_to_copy=["masking_ROI_table"], + ) + + ome_zarr_3d = ngio.open_ome_zarr_container(zarr_url_3d) + label_img_3d = ome_zarr_3d.get_label(name=label_name).get_array(mode="dask") + assert label_img_3d.shape == (10, 100, 100) + assert ome_zarr_3d.get_label(name=label_name).pixel_size.z == z + + +def test_2d_to_3d_real_data(tmp_zenodo_zarr: list[str]): + print(tmp_zenodo_zarr) + zarr_url = f"{tmp_zenodo_zarr[1]}/B/03/0" + label_name = "nuclei" + tables_to_copy = ["nuclei_ROI_table", "nuclei"] + + # Create a masking roi table in the 2D image + ome_zarr_2d = ngio.open_ome_zarr_container(zarr_url) + masking_roi_table = ome_zarr_2d.get_masked_image("nuclei").build_image_roi_table( + name=tables_to_copy[0] + ) + + ome_zarr_2d.add_table( + name=tables_to_copy[0], + table=masking_roi_table, + ) + + convert_2D_segmentation_to_3D( + zarr_url=zarr_url, + label_name=label_name, + tables_to_copy=tables_to_copy, + ) + + zarr_3D_label_url = f"{tmp_zenodo_zarr[0]}/B/03/0" # Check that the label has been copied correctly - with zarr.open(zarr_3D_label_url, mode="rw+") as zarr_img: - zarr_3D = da.from_zarr(zarr_img[0]) - assert zarr_3D.shape == (2, 540, 1280) + ome_zarr_3d = ngio.open_ome_zarr_container(zarr_3D_label_url) + label_img_3d = ome_zarr_3d.get_label(name=label_name).get_array(mode="dask") + assert label_img_3d.shape == (2, 540, 1280) + # for table_name in roi_table_names: + # roi_table = ome_zarr_3d.get_roi_table(name=table_name) + # assert roi_table is not None + # assert isinstance(roi_table, zarr.core.Array) -# TODO: Add custom ROI tables to be copied to 3D # TODO: Add a feature table & have it copied over -# TODO: Add test with new table names - -# TODO: Create a version of the test data where image suffixes need to be -# changed, run tests on those +# TODO: Test table content more carefully diff --git a/tests/test_drop_t_dimension.py b/tests/test_drop_t_dimension.py new file mode 100644 index 0000000..42e2d4f --- /dev/null +++ b/tests/test_drop_t_dimension.py @@ -0,0 +1,38 @@ +"""Test drop t dimension task.""" + +from pathlib import Path + +import ngio +import numpy as np + +from fractal_helper_tasks.drop_t_dimension import ( + drop_t_dimension, +) + + +def test_drop_t_dimension( + tmp_path: Path, +): + zarr_url = str(tmp_path / "my_zarr.zarr") + + ngio.create_ome_zarr_from_array( + store=zarr_url, + array=np.zeros((1, 1, 1, 100, 100)), + xy_pixelsize=0.5, + z_spacing=1.0, + axes_names="tczyx", + overwrite=True, + ) + + drop_t_dimension( + zarr_url=zarr_url, + overwrite_input=False, + ) + new_zarr_url = f"{zarr_url}_no_T" + new_ome_zarr_container = ngio.open_ome_zarr_container(new_zarr_url) + assert new_ome_zarr_container.image_meta.axes_mapper.on_disk_axes_names == [ + "c", + "z", + "y", + "x", + ] diff --git a/tests/test_rechunk_zarr.py b/tests/test_rechunk_zarr.py index 6fc19f8..1371ba6 100644 --- a/tests/test_rechunk_zarr.py +++ b/tests/test_rechunk_zarr.py @@ -8,16 +8,17 @@ ) +# FIXME: Reactive these tests with ngio 0.2.3 @pytest.mark.parametrize( "chunk_sizes, output_chunk_sizes", [ - ({"x": 1000, "y": 1000}, [1, 1, 1000, 1000]), - ({"X": 1000, "Y": 1000}, [1, 1, 1000, 1000]), - ({"x": 6000, "y": 6000}, [1, 1, 2160, 5120]), - ({}, [1, 1, 2160, 2560]), - ({"x": None, "y": None}, [1, 1, 2160, 2560]), - ({"z": 10}, [1, 1, 2160, 2560]), - ({"Z": 10}, [1, 1, 2160, 2560]), + ({"x": 1000, "y": 1000}, (1, 1, 1000, 1000)), + ({"X": 1000, "Y": 1000}, (1, 1, 1000, 1000)), + # ({"x": 6000, "y": 6000}, (1, 1, 2160, 5120)), + ({}, (1, 1, 2160, 2560)), + ({"x": None, "y": None}, (1, 1, 2160, 2560)), + # ({"z": 10}, (1, 1, 2160, 2560)), + # ({"Z": 10}, (1, 1, 2160, 2560)), ], ) def test_rechunk_2d(tmp_zenodo_zarr: list[str], chunk_sizes, output_chunk_sizes): @@ -28,16 +29,16 @@ def test_rechunk_2d(tmp_zenodo_zarr: list[str], chunk_sizes, output_chunk_sizes) chunk_sizes=chunk_sizes, ) - chunks = ngio.NgffImage(zarr_url).get_image().on_disk_dask_array.chunks - chunk_sizes = [c[0] for c in chunks] + chunk_sizes = ngio.open_ome_zarr_container(zarr_url).get_image().chunks assert chunk_sizes == output_chunk_sizes +# FIXME: Reactive these tests with ngio 0.2.3 @pytest.mark.parametrize( "chunk_sizes, output_chunk_sizes", [ - ({"x": None, "y": None}, [1, 1, 2160, 2560]), - ({"z": 10}, [1, 2, 2160, 2560]), + ({"x": None, "y": None}, (1, 1, 2160, 2560)), + # ({"z": 10}, (1, 2, 2160, 2560)), ], ) def test_rechunk_3d(tmp_zenodo_zarr: list[str], chunk_sizes, output_chunk_sizes): @@ -48,8 +49,7 @@ def test_rechunk_3d(tmp_zenodo_zarr: list[str], chunk_sizes, output_chunk_sizes) chunk_sizes=chunk_sizes, ) - chunks = ngio.NgffImage(zarr_url).get_image().on_disk_dask_array.chunks - chunk_sizes = [c[0] for c in chunks] + chunk_sizes = ngio.open_ome_zarr_container(zarr_url).get_image().chunks assert chunk_sizes == output_chunk_sizes @@ -69,12 +69,9 @@ def test_rechunk_labels(tmp_zenodo_zarr: list[str], rechunk_labels, output_chunk chunk_sizes=chunk_sizes, rechunk_labels=rechunk_labels, ) - chunks = ( - ngio.NgffImage(zarr_url) - .labels.get_label(name="nuclei", path="0") - .on_disk_dask_array.chunks + chunk_sizes = list( + ngio.open_ome_zarr_container(zarr_url).get_label(name="nuclei", path="0").chunks ) - chunk_sizes = [c[0] for c in chunks] assert chunk_sizes == output_chunk_sizes @@ -102,8 +99,8 @@ def test_rechunk_no_overwrite_input(tmp_zenodo_zarr: list[str]): new_zarr_url = f"{zarr_url}_{suffix}" overwrite_input = False chunk_sizes = {"x": 1000, "y": 1000} - output_chunk_sizes = [1, 1, 1000, 1000] - original_chunk_sizes = [1, 1, 2160, 2560] + output_chunk_sizes = (1, 1, 1000, 1000) + original_chunk_sizes = (1, 1, 2160, 2560) output = rechunk_zarr( zarr_url=zarr_url, @@ -124,10 +121,8 @@ def test_rechunk_no_overwrite_input(tmp_zenodo_zarr: list[str]): # Existing zarr should be unchanged, but new zarr should have # expected chunking - chunks = ngio.NgffImage(zarr_url).get_image().on_disk_dask_array.chunks - chunk_sizes = [c[0] for c in chunks] + chunk_sizes = ngio.open_ome_zarr_container(zarr_url).get_image().chunks assert chunk_sizes == original_chunk_sizes - chunks = ngio.NgffImage(new_zarr_url).get_image().on_disk_dask_array.chunks - chunk_sizes = [c[0] for c in chunks] + chunk_sizes = ngio.open_ome_zarr_container(new_zarr_url).get_image().chunks assert chunk_sizes == output_chunk_sizes diff --git a/tests/test_valid_args_schemas.py b/tests/test_valid_args_schemas.py deleted file mode 100644 index c2ec92f..0000000 --- a/tests/test_valid_args_schemas.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -from pathlib import Path - -import pytest -from fractal_tasks_core.dev.lib_args_schemas import ( - create_schema_for_single_task, -) -from fractal_tasks_core.dev.lib_signature_constraints import ( - _extract_function, - _validate_function_signature, -) -from jsonschema.validators import ( - Draft7Validator, - Draft201909Validator, - Draft202012Validator, -) - -from . import TASK_LIST - - -def test_task_functions_have_valid_signatures(): - """ - Test that task functions have valid signatures. - """ - for task in TASK_LIST: - for key in ["executable_non_parallel", "executable_parallel"]: - executable = task.get(key) - if executable is None: - continue - function_name = Path(executable).with_suffix("").name - task_function = _extract_function( - executable, function_name, package_name="fractal_helper_tasks" - ) - _validate_function_signature(task_function) - - -def test_args_schemas_are_up_to_date(): - """ - Test that args_schema attributes in the manifest are up-to-date - """ - for task in TASK_LIST: - for kind in ["_non_parallel", "_parallel"]: - executable = task.get(f"executable_{kind}") - if executable is None: - continue - print(f"Now handling {executable}") - old_schema = task[f"args_schema_{kind}"] - new_schema = create_schema_for_single_task( - executable, package="fractal_helper_tasks" - ) - # The following step is required because some arguments may have a - # default which has a non-JSON type (e.g. a tuple), which we need - # to convert to JSON type (i.e. an array) before comparison. - new_schema = json.loads(json.dumps(new_schema)) - assert new_schema == old_schema - - -@pytest.mark.parametrize( - "jsonschema_validator", - [Draft7Validator, Draft201909Validator, Draft202012Validator], -) -def test_args_schema_comply_with_jsonschema_specs(jsonschema_validator): - """ - This test is actually useful, see - https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/564. - """ - for task in TASK_LIST: - for kind in ["_non_parallel", "_parallel"]: - executable = task.get(f"executable_{kind}") - if executable is None: - continue - print(f"Now handling {executable}") - schema = task[f"args_schema_{kind}"] - my_validator = jsonschema_validator(schema=schema) - my_validator.check_schema(my_validator.schema) - print( - f"Schema for task {task['executable']} is valid for " - f"{jsonschema_validator}." - ) diff --git a/tests/test_valid_manifest.py b/tests/test_valid_manifest.py deleted file mode 100644 index a894fdc..0000000 --- a/tests/test_valid_manifest.py +++ /dev/null @@ -1,26 +0,0 @@ -import json - -import requests -from jsonschema import validate - -from . import MANIFEST - - -def test_valid_manifest(tmp_path): - """ - NOTE: to avoid adding a fractal-server dependency, we simply download the - relevant file. - """ - # Download JSON Schema for ManifestV2 - url = ( - "https://raw.githubusercontent.com/fractal-analytics-platform/" - "fractal-server/main/" - "fractal_server/json_schemas/manifest_v2.json" - ) - r = requests.get(url) - with (tmp_path / "manifest_schema.json").open("wb") as f: - f.write(r.content) - with (tmp_path / "manifest_schema.json").open("r") as f: - manifest_schema = json.load(f) - - validate(instance=MANIFEST, schema=manifest_schema)