Skip to content

Commit ab8d952

Browse files
authored
Merge pull request #923 from fractal-analytics-platform/921-switch-to-fractal-task-tools
Switch to `fractal-task-tools` and deprecate current tools from `fractal-tasks-core`
2 parents 359d91d + e80cb19 commit ab8d952

36 files changed

+749
-165
lines changed

.github/workflows/ci_poetry.yml

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,7 @@ jobs:
9090
run: poetry install --with dev --without docs --no-interaction -E fractal-tasks
9191

9292
- name: Check if manifest has changed
93-
run: |
94-
poetry run python fractal_tasks_core/dev/create_manifest.py
95-
echo "*.json diff=json" >> .gitattributes && git config diff.json.textconv "jq --sort-keys '.' \$1"
96-
git diff ./fractal_tasks_core/__FRACTAL_MANIFEST__.json
97-
if [ -n "$(git diff --exit-code ./fractal_tasks_core/__FRACTAL_MANIFEST__.json)" ]; then
98-
echo "__FRACTAL_MANIFEST__.json has changed. Please run 'poetry run python fractal_tasks_core/dev/create_manifest.py' and commit the changes."
99-
exit 1
100-
else
101-
echo "__FRACTAL_MANIFEST__.json has not changed."
102-
fi
93+
run: poetry run fractal-manifest check --package fractal-tasks-core
10394

10495
- name: Cache Pooch folder
10596
id: cache-pooch-folder

CHANGELOG.md

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,22 @@
11
**Note**: Numbers like (\#123) point to closed Pull Requests on the fractal-tasks-core repository.
22

3+
4+
# 1.5.0 (a0 prerelease)
5+
6+
* Tasks:
7+
* Rely on `fractal-task-tools` for `run_fractal_task` wrapper (\#923).
8+
* Mark some existing tasks as _converter_ tasks(\#923).
9+
* Mark existing `run_fractal_task` wrapper as deprecated (\#923).
10+
* Manifest generation:
11+
* Rely on `fractal-task-tools` for manifest building (\#923).
12+
* Mark existing tools as deprecated (\#923).
13+
* Dependencies:
14+
* Add `fractal-task-tools==0.0.10` dependency (\#923).
15+
316
# 1.4.3
417

518
* Tasks:
6-
* Update apply_registration_to_image to handle race conditions better (\#919).
19+
* Update `apply_registration_to_image` to handle race conditions better (\#919).
720
* Testing
821
* Add `spherical-harmonics` extra to scmultiplex, for testing manifest build (\#899).
922
* Move manifest-creation CI to Python 3.11 (\#915).

examples/01_cardio_tiny_dataset/run_workflow.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262

6363
# Create zarr structure
6464
parallelization_list = cellvoyager_to_ome_zarr_init(
65-
zarr_urls=[],
6665
zarr_dir=zarr_dir,
6766
image_dirs=[img_path],
6867
allowed_channels=allowed_channels,

examples/02_cardio_tiny_mip/run_workflow.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767

6868
# Create zarr structure
6969
parallelization_list = cellvoyager_to_ome_zarr_init(
70-
zarr_urls=[],
7170
zarr_dir=zarr_dir,
7271
image_dirs=[img_path],
7372
allowed_channels=allowed_channels,

fractal_tasks_core/__FRACTAL_MANIFEST__.json

Lines changed: 10 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"3D"
1313
],
1414
"docs_info": "### Purpose\n- Converts **2D and 3D images from CellVoyager CV7000/8000** systems into OME-Zarr format, creating OME-Zarr HCS plates and combining all fields of view in a well into a single image.\n- Saves Fractal **region-of-interest (ROI) tables** for both individual fields of view and the entire well.\n- Handles overlapping fields of view by adjusting their positions to be non-overlapping while retaining the original position data as additional columns in the ROI tables.\n- Supports processing multiple plates in a single task.\n\n### Limitations\n- Currently, this task does not support time-resolved data and ignores the time fields in CellVoyager metadata.\n",
15+
"type": "converter_compound",
1516
"executable_non_parallel": "tasks/cellvoyager_to_ome_zarr_init.py",
1617
"executable_parallel": "tasks/cellvoyager_to_ome_zarr_compute.py",
1718
"meta_non_parallel": {
@@ -115,14 +116,6 @@
115116
},
116117
"additionalProperties": false,
117118
"properties": {
118-
"zarr_urls": {
119-
"items": {
120-
"type": "string"
121-
},
122-
"title": "Zarr Urls",
123-
"type": "array",
124-
"description": "List of paths or urls to the individual OME-Zarr image to be processed. Not used by the converter task. (standard argument for Fractal tasks, managed by Fractal server)."
125-
},
126119
"zarr_dir": {
127120
"title": "Zarr Dir",
128121
"type": "string",
@@ -191,7 +184,6 @@
191184
}
192185
},
193186
"required": [
194-
"zarr_urls",
195187
"zarr_dir",
196188
"image_dirs",
197189
"allowed_channels"
@@ -316,6 +308,7 @@
316308
"3D"
317309
],
318310
"docs_info": "### Purpose\n- Converts **multiplexed 2D and 3D images from CellVoyager CV7000/8000** systems into OME-Zarr format, storing each acquisition as a separate OME-Zarr image in the same OME-Zarr plate.\n- Creates **OME-Zarr HCS plates**, combining all fields of view for each acquisition in a well into a single image.\n- Saves Fractal **region-of-interest (ROI) tables** for both individual fields of view and the entire well.\n- Handles overlapping fields of view by adjusting their positions to be non-overlapping, while preserving the original position data as additional columns in the ROI tables.\n\n### Limitations\n- This task currently does not support time-resolved data and ignores the time fields in CellVoyager metadata.\n",
311+
"type": "converter_compound",
319312
"executable_non_parallel": "tasks/cellvoyager_to_ome_zarr_init_multiplex.py",
320313
"executable_parallel": "tasks/cellvoyager_to_ome_zarr_compute.py",
321314
"meta_non_parallel": {
@@ -443,14 +436,6 @@
443436
},
444437
"additionalProperties": false,
445438
"properties": {
446-
"zarr_urls": {
447-
"items": {
448-
"type": "string"
449-
},
450-
"title": "Zarr Urls",
451-
"type": "array",
452-
"description": "List of paths or urls to the individual OME-Zarr image to be processed. Not used by the converter task. (standard argument for Fractal tasks, managed by Fractal server)."
453-
},
454439
"zarr_dir": {
455440
"title": "Zarr Dir",
456441
"type": "string",
@@ -514,7 +499,6 @@
514499
}
515500
},
516501
"required": [
517-
"zarr_urls",
518502
"zarr_dir",
519503
"acquisitions"
520504
],
@@ -642,6 +626,7 @@
642626
"3D"
643627
],
644628
"docs_info": "### Purpose\n- Performs **Z-axis projection of intensity images** using a specified projection method.\n- **Generates a new OME-Zarr HCS plate** to store the projected data.\n\n### Limitations\n- Supports projections only for OME-Zarr HCS plates; other collections of OME-Zarrs are not yet supported.\n- Currently limited to data in the CZYX format.\n",
629+
"type": "compound",
645630
"executable_non_parallel": "tasks/copy_ome_zarr_hcs_plate.py",
646631
"executable_parallel": "tasks/projection.py",
647632
"meta_non_parallel": {
@@ -774,6 +759,7 @@
774759
"3D"
775760
],
776761
"docs_info": "### Purpose\n- **Corrects illumination** in OME-Zarr images using **pre-calculated flatfield profiles**.\n- Optionally performs **background subtraction**.\n\n### Limitations\n- Requires pre-calculated flatfield profiles in TIFF format.\n- Supports only fixed-value background subtraction; background subtraction profiles are not supported.\n",
762+
"type": "parallel",
777763
"executable_parallel": "tasks/illumination_correction.py",
778764
"meta_parallel": {
779765
"cpus_per_task": 1,
@@ -846,6 +832,7 @@
846832
"3D"
847833
],
848834
"docs_info": "### Purpose\n- **Segments images using Cellpose models**.\n- Supports both **built-in Cellpose models** (shipped with Cellpose) and **user-trained models**.\n- Accepts dual image input for segmentation.\n- Can process **arbitrary regions of interest (ROIs)**, including whole images, fields of view (FOVs), or masked outputs from prior segmentations, based on corresponding ROI tables.\n- Provides access to all advanced Cellpose parameters.\n- Allows custom rescaling options per channel, particularly useful for sparse images.\n\n### Limitations\n- Compatible only with Cellpose 2.x models; does not yet support 3.x models.\n",
835+
"type": "parallel",
849836
"executable_parallel": "tasks/cellpose_segmentation.py",
850837
"meta_parallel": {
851838
"cpus_per_task": 4,
@@ -1147,6 +1134,7 @@
11471134
"3D"
11481135
],
11491136
"docs_info": "### Purpose\n- **Computes image-based registration** transformations for acquisitions in **HCS** OME-Zarr datasets.\n- Processes images grouped by well, under the assumption that each well contains one image per acquisition.\n- Calculates transformations for **specified regions of interest (ROIs)** and stores the results in the corresponding ROI table.\n- Typically used as the first task in a workflow, followed by `Find Registration Consensus` and optionally `Apply Registration to Image`.\n\n### Limitations\n- Supports only HCS OME-Zarr datasets, leveraging their acquisition metadata and well-based image grouping.\n- Assumes each well contains a single image per acquisition.\n",
1137+
"type": "compound",
11501138
"executable_non_parallel": "tasks/image_based_registration_hcs_init.py",
11511139
"executable_parallel": "tasks/calculate_registration_image_based.py",
11521140
"meta_non_parallel": {
@@ -1285,6 +1273,7 @@
12851273
"3D"
12861274
],
12871275
"docs_info": "### Purpose\n- Determines the **consensus alignment** region across all selected acquisitions **within each well of an HCS OME-Zarr dataset**.\n- Generates a new ROI table for each image, defining consensus regions that are aligned across all acquisitions.\n- Typically used as the second task in a workflow, following `Calculate Registration (image-based)` and optionally preceding `Apply Registration to Image`.\n\n### Limitations\n- Supports only HCS OME-Zarr datasets, leveraging their acquisition metadata and well-based image grouping.\n",
1276+
"type": "compound",
12881277
"executable_non_parallel": "tasks/init_group_by_well_for_multiplexing.py",
12891278
"executable_parallel": "tasks/find_registration_consensus.py",
12901279
"meta_non_parallel": {
@@ -1394,6 +1383,7 @@
13941383
"3D"
13951384
],
13961385
"docs_info": "### Purpose\n- **Applies pre-calculated registration** transformations to images in an **HCS** OME-Zarr dataset, aligning all acquisitions to a specified reference acquisition.\n- **Masks regions not included** in the registered ROI table and aligns both intensity and label images.\n- Replaces the non-aligned image with the newly aligned image in the dataset if `overwrite input` is selected.\n- Typically used as the third task in a workflow, following `Calculate Registration (image-based)` and `Find Registration Consensus`.\n\n### Limitations\n- If `overwrite input` is selected, the non-aligned image is permanently deleted, which may impact workflows requiring access to the original images.\n",
1386+
"type": "parallel",
13971387
"executable_parallel": "tasks/apply_registration_to_image.py",
13981388
"meta_parallel": {
13991389
"cpus_per_task": 1,
@@ -1441,18 +1431,11 @@
14411431
"3D"
14421432
],
14431433
"docs_info": "### Purpose\n- Imports a **single OME-Zarr dataset** into the Fractal framework for further processing.\n- Supports importing either a **full OME-Zarr HCS plate** or an **individual OME-Zarr image**.\n- Ensures the OME-Zarr dataset is located in the `zarr_dir` specified by the dataset.\n- Generates the necessary **image list metadata** required for processing the OME-Zarr with Fractal.\n- Optionally **adds new ROI tables** to the existing OME-Zarr, enabling compatibility with many other tasks.\n\n### Limitations\n- Supports only OME-Zarr datasets already present in the `zarr_dir` of the corresponding dataset.\n- Assumes the input OME-Zarr is correctly structured and formatted for compatibility with the Fractal framework.\n",
1434+
"type": "converter_non_parallel",
14441435
"executable_non_parallel": "tasks/import_ome_zarr.py",
14451436
"args_schema_non_parallel": {
14461437
"additionalProperties": false,
14471438
"properties": {
1448-
"zarr_urls": {
1449-
"items": {
1450-
"type": "string"
1451-
},
1452-
"title": "Zarr Urls",
1453-
"type": "array",
1454-
"description": "List of paths or urls to the individual OME-Zarr image to be processed. Not used. (standard argument for Fractal tasks, managed by Fractal server)."
1455-
},
14561439
"zarr_dir": {
14571440
"title": "Zarr Dir",
14581441
"type": "string",
@@ -1501,7 +1484,6 @@
15011484
}
15021485
},
15031486
"required": [
1504-
"zarr_urls",
15051487
"zarr_dir",
15061488
"zarr_name"
15071489
],
@@ -1518,6 +1500,7 @@
15181500
"3D"
15191501
],
15201502
"docs_info": "### Purpose\n- Executes a **napari workflow** on the regions of interest (ROIs) within a single OME-NGFF image.\n- Processes specified images and labels as inputs to the workflow, producing outputs such as new labels and data tables.\n- Offers **flexibility in defining input and output** specifications to customize the workflow for specific datasets and analysis needs.\n\n### Limitations\n- Currently supports only Napari workflows that utilize functions from the `napari-segment-blobs-and-things-with-membranes` module. Other Napari-compatible modules are not supported.\n\n### Input Specifications\nNapari workflows require explicit definitions of input and output data.\nExample of valid `input_specs`:\n```json\n{\n \"in_1\": {\"type\": \"image\", \"channel\": {\"wavelength_id\": \"A01_C02\"}},\n \"in_2\": {\"type\": \"image\", \"channel\": {\"label\": \"DAPI\"}},\n \"in_3\": {\"type\": \"label\", \"label_name\": \"label_DAPI\"}\n}\n```\n\nExample of valid `output_specs`:\n```json\n{\n \"out_1\": {\"type\": \"label\", \"label_name\": \"label_DAPI_new\"},\n \"out_2\": {\"type\": \"dataframe\", \"table_name\": \"measurements\"},\n}\n```\n",
1503+
"type": "parallel",
15211504
"executable_parallel": "tasks/napari_workflows_wrapper.py",
15221505
"meta_parallel": {
15231506
"cpus_per_task": 8,

fractal_tasks_core/dev/create_manifest.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,22 @@
1818
from pathlib import Path
1919
from typing import Optional
2020

21+
from typing_extensions import deprecated
22+
23+
from fractal_tasks_core.dev.deprecation_message import DEPRECATION_MSG_FUN
2124
from fractal_tasks_core.dev.lib_args_schemas import (
2225
create_schema_for_single_task,
2326
)
2427
from fractal_tasks_core.dev.lib_task_docs import create_docs_info
2528
from fractal_tasks_core.dev.lib_task_docs import read_docs_info_from_file
2629

27-
2830
logging.basicConfig(level=logging.INFO)
2931

3032

3133
ARGS_SCHEMA_VERSION = "pydantic_v2"
3234

3335

36+
@deprecated(DEPRECATION_MSG_FUN)
3437
def create_manifest(
3538
package: str = "fractal_tasks_core",
3639
authors: Optional[str] = None,
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
DEPRECATION_MSG_FUN = (
2+
"This function is deprecated and will be removed in a future release of "
3+
"`fractal-tasks-core`. Please migrate to `fractal-task-tools` for "
4+
"manifest building and task wrappers. See "
5+
"https://fractal-analytics-platform.github.io/fractal-task-tools/usage/"
6+
"#migrate-from-fractal-tasks-core."
7+
)
8+
9+
10+
DEPRECATION_MSG_CLASS = (
11+
"This class is deprecated and will be removed in a future release of "
12+
"`fractal-tasks-core`. Please migrate to `fractal-task-tools` for "
13+
"manifest building and task wrappers. See "
14+
"https://fractal-analytics-platform.github.io/fractal-task-tools/usage/"
15+
"#migrate-from-fractal-tasks-core."
16+
)

fractal_tasks_core/dev/lib_args_schemas.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
from pydantic._internal import _generate_schema
2525
from pydantic._internal import _typing_extra
2626
from pydantic._internal._config import ConfigWrapper
27+
from typing_extensions import deprecated
2728

29+
from fractal_tasks_core.dev.deprecation_message import DEPRECATION_MSG_FUN
2830
from fractal_tasks_core.dev.lib_descriptions import (
2931
_get_class_attrs_descriptions,
3032
)
@@ -145,6 +147,7 @@ def _create_schema_for_function(function: Callable) -> _Schema:
145147
return json_schema
146148

147149

150+
@deprecated(DEPRECATION_MSG_FUN)
148151
def create_schema_for_single_task(
149152
executable: str,
150153
package: Optional[str] = "fractal_tasks_core",

fractal_tasks_core/dev/lib_pydantic_generatejsonschema.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,14 @@
1515
from pydantic.json_schema import GenerateJsonSchema
1616
from pydantic.json_schema import JsonSchemaValue
1717
from pydantic_core.core_schema import WithDefaultSchema
18+
from typing_extensions import deprecated
19+
20+
from fractal_tasks_core.dev.deprecation_message import DEPRECATION_MSG_CLASS
1821

1922
logger = logging.getLogger("CustomGenerateJsonSchema")
2023

2124

25+
@deprecated(DEPRECATION_MSG_CLASS)
2226
class CustomGenerateJsonSchema(GenerateJsonSchema):
2327
def get_flattened_anyof(
2428
self, schemas: list[JsonSchemaValue]

fractal_tasks_core/dev/task_list.py

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,41 @@
1111
"""
1212
Fractal task list.
1313
"""
14-
from fractal_tasks_core.dev.task_models import CompoundTask
15-
from fractal_tasks_core.dev.task_models import NonParallelTask
16-
from fractal_tasks_core.dev.task_models import ParallelTask
14+
from fractal_task_tools.task_models import CompoundTask
15+
from fractal_task_tools.task_models import ConverterCompoundTask
16+
from fractal_task_tools.task_models import ConverterNonParallelTask
17+
from fractal_task_tools.task_models import ParallelTask
18+
19+
AUTHORS = "Fractal Core Team"
20+
DOCS_LINK = "https://fractal-analytics-platform.github.io/fractal-tasks-core"
21+
INPUT_MODELS = [
22+
["fractal_tasks_core", "channels.py", "OmeroChannel"],
23+
["fractal_tasks_core", "channels.py", "Window"],
24+
["fractal_tasks_core", "channels.py", "ChannelInputModel"],
25+
["fractal_tasks_core", "tasks/io_models.py", "NapariWorkflowsInput"],
26+
["fractal_tasks_core", "tasks/io_models.py", "NapariWorkflowsOutput"],
27+
[
28+
"fractal_tasks_core",
29+
"tasks/cellpose_utils.py",
30+
"CellposeCustomNormalizer",
31+
],
32+
[
33+
"fractal_tasks_core",
34+
"tasks/cellpose_utils.py",
35+
"CellposeChannel1InputModel",
36+
],
37+
[
38+
"fractal_tasks_core",
39+
"tasks/cellpose_utils.py",
40+
"CellposeChannel2InputModel",
41+
],
42+
["fractal_tasks_core", "tasks/cellpose_utils.py", "CellposeModelParams"],
43+
["fractal_tasks_core", "tasks/io_models.py", "MultiplexingAcquisition"],
44+
]
45+
1746

1847
TASK_LIST = [
19-
CompoundTask(
48+
ConverterCompoundTask(
2049
name="Convert Cellvoyager to OME-Zarr",
2150
executable_init="tasks/cellvoyager_to_ome_zarr_init.py",
2251
executable="tasks/cellvoyager_to_ome_zarr_compute.py",
@@ -27,7 +56,7 @@
2756
tags=["Yokogawa", "Cellvoyager", "2D", "3D"],
2857
docs_info="file:task_info/convert_cellvoyager_to_ome_zarr.md",
2958
),
30-
CompoundTask(
59+
ConverterCompoundTask(
3160
name="Convert Cellvoyager Multiplexing to OME-Zarr",
3261
executable_init="tasks/cellvoyager_to_ome_zarr_init_multiplex.py",
3362
executable="tasks/cellvoyager_to_ome_zarr_compute.py",
@@ -108,7 +137,7 @@
108137
tags=["Multiplexing", "2D", "3D"],
109138
docs_info="file:task_info/apply_registration_to_image.md",
110139
),
111-
NonParallelTask(
140+
ConverterNonParallelTask(
112141
name="Import OME-Zarr",
113142
executable="tasks/import_ome_zarr.py",
114143
docs_info="file:task_info/import_ome_zarr.md",

0 commit comments

Comments
 (0)