support zarrv3 (#985)

* changed pyproject to zarr>=3

* adjust dependencies

* wip

* more fixes

* update ome-zarr dep

* Add zarr v3 formats

* refactoring formats

* wip replace parse_url() with _open_zarr_store()

* update way of writing transforms, update typehints

* attempt fix consolidated metadata, fix channel names write

* fix partial read tests

* fix path

* access name in group directly

* fix read in consolidated metadata

* initial ugly fix groups and consolidated metadata when deleting

* fix reading back table

* open group without using consolidated metadata when writing

* revert adding labels

* fix read / write issues with consolidated metadata

* use_consolidated False when writing transforms

* add ome_format arg

* check valid element formats in container format

* uncomment code arrayNotFoundError

* remove future annotations import

* update workflow mac version

* drop python 3.10

version is not supported by zarr v3.0.8

* change target python version and readthedocs python version

* minor updates

* update Self import

* fix ome errors

* add windows workflow

* prevent consolidation labels group when deleting element

* add shapes test

* fix shape conversion

* update dask dependency because of zarr v3

* fix multipoly

* fix invalid read name test

* use UPath

* fix paths

* resolve paths

* refactor overwrite transformations

* refactor

* further refactor, add docstrings

* refactor io_raster

* several refactors io

* war on warnings

* checks backward compatibility

* correct test

* further reduce warnings

* remove log with no useful info

* remove log as it is stated in doc string

* move log to info in docstring

* remove deprecated warning

* get rid of categorical and str casting warnings

* below 1000 warnings

* further reduction of warnings

* remove deprecated code

* correct location for storing transforms

* consistent naming

* update according to ome-zarr-py

* correct docstring

* update docstring

* remove todo

* remove unassigned function call

* refactor new _open_zarr_store to _resolve_zarr_store

* silence zarr parquet warnings

* change overwriting warning, silence in tests

* silence overwriting warnings

* silence chunk warning

* remove argument from docstring, update typehint

* update typehint

* small fixes

* fail if not root does not exist

* write out function name

* small fixes

* initial replacement parse_url

* alter docstring

* adjust argument docstring

* remove overwrite warnings

* fix test

* replace parse_url

* change version

* remove type hints from docstrings

* refactor to one function

* change typehint

* remove typehints from docstring

* remove type hint return in docstring

* remove comment

* ensure comment added back

* fix channel metadata

* get rid of TableValidateMixin

* code fixes

* fix

* remove format without effect

* remove unnecessary catch warnings

* add todo

* fixes

* remove unnecessary .get('ome')

* add clarifying comment

* wip tests readwrite across formats

* return None instead of AnnData

* remove TODO

* remove invalid characters from test

* remove unused fixture and commented code

* almost completed extending readwrite tests to all container versions

* add OSError

* partial fix writing empty spatialdata

* fix type

* fix overwrite when no zarr store

* fix write element to empty directory location

* correct no zarr store write test

* remove unnecessary code

* fix write_element

* delete group isntead of .zattrs

* remove parse_url

* removed logger.ingo() and most of remaining warnings from tests

* addressing review comments

* addressed consolidate metadata comment

* make full coverage of _validate_can_safely_write_to_path() easier to understand

* restore partial read/write tests

* minor changes in test_partial_read(); code review finished

* support ome-zarr-py master

* fix docs

* ensure multiscales written correctly

* refactor read_zarr (#982)

* refactor read_zarr

* remove unneccesary checks

* Refactor zarrv3 (#986)

* refactor read_zarr

* remove unneccesary checks

* emit warning with old spatialdata storage version detected

* simplify code for writing (code review changes)

* finished addressing review

---------

Co-authored-by: Luca Marconato <[email protected]>

Author: melonora

.github/workflows/test.yaml

@@ -18,15 +18,17 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                python: ["3.10", "3.12"]
+                python: ["3.11", "3.13"]
                 os: [ubuntu-latest]
                 include:
                     - os: macos-latest
-                      python: "3.10"
+                      python: "3.11"
                     - os: macos-latest
                       python: "3.12"
                       pip-flags: "--pre"
                       name: "Python 3.12 (pre-release)"
+                    - os: windows-latest
+                      python: "3.11"
 
         env:
             OS: ${{ matrix.os }}

.mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.10
+python_version = 3.11
 
 ignore_errors = False
 warn_redundant_casts = True

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
     os: ubuntu-20.04
     tools:
-        python: "3.10"
+        python: "3.11"
 sphinx:
     configuration: docs/conf.py
     fail_on_warning: true

docs/api/data_formats.md

@@ -1,7 +1,7 @@
 # Data formats (advanced)
 
-The SpatialData format is defined as a set of versioned subclasses of `spatialdata._io.format.SpatialDataFormat`, one per type of element.
-These classes are useful to ensure backward compatibility whenever a major version change is introduced. We also provide pointers to the latest format.
+The SpatialData format is defined as a set of versioned subclasses of `ome_zarr.format.Format`, one per type of element. The `spatialdata.SpatialDataFormatType` is a union type encompassing the possible valid formats.
+These format subclasses are useful to ensure backward compatibility whenever a major version change is introduced. We also provide pointers to the latest format.
 
 ```{eval-rst}
 .. currentmodule:: spatialdata._io.format

docs/api/io.md

@@ -7,6 +7,5 @@ use any of the [spatialdata-io readers](https://spatialdata.scverse.org/projects
 .. currentmodule:: spatialdata
 
 .. autofunction:: read_zarr
-.. autofunction:: save_transformations
 .. autofunction:: get_dask_backing_files
 ```

pyproject.toml

@@ -15,7 +15,7 @@ maintainers = [
 urls.Documentation = "https://spatialdata.scverse.org/en/latest"
 urls.Source = "https://github.com/scverse/spatialdata.git"
 urls.Home-page = "https://github.com/scverse/spatialdata.git"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dynamic= [
   "version" # allow version to be set by git tags
 ]
@@ -25,15 +25,15 @@ dependencies = [
     "anndata>=0.9.1",
     "click",
     "dask-image",
-    "dask>=2024.4.1,<=2024.11.2",
+    "dask>=2024.10.0,<=2024.11.2",
     "datashader",
-    "fsspec",
+    "fsspec[s3,http]",
     "geopandas>=0.14",
     "multiscale_spatial_image>=2.0.3",
     "networkx",
     "numba>=0.55.0",
     "numpy",
-    "ome_zarr>=0.8.4",
+    "ome_zarr>=0.12.2",
     "pandas",
     "pooch",
     "pyarrow",
@@ -44,15 +44,17 @@ dependencies = [
     "scikit-image",
     "scipy",
     "typing_extensions>=4.8.0",
+    "universal_pathlib>=0.2.6",
     "xarray>=2024.10.0",
     "xarray-schema",
     "xarray-spatial>=0.3.5",
-    "zarr<3",
+    "zarr>=3.0.0",
 ]
 
 [project.optional-dependencies]
 dev = [
     "bump2version",
+    "sentry-prevent-cli",
 ]
 test = [
     "pytest",
@@ -97,7 +99,13 @@ xfail_strict = true
 addopts = [
 #    "-Werror",  # if 3rd party libs raise DeprecationWarnings, just use filterwarnings below
     "--import-mode=importlib",  # allow using test files with same name
-    "-s"  # print output from tests
+    "-s",  # print output from tests
+]
+# These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "gpu: run test on GPU using CuPY.",
+    "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
 ]
 # info on how to use this https://stackoverflow.com/questions/57925071/how-do-i-avoid-getting-deprecationwarning-from-inside-dependencies-with-pytest
 filterwarnings = [
@@ -131,7 +139,7 @@ exclude = [
 
 ]
 line-length = 120
-target-version = "py310"
+target-version = "py311"
 
 [tool.ruff.lint]
 ignore = [

src/spatialdata/__init__.py

@@ -42,10 +42,10 @@
     "SpatialData",
     "get_extent",
     "get_centroids",
+    "SpatialDataFormatType",
     "read_zarr",
     "unpad_raster",
     "get_pyramid_levels",
-    "save_transformations",
     "get_dask_backing_files",
     "are_extents_equal",
     "relabel_sequential",
@@ -79,7 +79,7 @@
 )
 from spatialdata._core.query.spatial_query import bounding_box_query, polygon_query
 from spatialdata._core.spatialdata import SpatialData
-from spatialdata._io._utils import get_dask_backing_files, save_transformations
-from spatialdata._io.format import SpatialDataFormat
+from spatialdata._io._utils import get_dask_backing_files
+from spatialdata._io.format import SpatialDataFormatType
 from spatialdata._io.io_zarr import read_zarr
 from spatialdata._utils import get_pyramid_levels, unpad_raster

src/spatialdata/_core/_deepcopy.py

@@ -46,7 +46,7 @@ def _(sdata: SpatialData) -> SpatialData:
     for _, element_name, element in sdata.gen_elements():
         elements_dict[element_name] = deepcopy(element)
     deepcopied_attrs = _deepcopy(sdata.attrs)
-    return SpatialData.from_elements_dict(elements_dict, attrs=deepcopied_attrs)
+    return SpatialData.init_from_elements(elements_dict, attrs=deepcopied_attrs)
 
 
 @deepcopy.register(DataArray)

src/spatialdata/_core/_elements.py

@@ -3,7 +3,6 @@
 from collections import UserDict
 from collections.abc import Iterable, KeysView, ValuesView
 from typing import TypeVar
-from warnings import warn
 
 from anndata import AnnData
 from dask.dataframe import DataFrame as DaskDataFrame
@@ -41,9 +40,7 @@ def _remove_shared_key(self, key: str) -> None:
     @staticmethod
     def _check_key(key: str, element_keys: Iterable[str], shared_keys: set[str | None]) -> None:
         check_valid_name(key)
-        if key in element_keys:
-            warn(f"Key `{key}` already exists. Overwriting it in-memory.", UserWarning, stacklevel=2)
-        else:
+        if key not in element_keys:
             try:
                 check_key_is_case_insensitively_unique(key, shared_keys)
             except ValueError as e:

src/spatialdata/_core/concatenate.py

@@ -68,8 +68,7 @@ def _concatenate_tables(
         TableModel.INSTANCE_KEY: instance_key,
     }
     merged_table.uns[TableModel.ATTRS_KEY] = attrs
-
-    return TableModel().validate(merged_table)
+    return TableModel().parse(merged_table)
 
 
 def concatenate(
@@ -252,6 +251,8 @@ def _fix_ensure_unique_element_names(
         tables_by_sdata.append(tables)
     sdatas_fixed = []
     for elements, tables in zip(elements_by_sdata, tables_by_sdata, strict=True):
-        sdata = SpatialData.init_from_elements(elements, tables=tables)
+        if tables is not None:
+            elements.update(tables)
+        sdata = SpatialData.init_from_elements(elements)
         sdatas_fixed.append(sdata)
     return sdatas_fixed