From 6b3e4e1c604c05fc8e95b5bec026789f27b7ff1c Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 13:08:00 +0100 Subject: [PATCH 1/9] Don't warn when order passed for v2 arrays --- src/zarr/api/asynchronous.py | 2 -- tests/test_api.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index b262ced29b..f568735b46 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1010,8 +1010,6 @@ async def create( warnings.warn("object_codec is not yet implemented", RuntimeWarning, stacklevel=2) if read_only is not None: warnings.warn("read_only is not yet implemented", RuntimeWarning, stacklevel=2) - if order is not None: - _warn_order_kwarg() if write_empty_chunks is not None: _warn_write_empty_chunks_kwarg() diff --git a/tests/test_api.py b/tests/test_api.py index 8cd4ab6b60..54fc4c9ad0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1339,3 +1339,23 @@ def test_auto_chunks(f: Callable[..., Array]) -> None: a = f(**kwargs) assert a.chunks == (500, 500) + + +def test_order_warning() -> None: + # Passing order shouldn't warn for v2 + zarr.create( + (1,), + store={}, + order="F", + zarr_format=2, + ) + # Passing order should warn for v3 + with pytest.warns( + RuntimeWarning, match="The `order` keyword argument has no effect for Zarr format 3 arrays" + ): + zarr.create( + (1,), + store={}, + order="F", + zarr_format=3, + ) From e0f48cbe5038992d4a5f77167fe9f471a3b7b72e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 13:15:11 +0100 Subject: [PATCH 2/9] Clean up other instances of warning on order kwarg --- src/zarr/api/asynchronous.py | 3 --- src/zarr/core/array.py | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index f568735b46..cdbbbb16d2 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -27,7 +27,6 @@ MemoryOrder, ZarrFormat, _default_zarr_format, - _warn_order_kwarg, _warn_write_empty_chunks_kwarg, parse_dtype, ) @@ -1243,8 +1242,6 @@ async def open_array( zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - if "order" in kwargs: - _warn_order_kwarg() if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index b4e8ac0ff6..39e4463875 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4269,8 +4269,10 @@ async def init_array( if config is None: config = {} - if order is not None and isinstance(config, dict): - config["order"] = config.get("order", order) + if order is not None: + _warn_order_kwarg() + if isinstance(config, dict): + config["order"] = config.get("order", order) meta = AsyncArray._create_metadata_v3( shape=shape_parsed, From b8218aab2f444f5d38d3412854487f708f9eb540 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 13:16:25 +0100 Subject: [PATCH 3/9] Fix test --- tests/test_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 54fc4c9ad0..e8507b9172 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -355,8 +355,12 @@ def test_array_order(zarr_format: ZarrFormat) -> None: @pytest.mark.parametrize("order", ["C", "F"]) def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: - with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): + if zarr_format == 3: + with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): + arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) + else: arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) + assert arr.order == order vals = np.asarray(arr) From 97542866033fa354e072d4fc77efaa593a43fb53 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 13:56:22 +0100 Subject: [PATCH 4/9] Fix default memory order arguments --- src/zarr/core/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 86cc6a3c6b..577c2703d0 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -2380,7 +2380,7 @@ def create_array( compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = 0, - order: MemoryOrder | None = "C", + order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNames = None, @@ -2774,7 +2774,7 @@ def array( compressor: CompressorLike = None, serializer: SerializerLike = "auto", fill_value: Any | None = 0, - order: MemoryOrder | None = "C", + order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNames = None, From 4d5cfc62c7ba1ccc13810dbb5f832c21c4b70b76 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 13:56:37 +0100 Subject: [PATCH 5/9] Fix array test --- tests/test_array.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/test_array.py b/tests/test_array.py index 3fc7b3938c..c444bb952a 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -5,6 +5,7 @@ import pickle import re import sys +from contextlib import nullcontext from itertools import accumulate from typing import TYPE_CHECKING, Any, Literal from unittest import mock @@ -1425,14 +1426,23 @@ def test_order( # Test without passing config parameter config = None - arr = zarr.create_array( - store=store, - shape=(2, 2), - zarr_format=zarr_format, - dtype="i4", - order=order, - config=config, - ) + if zarr_format == 3 and order is not None: + ctx = pytest.warns( + RuntimeWarning, + match="The `order` keyword argument has no effect for Zarr format 3 arrays", + ) + else: + ctx = nullcontext() + + with ctx: + arr = zarr.create_array( + store=store, + shape=(2, 2), + zarr_format=zarr_format, + dtype="i4", + order=order, + config=config, + ) assert arr.order == expected if zarr_format == 2: assert arr.metadata.zarr_format == 2 From 4683f6472e4e5dbb5afd1dc2e893d9ea31c084c0 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 14:22:13 +0100 Subject: [PATCH 6/9] Add note to .order property --- src/zarr/core/array.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 39e4463875..d0e40b6dc2 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1041,6 +1041,14 @@ def order(self) -> MemoryOrder: ------- bool Memory order of the array + + Notes + ----- + For Zarr format 2 arrays this is the memory order in which + data is stored and returned when read in to a NumPy array. + + For Zarr format 3 arrays this is just the memory order + in which data is returned when read into a NumPy array. """ if self.metadata.zarr_format == 2: return self.metadata.order From 9f37bfec8d54802c713ded9f6c9033d46f2c8646 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 21 May 2025 14:23:31 +0100 Subject: [PATCH 7/9] Only copy array order for v2 arrays --- src/zarr/core/array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index d0e40b6dc2..8e935cdb99 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4531,7 +4531,7 @@ def _parse_keep_array_attr( serializer = "auto" if fill_value is None: fill_value = data.fill_value - if order is None: + if order is None and zarr_format == 2: order = data.order if chunk_key_encoding is None and zarr_format == data.metadata.zarr_format: if isinstance(data.metadata, ArrayV2Metadata): From 120aac43531c0764420da4fceca6261ccb4e52b4 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 31 May 2025 09:28:06 +0100 Subject: [PATCH 8/9] Move config warning next to code that ignores order --- src/zarr/api/asynchronous.py | 26 +++----------------- src/zarr/core/array.py | 46 ++++++++++++++++++++++++------------ tests/test_api.py | 33 +++++++++++++++----------- 3 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index cdbbbb16d2..9906b4af62 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -17,7 +17,6 @@ from_array, get_array_metadata, ) -from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams from zarr.core.buffer import NDArrayLike from zarr.core.common import ( JSON, @@ -45,6 +44,7 @@ from collections.abc import Iterable from zarr.abc.codec import Codec + from zarr.core.array_spec import ArrayConfigLike from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.storage import StoreLike @@ -1020,27 +1020,6 @@ async def create( mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - config_dict: ArrayConfigParams = {} - - if write_empty_chunks is not None: - if config is not None: - msg = ( - "Both write_empty_chunks and config keyword arguments are set. " - "This is redundant. When both are set, write_empty_chunks will be ignored and " - "config will be used." - ) - warnings.warn(UserWarning(msg), stacklevel=1) - config_dict["write_empty_chunks"] = write_empty_chunks - if order is not None and config is not None: - msg = ( - "Both order and config keyword arguments are set. " - "This is redundant. When both are set, order will be ignored and " - "config will be used." - ) - warnings.warn(UserWarning(msg), stacklevel=1) - - config_parsed = ArrayConfig.from_dict(config_dict) - return await AsyncArray._create( store_path, shape=shape, @@ -1057,8 +1036,9 @@ async def create( chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, + write_empty_chunks=write_empty_chunks, attributes=attributes, - config=config_parsed, + config=config, **kwargs, ) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 8e935cdb99..5e79280f5e 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -31,7 +31,7 @@ from zarr.abc.store import Store, set_or_delete from zarr.codecs._v2 import V2Codec from zarr.core._info import ArrayInfo -from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, parse_array_config +from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams, parse_array_config from zarr.core.attributes import Attributes from zarr.core.buffer import ( BufferPrototype, @@ -62,7 +62,6 @@ _warn_order_kwarg, concurrent_map, parse_dtype, - parse_order, parse_shapelike, product, ) @@ -564,6 +563,7 @@ async def _create( filters: list[dict[str, JSON]] | None = None, compressor: CompressorLike = "auto", # runtime + write_empty_chunks: bool | None = None, overwrite: bool = False, data: npt.ArrayLike | None = None, config: ArrayConfigLike | None = None, @@ -584,7 +584,34 @@ async def _create( _chunks = normalize_chunks(chunks, shape, dtype_parsed.itemsize) else: _chunks = normalize_chunks(chunk_shape, shape, dtype_parsed.itemsize) - config_parsed = parse_array_config(config) + + if config is not None: + config_parsed = parse_array_config(config) + if write_empty_chunks is not None: + msg = ( + "Both write_empty_chunks and config keyword arguments are set. " + "This is redundant. When both are set, write_empty_chunks will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) + if order is not None: + msg = ( + "Both order and config keyword arguments are set. " + "This is redundant. When both are set, order will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) + else: + config_dict: ArrayConfigParams = {} + if write_empty_chunks is not None: + config_dict["write_empty_chunks"] = write_empty_chunks + if order is not None: + if zarr_format == 3: + _warn_order_kwarg() + else: + config_dict["order"] = order + + config_parsed = ArrayConfig.from_dict(config_dict) result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] if zarr_format == 3: @@ -601,10 +628,6 @@ async def _create( "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead." ) - if order is not None: - _warn_order_kwarg() - config_parsed = replace(config_parsed, order=order) - result = await cls._create_v3( store_path, shape=shape, @@ -630,11 +653,6 @@ async def _create( if dimension_names is not None: raise ValueError("dimension_names cannot be used for arrays with zarr_format 2.") - if order is None: - order_parsed = parse_order(zarr_config.get("array.order")) - else: - order_parsed = order - result = await cls._create_v2( store_path, shape=shape, @@ -642,7 +660,6 @@ async def _create( chunks=_chunks, dimension_separator=dimension_separator, fill_value=fill_value, - order=order_parsed, config=config_parsed, filters=filters, compressor=compressor, @@ -788,7 +805,6 @@ async def _create_v2( shape: ChunkCoords, dtype: np.dtype[Any], chunks: ChunkCoords, - order: MemoryOrder, config: ArrayConfig, dimension_separator: Literal[".", "/"] | None = None, fill_value: float | None = None, @@ -820,7 +836,7 @@ async def _create_v2( shape=shape, dtype=dtype, chunks=chunks, - order=order, + order=config.order, dimension_separator=dimension_separator, fill_value=fill_value, filters=filters, diff --git a/tests/test_api.py b/tests/test_api.py index e8507b9172..5283d36c87 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -353,23 +353,28 @@ def test_array_order(zarr_format: ZarrFormat) -> None: raise AssertionError +@pytest.mark.parametrize("config_order", ["C", "F"]) @pytest.mark.parametrize("order", ["C", "F"]) -def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: - if zarr_format == 3: - with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): +def test_array_order_warns( + order: MemoryOrder, config_order: MemoryOrder, zarr_format: ZarrFormat +) -> None: + with zarr.config.set({"array.order": config_order}): + if zarr_format == 3: + with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): + arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) + # For zarr v3, order should be ignored and taken from config + assert arr.order == config_order + else: arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) - else: - arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) + assert arr.order == order - assert arr.order == order - - vals = np.asarray(arr) - if order == "C": - assert vals.flags.c_contiguous - elif order == "F": - assert vals.flags.f_contiguous - else: - raise AssertionError + vals = np.asarray(arr) + if arr.order == "C": + assert vals.flags.c_contiguous + elif arr.order == "F": + assert vals.flags.f_contiguous + else: + raise AssertionError # def test_lazy_loader(): From 33f1b8185ce570bb3b62ebe8f1ae3a9898617bc2 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 31 May 2025 09:32:29 +0100 Subject: [PATCH 9/9] Don't set order for v3 --- src/zarr/core/array.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 5e79280f5e..46961ad17a 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4263,6 +4263,7 @@ async def init_array( attributes=attributes, ) else: + # zarr_format == 3 array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( compressors=compressors, filters=filters, @@ -4295,8 +4296,6 @@ async def init_array( config = {} if order is not None: _warn_order_kwarg() - if isinstance(config, dict): - config["order"] = config.get("order", order) meta = AsyncArray._create_metadata_v3( shape=shape_parsed,