Merge remote-tracking branch 'upstream/v3' into tom/fix/open-fallback

Author: TomAugspurger

.pre-commit-config.yaml

@@ -49,3 +49,7 @@ repos:
     hooks:
       - id: rst-directive-colons
       - id: rst-inline-touching-normal
+  - repo: https://github.com/numpy/numpydoc
+    rev: v1.8.0
+    hooks:
+      - id: numpydoc-validation

pyproject.toml

@@ -319,3 +319,7 @@ ignore = [
 	"PC111",  # fix Python code in documentation - enable later
 	"PC180",  # for JavaScript - not interested
 ]
+
+[tool.numpydoc_validation]
+# See https://numpydoc.readthedocs.io/en/latest/validation.html#built-in-validation-checks for list of checks
+checks = ["GL06", "GL07", "GL10", "PR03", "PR05", "PR06"]

src/zarr/abc/codec.py

@@ -20,11 +20,11 @@
     from zarr.core.indexing import SelectorTuple
 
 __all__ = [
-    "BaseCodec",
     "ArrayArrayCodec",
     "ArrayBytesCodec",
     "ArrayBytesCodecPartialDecodeMixin",
     "ArrayBytesCodecPartialEncodeMixin",
+    "BaseCodec",
     "BytesBytesCodec",
     "CodecInput",
     "CodecOutput",

src/zarr/abc/store.py

@@ -43,7 +43,7 @@ class Store(ABC):
     _mode: AccessMode
     _is_open: bool
 
-    def __init__(self, mode: AccessModeLiteral = "r", *args: Any, **kwargs: Any) -> None:
+    def __init__(self, *args: Any, mode: AccessModeLiteral = "r", **kwargs: Any) -> None:
         self._is_open = False
         self._mode = AccessMode.from_literal(mode)
 
@@ -69,13 +69,10 @@ def __exit__(
     async def _open(self) -> None:
         if self._is_open:
             raise ValueError("store is already open")
-        if not await self.empty():
-            if self.mode.update or self.mode.readonly:
-                pass
-            elif self.mode.overwrite:
-                await self.clear()
-            else:
-                raise FileExistsError("Store already exists")
+        if self.mode.str == "w":
+            await self.clear()
+        elif self.mode.str == "w-" and not await self.empty():
+            raise FileExistsError("Store already exists")
         self._is_open = True
 
     async def _ensure_open(self) -> None:

src/zarr/api/asynchronous.py

@@ -160,7 +160,7 @@ async def load(
 
     Parameters
     ----------
-    store : Store or string
+    store : Store or str
         Store or path to directory in file system or name of zip file.
     path : str or None, optional
         The path within the store from which to load.
@@ -204,7 +204,7 @@ async def open(
 
     Parameters
     ----------
-    store : Store or string, optional
+    store : Store or str, optional
         Store or path to directory in file system or name of zip file.
     mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
         Persistence mode: 'r' means read only (must exist); 'r+' means
@@ -271,7 +271,7 @@ async def save(
 
     Parameters
     ----------
-    store : Store or string
+    store : Store or str
         Store or path to directory in file system or name of zip file.
     args : ndarray
         NumPy arrays with data to save.
@@ -307,7 +307,7 @@ async def save_array(
 
     Parameters
     ----------
-    store : Store or string
+    store : Store or str
         Store or path to directory in file system or name of zip file.
     arr : ndarray
         NumPy array with data to save.
@@ -355,7 +355,7 @@ async def save_group(
 
     Parameters
     ----------
-    store : Store or string
+    store : Store or str
         Store or path to directory in file system or name of zip file.
     args : ndarray
         NumPy arrays with data to save.
@@ -471,7 +471,7 @@ async def group(
 
     Parameters
     ----------
-    store : Store or string, optional
+    store : Store or str, optional
         Store or path to directory in file system.
     overwrite : bool, optional
         If True, delete any pre-existing data in `store` at `path` before
@@ -485,7 +485,7 @@ async def group(
         to all attribute read operations.
     synchronizer : object, optional
         Array synchronizer.
-    path : string, optional
+    path : str, optional
         Group path within store.
     meta_array : array-like, optional
         An array instance to use for determining arrays to create and return
@@ -551,7 +551,7 @@ async def open_group(
 
     Parameters
     ----------
-    store : Store, string, or mapping, optional
+    store : Store, str, or mapping, optional
         Store or path to directory in file system or name of zip file.
 
         Strings are interpreted as paths on the local file system
@@ -574,9 +574,9 @@ async def open_group(
         to all attribute read operations.
     synchronizer : object, optional
         Array synchronizer.
-    path : string, optional
+    path : str, optional
         Group path within store.
-    chunk_store : Store or string, optional
+    chunk_store : Store or str, optional
         Store or path to directory in file system or name of zip file.
     storage_options : dict
         If using an fsspec URL to create the store, these will be passed to
@@ -670,22 +670,22 @@ async def create(
         False, will be set to `shape`, i.e., single chunk for the whole array.
         If an int, the chunk size in each dimension will be given by the value
         of `chunks`. Default is True.
-    dtype : string or dtype, optional
+    dtype : str or dtype, optional
         NumPy dtype.
     compressor : Codec, optional
         Primary compressor.
     fill_value : object
         Default value to use for uninitialized portions of the array.
     order : {'C', 'F'}, optional
         Memory layout to be used within each chunk.
-    store : Store or string
+    store : Store or str
         Store or path to directory in file system or name of zip file.
     synchronizer : object, optional
         Array synchronizer.
     overwrite : bool, optional
         If True, delete all pre-existing data in `store` at `path` before
         creating the array.
-    path : string, optional
+    path : str, optional
         Path under which array is stored.
     chunk_store : MutableMapping, optional
         Separate storage for chunks. If not provided, `store` will be used
@@ -943,11 +943,11 @@ async def open_array(
 
     Parameters
     ----------
-    store : Store or string
+    store : Store or str
         Store or path to directory in file system or name of zip file.
     zarr_format : {2, 3, None}, optional
         The zarr format to use when saving.
-    path : string, optional
+    path : str, optional
         Path in store to array.
     storage_options : dict
         If using an fsspec URL to create the store, these will be passed to

src/zarr/codecs/__init__.py

@@ -28,8 +28,8 @@
     "ShardingCodec",
     "ShardingCodecIndexLocation",
     "TransposeCodec",
-    "VLenUTF8Codec",
     "VLenBytesCodec",
+    "VLenUTF8Codec",
     "ZstdCodec",
 ]
 

src/zarr/codecs/pipeline.py

@@ -17,6 +17,7 @@
 from zarr.core.common import ChunkCoords, concurrent_map
 from zarr.core.config import config
 from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice
+from zarr.core.metadata.v2 import _default_fill_value
 from zarr.registry import register_pipeline
 
 if TYPE_CHECKING:
@@ -247,7 +248,17 @@ async def read_batch(
                 if chunk_array is not None:
                     out[out_selection] = chunk_array
                 else:
-                    out[out_selection] = chunk_spec.fill_value
+                    fill_value = chunk_spec.fill_value
+
+                    if fill_value is None:
+                        # Zarr V2 allowed `fill_value` to be null in the metadata.
+                        # Zarr V3 requires it to be set. This has already been
+                        # validated when decoding the metadata, but we support reading
+                        # Zarr V2 data and need to support the case where fill_value
+                        # is None.
+                        fill_value = _default_fill_value(dtype=chunk_spec.dtype)
+
+                    out[out_selection] = fill_value
         else:
             chunk_bytes_batch = await concurrent_map(
                 [
@@ -274,7 +285,10 @@ async def read_batch(
                         tmp = tmp.squeeze(axis=drop_axes)
                     out[out_selection] = tmp
                 else:
-                    out[out_selection] = chunk_spec.fill_value
+                    fill_value = chunk_spec.fill_value
+                    if fill_value is None:
+                        fill_value = _default_fill_value(dtype=chunk_spec.dtype)
+                    out[out_selection] = fill_value
 
     def _merge_chunk_array(
         self,

src/zarr/core/array.py

@@ -380,7 +380,7 @@ async def _create_v2(
             chunks=chunks,
             order=order,
             dimension_separator=dimension_separator,
-            fill_value=0 if fill_value is None else fill_value,
+            fill_value=fill_value,
             compressor=compressor,
             filters=filters,
             attributes=attributes,
@@ -1288,11 +1288,11 @@ def get_basic_selection(
             array. May be any combination of int and/or slice or ellipsis for multidimensional arrays.
         out : NDBuffer, optional
             If given, load the selected data directly into this buffer.
+        prototype : BufferPrototype, optional
+            The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used.
         fields : str or sequence of str, optional
             For arrays with a structured dtype, one or more fields can be specified to
             extract data for.
-        prototype : BufferPrototype, optional
-            The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used.
 
         Returns
         -------
@@ -2284,6 +2284,17 @@ def resize(self, new_shape: ChunkCoords) -> Array:
         This method does not modify the original Array object. Instead, it returns a new Array
         with the specified shape.
 
+        Notes
+        -----
+        When resizing an array, the data are not rearranged in any way.
+
+        If one or more dimensions are shrunk, any chunks falling outside the
+        new array shape will be deleted from the underlying store.
+        However, it is noteworthy that the chunks partially falling inside the new array
+        (i.e. boundary chunks) will remain intact, and therefore,
+        the data falling outside the new array but inside the boundary chunks
+        would be restored by a subsequent resize operation that grows the array size.
+
         Examples
         --------
         >>> import zarr
@@ -2301,17 +2312,6 @@ def resize(self, new_shape: ChunkCoords) -> Array:
         (20000, 1000)
         >>> z2.shape
         (50, 50)
-
-        Notes
-        -----
-        When resizing an array, the data are not rearranged in any way.
-
-        If one or more dimensions are shrunk, any chunks falling outside the
-        new array shape will be deleted from the underlying store.
-        However, it is noteworthy that the chunks partially falling inside the new array
-        (i.e. boundary chunks) will remain intact, and therefore,
-        the data falling outside the new array but inside the boundary chunks
-        would be restored by a subsequent resize operation that grows the array size.
         """
         return type(self)(
             sync(

src/zarr/core/group.py

@@ -299,7 +299,7 @@ async def get(
 
         Parameters
         ----------
-        key : string
+        key : str
             Group member name.
         default : object
             Default value to return if key is not found (default: None).
@@ -396,7 +396,7 @@ async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup:
 
         Parameters
         ----------
-        name : string
+        name : str
             Group name.
         overwrite : bool, optional
             Overwrite any existing group with given `name` if present.
@@ -525,7 +525,7 @@ async def create_dataset(self, name: str, **kwargs: Any) -> AsyncArray:
 
         Parameters
         ----------
-        name : string
+        name : str
             Array name.
         kwargs : dict
             Additional arguments passed to :func:`zarr.AsyncGroup.create_array`.
@@ -558,11 +558,11 @@ async def require_dataset(
 
         Parameters
         ----------
-        name : string
+        name : str
             Array name.
         shape : int or tuple of ints
             Array shape.
-        dtype : string or dtype, optional
+        dtype : str or dtype, optional
             NumPy dtype.
         exact : bool, optional
             If True, require `dtype` to match exactly. If false, require
@@ -592,11 +592,11 @@ async def require_array(
 
         Parameters
         ----------
-        name : string
+        name : str
             Array name.
         shape : int or tuple of ints
             Array shape.
-        dtype : string or dtype, optional
+        dtype : str or dtype, optional
             NumPy dtype.
         exact : bool, optional
             If True, require `dtype` to match exactly. If false, require
@@ -857,7 +857,7 @@ def get(self, path: str, default: DefaultT | None = None) -> Array | Group | Def
 
         Parameters
         ----------
-        key : string
+        key : str
             Group member name.
         default : object
             Default value to return if key is not found (default: None).
@@ -1003,7 +1003,7 @@ def require_group(self, name: str, **kwargs: Any) -> Group:
 
         Parameters
         ----------
-        name : string
+        name : str
             Group name.
         overwrite : bool, optional
             Overwrite any existing group with given `name` if present.
@@ -1125,7 +1125,7 @@ def create_dataset(self, name: str, **kwargs: Any) -> Array:
 
         Parameters
         ----------
-        name : string
+        name : str
             Array name.
         kwargs : dict
             Additional arguments passed to :func:`zarr.Group.create_array`
@@ -1150,11 +1150,11 @@ def require_dataset(self, name: str, **kwargs: Any) -> Array:
 
         Parameters
         ----------
-        name : string
+        name : str
             Array name.
         shape : int or tuple of ints
             Array shape.
-        dtype : string or dtype, optional
+        dtype : str or dtype, optional
             NumPy dtype.
         exact : bool, optional
             If True, require `dtype` to match exactly. If false, require
@@ -1177,11 +1177,11 @@ def require_array(self, name: str, **kwargs: Any) -> Array:
 
         Parameters
         ----------
-        name : string
+        name : str
             Array name.
         shape : int or tuple of ints
             Array shape.
-        dtype : string or dtype, optional
+        dtype : str or dtype, optional
             NumPy dtype.
         exact : bool, optional
             If True, require `dtype` to match exactly. If false, require