Merge remote-tracking branch 'upstream/main' into tom/feature/object-size

Author: TomAugspurger

.github/workflows/test.yml

@@ -33,15 +33,14 @@ jobs:
             numpy-version: '2.1'
             dependency-set: 'optional'
             os: 'macos-latest'
-          # https://github.com/zarr-developers/zarr-python/issues/2438
-          # - python-version: '3.11'
-          #   numpy-version: '1.25'
-          #   dependency-set: 'optional'
-          #   os: 'windows-latest'
-          # - python-version: '3.13'
-          #   numpy-version: '2.1'
-          #   dependency-set: 'optional'
-          #   os: 'windows-latest'
+          - python-version: '3.11'
+            numpy-version: '1.25'
+            dependency-set: 'optional'
+            os: 'windows-latest'
+          - python-version: '3.13'
+            numpy-version: '2.1'
+            dependency-set: 'optional'
+            os: 'windows-latest'
     runs-on: ${{ matrix.os }}
 
     steps:

src/zarr/abc/store.py

@@ -336,17 +336,16 @@ def supports_listing(self) -> bool:
         ...
 
     @abstractmethod
-    def list(self) -> AsyncGenerator[str, None]:
+    def list(self) -> AsyncGenerator[str]:
         """Retrieve all keys in the store.
 
         Returns
         -------
         AsyncGenerator[str, None]
         """
-        ...
 
     @abstractmethod
-    def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         """
         Retrieve all keys in the store that begin with a given prefix. Keys are returned relative
         to the root of the store.
@@ -359,10 +358,9 @@ def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
         -------
         AsyncGenerator[str, None]
         """
-        ...
 
     @abstractmethod
-    def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         """
         Retrieve all keys and prefixes with a given prefix and which do not contain the character
         “/” after the given prefix.
@@ -375,7 +373,6 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
         -------
         AsyncGenerator[str, None]
         """
-        ...
 
     async def delete_dir(self, prefix: str) -> None:
         """

src/zarr/core/group.py

@@ -967,7 +967,7 @@ async def create_array(
 
     @deprecated("Use AsyncGroup.create_array instead.")
     async def create_dataset(
-        self, name: str, **kwargs: Any
+        self, name: str, *, shape: ShapeLike, **kwargs: Any
     ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
         """Create an array.
 
@@ -988,7 +988,7 @@ async def create_dataset(
         .. deprecated:: 3.0.0
             The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.create_array` instead.
         """
-        return await self.create_array(name, **kwargs)
+        return await self.create_array(name, shape=shape, **kwargs)
 
     @deprecated("Use AsyncGroup.require_array instead.")
     async def require_dataset(
@@ -1666,7 +1666,7 @@ def create_dataset(self, name: str, **kwargs: Any) -> Array:
         return Array(self._sync(self._async_group.create_dataset(name, **kwargs)))
 
     @deprecated("Use Group.require_array instead.")
-    def require_dataset(self, name: str, **kwargs: Any) -> Array:
+    def require_dataset(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array:
         """Obtain an array, creating if it doesn't exist.
 
         Arrays are known as "datasets" in HDF5 terminology. For compatibility
@@ -1688,9 +1688,9 @@ def require_dataset(self, name: str, **kwargs: Any) -> Array:
         .. deprecated:: 3.0.0
             The h5py compatibility methods will be removed in 3.1.0. Use `Group.require_array` instead.
         """
-        return Array(self._sync(self._async_group.require_array(name, **kwargs)))
+        return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs)))
 
-    def require_array(self, name: str, **kwargs: Any) -> Array:
+    def require_array(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array:
         """Obtain an array, creating if it doesn't exist.
 
 
@@ -1707,7 +1707,7 @@ def require_array(self, name: str, **kwargs: Any) -> Array:
         -------
         a : Array
         """
-        return Array(self._sync(self._async_group.require_array(name, **kwargs)))
+        return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs)))
 
     @_deprecate_positional_args
     def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array:

src/zarr/storage/local.py

@@ -133,7 +133,7 @@ def with_mode(self, mode: AccessModeLiteral) -> Self:
         return type(self)(root=self.root, mode=mode)
 
     def __str__(self) -> str:
-        return f"file://{self.root}"
+        return f"file://{self.root.as_posix()}"
 
     def __repr__(self) -> str:
         return f"LocalStore({str(self)!r})"
@@ -220,30 +220,28 @@ async def exists(self, key: str) -> bool:
         path = self.root / key
         return await asyncio.to_thread(path.is_file)
 
-    async def list(self) -> AsyncGenerator[str, None]:
+    async def list(self) -> AsyncGenerator[str]:
         # docstring inherited
         to_strip = self.root.as_posix() + "/"
         for p in list(self.root.rglob("*")):
             if p.is_file():
                 yield p.as_posix().replace(to_strip, "")
 
-    async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         to_strip = self.root.as_posix() + "/"
         prefix = prefix.rstrip("/")
         for p in (self.root / prefix).rglob("*"):
             if p.is_file():
                 yield p.as_posix().replace(to_strip, "")
 
-    async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         base = self.root / prefix
-        to_strip = str(base) + "/"
-
         try:
             key_iter = base.iterdir()
             for key in key_iter:
-                yield key.as_posix().replace(to_strip, "")
+                yield key.relative_to(base).as_posix()
         except (FileNotFoundError, NotADirectoryError):
             pass
 

src/zarr/storage/logging.py

@@ -55,7 +55,7 @@ def _configure_logger(
         self, log_level: str = "DEBUG", log_handler: logging.Handler | None = None
     ) -> None:
         self.log_level = log_level
-        self.logger = logging.getLogger(f"LoggingStore({self._store!s})")
+        self.logger = logging.getLogger(f"LoggingStore({self._store})")
         self.logger.setLevel(log_level)
 
         if not self.logger.hasHandlers():
@@ -147,7 +147,7 @@ async def clear(self) -> None:
             return await self._store.clear()
 
     def __str__(self) -> str:
-        return f"logging-{self._store!s}"
+        return f"logging-{self._store}"
 
     def __repr__(self) -> str:
         return f"LoggingStore({repr(self._store)!r})"
@@ -204,19 +204,19 @@ async def set_partial_values(
         with self.log(keys):
             return await self._store.set_partial_values(key_start_values=key_start_values)
 
-    async def list(self) -> AsyncGenerator[str, None]:
+    async def list(self) -> AsyncGenerator[str]:
         # docstring inherited
         with self.log():
             async for key in self._store.list():
                 yield key
 
-    async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         with self.log(prefix):
             async for key in self._store.list_prefix(prefix=prefix):
                 yield key
 
-    async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         with self.log(prefix):
             async for key in self._store.list_dir(prefix=prefix):

src/zarr/storage/memory.py

@@ -150,19 +150,19 @@ async def set_partial_values(self, key_start_values: Iterable[tuple[str, int, by
         # docstring inherited
         raise NotImplementedError
 
-    async def list(self) -> AsyncGenerator[str, None]:
+    async def list(self) -> AsyncGenerator[str]:
         # docstring inherited
         for key in self._store_dict:
             yield key
 
-    async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         # note: we materialize all dict keys into a list here so we can mutate the dict in-place (e.g. in delete_prefix)
         for key in list(self._store_dict):
             if key.startswith(prefix):
                 yield key
 
-    async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         prefix = prefix.rstrip("/")
 

src/zarr/storage/remote.py

@@ -325,13 +325,13 @@ async def set_partial_values(
         # docstring inherited
         raise NotImplementedError
 
-    async def list(self) -> AsyncGenerator[str, None]:
+    async def list(self) -> AsyncGenerator[str]:
         # docstring inherited
         allfiles = await self.fs._find(self.path, detail=False, withdirs=False)
         for onefile in (a.replace(self.path + "/", "") for a in allfiles):
             yield onefile
 
-    async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         prefix = f"{self.path}/{prefix.rstrip('/')}"
         try:
@@ -341,7 +341,7 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
         for onefile in (a.replace(prefix + "/", "") for a in allfiles):
             yield onefile.removeprefix(self.path).removeprefix("/")
 
-    async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         for onefile in await self.fs._find(
             f"{self.path}/{prefix}", detail=False, maxdepth=None, withdirs=False

src/zarr/storage/zip.py

@@ -237,19 +237,19 @@ async def exists(self, key: str) -> bool:
             else:
                 return True
 
-    async def list(self) -> AsyncGenerator[str, None]:
+    async def list(self) -> AsyncGenerator[str]:
         # docstring inherited
         with self._lock:
             for key in self._zf.namelist():
                 yield key
 
-    async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         async for key in self.list():
             if key.startswith(prefix):
                 yield key
 
-    async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         # docstring inherited
         prefix = prefix.rstrip("/")
 

tests/test_store/test_local.py

@@ -31,7 +31,7 @@ def store_kwargs(self, tmpdir) -> dict[str, str]:
         return {"root": str(tmpdir), "mode": "r+"}
 
     def test_store_repr(self, store: LocalStore) -> None:
-        assert str(store) == f"file://{store.root!s}"
+        assert str(store) == f"file://{store.root.as_posix()}"
 
     def test_store_supports_writes(self, store: LocalStore) -> None:
         assert store.supports_writes

tests/test_store/test_zip.py

@@ -53,7 +53,7 @@ async def test_not_writable_store_raises(self, store_kwargs: dict[str, Any]) ->
             await store.set("foo", cpu.Buffer.from_bytes(b"bar"))
 
     def test_store_repr(self, store: ZipStore) -> None:
-        assert str(store) == f"zip://{store.path!s}"
+        assert str(store) == f"zip://{store.path}"
 
     def test_store_supports_writes(self, store: ZipStore) -> None:
         assert store.supports_writes