Merge remote-tracking branch 'upstream/main'

Author: brokkoli71

.pre-commit-config.yaml

@@ -37,6 +37,7 @@ repos:
           - obstore>=0.5.1
           # Tests
           - pytest
+          - hypothesis
   - repo: https://github.com/scientific-python/cookie
     rev: 2025.01.22
     hooks:

changes/2962.fix.rst

@@ -0,0 +1 @@
+Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels.

changes/2978.bugfix.rst

@@ -0,0 +1 @@
+Fixed sharding with GPU buffers.

changes/3045.bugfix.rst

@@ -0,0 +1 @@
+Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`.

docs/user-guide/arrays.rst

@@ -209,8 +209,8 @@ prints additional diagnostics, e.g.::
    Serializer         : BytesCodec(endian=<Endian.little: 'little'>)
    Compressors        : (BloscCodec(typesize=4, cname=<BloscCname.zstd: 'zstd'>, clevel=3, shuffle=<BloscShuffle.bitshuffle: 'bitshuffle'>, blocksize=0),)
    No. bytes          : 400000000 (381.5M)
-   No. bytes stored   : 9696520
-   Storage ratio      : 41.3
+   No. bytes stored   : 3558573
+   Storage ratio      : 112.4
    Chunks Initialized : 100
 
 .. note::

pyproject.toml

@@ -83,7 +83,7 @@ test = [
 remote_tests = [
     'zarr[remote]',
     "botocore",
-    "s3fs",
+    "s3fs>=2023.10.0",
     "moto[s3,server]",
     "requests",
 ]
@@ -104,7 +104,7 @@ docs = [
     # Optional dependencies to run examples
     'numcodecs[msgpack]',
     'rich',
-    's3fs',
+    's3fs>=2023.10.0',
     'astroid<4'
 ]
 
@@ -348,27 +348,27 @@ python_version = "3.11"
 ignore_missing_imports = true
 namespace_packages = false
 
-
 strict = true
 warn_unreachable = true
-
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
 
+
 [[tool.mypy.overrides]]
 module = [
-    "zarr.v2.*",
+    "tests.package_with_entrypoint.*",
+    "zarr.testing.stateful",
+    "tests.test_codecs.test_transpose",
+    "tests.test_config"
 ]
-ignore_errors = true
+strict = false
 
+# TODO: Move the next modules up to the strict = false section
+# and fix the errors
 [[tool.mypy.overrides]]
 module = [
-    "zarr.testing.stateful",  # lots of hypothesis decorator errors
-    "tests.package_with_entrypoint.*",
     "tests.test_codecs.test_codecs",
-    "tests.test_codecs.test_transpose",
     "tests.test_metadata.*",
     "tests.test_store.*",
-    "tests.test_config",
     "tests.test_group",
     "tests.test_indexing",
     "tests.test_properties",

src/zarr/api/asynchronous.py

@@ -16,6 +16,7 @@
     JSON,
     AccessModeLiteral,
     ChunkCoords,
+    DimensionNames,
     MemoryOrder,
     ZarrFormat,
     _default_zarr_format,
@@ -865,7 +866,7 @@ async def create(
         | None
     ) = None,
     codecs: Iterable[Codec | dict[str, JSON]] | None = None,
-    dimension_names: Iterable[str] | None = None,
+    dimension_names: DimensionNames = None,
     storage_options: dict[str, Any] | None = None,
     config: ArrayConfigLike | None = None,
     **kwargs: Any,

src/zarr/api/synchronous.py

@@ -33,6 +33,7 @@
         JSON,
         AccessModeLiteral,
         ChunkCoords,
+        DimensionNames,
         MemoryOrder,
         ShapeLike,
         ZarrFormat,
@@ -626,7 +627,7 @@ def create(
         | None
     ) = None,
     codecs: Iterable[Codec | dict[str, JSON]] | None = None,
-    dimension_names: Iterable[str] | None = None,
+    dimension_names: DimensionNames = None,
     storage_options: dict[str, Any] | None = None,
     config: ArrayConfigLike | None = None,
     **kwargs: Any,
@@ -761,7 +762,7 @@ def create_array(
     zarr_format: ZarrFormat | None = 3,
     attributes: dict[str, JSON] | None = None,
     chunk_key_encoding: ChunkKeyEncodingLike | None = None,
-    dimension_names: Iterable[str] | None = None,
+    dimension_names: DimensionNames = None,
     storage_options: dict[str, Any] | None = None,
     overwrite: bool = False,
     config: ArrayConfigLike | None = None,
@@ -926,7 +927,7 @@ def from_array(
     zarr_format: ZarrFormat | None = None,
     attributes: dict[str, JSON] | None = None,
     chunk_key_encoding: ChunkKeyEncodingLike | None = None,
-    dimension_names: Iterable[str] | None = None,
+    dimension_names: DimensionNames = None,
     storage_options: dict[str, Any] | None = None,
     overwrite: bool = False,
     config: ArrayConfigLike | None = None,

src/zarr/codecs/blosc.py

@@ -8,6 +8,7 @@
 
 import numcodecs
 from numcodecs.blosc import Blosc
+from packaging.version import Version
 
 from zarr.abc.codec import BytesBytesCodec
 from zarr.core.buffer.cpu import as_numpy_array_wrapper
@@ -163,6 +164,9 @@ def _blosc_codec(self) -> Blosc:
             "shuffle": map_shuffle_str_to_int[self.shuffle],
             "blocksize": self.blocksize,
         }
+        # See https://github.com/zarr-developers/numcodecs/pull/713
+        if Version(numcodecs.__version__) >= Version("0.16.0"):
+            config_dict["typesize"] = self.typesize
         return Blosc.from_config(config_dict)
 
     async def _decode_single(

src/zarr/codecs/sharding.py

@@ -683,7 +683,7 @@ def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec:
             config=ArrayConfig(
                 order="C", write_empty_chunks=False
             ),  # Note: this is hard-coded for simplicity -- it is not surfaced into user code,
-            prototype=numpy_buffer_prototype(),
+            prototype=default_buffer_prototype(),
         )
 
     def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: