Merge branch 'main' of github.com:zarr-developers/zarr-python into feat/store-paths

Author: d-v-b

.github/workflows/gpu_test.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.11']
-        numpy-version: ['2.0']
+        numpy-version: ['2.1']
         dependency-set: ["minimal"]
 
     steps:

.github/workflows/hypothesis.yaml

@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.11']
-        numpy-version: ['1.26']
+        numpy-version: ['2.1']
         dependency-set: ["optional"]
 
     steps:

.github/workflows/test.yml

@@ -16,14 +16,33 @@ concurrency:
 
 jobs:
   test:
-    name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
+    name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
 
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ['3.11', '3.12', '3.13']
-        numpy-version: ['1.25', '1.26', '2.0']
+        numpy-version: ['1.25', '2.1']
         dependency-set: ["minimal", "optional"]
+        os: ["ubuntu-latest"]
+        include:
+          - python-version: '3.11'
+            numpy-version: '1.25'
+            dependency-set: 'optional'
+            os: 'macos-latest'
+          - python-version: '3.13'
+            numpy-version: '2.1'
+            dependency-set: 'optional'
+            os: 'macos-latest'
+          # https://github.com/zarr-developers/zarr-python/issues/2438
+          # - python-version: '3.11'
+          #   numpy-version: '1.25'
+          #   dependency-set: 'optional'
+          #   os: 'windows-latest'
+          # - python-version: '3.13'
+          #   numpy-version: '2.1'
+          #   dependency-set: 'optional'
+          #   os: 'windows-latest'
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
@@ -43,3 +62,35 @@ jobs:
     - name: Run Tests
       run: |
         hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run
+
+  test-upstream-and-min-deps:
+    name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }}
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11', "3.13"]
+        dependency-set: ["upstream", "min_deps"]
+        exclude:
+          - python-version: "3.13"
+            dependency-set: min_deps
+          - python-version: "3.11"
+            dependency-set: upstream
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install Hatch
+      run: |
+        python -m pip install --upgrade pip 
+        pip install hatch
+    - name: Set Up Hatch Env
+      run: |
+        hatch env create ${{ matrix.dependency-set }}
+        hatch env run -e ${{ matrix.dependency-set }} list-env
+    - name: Run Tests
+      run: |
+        hatch env run --env ${{ matrix.dependency-set }} run

.pep8speaks.yml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.9
+    rev: v0.7.0
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -22,7 +22,7 @@ repos:
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.12.1
     hooks:
       - id: mypy
         files: src|tests
@@ -37,8 +37,6 @@ repos:
           - universal-pathlib
           # Tests
           - pytest
-          # Zarr v2
-          - types-redis
   - repo: https://github.com/scientific-python/cookie
     rev: 2024.08.19
     hooks:

.pre-commit-config.yaml

@@ -72,7 +72,7 @@ that implements the `AbstractFileSystem` API,
 .. code-block:: python
 
    >>> import zarr
-   >>> store = zarr.storage.RemoteStore("gs://foo/bar", mode="r")
+   >>> store = zarr.storage.RemoteStore.from_url("gs://foo/bar", mode="r")
    >>> zarr.open(store=store)
    <Array <RemoteStore(GCSFileSystem, foo/bar)> shape=(10, 20) dtype=float32>
 

docs/guide/storage.rst

@@ -28,12 +28,13 @@ requires-python = ">=3.11"
 dependencies = [
     'asciitree',
     'numpy>=1.25',
-    'numcodecs>=0.10.2',
-    'fsspec>2024',
-    'crc32c',
-    'typing_extensions',
-    'donfig',
+    'numcodecs>=0.13',
+    'fsspec>=2022.10.0',
+    'crc32c>=2.3',
+    'typing_extensions>=4.6',
+    'donfig>=0.8',
 ]
+
 dynamic = [
   "version",
 ]
@@ -60,7 +61,6 @@ test = [
     "pytest",
     "pytest-cov",
     "msgpack",
-    "lmdb",
     "s3fs",
     "pytest-asyncio",
     "moto[s3]",
@@ -83,22 +83,20 @@ gpu = [
 docs = [
     'sphinx==8.1.3',
     'sphinx-autobuild>=2021.3.14',
-    'sphinx-autoapi==3.3.2',
+    'sphinx-autoapi==3.3.3',
     'sphinx_design',
     'sphinx-issues',
     'sphinx-copybutton',
     'pydata-sphinx-theme',
     'numpydoc',
     'numcodecs[msgpack]',
     'msgpack',
-    'lmdb',
 ]
 extra = [
     'msgpack',
 ]
 optional = [
-    'lmdb',
-    'universal-pathlib',
+    'universal-pathlib>=0.0.22',
 ]
 
 [project.urls]
@@ -134,17 +132,17 @@ features = ["test", "extra"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 version = ["minimal"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 features = ["optional"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 features = ["gpu"]
 
 [tool.hatch.envs.test.scripts]
@@ -165,7 +163,7 @@ features = ["test", "extra", "gpu"]
 
 [[tool.hatch.envs.gputest.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 version = ["minimal"]
 
 [tool.hatch.envs.gputest.scripts]
@@ -183,6 +181,65 @@ features = ['docs']
 build = "cd docs && make html"
 serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
 
+[tool.hatch.envs.upstream]
+python = "3.13"
+dependencies = [
+    'numpy',  # from scientific-python-nightly-wheels
+    'numcodecs @ git+https://github.com/zarr-developers/numcodecs',
+    'fsspec @ git+https://github.com/fsspec/filesystem_spec',
+    's3fs @ git+https://github.com/fsspec/s3fs',
+    'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib',
+    'crc32c @ git+https://github.com/ICRAR/crc32c',
+    'typing_extensions @ git+https://github.com/python/typing_extensions',
+    'donfig @ git+https://github.com/pytroll/donfig',
+    # test deps
+    'hypothesis',
+    'pytest',
+    'pytest-cov',
+    'pytest-asyncio',
+    'moto[s3]',
+]
+
+[tool.hatch.envs.upstream.env-vars]
+PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/"
+PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/"
+PIP_PRE = "1"
+
+[tool.hatch.envs.upstream.scripts]
+run = "pytest --verbose"
+run-mypy = "mypy src"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+list-env = "pip list"
+
+[tool.hatch.envs.min_deps]
+description = """Test environment for minimum supported dependencies
+
+See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/
+"""
+python = "3.11"
+dependencies = [
+    'numpy==1.25.*',
+    'numcodecs==0.13.*',  # 0.13 needed for? (should be 0.11)
+    'fsspec==2022.10.0',
+    's3fs==2022.10.0',
+    'universal_pathlib==0.0.22',
+    'crc32c==2.3.*',
+    'typing_extensions==4.6.*',  # 4.5 needed for @deprecated, 4.6 for Buffer
+    'donfig==0.8.*',
+    # test deps
+    'hypothesis',
+    'pytest',
+    'pytest-cov',
+    'pytest-asyncio',
+    'moto[s3]',
+]
+
+[tool.hatch.envs.min_deps.scripts]
+run = "pytest --verbose"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+list-env = "pip list"
+
+
 [tool.ruff]
 line-length = 100
 force-exclude = true

pyproject.toml

@@ -184,7 +184,7 @@ def with_mode(self, mode: AccessModeLiteral) -> Self:
 
         Returns
         -------
-        store:
+        store
             A new store of the same type with the new mode.
 
         Examples

src/zarr/abc/store.py

@@ -396,12 +396,16 @@ async def save_array(
 
     mode = kwargs.pop("mode", None)
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
+    if np.isscalar(arr):
+        arr = np.array(arr)
+    shape = arr.shape
+    chunks = getattr(arr, "chunks", None)  # for array-likes with chunks attribute
     new = await AsyncArray.create(
         store_path,
         zarr_format=zarr_format,
-        shape=arr.shape,
+        shape=shape,
         dtype=arr.dtype,
-        chunks=arr.shape,
+        chunks=chunks,
         **kwargs,
     )
     await new.setitem(slice(None), arr)
@@ -712,7 +716,7 @@ async def create(
     dtype: npt.DTypeLike | None = None,
     compressor: dict[str, JSON] | None = None,  # TODO: default and type change
     fill_value: Any | None = 0,  # TODO: need type
-    order: MemoryOrder | None = None,  # TODO: default change
+    order: MemoryOrder | None = None,
     store: str | StoreLike | None = None,
     synchronizer: Any | None = None,
     overwrite: bool = False,
@@ -761,6 +765,7 @@ async def create(
         Default value to use for uninitialized portions of the array.
     order : {'C', 'F'}, optional
         Memory layout to be used within each chunk.
+        Default is set in Zarr's config (`array.order`).
     store : Store or str
         Store or path to directory in file system or name of zip file.
     synchronizer : object, optional
@@ -834,12 +839,6 @@ async def create(
         else:
             chunk_shape = shape
 
-    if order is not None:
-        warnings.warn(
-            "order is deprecated, use config `array.order` instead",
-            DeprecationWarning,
-            stacklevel=2,
-        )
     if synchronizer is not None:
         warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2)
     if chunk_store is not None:
@@ -889,6 +888,7 @@ async def create(
         codecs=codecs,
         dimension_names=dimension_names,
         attributes=attributes,
+        order=order,
         **kwargs,
     )