(fix): check for T data type kind to fall back to python (#92)

* (fix): add T data type

* (chore): tests

* (fix): bump ubuntu on readthedocs.yml

* (fix): add clang package specifically for readthedocs

* (fix): try clang

* (fix): handle uninstalled clang

* (fix): right, in a docker container

* (style): match indent

* (fix): use standard api, update test to latest version

* (fix): latest fixed ubuntu

Author: ilan-gold

.github/workflows/cd.yml

@@ -59,6 +59,10 @@ jobs:
             # If we're running on rhel centos, install needed packages.
             if command -v yum &> /dev/null; then
                 yum update -y && yum install -y perl-core
+                # https://github.com/PyO3/maturin-action/discussions/152
+                if [[ "${{ matrix.os }}" == "linux" && "${{ matrix.target }}" == "x86_64" && "${{ matrix.manylinux }}" == "2_28" ]]; then
+                    yum update -y && yum install -y clang
+                fi
             fi
       - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} dist/
       - run: twine check --strict dist/*

.readthedocs.yml

@@ -1,6 +1,8 @@
 version: 2
 build:
-  os: ubuntu-20.04
+  os: ubuntu-24.04
+  apt_packages:
+    - clang
   tools:
     python: "3.12"
     rust: "latest"

pyproject.toml

@@ -83,6 +83,7 @@ filterwarnings = [
     "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
     "ignore:Creating a zarr.buffer.gpu.*:UserWarning",
     "ignore:Duplicate name:UserWarning",                                            # from ZipFile
+    "ignore:.*not part in the Zarr format 3.*:UserWarning",
 ]
 markers = ["gpu: mark a test as requiring CuPy and GPU"]
 

python/zarrs/pipeline.py

@@ -233,9 +233,9 @@ def _raise_error_on_unsupported_batch_dtype(
         ],
     ):
         # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
-        # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object is also out
+        # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
         if any(
-            info.dtype.kind in {"V", "S", "U", "M", "m", "O"}
+            info.dtype.kind in {"V", "S", "U", "M", "m", "O", "T"}
             for (_, info, _, _, _) in batch_info
         ):
             raise UnsupportedDataTypeError()

tests/test_vlen.py

@@ -2,16 +2,14 @@
 
 import numpy as np
 import pytest
-from zarr import Array
+import zarr
 from zarr.abc.codec import Codec
 from zarr.abc.store import Store
-from zarr.codecs import VLenBytesCodec, VLenUTF8Codec, ZstdCodec
+from zarr.codecs import ZstdCodec
 from zarr.core.metadata.v3 import ArrayV3Metadata, DataType
 from zarr.core.strings import _NUMPY_SUPPORTS_VLEN_STRING
 from zarr.storage import StorePath
 
-pytest.skip(allow_module_level=True)
-
 numpy_str_dtypes: list[type | str | None] = [None, str, "str", np.dtypes.StrDType]
 expected_zarr_string_dtype: np.dtype[Any]
 if _NUMPY_SUPPORTS_VLEN_STRING:
@@ -21,30 +19,28 @@
     expected_zarr_string_dtype = np.dtype("O")
 
 
-@pytest.mark.parametrize("store", ["memory", "local"], indirect=["store"])
+@pytest.mark.parametrize("store", ["local"], indirect=["store"])
 @pytest.mark.parametrize("dtype", numpy_str_dtypes)
 @pytest.mark.parametrize("as_object_array", [False, True])
-@pytest.mark.parametrize(
-    "codecs", [None, [VLenUTF8Codec()], [VLenUTF8Codec(), ZstdCodec()]]
-)
+@pytest.mark.parametrize("compressor", [None, ZstdCodec()])
 def test_vlen_string(
-    *,
     store: Store,
-    dtype: None | np.dtype[Any],
+    dtype: np.dtype[Any] | None,
+    *,
     as_object_array: bool,
-    codecs: None | list[Codec],
+    compressor: Codec | None,
 ) -> None:
     strings = ["hello", "world", "this", "is", "a", "test"]
     data = np.array(strings, dtype=dtype).reshape((2, 3))
 
     sp = StorePath(store, path="string")
-    a = Array.create(
+    a = zarr.create_array(
         sp,
         shape=data.shape,
-        chunk_shape=data.shape,
+        chunks=data.shape,
         dtype=data.dtype,
         fill_value="",
-        codecs=codecs,
+        compressors=compressor,
     )
     assert isinstance(a.metadata, ArrayV3Metadata)  # needed for mypy
 
@@ -59,33 +55,31 @@ def test_vlen_string(
     assert a.dtype == expected_zarr_string_dtype
 
     # test round trip
-    b = Array.open(sp)
+    b = zarr.open(sp)
     assert isinstance(b.metadata, ArrayV3Metadata)  # needed for mypy
     assert np.array_equal(data, b[:, :])
     assert b.metadata.data_type == DataType.string
     assert a.dtype == expected_zarr_string_dtype
 
 
-@pytest.mark.parametrize("store", ["memory", "local"], indirect=["store"])
+@pytest.mark.parametrize("store", ["local"], indirect=["store"])
 @pytest.mark.parametrize("as_object_array", [False, True])
-@pytest.mark.parametrize(
-    "codecs", [None, [VLenBytesCodec()], [VLenBytesCodec(), ZstdCodec()]]
-)
+@pytest.mark.parametrize("compressor", [None, ZstdCodec()])
 def test_vlen_bytes(
-    *, store: Store, as_object_array: bool, codecs: None | list[Codec]
+    store: Store, *, as_object_array: bool, compressor: Codec | None
 ) -> None:
     bstrings = [b"hello", b"world", b"this", b"is", b"a", b"test"]
     data = np.array(bstrings).reshape((2, 3))
     assert data.dtype == "|S5"
 
     sp = StorePath(store, path="string")
-    a = Array.create(
+    a = zarr.create_array(
         sp,
         shape=data.shape,
-        chunk_shape=data.shape,
+        chunks=data.shape,
         dtype=data.dtype,
         fill_value=b"",
-        codecs=codecs,
+        compressors=compressor,
     )
     assert isinstance(a.metadata, ArrayV3Metadata)  # needed for mypy
 
@@ -99,7 +93,7 @@ def test_vlen_bytes(
     assert a.dtype == "O"
 
     # test round trip
-    b = Array.open(sp)
+    b = zarr.open(sp)
     assert isinstance(b.metadata, ArrayV3Metadata)  # needed for mypy
     assert np.array_equal(data, b[:, :])
     assert b.metadata.data_type == DataType.bytes