get json de/serialization largely working, and start making tests pass

Author: d-v-b

src/zarr/api/asynchronous.py

@@ -982,7 +982,7 @@ async def create(
     if zarr_format == 2:
         if chunks is None:
             chunks = shape
-        dtype = parse_dtype(dtype, zarr_format)
+        dtype = parse_dtype(dtype, zarr_format=zarr_format)
         if not filters:
             filters = _default_filters(dtype)
         if not compressor:

src/zarr/codecs/sharding.py

@@ -355,9 +355,10 @@ def __init__(
         object.__setattr__(self, "index_location", index_location_parsed)
 
         # Use instance-local lru_cache to avoid memory leaks
-        object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
-        object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
-        object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
+        # TODO: fix these when we don't get hashability errors for certain numpy dtypes
+        # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
+        # object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
+        # object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
 
     # todo: typedict return type
     def __getstate__(self) -> dict[str, Any]:

src/zarr/core/array.py

@@ -580,7 +580,7 @@ async def _create(
         """
         store_path = await make_store_path(store)
 
-        dtype_parsed = parse_dtype(dtype, zarr_format)
+        dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format)
         shape = parse_shapelike(shape)
 
         if chunks is not None and chunk_shape is not None:
@@ -693,13 +693,23 @@ def _create_metadata_v3(
                 category=UserWarning,
                 stacklevel=2,
             )
+
+        # resolve the numpy dtype into zarr v3 datatype
+        zarr_data_type = get_data_type_from_numpy(dtype)
+
+        if fill_value is None:
+            # v3 spec will not allow a null fill value
+            fill_value_parsed = dtype.type(zarr_data_type.default)
+        else:
+            fill_value_parsed = fill_value
+
         chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape)
         return ArrayV3Metadata(
             shape=shape,
-            data_type=dtype,
+            data_type=zarr_data_type,
             chunk_grid=chunk_grid_parsed,
             chunk_key_encoding=chunk_key_encoding_parsed,
-            fill_value=fill_value,
+            fill_value=fill_value_parsed,
             codecs=codecs,
             dimension_names=tuple(dimension_names) if dimension_names else None,
             attributes=attributes or {},
@@ -3909,7 +3919,7 @@ async def init_array(
 
     from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation
 
-    dtype_parsed = parse_dtype(dtype)
+    dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format)
     shape_parsed = parse_shapelike(shape)
     chunk_key_encoding_parsed = _parse_chunk_key_encoding(
         chunk_key_encoding, zarr_format=zarr_format

src/zarr/core/common.py

@@ -19,6 +19,7 @@
 import numpy as np
 
 from zarr.core.config import config as zarr_config
+from zarr.core.strings import _VLEN_STRING_DTYPE
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable, Iterator
@@ -166,7 +167,13 @@ def parse_bool(data: Any) -> bool:
     raise ValueError(f"Expected bool, got {data} instead.")
 
 
-def parse_dtype(dtype: Any) -> np.dtype[Any]:
+def parse_dtype(dtype: Any, zarr_format: ZarrFormat) -> np.dtype[Any]:
+    if dtype is str or dtype == "str":
+        if zarr_format == 2:
+            # special case as object
+            return np.dtype("object")
+        else:
+            return _VLEN_STRING_DTYPE
     return np.dtype(dtype)
 
 

src/zarr/core/config.py

@@ -88,13 +88,17 @@ def enable_gpu(self) -> ConfigSet:
                     "bytes": [{"id": "vlen-bytes"}],
                     "raw": None,
                 },
-                "v3_default_filters": {"numeric": [], "string": [], "bytes": []},
+                "v3_default_filters": {"boolean": [], "numeric": [], "string": [], "bytes": []},
                 "v3_default_serializer": {
+                    "boolean": {"name": "bytes", "configuration": {"endian": "little"}},
                     "numeric": {"name": "bytes", "configuration": {"endian": "little"}},
                     "string": {"name": "vlen-utf8"},
                     "bytes": {"name": "vlen-bytes"},
                 },
                 "v3_default_compressors": {
+                    "boolean": [
+                        {"name": "zstd", "configuration": {"level": 0, "checksum": False}},
+                    ],
                     "numeric": [
                         {"name": "zstd", "configuration": {"level": 0, "checksum": False}},
                     ],