Merge pull request #61 from p2p-ld/zero-length-dimensinos

Support leading zero length dimensions

Author: sneakers-the-rat

docs/changelog.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## Upcoming
+
+**Bugfix**
+
+- [#60](https://github.com/p2p-ld/numpydantic/issues/60), [#61](https://github.com/p2p-ld/numpydantic/pulls/61) - 
+  Support the edge case of leading zero-length dimensions while round tripping to JSON
+
+**Added*
+
+- [#61](https://github.com/p2p-ld/numpydantic/pulls/61) -
+  Serialize `shape` in `round_trip` mode, and attempt to reshape when validating from round trip.
+  This supports the edge case of a leading zero-length dimension.
+- Since the `dtype` of an array of objects can't be detected when there are no objects,
+  allow `Any` to be used in validation to indicate that the dtype could be anything/is unknowable.
+
+**Testing**
+
+- Use static lists rather than generators in test case `merged_product` collections
+  rather than a generator to avoid the footgun of exhausting a generator once,
+  and then on the next iteration it appears as an empty collection,
+  which would cause tests to invisibly not be run.
+
 ## 1.*
 
 ### 1.6.*

docs/serialization.md

@@ -135,6 +135,23 @@ DaskJsonDict.is_valid(roundtrip_json)
 NumpyJsonDict.is_valid(roundtrip_json)
 ```
 
+````{note} Zero-length dimensions
+Zero-length dimensions are a special case in many array frameworks.
+When serializing to JSON, in most cases, zero-length dimensions are lost.
+
+Numpy has some odd edge cases with zero-length arrays,
+e.g. an empty array can have a non-zero shape:
+
+```python
+>>> np.random.rand(0,3)
+array([], shape=(0, 3), dtype=float64)
+```
+
+When round-tripping, if the array framework supports reshape operations,
+the `shape` of the array will be preserved in the JSON form,
+and during validation the interface will attempt to reshape the input array to the stored shape.
+````
+
 #### Controlling paths
 
 When possible, the full content of the array is omitted in favor

pdm.lock

@@ -147,6 +147,7 @@ markers = [
     "union: union dtypes",
     "pipe_union: union dtypes specified with a pipe",
     "scalar: scalar values",
+    "model: dtype annotatiosn are a pydantic model"
 ]
 
 [tool.black]
@@ -190,7 +191,7 @@ select = [
     "T100",
 ]
 ignore = [
-    "ANN101", "ANN102", "ANN401", "ANN204",
+    "ANN401", "ANN204",
     # explicit strict arg for zip
     "B905",
     # builtin type annotations

pyproject.toml

@@ -33,11 +33,14 @@ class DaskJsonDict(JsonDict):
     name: str
     chunks: Iterable[tuple[int, ...]]
     dtype: str
+    shape: Union[tuple[int, ...], None] = None
     value: list
 
     def to_array_input(self) -> DaskArray:
         """Construct a dask array"""
         np_array = np.array(self.value, dtype=self.dtype)
+        if self.shape is not None and np_array.shape != self.shape:
+            np_array = self.reshape_input(np_array, self.shape)
         array = from_array(
             np_array,
             name=self.name,
@@ -75,28 +78,37 @@ def before_validation(self, array: DaskArray) -> NDArrayType:
         Try and coerce dicts that should be model objects into the model objects
         """
         try:
-            if issubclass(self.dtype, BaseModel) and isinstance(
-                array.reshape(-1)[0].compute(), dict
-            ):
+            if issubclass(self.dtype, BaseModel):
+                flat_array = array.reshape(-1)
+                if len(flat_array) == 0:
+                    return array
 
-                def _chunked_to_model(array: np.ndarray) -> np.ndarray:
-                    def _vectorized_to_model(item: Union[dict, BaseModel]) -> BaseModel:
-                        if not isinstance(item, self.dtype):
-                            return self.dtype(**item)
-                        else:  # pragma: no cover
-                            return item
+                if isinstance(flat_array[0].compute(), dict):
 
-                    return np.vectorize(_vectorized_to_model)(array)
+                    def _chunked_to_model(array: np.ndarray) -> np.ndarray:
+                        def _vectorized_to_model(
+                            item: Union[dict, BaseModel],
+                        ) -> BaseModel:
+                            if not isinstance(item, self.dtype):
+                                return self.dtype(**item)
+                            else:  # pragma: no cover
+                                return item
 
-                array = array.map_blocks(_chunked_to_model, dtype=self.dtype)
+                        return np.vectorize(_vectorized_to_model)(array)
+
+                    array = array.map_blocks(_chunked_to_model, dtype=self.dtype)
         except TypeError:
             # fine, dtype isn't a type
             pass
         return array
 
     def get_object_dtype(self, array: NDArrayType) -> DtypeType:
         """Dask arrays require a compute() call to retrieve a single value"""
-        return type(array.reshape(-1)[0].compute())
+        flat_array = array.reshape(-1)
+        if len(flat_array) == 0:
+            return Any
+        else:
+            return type(flat_array[0].compute())
 
     @classmethod
     def enabled(cls) -> bool:
@@ -121,12 +133,15 @@ def to_json(
         """
         np_array = np.array(array)
         as_json = np_array.tolist()
+        if not isinstance(as_json, list):
+            as_json = [as_json]
         if info.round_trip:
             as_json = DaskJsonDict(
                 type=cls.name,
                 value=as_json,
                 name=array.name,
                 chunks=array.chunks,
                 dtype=str(np_array.dtype),
+                shape=array.shape,
             )
         return as_json

src/numpydantic/interface/dask.py

@@ -137,6 +137,25 @@ def handle_input(cls: Type[U], value: Union[dict, U, W]) -> Union[V, W]:
             value = value.to_array_input()
         return value
 
+    @staticmethod
+    def reshape_input(value: T, shape: tuple[int, ...]) -> T:
+        """
+        If a `reshape` value is present on the array, and the array shape doesn't match,
+        attempt to reshape it.
+        """
+        if value.shape != shape:
+            try:
+                value = value.reshape(shape)
+            except ValueError:
+                warnings.warn(
+                    f"Input data has shape {value.shape}, "
+                    f"but roundtrip form specifies {shape},"
+                    f"and {value.shape} can't be cast to {shape}. "
+                    f"Attempting to proceed with validation without reshaping.",
+                    stacklevel=1,
+                )
+        return value
+
 
 class MarkedJson(BaseModel):
     """
@@ -270,7 +289,7 @@ def before_validation(self, array: Any) -> NDArrayType:
 
     def get_dtype(self, array: NDArrayType) -> DtypeType:
         """
-        Get the dtype from the input array
+        Get the dtype from the input array.
         """
         if hasattr(array.dtype, "type") and array.dtype.type is np.object_:
             return self.get_object_dtype(array)
@@ -281,8 +300,15 @@ def get_object_dtype(self, array: NDArrayType) -> DtypeType:
         """
         When an array contains an object, get the dtype of the object contained
         by the array.
+
+        If this method returns `Any`, the dtype validation passes -
+        used for e.g. empty arrays for which the dtype of the array can't be determined
+        (since there are no objects).
         """
-        return type(array.ravel()[0])
+        try:
+            return type(array.ravel()[0])
+        except IndexError:
+            return Any
 
     def validate_dtype(self, dtype: DtypeType) -> bool:
         """

src/numpydantic/interface/interface.py

@@ -28,12 +28,17 @@ class NumpyJsonDict(JsonDict):
     type: Literal["numpy"]
     dtype: str
     value: list
+    # allow shape to be None for backwards compatibility.
+    shape: Union[tuple[int, ...], None] = None
 
     def to_array_input(self) -> ndarray:
         """
         Construct a numpy array
         """
-        return np.array(self.value, dtype=self.dtype)
+        array = np.array(self.value, dtype=self.dtype)
+        if self.shape is not None and array.shape != self.shape:
+            array = self.reshape_input(array, self.shape)
+        return array
 
 
 class NumpyInterface(Interface):
@@ -82,7 +87,11 @@ def before_validation(self, array: Any) -> ndarray:
             array = np.array(array)
 
         try:
-            if issubclass(self.dtype, BaseModel) and isinstance(array.flat[0], dict):
+            if (
+                issubclass(self.dtype, BaseModel)
+                and len(array) > 0
+                and isinstance(array.flat[0], dict)
+            ):
                 array = np.vectorize(lambda x: self.dtype(**x))(array)
         except TypeError:
             # fine, dtype isn't a type
@@ -110,6 +119,9 @@ def to_json(
 
         if info.round_trip:
             json_array = NumpyJsonDict(
-                type=cls.name, dtype=str(array.dtype), value=json_array
+                type=cls.name,
+                dtype=str(array.dtype),
+                value=json_array,
+                shape=array.shape,
             )
         return json_array

src/numpydantic/interface/numpy.py

@@ -137,7 +137,6 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
     References:
         https://github.com/python/cpython/blob/8a2baedc4bcb606da937e4e066b4b3a18961cace/Lib/pathlib/_abc.py#L244-L270
     """
-    # pdb.set_trace()
     if not isinstance(other, Path):  # pragma: no cover - ripped from cpython
         other = Path(other)
     self_parts = self.parts

src/numpydantic/serialization.py

@@ -96,6 +96,9 @@ class SubClass(BasicModel):
         marks={"scalar"},
         passes=False,
     ),
+    ValidationCase(
+        annotation_shape=("*", 3), shape=(0, 3), id="zero-length", passes=True
+    ),
 )
 """
 Base Shape cases
@@ -162,16 +165,32 @@ class SubClass(BasicModel):
         marks={"np_str", "str", "tuple"},
     ),
     ValidationCase(
-        annotation_dtype=BasicModel, dtype=BasicModel, passes=True, id="model-model"
+        annotation_dtype=BasicModel,
+        dtype=BasicModel,
+        passes=True,
+        id="model-model",
+        marks={"model"},
     ),
     ValidationCase(
-        annotation_dtype=BasicModel, dtype=BadModel, passes=False, id="model-badmodel"
+        annotation_dtype=BasicModel,
+        dtype=BadModel,
+        passes=False,
+        id="model-badmodel",
+        marks={"model"},
     ),
     ValidationCase(
-        annotation_dtype=BasicModel, dtype=int, passes=False, id="model-int"
+        annotation_dtype=BasicModel,
+        dtype=int,
+        passes=False,
+        id="model-int",
+        marks={"model"},
     ),
     ValidationCase(
-        annotation_dtype=BasicModel, dtype=SubClass, passes=True, id="model-subclass"
+        annotation_dtype=BasicModel,
+        dtype=SubClass,
+        passes=True,
+        id="model-subclass",
+        marks={"model"},
     ),
     ValidationCase(
         annotation_dtype=UNION_TYPE,
@@ -305,3 +324,5 @@ class SubClass(BasicModel):
 """
 Merged product of all cases, but only those that pass
 """
+
+ZERO_LENGTH_CASES_PASSING = [c for c in ALL_CASES_PASSING if "zero-length" in c.id]

src/numpydantic/testing/cases.py

@@ -6,7 +6,6 @@
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
-    Generator,
     List,
     Literal,
     Optional,
@@ -303,7 +302,7 @@ def merge_cases(*args: ValidationCase) -> ValidationCase:
 
 def merged_product(
     *args: Sequence[ValidationCase], conditions: dict = None
-) -> Generator[ValidationCase, None, None]:
+) -> list[ValidationCase]:
     """
     Generator for the product of the iterators of validation cases,
     merging each tuple, and respecting if they should be :meth:`.ValidationCase.skip`
@@ -341,6 +340,7 @@ def merged_product(
 
     """
     iterator = product(*args)
+    cases = []
     for case_tuple in iterator:
         case = merge_cases(*case_tuple)
         if case.skip():
@@ -349,4 +349,5 @@ def merged_product(
             matching = all([getattr(case, k, None) == v for k, v in conditions.items()])
             if not matching:
                 continue
-        yield case
+        cases.append(case)
+    return cases