Encode/Decode complex fill values (#363)

* Test that Zarr Arrays can contain complex data

Implement a simple test to try constructing Zarr Arrays with complex
data to see what issues crop up so as to fix them.

* Test storing complex fill values in Zarr Arrays

Provides a simple test to see if Zarr Arrays can handle storing a
variety of complex typed fill values.

* Handle encoding/decoding of complex fill values

Simply places the real and imaginary values into a list with each one
being encoded/decoded using the floating type used to represent them
independently. This keeps the encoded complex fill values human
readable, follows the existing floating value encoding/decoding rules,
and represents the data in a way that JSON can easily represent it.

* Note support of complex data in Zarr Arrays

Author: jakirkham

docs/release.rst

@@ -58,6 +58,9 @@ Bug fixes
 * Ensure ``DictStore`` contains only ``bytes`` to facilitate comparisons and protect against writes.
   By :user:`John Kirkham <jakirkham>`, :issue:`350`
 
+* Test and fix an issue (w.r.t. fill values) when storing complex data to ``Array``.
+  By :user:`John Kirkham <jakirkham>`, :issue:`363`
+
 * Always use a ``tuple`` when indexing a NumPy ``ndarray``.
   By :user:`John Kirkham <jakirkham>`, :issue:`376`
 

zarr/meta.py

@@ -163,6 +163,11 @@ def decode_fill_value(v, dtype):
             return np.NINF
         else:
             return np.array(v, dtype=dtype)[()]
+    elif dtype.kind in 'c':
+        v = (decode_fill_value(v[0], dtype.type().real.dtype),
+             decode_fill_value(v[1], dtype.type().imag.dtype))
+        v = v[0] + 1j * v[1]
+        return np.array(v, dtype=dtype)[()]
     elif dtype.kind == 'S':
         # noinspection PyBroadException
         try:
@@ -201,6 +206,10 @@ def encode_fill_value(v, dtype):
         return int(v)
     elif dtype.kind == 'b':
         return bool(v)
+    elif dtype.kind in 'c':
+        v = (encode_fill_value(v.real, dtype.type().real.dtype),
+             encode_fill_value(v.imag, dtype.type().imag.dtype))
+        return v
     elif dtype.kind in 'SV':
         v = base64.standard_b64encode(v)
         if not PY2:  # pragma: py2 no cover

zarr/tests/test_core.py

@@ -993,6 +993,15 @@ def test_dtypes(self):
             z[:] = a
             assert_array_almost_equal(a, z[:])
 
+        # complex
+        for dtype in 'c8', 'c16':
+            z = self.create_array(shape=10, chunks=3, dtype=dtype)
+            assert z.dtype == np.dtype(dtype)
+            a = np.linspace(0, 1, z.shape[0], dtype=dtype)
+            a -= 1j * a
+            z[:] = a
+            assert_array_almost_equal(a, z[:])
+
         # datetime, timedelta
         for base_type in 'Mm':
             for resolution in 'D', 'us', 'ns':

zarr/tests/test_meta.py

@@ -116,6 +116,60 @@ def test_encode_decode_array_2():
     assert [df.get_config()] == meta_dec['filters']
 
 
+def test_encode_decode_array_complex():
+
+    # some variations
+    for k in ['c8', 'c16']:
+        compressor = Blosc(cname='lz4', clevel=3, shuffle=2)
+        dtype = np.dtype(k)
+        fill_value = dtype.type(np.nan-1j)
+        meta = dict(
+            shape=(100, 100),
+            chunks=(10, 10),
+            dtype=dtype,
+            compressor=compressor.get_config(),
+            fill_value=fill_value,
+            order=dtype.char,
+            filters=[]
+        )
+
+        meta_json = '''{
+            "chunks": [10, 10],
+            "compressor": {
+                "id": "blosc",
+                "clevel": 3,
+                "cname": "lz4",
+                "shuffle": 2,
+                "blocksize": 0
+            },
+            "dtype": "%s",
+            "fill_value": ["NaN", -1.0],
+            "filters": [],
+            "order": "%s",
+            "shape": [100, 100],
+            "zarr_format": %s
+        }''' % (dtype.str, dtype.char, ZARR_FORMAT)
+
+        # test encoding
+        meta_enc = encode_array_metadata(meta)
+        assert_json_equal(meta_json, meta_enc)
+
+        # test decoding
+        meta_dec = decode_array_metadata(meta_enc)
+        assert ZARR_FORMAT == meta_dec['zarr_format']
+        assert meta['shape'] == meta_dec['shape']
+        assert meta['chunks'] == meta_dec['chunks']
+        assert meta['dtype'] == meta_dec['dtype']
+        assert meta['compressor'] == meta_dec['compressor']
+        assert meta['order'] == meta_dec['order']
+        # Based off of this SO answer: https://stackoverflow.com/a/49972198
+        assert np.all(
+            fill_value.view((np.uint8, fill_value.itemsize)) ==
+            meta_dec['fill_value'].view((np.uint8, meta_dec['fill_value'].itemsize))
+        )
+        assert [] == meta_dec['filters']
+
+
 def test_encode_decode_array_datetime_timedelta():
 
     # some variations