Cast datetime and timedelta to signed 64-bit int (#344)

* Cast datetime and timedelta to signed 64-bit int

The `NaT` type is represented as `-0`. As a result, casting to an
unsigned integral fails and throws an error. However casting to a signed
integral type does not have this problem and proceeds without issues.

* Update datetime/timedelta test to use signed ints

* Test encode/decode of a datetime/timedelta array

Use a structured array with datetime and timedelta values and a fill
value of NaT to test a bunch of different workarounds for encoding and
decoding datetime and timedelta values and array.

* Note improved `NaT` handling in the changelog

* Use `dtype` to cast `NaT` explicitly

* test NaT as fill_value for zarr.full()

* Drop extra whitespace

Should fix the flake8 error seen on CI.

ref: https://travis-ci.org/zarr-developers/zarr/jobs/465075634

Author: jakirkham

docs/release.rst

@@ -44,6 +44,11 @@ Maintenance
 * CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and
   upgrade all pinned package requirements. :issue:`308`.
 
+* Corrects handling of ``NaT`` in ``datetime64`` and ``timedelta64`` in various
+  compressors (by :user:`John Kirkham <jakirkham>`; :issue:`344`).
+
+Acknowledgments
+~~~~~~~~~~~~~~~
 
 .. _release_2.2.0:
 

zarr/meta.py

@@ -209,6 +209,6 @@ def encode_fill_value(v, dtype):
     elif dtype.kind == 'U':
         return v
     elif dtype.kind in 'mM':
-        return int(v.view('u8'))
+        return int(v.view('i8'))
     else:
         return v

zarr/tests/test_core.py

@@ -955,8 +955,9 @@ def test_dtypes(self):
                 dtype = '{}8[{}]'.format(base_type, resolution)
                 z = self.create_array(shape=100, dtype=dtype, fill_value=0)
                 assert z.dtype == np.dtype(dtype)
-                a = np.random.randint(0, np.iinfo('u8').max, size=z.shape[0],
-                                      dtype='u8').view(dtype)
+                a = np.random.randint(np.iinfo('i8').min, np.iinfo('i8').max,
+                                      size=z.shape[0],
+                                      dtype='i8').view(dtype)
                 z[:] = a
                 assert_array_equal(a, z[:])
 

zarr/tests/test_creation.py

@@ -135,6 +135,12 @@ def test_full():
     z = full(100, chunks=10, fill_value=np.nan, dtype='f8')
     assert np.all(np.isnan(z[:]))
 
+    # NaT
+    z = full(100, chunks=10, fill_value='NaT', dtype='M8[s]')
+    assert np.all(np.isnat(z[:]))
+    z = full(100, chunks=10, fill_value='NaT', dtype='m8[s]')
+    assert np.all(np.isnat(z[:]))
+
     # byte string dtype
     v = b'xxx'
     z = full(100, chunks=10, fill_value=v, dtype='S3')

zarr/tests/test_meta.py

@@ -116,6 +116,60 @@ def test_encode_decode_array_2():
     assert [df.get_config()] == meta_dec['filters']
 
 
+def test_encode_decode_array_datetime_timedelta():
+
+    # some variations
+    for k in ['m8[s]', 'M8[s]']:
+        compressor = Blosc(cname='lz4', clevel=3, shuffle=2)
+        dtype = np.dtype(k)
+        fill_value = dtype.type("NaT")
+        meta = dict(
+            shape=(100, 100),
+            chunks=(10, 10),
+            dtype=dtype,
+            compressor=compressor.get_config(),
+            fill_value=fill_value,
+            order=dtype.char,
+            filters=[]
+        )
+
+        meta_json = '''{
+            "chunks": [10, 10],
+            "compressor": {
+                "id": "blosc",
+                "clevel": 3,
+                "cname": "lz4",
+                "shuffle": 2,
+                "blocksize": 0
+            },
+            "dtype": "%s",
+            "fill_value": -9223372036854775808,
+            "filters": [],
+            "order": "%s",
+            "shape": [100, 100],
+            "zarr_format": %s
+        }''' % (dtype.str, dtype.char, ZARR_FORMAT)
+
+        # test encoding
+        meta_enc = encode_array_metadata(meta)
+        assert_json_equal(meta_json, meta_enc)
+
+        # test decoding
+        meta_dec = decode_array_metadata(meta_enc)
+        assert ZARR_FORMAT == meta_dec['zarr_format']
+        assert meta['shape'] == meta_dec['shape']
+        assert meta['chunks'] == meta_dec['chunks']
+        assert meta['dtype'] == meta_dec['dtype']
+        assert meta['compressor'] == meta_dec['compressor']
+        assert meta['order'] == meta_dec['order']
+        # Based off of this SO answer: https://stackoverflow.com/a/49972198
+        assert np.all(
+            fill_value.view((np.uint8, fill_value.itemsize)) ==
+            meta_dec['fill_value'].view((np.uint8, meta_dec['fill_value'].itemsize))
+        )
+        assert [] == meta_dec['filters']
+
+
 def test_encode_decode_array_dtype_shape():
 
     meta = dict(