Merge pull request #239 from zarr-developers/npy114-20180130-alimanfoo

Work around numpy 1.14 structured array changes

Author: alimanfoo

appveyor.yml

@@ -14,29 +14,45 @@ environment:
 
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7"
+      NUMPY_VERSION: "1.13.3"
 
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "2.7"
+      NUMPY_VERSION: "1.13.3"
       DISTUTILS_USE_SDK: "1"
 
     - PYTHON: "C:\\Python34"
+      NUMPY_VERSION: "1.13.3"
       PYTHON_VERSION: "3.4"
 
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4"
+      NUMPY_VERSION: "1.13.3"
       DISTUTILS_USE_SDK: "1"
 
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5"
+      NUMPY_VERSION: "1.13.3"
 
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5"
+      NUMPY_VERSION: "1.13.3"
 
     - PYTHON: "C:\\Python36"
       PYTHON_VERSION: "3.6"
+      NUMPY_VERSION: "1.13.3"
 
     - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6"
+      NUMPY_VERSION: "1.13.3"
+
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6"
+      NUMPY_VERSION: "1.14.0"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6"
+      NUMPY_VERSION: "1.14.0"
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -45,6 +61,8 @@ build: off
 
 test_script:
   - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel"
+  - "%CMD_IN_ENV% python -m pip install numpy==%NUMPY_VERSION%"
   - "%CMD_IN_ENV% python -m pip install -rrequirements_dev.txt"
-  - "%CMD_IN_ENV% python setup.py bdist_wheel"
-  - "%CMD_IN_ENV% python -m pytest -v zarr"
+  - "%CMD_IN_ENV% python setup.py install"
+  - "%CMD_IN_ENV% python -m pytest -v --pyargs zarr"
+  

docs/release.rst

@@ -6,7 +6,7 @@ Release notes
 2.2.0 (release candidate)
 -------------------------
 
-Version 2.2.0 is currently at the release candidate stage. To install the latest release 
+Version 2.2.0 is currently at the release candidate stage. To install the latest release
 candidate version::
 
     $ pip install --pre zarr
@@ -190,9 +190,12 @@ Maintenance
 
 * A data fixture has been included in the test suite to ensure data format
   compatibility is maintained; :issue:`83`, :issue:`146`.
-* The test suite has been migrated from nosetests to pytest; :issue:`189`, :issue:`225`, :issue:`237`.
+* The test suite has been migrated from nosetests to pytest; :issue:`189`, :issue:`225`.
 * Various continuous integration updates and improvements; :issue:`118`, :issue:`124`,
   :issue:`125`, :issue:`126`, :issue:`109`, :issue:`114`, :issue:`171`.
+* Bump numcodecs dependency to 0.5.3, completely remove nose dependency, :issue:`237`.
+* Fix compatibility issues with NumPy 1.14 regarding fill values for structured arrays,
+  :issue:`222`, :issue:`238`, :issue:`239`.
 
 Acknowledgments
 ~~~~~~~~~~~~~~~

requirements_dev.txt

@@ -16,7 +16,6 @@ mccabe==0.6.1
 monotonic==1.3
 msgpack-python==0.4.8
 numcodecs==0.5.3
-numpy==1.13.3
 packaging==16.8
 pkginfo==1.4.1
 pluggy==0.5.2

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py34, py35, py36, docs
+envlist = py27, py34, py35, py36-npy{113,114}, docs
 
 [testenv]
 setenv =
@@ -15,17 +15,20 @@ setenv =
 commands =
     python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]'
     py27,py34,py35: pytest -v --cov=zarr zarr
-    py36: pytest -v --cov=zarr --doctest-modules zarr
-    coverage report -m
-    py36: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
-    py36: flake8 --max-line-length=100 zarr
+    # don't run py36-npy114 with coverage because it is run together with py35-npy113 on travis
+    py36-npy114: pytest -v zarr
+    py36-npy113: pytest -v --cov=zarr --doctest-modules zarr
+    py27,py34,py35,py36-npy113: coverage report -m
+    py36-npy113: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
+    py36-npy113: flake8 --max-line-length=100 zarr
 deps =
     py27: backports.lzma
+    py27,py34,py35,py36-npy113: numpy==1.13.3
+    py36-npy114: numpy==1.14.0
     -rrequirements_dev.txt
     # linux only
     -rrequirements_dev_optional.txt
 
-
 [testenv:docs]
 basepython = python2.7
 changedir = docs

zarr/meta.py

@@ -138,16 +138,20 @@ def decode_fill_value(v, dtype):
             return np.NINF
         else:
             return np.array(v, dtype=dtype)[()]
-    elif dtype.kind in 'SV':
+    elif dtype.kind == 'S':
         # noinspection PyBroadException
         try:
             v = base64.standard_b64decode(v)
-            v = np.array(v, dtype=dtype)[()]
-            return v
         except Exception:
             # be lenient, allow for other values that may have been used before base64
             # encoding and may work as fill values, e.g., the number 0
-            return v
+            pass
+        v = np.array(v, dtype=dtype)[()]
+        return v
+    elif dtype.kind == 'V':
+        v = base64.standard_b64decode(v)
+        v = np.array(v, dtype=dtype.str).view(dtype)[()]
+        return v
     elif dtype.kind == 'U':
         # leave as-is
         return v
@@ -174,7 +178,7 @@ def encode_fill_value(v, dtype):
         return bool(v)
     elif dtype.kind in 'SV':
         v = base64.standard_b64encode(v)
-        if not PY2:
+        if not PY2:  # pragma: py2 no cover
             v = str(v, 'ascii')
         return v
     elif dtype.kind == 'U':

zarr/tests/test_core.py

@@ -834,12 +834,15 @@ def test_structured_array(self):
                       (b'ccc', 3, 12.6)],
                      dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')])
         for a in (d, d[:0]):
-            for fill_value in None, b'', (b'zzz', 0, 0.0):
-                z = self.create_array(shape=a.shape, chunks=2, dtype=a.dtype,
-                                      fill_value=fill_value)
+            for fill_value in None, b'', (b'zzz', 42, 16.8):
+                z = self.create_array(shape=a.shape, chunks=2, dtype=a.dtype, fill_value=fill_value)
                 assert len(a) == len(z)
                 if fill_value is not None:
-                    np_fill_value = np.array(fill_value, dtype=a.dtype)[()]
+                    if fill_value == b'':
+                        # numpy 1.14 compatibility
+                        np_fill_value = np.array(fill_value, dtype=a.dtype.str).view(a.dtype)[()]
+                    else:
+                        np_fill_value = np.array(fill_value, dtype=a.dtype)[()]
                     assert np_fill_value == z.fill_value
                     if len(z):
                         assert np_fill_value == z[0]
@@ -852,10 +855,6 @@ def test_structured_array(self):
                 assert_array_equal(a['bar'], z['bar'])
                 assert_array_equal(a['baz'], z['baz'])
 
-            with pytest.raises(ValueError):
-                # dodgy fill value
-                self.create_array(shape=a.shape, chunks=2, dtype=a.dtype, fill_value=42)
-
     def test_dtypes(self):
 
         # integers

zarr/tests/test_meta.py

@@ -69,12 +69,14 @@ def test_encode_decode_array_2():
     # some variations
     df = Delta(astype='u2', dtype='V14')
     compressor = Blosc(cname='lz4', clevel=3, shuffle=2)
+    dtype = np.dtype([('a', 'i4'), ('b', 'S10')])
+    fill_value = np.zeros((), dtype=dtype)[()]
     meta = dict(
         shape=(100, 100),
         chunks=(10, 10),
-        dtype=np.dtype([('a', 'i4'), ('b', 'S10')]),
+        dtype=dtype,
         compressor=compressor.get_config(),
-        fill_value=b'',
+        fill_value=fill_value,
         order='F',
         filters=[df.get_config()]
     )
@@ -89,7 +91,7 @@ def test_encode_decode_array_2():
             "blocksize": 0
         },
         "dtype": [["a", "<i4"], ["b", "|S10"]],
-        "fill_value": "",
+        "fill_value": "AAAAAAAAAAAAAAAAAAA=",
         "filters": [
             {"id": "delta", "astype": "<u2", "dtype": "|V14"}
         ],
@@ -110,8 +112,7 @@ def test_encode_decode_array_2():
     assert meta['dtype'] == meta_dec['dtype']
     assert meta['compressor'] == meta_dec['compressor']
     assert meta['order'] == meta_dec['order']
-    np_fill_value = np.array(meta['fill_value'], dtype=meta['dtype'])[()]
-    assert np_fill_value == meta_dec['fill_value']
+    assert fill_value == meta_dec['fill_value']
     assert [df.get_config()] == meta_dec['filters']
 
 

zarr/tests/test_util.py

@@ -104,7 +104,9 @@ def test_normalize_order():
 
 def test_normalize_fill_value():
     assert b'' == normalize_fill_value(0, dtype=np.dtype('S1'))
-    assert b'' == normalize_fill_value(0, dtype=np.dtype([('foo', 'i4'), ('bar', 'f8')]))
+    structured_dtype = np.dtype([('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')])
+    expect = np.array((b'', 0, 0.), dtype=structured_dtype)[()]
+    assert expect == normalize_fill_value(0, dtype=structured_dtype)
     assert '' == normalize_fill_value(0, dtype=np.dtype('U1'))
 
 

zarr/util.py

@@ -235,19 +235,16 @@ def normalize_fill_value(fill_value, dtype):
         # no fill value
         pass
 
-    elif fill_value == 0 and dtype.kind in 'SV':
-        # special case because 0 used as default, but cannot be used for structured arrays
-        fill_value = b''
+    elif fill_value == 0:
+        # this should be compatible across numpy versions for any array type, including
+        # structured arrays
+        fill_value = np.zeros((), dtype=dtype)[()]
 
     elif dtype.kind == 'U':
-        # special case unicode because of encoding issues on Windows if passed through
-        # numpy
+        # special case unicode because of encoding issues on Windows if passed through numpy
         # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713
 
-        if fill_value == 0:
-            fill_value = ''
-
-        elif PY2 and isinstance(fill_value, binary_type):  # pragma: py3 no cover
+        if PY2 and isinstance(fill_value, binary_type):  # pragma: py3 no cover
             # this is OK on PY2, can be written as JSON
             pass
 
@@ -257,11 +254,17 @@ def normalize_fill_value(fill_value, dtype):
 
     else:
         try:
-            fill_value = np.array(fill_value, dtype=dtype)[()]
+            if isinstance(fill_value, binary_type) and dtype.kind == 'V':
+                # special case for numpy 1.14 compatibility
+                fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()]
+            else:
+                fill_value = np.array(fill_value, dtype=dtype)[()]
+
         except Exception as e:
             # re-raise with our own error message to be helpful
             raise ValueError('fill_value {!r} is not valid for dtype {}; nested '
                              'exception: {}'.format(fill_value, dtype, e))
+
     return fill_value