FIX: Require h5py 2.10 for Windows + Python < 3.6

.azure-pipelines/windows.yml

@@ -10,6 +10,7 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   variables:
     EXTRA_WHEELS: "https://5cf40426d9f06eb7461d-6fe47d9331aba7cd62fc36c7196769e4.ssl.cf2.rackcdn.com"
+    DEPENDS: numpy scipy matplotlib h5py pydicom
   strategy:
     matrix:
       ${{ insert }}: ${{ parameters.matrix }}
@@ -20,11 +21,14 @@ jobs:
         versionSpec: '$(PYTHON_VERSION)'
         addToPath: true
         architecture: '$(PYTHON_ARCH)'
+    - script: |
+        echo %PYTHONHASHSEED%
+      displayName: 'Display hash seed'
     - script: |
         python -m pip install --upgrade pip setuptools>=30.3.0 wheel
       displayName: 'Update build tools'
     - script: |
-        python -m pip install --find-links %EXTRA_WHEELS% numpy scipy matplotlib h5py pydicom
+        python -m pip install --find-links %EXTRA_WHEELS% %DEPENDS%
         python -m pip install nose mock coverage codecov
       displayName: 'Install dependencies'
     - script: |

azure-pipelines.yml

@@ -11,6 +11,11 @@ jobs:
       py35-x64:
         PYTHON_VERSION: '3.5'
         PYTHON_ARCH: 'x64'
+      py35-h5py-check:
+        PYTHON_VERSION: '3.5'
+        PYTHON_ARCH: 'x64'
+        PYTHONHASHSEED: 283137131
+        DEPENDS: "h5py==2.9.0"
       py36-x86:
         PYTHON_VERSION: '3.6'
         PYTHON_ARCH: 'x86'

nibabel/_h5py_compat.py

@@ -0,0 +1,12 @@
+import sys
+import os
+from .optpkg import optional_package
+
+# PY35: A bug affected Windows installations of h5py in Python3 versions <3.6
+# due to random dictionary ordering, causing float64 data arrays to sometimes be
+# loaded as longdouble (also 64 bit on Windows). This caused stochastic failures
+# to correctly handle data caches, and possibly other subtle bugs we never
+# caught. This was fixed in h5py 2.10.
+# Please see https://github.com/nipy/nibabel/issues/665 for details.
+min_h5py = '2.10' if os.name == 'nt' and (3,) <= sys.version_info < (3, 6) else None
+h5py, have_h5py, setup_module = optional_package('h5py', min_version=min_h5py)

nibabel/minc1.py

@@ -173,7 +173,7 @@ def _normalize(self, data, sliceobj=()):
             applied to `data`
         """
         ddt = self.get_data_dtype()
-        if ddt.type in np.sctypes['float']:
+        if np.issubdtype(ddt.type, np.floating):
             return data
         image_max = self._image_max
         image_min = self._image_min

nibabel/minc2.py

@@ -28,8 +28,7 @@
 import numpy as np
 
 from .keywordonly import kw_only_meth
-from .optpkg import optional_package
-h5py, have_h5py, setup_module = optional_package('h5py')
+from ._h5py_compat import h5py
 
 from .minc1 import Minc1File, MincHeader, Minc1Image, MincError
 

nibabel/tests/test_h5py_compat.py

@@ -0,0 +1,44 @@
+"""
+These tests are almost certainly overkill, but serve to verify that
+the behavior of _h5py_compat is pass-through in all but a small set of
+well-defined cases
+"""
+import sys
+import os
+from distutils.version import LooseVersion
+import numpy as np
+
+from ..optpkg import optional_package
+from .. import _h5py_compat as compat
+from ..testing import assert_equal, assert_true, assert_false, assert_not_equal
+
+h5py, have_h5py, _ = optional_package('h5py')
+
+
+def test_optpkg_equivalence():
+    # No effect on Linux/OSX
+    if os.name == 'posix':
+        assert_equal(have_h5py, compat.have_h5py)
+    # No effect on Python 2.7 or 3.6+
+    if sys.version_info >= (3, 6) or sys.version_info < (3,):
+        assert_equal(have_h5py, compat.have_h5py)
+    # Available in a strict subset of cases
+    if not have_h5py:
+        assert_false(compat.have_h5py)
+    # Available when version is high enough
+    elif LooseVersion(h5py.__version__) >= '2.10':
+        assert_true(compat.have_h5py)
+
+
+def test_disabled_h5py_cases():
+    # On mismatch
+    if have_h5py and not compat.have_h5py:
+        # Recapitulate min_h5py conditions from _h5py_compat
+        assert_equal(os.name, 'nt')
+        assert_true((3,) <= sys.version_info < (3, 6))
+        assert_true(LooseVersion(h5py.__version__) < '2.10')
+        # Verify that the root cause is present
+        # If any tests fail, they will likely be these, so they may be
+        # ill-advised...
+        assert_equal(str(np.longdouble), str(np.float64))
+        assert_not_equal(np.longdouble, np.float64)

nibabel/tests/test_image_api.py

@@ -32,7 +32,7 @@
 
 from ..optpkg import optional_package
 _, have_scipy, _ = optional_package('scipy')
-_, have_h5py, _ = optional_package('h5py')
+from .._h5py_compat import have_h5py
 
 from .. import (AnalyzeImage, Spm99AnalyzeImage, Spm2AnalyzeImage,
                 Nifti1Pair, Nifti1Image, Nifti2Pair, Nifti2Image,

nibabel/tests/test_imageclasses.py

@@ -6,12 +6,11 @@
 
 import numpy as np
 
-from nibabel.optpkg import optional_package
-
 import nibabel as nib
 from nibabel.analyze import AnalyzeImage
 from nibabel.nifti1 import Nifti1Image
 from nibabel.nifti2 import Nifti2Image
+from .._h5py_compat import have_h5py
 
 from nibabel import imageclasses
 from nibabel.imageclasses import spatial_axes_first, class_map, ext_map
@@ -23,8 +22,6 @@
 
 DATA_DIR = pjoin(dirname(__file__), 'data')
 
-have_h5py = optional_package('h5py')[1]
-
 MINC_3DS = ('minc1_1_scale.mnc',)
 MINC_4DS = ('minc1_4d.mnc',)
 if have_h5py:

nibabel/tests/test_minc2.py

@@ -11,12 +11,9 @@
 
 import numpy as np
 
-from ..optpkg import optional_package
-
-h5py, have_h5py, setup_module = optional_package('h5py')
-
 from .. import minc2
 from ..minc2 import Minc2File, Minc2Image
+from .._h5py_compat import h5py, have_h5py, setup_module
 
 from nose.tools import (assert_true, assert_equal, assert_false, assert_raises)
 

nibabel/tests/test_minc2_data.py

@@ -14,9 +14,7 @@
 
 import numpy as np
 
-from nibabel.optpkg import optional_package
-
-h5py, have_h5py, setup_module = optional_package('h5py')
+from .._h5py_compat import h5py, have_h5py, setup_module
 
 from .nibabel_data import get_nibabel_data, needs_nibabel_data
 from .. import load as top_load, Nifti1Image

nibabel/tests/test_proxy_api.py

@@ -44,8 +44,7 @@
 from .. import minc1
 from ..externals.netcdf import netcdf_file
 from .. import minc2
-from ..optpkg import optional_package
-h5py, have_h5py, _ = optional_package('h5py')
+from .._h5py_compat import h5py, have_h5py
 from .. import ecat
 from .. import parrec