diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml index ad1b72eb77..0cda80d6ee 100644 --- a/.azure-pipelines/windows.yml +++ b/.azure-pipelines/windows.yml @@ -10,6 +10,7 @@ jobs: vmImage: ${{ parameters.vmImage }} variables: EXTRA_WHEELS: "https://5cf40426d9f06eb7461d-6fe47d9331aba7cd62fc36c7196769e4.ssl.cf2.rackcdn.com" + DEPENDS: numpy scipy matplotlib h5py pydicom strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} @@ -20,11 +21,14 @@ jobs: versionSpec: '$(PYTHON_VERSION)' addToPath: true architecture: '$(PYTHON_ARCH)' + - script: | + echo %PYTHONHASHSEED% + displayName: 'Display hash seed' - script: | python -m pip install --upgrade pip setuptools>=30.3.0 wheel displayName: 'Update build tools' - script: | - python -m pip install --find-links %EXTRA_WHEELS% numpy scipy matplotlib h5py pydicom + python -m pip install --find-links %EXTRA_WHEELS% %DEPENDS% python -m pip install nose mock coverage codecov displayName: 'Install dependencies' - script: | diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cb2612d5c3..f595ec35b7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,6 +11,11 @@ jobs: py35-x64: PYTHON_VERSION: '3.5' PYTHON_ARCH: 'x64' + py35-h5py-check: + PYTHON_VERSION: '3.5' + PYTHON_ARCH: 'x64' + PYTHONHASHSEED: 283137131 + DEPENDS: "h5py==2.9.0" py36-x86: PYTHON_VERSION: '3.6' PYTHON_ARCH: 'x86' diff --git a/nibabel/_h5py_compat.py b/nibabel/_h5py_compat.py new file mode 100644 index 0000000000..2c0b0eb2c0 --- /dev/null +++ b/nibabel/_h5py_compat.py @@ -0,0 +1,12 @@ +import sys +import os +from .optpkg import optional_package + +# PY35: A bug affected Windows installations of h5py in Python3 versions <3.6 +# due to random dictionary ordering, causing float64 data arrays to sometimes be +# loaded as longdouble (also 64 bit on Windows). This caused stochastic failures +# to correctly handle data caches, and possibly other subtle bugs we never +# caught. This was fixed in h5py 2.10. +# Please see https://github.com/nipy/nibabel/issues/665 for details. +min_h5py = '2.10' if os.name == 'nt' and (3,) <= sys.version_info < (3, 6) else None +h5py, have_h5py, setup_module = optional_package('h5py', min_version=min_h5py) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index a8535eec05..1ca25eaf9c 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -173,7 +173,7 @@ def _normalize(self, data, sliceobj=()): applied to `data` """ ddt = self.get_data_dtype() - if ddt.type in np.sctypes['float']: + if np.issubdtype(ddt.type, np.floating): return data image_max = self._image_max image_min = self._image_min diff --git a/nibabel/minc2.py b/nibabel/minc2.py index 40f38f97b3..37821409c4 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -28,8 +28,7 @@ import numpy as np from .keywordonly import kw_only_meth -from .optpkg import optional_package -h5py, have_h5py, setup_module = optional_package('h5py') +from ._h5py_compat import h5py from .minc1 import Minc1File, MincHeader, Minc1Image, MincError diff --git a/nibabel/tests/test_h5py_compat.py b/nibabel/tests/test_h5py_compat.py new file mode 100644 index 0000000000..26d70b6e55 --- /dev/null +++ b/nibabel/tests/test_h5py_compat.py @@ -0,0 +1,44 @@ +""" +These tests are almost certainly overkill, but serve to verify that +the behavior of _h5py_compat is pass-through in all but a small set of +well-defined cases +""" +import sys +import os +from distutils.version import LooseVersion +import numpy as np + +from ..optpkg import optional_package +from .. import _h5py_compat as compat +from ..testing import assert_equal, assert_true, assert_false, assert_not_equal + +h5py, have_h5py, _ = optional_package('h5py') + + +def test_optpkg_equivalence(): + # No effect on Linux/OSX + if os.name == 'posix': + assert_equal(have_h5py, compat.have_h5py) + # No effect on Python 2.7 or 3.6+ + if sys.version_info >= (3, 6) or sys.version_info < (3,): + assert_equal(have_h5py, compat.have_h5py) + # Available in a strict subset of cases + if not have_h5py: + assert_false(compat.have_h5py) + # Available when version is high enough + elif LooseVersion(h5py.__version__) >= '2.10': + assert_true(compat.have_h5py) + + +def test_disabled_h5py_cases(): + # On mismatch + if have_h5py and not compat.have_h5py: + # Recapitulate min_h5py conditions from _h5py_compat + assert_equal(os.name, 'nt') + assert_true((3,) <= sys.version_info < (3, 6)) + assert_true(LooseVersion(h5py.__version__) < '2.10') + # Verify that the root cause is present + # If any tests fail, they will likely be these, so they may be + # ill-advised... + assert_equal(str(np.longdouble), str(np.float64)) + assert_not_equal(np.longdouble, np.float64) diff --git a/nibabel/tests/test_image_api.py b/nibabel/tests/test_image_api.py index 979b8777f9..8792fe938e 100644 --- a/nibabel/tests/test_image_api.py +++ b/nibabel/tests/test_image_api.py @@ -32,7 +32,7 @@ from ..optpkg import optional_package _, have_scipy, _ = optional_package('scipy') -_, have_h5py, _ = optional_package('h5py') +from .._h5py_compat import have_h5py from .. import (AnalyzeImage, Spm99AnalyzeImage, Spm2AnalyzeImage, Nifti1Pair, Nifti1Image, Nifti2Pair, Nifti2Image, diff --git a/nibabel/tests/test_imageclasses.py b/nibabel/tests/test_imageclasses.py index 3c3c437136..12232c42e4 100644 --- a/nibabel/tests/test_imageclasses.py +++ b/nibabel/tests/test_imageclasses.py @@ -6,12 +6,11 @@ import numpy as np -from nibabel.optpkg import optional_package - import nibabel as nib from nibabel.analyze import AnalyzeImage from nibabel.nifti1 import Nifti1Image from nibabel.nifti2 import Nifti2Image +from .._h5py_compat import have_h5py from nibabel import imageclasses from nibabel.imageclasses import spatial_axes_first, class_map, ext_map @@ -23,8 +22,6 @@ DATA_DIR = pjoin(dirname(__file__), 'data') -have_h5py = optional_package('h5py')[1] - MINC_3DS = ('minc1_1_scale.mnc',) MINC_4DS = ('minc1_4d.mnc',) if have_h5py: diff --git a/nibabel/tests/test_minc2.py b/nibabel/tests/test_minc2.py index d456de0eec..f4367cbc11 100644 --- a/nibabel/tests/test_minc2.py +++ b/nibabel/tests/test_minc2.py @@ -11,12 +11,9 @@ import numpy as np -from ..optpkg import optional_package - -h5py, have_h5py, setup_module = optional_package('h5py') - from .. import minc2 from ..minc2 import Minc2File, Minc2Image +from .._h5py_compat import h5py, have_h5py, setup_module from nose.tools import (assert_true, assert_equal, assert_false, assert_raises) diff --git a/nibabel/tests/test_minc2_data.py b/nibabel/tests/test_minc2_data.py index 0471f87e7e..57146171e9 100644 --- a/nibabel/tests/test_minc2_data.py +++ b/nibabel/tests/test_minc2_data.py @@ -14,9 +14,7 @@ import numpy as np -from nibabel.optpkg import optional_package - -h5py, have_h5py, setup_module = optional_package('h5py') +from .._h5py_compat import h5py, have_h5py, setup_module from .nibabel_data import get_nibabel_data, needs_nibabel_data from .. import load as top_load, Nifti1Image diff --git a/nibabel/tests/test_proxy_api.py b/nibabel/tests/test_proxy_api.py index f1a1248c61..58ad5fa5d2 100644 --- a/nibabel/tests/test_proxy_api.py +++ b/nibabel/tests/test_proxy_api.py @@ -44,8 +44,7 @@ from .. import minc1 from ..externals.netcdf import netcdf_file from .. import minc2 -from ..optpkg import optional_package -h5py, have_h5py, _ = optional_package('h5py') +from .._h5py_compat import h5py, have_h5py from .. import ecat from .. import parrec