From 52c85270f61804f8ebedf05502ee38beb5b117d0 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 10:39:22 -0400 Subject: [PATCH 1/9] CI: Add expected failing test --- appveyor.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 796c74a7da..9ab441f7fa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,7 @@ # CI on Windows via appveyor environment: + DEPENDS: numpy scipy matplotlib h5py pydicom matrix: - PYTHON: C:\Python27 @@ -10,6 +11,9 @@ environment: - PYTHON: C:\Python34-x64 - PYTHON: C:\Python35 - PYTHON: C:\Python35-x64 + - PYTHON: C:\Python35-x64 + PYTHONHASHSEED: 283137131 + DEPENDS: "h5py==2.9.0" - PYTHON: C:\Python36 - PYTHON: C:\Python36-x64 - PYTHON: C:\Python37 @@ -28,7 +32,7 @@ install: # Install the dependencies of the project. - pip install --upgrade pip setuptools>=27.0 wheel - - pip install numpy scipy matplotlib h5py pydicom + - pip install %DEPENDS% - pip install nose mock coverage codecov - pip install . - SET NIBABEL_DATA_DIR=%CD%\nibabel-data From 12b2ad1cd7564ff024e66af4f43aa92d245abfe8 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 11:07:06 -0400 Subject: [PATCH 2/9] CI: Temporarily drop all other tests --- appveyor.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9ab441f7fa..d946d4194b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,19 +5,9 @@ environment: DEPENDS: numpy scipy matplotlib h5py pydicom matrix: - - PYTHON: C:\Python27 - - PYTHON: C:\Python27-x64 - - PYTHON: C:\Python34 - - PYTHON: C:\Python34-x64 - - PYTHON: C:\Python35 - - PYTHON: C:\Python35-x64 - PYTHON: C:\Python35-x64 PYTHONHASHSEED: 283137131 DEPENDS: "h5py==2.9.0" - - PYTHON: C:\Python36 - - PYTHON: C:\Python36-x64 - - PYTHON: C:\Python37 - - PYTHON: C:\Python37-x64 install: # Prepend newly installed Python to the PATH of this build (this cannot be From 9dec496878544ad5d1d22e2132fe95dc8763909b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 11:07:46 -0400 Subject: [PATCH 3/9] ENH: Require h5py 2.10 for Windows + Python < 3.6 --- nibabel/minc2.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nibabel/minc2.py b/nibabel/minc2.py index 40f38f97b3..edf553de2e 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -25,11 +25,21 @@ mincstats my_funny.mnc """ +import sys +import os import numpy as np from .keywordonly import kw_only_meth from .optpkg import optional_package -h5py, have_h5py, setup_module = optional_package('h5py') + +# PY35: A bug affected Windows installations of h5py in Python3 versions <3.6 +# due to random dictionary ordering, causing float64 data arrays to sometimes be +# loaded as longdouble (also 64 bit on Windows). This caused stochastic failures +# to correctly handle data caches, and possibly other subtle bugs we never +# caught. This was fixed in h5py 2.10. +# Please see https://github.com/nipy/nibabel/issues/665 for details. +min_h5py = '2.10' if os.name == 'nt' and (3,) <= sys.version_info < (3, 6) else None +h5py, have_h5py, setup_module = optional_package('h5py', min_version=min_h5py) from .minc1 import Minc1File, MincHeader, Minc1Image, MincError From 0a29f4526598ae2a73ae20369c748272ca62cfe4 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 12:15:01 -0400 Subject: [PATCH 4/9] TEST: Import have_h5py and similar from nibabel.minc2 --- nibabel/tests/test_image_api.py | 2 +- nibabel/tests/test_imageclasses.py | 5 +---- nibabel/tests/test_minc2.py | 6 +----- nibabel/tests/test_minc2_data.py | 4 +--- nibabel/tests/test_proxy_api.py | 3 +-- 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/nibabel/tests/test_image_api.py b/nibabel/tests/test_image_api.py index ac2a2428c4..169660cc90 100644 --- a/nibabel/tests/test_image_api.py +++ b/nibabel/tests/test_image_api.py @@ -34,7 +34,7 @@ from ..optpkg import optional_package _, have_scipy, _ = optional_package('scipy') -_, have_h5py, _ = optional_package('h5py') +from ..minc2 import have_h5py from .. import (AnalyzeImage, Spm99AnalyzeImage, Spm2AnalyzeImage, Nifti1Pair, Nifti1Image, Nifti2Pair, Nifti2Image, diff --git a/nibabel/tests/test_imageclasses.py b/nibabel/tests/test_imageclasses.py index 3c3c437136..5060e23e05 100644 --- a/nibabel/tests/test_imageclasses.py +++ b/nibabel/tests/test_imageclasses.py @@ -6,12 +6,11 @@ import numpy as np -from nibabel.optpkg import optional_package - import nibabel as nib from nibabel.analyze import AnalyzeImage from nibabel.nifti1 import Nifti1Image from nibabel.nifti2 import Nifti2Image +from ..minc2 import have_h5py from nibabel import imageclasses from nibabel.imageclasses import spatial_axes_first, class_map, ext_map @@ -23,8 +22,6 @@ DATA_DIR = pjoin(dirname(__file__), 'data') -have_h5py = optional_package('h5py')[1] - MINC_3DS = ('minc1_1_scale.mnc',) MINC_4DS = ('minc1_4d.mnc',) if have_h5py: diff --git a/nibabel/tests/test_minc2.py b/nibabel/tests/test_minc2.py index c4cb9341ca..23eff702b3 100644 --- a/nibabel/tests/test_minc2.py +++ b/nibabel/tests/test_minc2.py @@ -12,12 +12,8 @@ import numpy as np -from ..optpkg import optional_package - -h5py, have_h5py, setup_module = optional_package('h5py') - from .. import minc2 -from ..minc2 import Minc2File, Minc2Image +from ..minc2 import Minc2File, Minc2Image, h5py, have_h5py, setup_module from nose.tools import (assert_true, assert_equal, assert_false, assert_raises) diff --git a/nibabel/tests/test_minc2_data.py b/nibabel/tests/test_minc2_data.py index 1ec4999a43..8b6fdfb78f 100644 --- a/nibabel/tests/test_minc2_data.py +++ b/nibabel/tests/test_minc2_data.py @@ -15,9 +15,7 @@ import numpy as np -from nibabel.optpkg import optional_package - -h5py, have_h5py, setup_module = optional_package('h5py') +from ..minc2 import h5py, have_h5py, setup_module from .nibabel_data import get_nibabel_data, needs_nibabel_data from .. import load as top_load, Nifti1Image diff --git a/nibabel/tests/test_proxy_api.py b/nibabel/tests/test_proxy_api.py index 7280c5552d..1e329ae230 100644 --- a/nibabel/tests/test_proxy_api.py +++ b/nibabel/tests/test_proxy_api.py @@ -46,8 +46,7 @@ from .. import minc1 from ..externals.netcdf import netcdf_file from .. import minc2 -from ..optpkg import optional_package -h5py, have_h5py, _ = optional_package('h5py') +from ..minc2 import h5py, have_h5py from .. import ecat from .. import parrec From 622dfa6214043284f135509c97c4afa593c57d1e Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 12:22:40 -0400 Subject: [PATCH 5/9] FIX: More robust floating point dtype check --- nibabel/minc1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index a8535eec05..1ca25eaf9c 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -173,7 +173,7 @@ def _normalize(self, data, sliceobj=()): applied to `data` """ ddt = self.get_data_dtype() - if ddt.type in np.sctypes['float']: + if np.issubdtype(ddt.type, np.floating): return data image_max = self._image_max image_min = self._image_min From 9f940138d541c65d3373165b64a6c63462ad67c9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 14:44:10 -0400 Subject: [PATCH 6/9] Revert "CI: Temporarily drop all other tests" This reverts commit 12b2ad1cd7564ff024e66af4f43aa92d245abfe8. --- appveyor.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index d946d4194b..9ab441f7fa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,9 +5,19 @@ environment: DEPENDS: numpy scipy matplotlib h5py pydicom matrix: + - PYTHON: C:\Python27 + - PYTHON: C:\Python27-x64 + - PYTHON: C:\Python34 + - PYTHON: C:\Python34-x64 + - PYTHON: C:\Python35 + - PYTHON: C:\Python35-x64 - PYTHON: C:\Python35-x64 PYTHONHASHSEED: 283137131 DEPENDS: "h5py==2.9.0" + - PYTHON: C:\Python36 + - PYTHON: C:\Python36-x64 + - PYTHON: C:\Python37 + - PYTHON: C:\Python37-x64 install: # Prepend newly installed Python to the PATH of this build (this cannot be From 0032049ebf1421e4f8d8b1e1349dcd70a821cd95 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 9 Sep 2019 16:39:04 -0400 Subject: [PATCH 7/9] CI: Require binaries for 3.4 as dependencies stop building wheels --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 9ab441f7fa..aba8c60380 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,7 +8,9 @@ environment: - PYTHON: C:\Python27 - PYTHON: C:\Python27-x64 - PYTHON: C:\Python34 + DEPENDS: --prefer-binary numpy scipy matplotlib h5py pydicom - PYTHON: C:\Python34-x64 + DEPENDS: --prefer-binary numpy scipy matplotlib h5py pydicom - PYTHON: C:\Python35 - PYTHON: C:\Python35-x64 - PYTHON: C:\Python35-x64 From 4127cfccd495e856f3b39883fffa896f411fb6f7 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 10 Sep 2019 10:00:07 -0400 Subject: [PATCH 8/9] RF/TEST: Move h5py compatibility hacks into a private module --- nibabel/_h5py_compat.py | 12 ++++++++ nibabel/minc2.py | 13 +-------- nibabel/tests/test_h5py_compat.py | 44 ++++++++++++++++++++++++++++++ nibabel/tests/test_image_api.py | 2 +- nibabel/tests/test_imageclasses.py | 2 +- nibabel/tests/test_minc2.py | 3 +- nibabel/tests/test_minc2_data.py | 2 +- nibabel/tests/test_proxy_api.py | 2 +- 8 files changed, 63 insertions(+), 17 deletions(-) create mode 100644 nibabel/_h5py_compat.py create mode 100644 nibabel/tests/test_h5py_compat.py diff --git a/nibabel/_h5py_compat.py b/nibabel/_h5py_compat.py new file mode 100644 index 0000000000..2c0b0eb2c0 --- /dev/null +++ b/nibabel/_h5py_compat.py @@ -0,0 +1,12 @@ +import sys +import os +from .optpkg import optional_package + +# PY35: A bug affected Windows installations of h5py in Python3 versions <3.6 +# due to random dictionary ordering, causing float64 data arrays to sometimes be +# loaded as longdouble (also 64 bit on Windows). This caused stochastic failures +# to correctly handle data caches, and possibly other subtle bugs we never +# caught. This was fixed in h5py 2.10. +# Please see https://github.com/nipy/nibabel/issues/665 for details. +min_h5py = '2.10' if os.name == 'nt' and (3,) <= sys.version_info < (3, 6) else None +h5py, have_h5py, setup_module = optional_package('h5py', min_version=min_h5py) diff --git a/nibabel/minc2.py b/nibabel/minc2.py index edf553de2e..37821409c4 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -25,21 +25,10 @@ mincstats my_funny.mnc """ -import sys -import os import numpy as np from .keywordonly import kw_only_meth -from .optpkg import optional_package - -# PY35: A bug affected Windows installations of h5py in Python3 versions <3.6 -# due to random dictionary ordering, causing float64 data arrays to sometimes be -# loaded as longdouble (also 64 bit on Windows). This caused stochastic failures -# to correctly handle data caches, and possibly other subtle bugs we never -# caught. This was fixed in h5py 2.10. -# Please see https://github.com/nipy/nibabel/issues/665 for details. -min_h5py = '2.10' if os.name == 'nt' and (3,) <= sys.version_info < (3, 6) else None -h5py, have_h5py, setup_module = optional_package('h5py', min_version=min_h5py) +from ._h5py_compat import h5py from .minc1 import Minc1File, MincHeader, Minc1Image, MincError diff --git a/nibabel/tests/test_h5py_compat.py b/nibabel/tests/test_h5py_compat.py new file mode 100644 index 0000000000..26d70b6e55 --- /dev/null +++ b/nibabel/tests/test_h5py_compat.py @@ -0,0 +1,44 @@ +""" +These tests are almost certainly overkill, but serve to verify that +the behavior of _h5py_compat is pass-through in all but a small set of +well-defined cases +""" +import sys +import os +from distutils.version import LooseVersion +import numpy as np + +from ..optpkg import optional_package +from .. import _h5py_compat as compat +from ..testing import assert_equal, assert_true, assert_false, assert_not_equal + +h5py, have_h5py, _ = optional_package('h5py') + + +def test_optpkg_equivalence(): + # No effect on Linux/OSX + if os.name == 'posix': + assert_equal(have_h5py, compat.have_h5py) + # No effect on Python 2.7 or 3.6+ + if sys.version_info >= (3, 6) or sys.version_info < (3,): + assert_equal(have_h5py, compat.have_h5py) + # Available in a strict subset of cases + if not have_h5py: + assert_false(compat.have_h5py) + # Available when version is high enough + elif LooseVersion(h5py.__version__) >= '2.10': + assert_true(compat.have_h5py) + + +def test_disabled_h5py_cases(): + # On mismatch + if have_h5py and not compat.have_h5py: + # Recapitulate min_h5py conditions from _h5py_compat + assert_equal(os.name, 'nt') + assert_true((3,) <= sys.version_info < (3, 6)) + assert_true(LooseVersion(h5py.__version__) < '2.10') + # Verify that the root cause is present + # If any tests fail, they will likely be these, so they may be + # ill-advised... + assert_equal(str(np.longdouble), str(np.float64)) + assert_not_equal(np.longdouble, np.float64) diff --git a/nibabel/tests/test_image_api.py b/nibabel/tests/test_image_api.py index 169660cc90..9df362e657 100644 --- a/nibabel/tests/test_image_api.py +++ b/nibabel/tests/test_image_api.py @@ -34,7 +34,7 @@ from ..optpkg import optional_package _, have_scipy, _ = optional_package('scipy') -from ..minc2 import have_h5py +from .._h5py_compat import have_h5py from .. import (AnalyzeImage, Spm99AnalyzeImage, Spm2AnalyzeImage, Nifti1Pair, Nifti1Image, Nifti2Pair, Nifti2Image, diff --git a/nibabel/tests/test_imageclasses.py b/nibabel/tests/test_imageclasses.py index 5060e23e05..12232c42e4 100644 --- a/nibabel/tests/test_imageclasses.py +++ b/nibabel/tests/test_imageclasses.py @@ -10,7 +10,7 @@ from nibabel.analyze import AnalyzeImage from nibabel.nifti1 import Nifti1Image from nibabel.nifti2 import Nifti2Image -from ..minc2 import have_h5py +from .._h5py_compat import have_h5py from nibabel import imageclasses from nibabel.imageclasses import spatial_axes_first, class_map, ext_map diff --git a/nibabel/tests/test_minc2.py b/nibabel/tests/test_minc2.py index 23eff702b3..93bff42a47 100644 --- a/nibabel/tests/test_minc2.py +++ b/nibabel/tests/test_minc2.py @@ -13,7 +13,8 @@ import numpy as np from .. import minc2 -from ..minc2 import Minc2File, Minc2Image, h5py, have_h5py, setup_module +from ..minc2 import Minc2File, Minc2Image +from .._h5py_compat import h5py, have_h5py, setup_module from nose.tools import (assert_true, assert_equal, assert_false, assert_raises) diff --git a/nibabel/tests/test_minc2_data.py b/nibabel/tests/test_minc2_data.py index 8b6fdfb78f..d5b19fe9ce 100644 --- a/nibabel/tests/test_minc2_data.py +++ b/nibabel/tests/test_minc2_data.py @@ -15,7 +15,7 @@ import numpy as np -from ..minc2 import h5py, have_h5py, setup_module +from .._h5py_compat import h5py, have_h5py, setup_module from .nibabel_data import get_nibabel_data, needs_nibabel_data from .. import load as top_load, Nifti1Image diff --git a/nibabel/tests/test_proxy_api.py b/nibabel/tests/test_proxy_api.py index 1e329ae230..074f2de5b2 100644 --- a/nibabel/tests/test_proxy_api.py +++ b/nibabel/tests/test_proxy_api.py @@ -46,7 +46,7 @@ from .. import minc1 from ..externals.netcdf import netcdf_file from .. import minc2 -from ..minc2 import h5py, have_h5py +from .._h5py_compat import h5py, have_h5py from .. import ecat from .. import parrec From 920eefa4cd421a5d46a041149f457dd71c5e7d86 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 11 Sep 2019 10:15:19 -0400 Subject: [PATCH 9/9] CI: Add expected failing test to Azure --- .azure-pipelines/windows.yml | 6 +++++- azure-pipelines.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml index ad1b72eb77..0cda80d6ee 100644 --- a/.azure-pipelines/windows.yml +++ b/.azure-pipelines/windows.yml @@ -10,6 +10,7 @@ jobs: vmImage: ${{ parameters.vmImage }} variables: EXTRA_WHEELS: "https://5cf40426d9f06eb7461d-6fe47d9331aba7cd62fc36c7196769e4.ssl.cf2.rackcdn.com" + DEPENDS: numpy scipy matplotlib h5py pydicom strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} @@ -20,11 +21,14 @@ jobs: versionSpec: '$(PYTHON_VERSION)' addToPath: true architecture: '$(PYTHON_ARCH)' + - script: | + echo %PYTHONHASHSEED% + displayName: 'Display hash seed' - script: | python -m pip install --upgrade pip setuptools>=30.3.0 wheel displayName: 'Update build tools' - script: | - python -m pip install --find-links %EXTRA_WHEELS% numpy scipy matplotlib h5py pydicom + python -m pip install --find-links %EXTRA_WHEELS% %DEPENDS% python -m pip install nose mock coverage codecov displayName: 'Install dependencies' - script: | diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cb2612d5c3..f595ec35b7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,6 +11,11 @@ jobs: py35-x64: PYTHON_VERSION: '3.5' PYTHON_ARCH: 'x64' + py35-h5py-check: + PYTHON_VERSION: '3.5' + PYTHON_ARCH: 'x64' + PYTHONHASHSEED: 283137131 + DEPENDS: "h5py==2.9.0" py36-x86: PYTHON_VERSION: '3.6' PYTHON_ARCH: 'x86'