|
33 | 33 | @author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) |
34 | 34 | """ |
35 | 35 | import glob |
| 36 | +import json |
36 | 37 | import os |
37 | 38 | import re |
38 | 39 | import fileinput |
|
41 | 42 | from easybuild.tools import LooseVersion |
42 | 43 |
|
43 | 44 | import easybuild.tools.environment as env |
| 45 | +from easybuild.base import fancylogger |
44 | 46 | from easybuild.easyblocks.generic.configuremake import ConfigureMake |
45 | 47 | from easybuild.framework.easyconfig import CUSTOM |
46 | 48 | from easybuild.framework.easyconfig.templates import PYPI_SOURCE |
|
107 | 109 | """ % {'EBPYTHONPREFIXES': EBPYTHONPREFIXES} |
108 | 110 |
|
109 | 111 |
|
| 112 | +def det_pip_version(python_cmd='python'): |
| 113 | + """Determine version of currently active 'pip' module.""" |
| 114 | + |
| 115 | + pip_version = None |
| 116 | + log = fancylogger.getLogger('det_pip_version', fname=False) |
| 117 | + log.info("Determining pip version...") |
| 118 | + |
| 119 | + res = run_shell_cmd("%s -m pip --version" % python_cmd, hidden=True) |
| 120 | + out = res.output |
| 121 | + |
| 122 | + pip_version_regex = re.compile('^pip ([0-9.]+)') |
| 123 | + res = pip_version_regex.search(out) |
| 124 | + if res: |
| 125 | + pip_version = res.group(1) |
| 126 | + log.info("Found pip version: %s", pip_version) |
| 127 | + else: |
| 128 | + log.warning("Failed to determine pip version from '%s' using pattern '%s'", out, pip_version_regex.pattern) |
| 129 | + |
| 130 | + return pip_version |
| 131 | + |
| 132 | + |
| 133 | +def det_installed_python_packages(names_only=True, python_cmd=None): |
| 134 | + """ |
| 135 | + Return list of Python packages that are installed |
| 136 | +
|
| 137 | + Note that the names are reported by pip and might be different to the name that need to be used to import it. |
| 138 | +
|
| 139 | + :param names_only: boolean indicating whether only names or full info from `pip list` should be returned |
| 140 | + :param python_cmd: Python command to use (if None, 'python' is used) |
| 141 | + """ |
| 142 | + log = fancylogger.getLogger('det_installed_python_packages', fname=False) |
| 143 | + |
| 144 | + if python_cmd is None: |
| 145 | + python_cmd = 'python' |
| 146 | + |
| 147 | + # Check installed Python packages |
| 148 | + cmd = ' '.join([ |
| 149 | + python_cmd, '-m', 'pip', |
| 150 | + 'list', |
| 151 | + '--isolated', |
| 152 | + '--disable-pip-version-check', |
| 153 | + '--format', 'json', |
| 154 | + ]) |
| 155 | + res = run_shell_cmd(cmd, fail_on_error=False, hidden=True) |
| 156 | + if res.exit_code: |
| 157 | + raise EasyBuildError(f'Failed to determine installed python packages: {res.output}') |
| 158 | + |
| 159 | + # only check stdout, not stderr which might contain user facing warnings |
| 160 | + log.info(f'Got list of installed Python packages: {res.output}') |
| 161 | + pkgs = json.loads(res.output.strip()) |
| 162 | + return [pkg['name'] for pkg in pkgs] if names_only else pkgs |
| 163 | + |
| 164 | + |
| 165 | +def run_pip_check(python_cmd=None, unversioned_packages=None): |
| 166 | + """ |
| 167 | + Check installed Python packages using 'pip check' |
| 168 | +
|
| 169 | + :param unversioned_packages: list of Python packages to exclude in the version existence check |
| 170 | + :param python_cmd: Python command to use (if None, 'python' is used) |
| 171 | + """ |
| 172 | + log = fancylogger.getLogger('det_installed_python_packages', fname=False) |
| 173 | + |
| 174 | + if python_cmd is None: |
| 175 | + python_cmd = 'python' |
| 176 | + if unversioned_packages is None: |
| 177 | + unversioned_packages = [] |
| 178 | + |
| 179 | + pip_check_cmd = f"{python_cmd} -m pip check" |
| 180 | + |
| 181 | + pip_version = det_pip_version(python_cmd=python_cmd) |
| 182 | + if not pip_version: |
| 183 | + raise EasyBuildError("Failed to determine pip version!") |
| 184 | + min_pip_version = LooseVersion('9.0.0') |
| 185 | + if LooseVersion(pip_version) < min_pip_version: |
| 186 | + raise EasyBuildError(f"pip >= {min_pip_version} is required for '{pip_check_cmd}', found {pip_version}") |
| 187 | + |
| 188 | + pip_check_errors = [] |
| 189 | + |
| 190 | + res = run_shell_cmd(pip_check_cmd, fail_on_error=False) |
| 191 | + if res.exit_code: |
| 192 | + pip_check_errors.append(f"`{pip_check_cmd}` failed:\n{res.output}") |
| 193 | + else: |
| 194 | + log.info(f"`{pip_check_cmd}` passed successfully") |
| 195 | + |
| 196 | + # Also check for a common issue where the package version shows up as 0.0.0 often caused |
| 197 | + # by using setup.py as the installation method for a package which is released as a generic wheel |
| 198 | + # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code |
| 199 | + # so it will contain a version, but the raw tar.gz does not. |
| 200 | + pkgs = det_installed_python_packages(names_only=False, python_cmd=python_cmd) |
| 201 | + faulty_version = '0.0.0' |
| 202 | + faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] |
| 203 | + |
| 204 | + for unversioned_package in unversioned_packages: |
| 205 | + try: |
| 206 | + faulty_pkg_names.remove(unversioned_package) |
| 207 | + log.debug(f"Excluding unversioned package '{unversioned_package}' from check") |
| 208 | + except ValueError: |
| 209 | + try: |
| 210 | + version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) |
| 211 | + except StopIteration: |
| 212 | + msg = f"Package '{unversioned_package}' in unversioned_packages was not found in " |
| 213 | + msg += "the installed packages. Check that the name from `python -m pip list` is used " |
| 214 | + msg += "which may be different than the module name." |
| 215 | + else: |
| 216 | + msg = f"Package '{unversioned_package}' in unversioned_packages has a version of {version} " |
| 217 | + msg += "which is valid. Please remove it from unversioned_packages." |
| 218 | + pip_check_errors.append(msg) |
| 219 | + |
| 220 | + log.info("Found %s invalid packages out of %s packages", len(faulty_pkg_names), len(pkgs)) |
| 221 | + if faulty_pkg_names: |
| 222 | + faulty_pkg_names_str = '\n'.join(faulty_pkg_names) |
| 223 | + msg = "The following Python packages were likely not installed correctly because they show a " |
| 224 | + msg += f"version of '{faulty_version}':\n{faulty_pkg_names_str}\n" |
| 225 | + msg += "This may be solved by using a *-none-any.whl file as the source instead. " |
| 226 | + msg += "See e.g. the SOURCE*_WHL templates.\n" |
| 227 | + msg += "Otherwise you could check if the package provides a version at all or if e.g. poetry is " |
| 228 | + msg += "required (check the source for a pyproject.toml and see PEP517 for details on that)." |
| 229 | + pip_check_errors.append(msg) |
| 230 | + |
| 231 | + if pip_check_errors: |
| 232 | + raise EasyBuildError('\n'.join(pip_check_errors)) |
| 233 | + |
| 234 | + |
110 | 235 | class EB_Python(ConfigureMake): |
111 | 236 | """Support for building/installing Python |
112 | 237 | - default configure/build_step/make install works fine |
@@ -151,8 +276,8 @@ def __init__(self, *args, **kwargs): |
151 | 276 | # which is voluntarily or accidentally installed multiple times. |
152 | 277 | # Example: Upgrading to a higher version after installing new dependencies. |
153 | 278 | 'pip_ignore_installed': False, |
154 | | - # Python installations must be clean. Requires pip >= 9 |
155 | | - 'sanity_pip_check': LooseVersion(self._get_pip_ext_version() or '0.0') >= LooseVersion('9.0'), |
| 279 | + # disable per-extension 'pip check', since it's a global check done in sanity check step of Python easyblock |
| 280 | + 'sanity_pip_check': False, |
156 | 281 | # EasyBuild 5 |
157 | 282 | 'use_pip': True, |
158 | 283 | } |
@@ -558,6 +683,9 @@ def sanity_check_step(self): |
558 | 683 | except EasyBuildError as err: |
559 | 684 | raise EasyBuildError("Loading fake module failed: %s", err) |
560 | 685 |
|
| 686 | + # global 'pip check' to verify that version requirements are met for Python packages installed as extensions |
| 687 | + run_pip_check(python_cmd='python') |
| 688 | + |
561 | 689 | abiflags = '' |
562 | 690 | if LooseVersion(self.version) >= LooseVersion("3"): |
563 | 691 | run_shell_cmd("command -v python", hidden=True) |
|
0 commit comments