diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index e0a2f20c14..7297238cb4 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -221,6 +221,18 @@ bzl_library( ], ) +bzl_library( + name = "pep508_deps_bzl", + srcs = ["pep508_deps.bzl"], + deps = [ + ":pep508_env_bzl", + ":pep508_evaluate_bzl", + ":pep508_platform_bzl", + ":pep508_requirement_bzl", + "//python/private:normalize_name_bzl", + ], +) + bzl_library( name = "pep508_env_bzl", srcs = ["pep508_env.bzl"], @@ -368,7 +380,9 @@ bzl_library( ":generate_whl_library_build_bazel_bzl", ":parse_whl_name_bzl", ":patch_whl_bzl", + ":pep508_deps_bzl", ":pypi_repo_utils_bzl", + ":whl_metadata_bzl", ":whl_target_platforms_bzl", "//python/private:auth_bzl", "//python/private:envsubst_bzl", @@ -377,6 +391,11 @@ bzl_library( ], ) +bzl_library( + name = "whl_metadata_bzl", + srcs = ["whl_metadata.bzl"], +) + bzl_library( name = "whl_repo_name_bzl", srcs = ["whl_repo_name.bzl"], diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl new file mode 100644 index 0000000000..af0a75362b --- /dev/null +++ b/python/private/pypi/pep508_deps.bzl @@ -0,0 +1,351 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module is for implementing PEP508 compliant METADATA deps parsing. +""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_env.bzl", "env") +load(":pep508_evaluate.bzl", "evaluate") +load(":pep508_platform.bzl", "platform", "platform_from_str") +load(":pep508_requirement.bzl", "requirement") + +_ALL_OS_VALUES = [ + "windows", + "osx", + "linux", +] +_ALL_ARCH_VALUES = [ + "aarch64", + "ppc64", + "ppc64le", + "s390x", + "x86_32", + "x86_64", +] + +def deps(name, *, requires_dist, platforms = [], extras = [], host_python_version = None): + """Parse the RequiresDist from wheel METADATA + + Args: + name: {type}`str` the name of the wheel. + requires_dist: {type}`list[str]` the list of RequiresDist lines from the + METADATA file. + extras: {type}`list[str]` the requested extras to generate targets for. + platforms: {type}`list[str]` the list of target platform strings. + host_python_version: {type}`str` the host python version. + + Returns: + A struct with attributes: + * deps: {type}`list[str]` dependencies to include unconditionally. + * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular + subset of target platforms. + """ + reqs = sorted( + [requirement(r) for r in requires_dist], + key = lambda x: "{}:{}:".format(x.name, sorted(x.extras), x.marker), + ) + deps = {} + deps_select = {} + name = normalize_name(name) + want_extras = _resolve_extras(name, reqs, extras) + + # drop self edges + reqs = [r for r in reqs if r.name != name] + + platforms = [ + platform_from_str(p, python_version = host_python_version) + for p in platforms + ] or [ + platform_from_str("", python_version = host_python_version), + ] + + abis = sorted({p.abi: True for p in platforms if p.abi}) + if host_python_version and len(abis) > 1: + _, _, minor_version = host_python_version.partition(".") + minor_version, _, _ = minor_version.partition(".") + default_abi = "cp3" + minor_version + elif len(abis) > 1: + fail( + "all python versions need to be specified explicitly, got: {}".format(platforms), + ) + else: + default_abi = None + + for req in reqs: + _add_req( + deps, + deps_select, + req, + extras = want_extras, + platforms = platforms, + default_abi = default_abi, + ) + + return struct( + deps = sorted(deps), + deps_select = { + _platform_str(p): sorted(deps) + for p, deps in deps_select.items() + }, + ) + +def _platform_str(self): + if self.abi == None: + if not self.os and not self.arch: + return "//conditions:default" + elif not self.arch: + return "@platforms//os:{}".format(self.os) + else: + return "{}_{}".format(self.os, self.arch) + + minor_version = self.abi[3:] + if self.arch == None and self.os == None: + return str(Label("//python/config_settings:is_python_3.{}".format(minor_version))) + + return "cp3{}_{}_{}".format( + minor_version, + self.os or "anyos", + self.arch or "anyarch", + ) + +def _platform_specializations(self, cpu_values = _ALL_ARCH_VALUES, os_values = _ALL_OS_VALUES): + """Return the platform itself and all its unambiguous specializations. + + For more info about specializations see + https://bazel.build/docs/configurable-attributes + """ + specializations = [] + specializations.append(self) + if self.arch == None: + specializations.extend([ + platform(os = self.os, arch = arch, abi = self.abi) + for arch in cpu_values + ]) + if self.os == None: + specializations.extend([ + platform(os = os, arch = self.arch, abi = self.abi) + for os in os_values + ]) + if self.os == None and self.arch == None: + specializations.extend([ + platform(os = os, arch = arch, abi = self.abi) + for os in os_values + for arch in cpu_values + ]) + return specializations + +def _add(deps, deps_select, dep, platform): + dep = normalize_name(dep) + + if platform == None: + deps[dep] = True + + # If the dep is in the platform-specific list, remove it from the select. + pop_keys = [] + for p, _deps in deps_select.items(): + if dep not in _deps: + continue + + _deps.pop(dep) + if not _deps: + pop_keys.append(p) + + for p in pop_keys: + deps_select.pop(p) + return + + if dep in deps: + # If the dep is already in the main dependency list, no need to add it in the + # platform-specific dependency list. + return + + # Add the platform-specific branch + deps_select.setdefault(platform, {}) + + # Add the dep to specializations of the given platform if they + # exist in the select statement. + for p in _platform_specializations(platform): + if p not in deps_select: + continue + + deps_select[p][dep] = True + + if len(deps_select[platform]) == 1: + # We are adding a new item to the select and we need to ensure that + # existing dependencies from less specialized platforms are propagated + # to the newly added dependency set. + for p, _deps in deps_select.items(): + # Check if the existing platform overlaps with the given platform + if p == platform or platform not in _platform_specializations(p): + continue + + deps_select[platform].update(_deps) + +def _maybe_add_common_dep(deps, deps_select, platforms, dep): + abis = sorted({p.abi: True for p in platforms if p.abi}) + if len(abis) < 2: + return + + platforms = [platform()] + [ + platform(abi = abi) + for abi in abis + ] + + # If the dep is targeting all target python versions, lets add it to + # the common dependency list to simplify the select statements. + for p in platforms: + if p not in deps_select: + return + + if dep not in deps_select[p]: + return + + # All of the python version-specific branches have the dep, so lets add + # it to the common deps. + deps[dep] = True + for p in platforms: + deps_select[p].pop(dep) + if not deps_select[p]: + deps_select.pop(p) + +def _resolve_extras(self_name, reqs, extras): + """Resolve extras which are due to depending on self[some_other_extra]. + + Some packages may have cyclic dependencies resulting from extras being used, one example is + `etils`, where we have one set of extras as aliases for other extras + and we have an extra called 'all' that includes all other extras. + + Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32. + + When the `requirements.txt` is generated by `pip-tools`, then it is likely that + this step is not needed, but for other `requirements.txt` files this may be useful. + + NOTE @aignas 2023-12-08: the extra resolution is not platform dependent, + but in order for it to become platform dependent we would have to have + separate targets for each extra in extras. + """ + + # Resolve any extra extras due to self-edges, empty string means no + # extras The empty string in the set is just a way to make the handling + # of no extras and a single extra easier and having a set of {"", "foo"} + # is equivalent to having {"foo"}. + extras = extras or [""] + + self_reqs = [] + for req in reqs: + if req.name != self_name: + continue + + if req.marker == None: + # I am pretty sure we cannot reach this code as it does not + # make sense to specify packages in this way, but since it is + # easy to handle, lets do it. + # + # TODO @aignas 2023-12-08: add a test + extras = extras + req.extras + else: + # process these in a separate loop + self_reqs.append(req) + + # A double loop is not strictly optimal, but always correct without recursion + for req in self_reqs: + if [True for extra in extras if evaluate(req.marker, env = {"extra": extra})]: + extras = extras + req.extras + else: + continue + + # Iterate through all packages to ensure that we include all of the extras from previously + # visited packages. + for req_ in self_reqs: + if [True for extra in extras if evaluate(req.marker, env = {"extra": extra})]: + extras = extras + req_.extras + + # Poor mans set + return sorted({x: None for x in extras}) + +def _add_req(deps, deps_select, req, *, extras, platforms, default_abi = None): + if not req.marker: + _add(deps, deps_select, req.name, None) + return + + # NOTE @aignas 2023-12-08: in order to have reasonable select statements + # we do have to have some parsing of the markers, so it begs the question + # if packaging should be reimplemented in Starlark to have the best solution + # for now we will implement it in Python and see what the best parsing result + # can be before making this decision. + match_os = len([ + tag + for tag in [ + "os_name", + "sys_platform", + "platform_system", + ] + if tag in req.marker + ]) > 0 + match_arch = "platform_machine" in req.marker + match_version = "version" in req.marker + + if not (match_os or match_arch or match_version): + if [ + True + for extra in extras + for p in platforms + if evaluate( + req.marker, + env = env( + target_platform = p, + extra = extra, + ), + ) + ]: + _add(deps, deps_select, req.name, None) + return + + for plat in platforms: + if not [ + True + for extra in extras + if evaluate( + req.marker, + env = env( + target_platform = plat, + extra = extra, + ), + ) + ]: + continue + + if match_arch and default_abi: + _add(deps, deps_select, req.name, plat) + if plat.abi == default_abi: + _add(deps, deps_select, req.name, platform(os = plat.os, arch = plat.arch)) + elif match_arch: + _add(deps, deps_select, req.name, platform(os = plat.os, arch = plat.arch)) + elif match_os and default_abi: + _add(deps, deps_select, req.name, platform(os = plat.os, abi = plat.abi)) + if plat.abi == default_abi: + _add(deps, deps_select, req.name, platform(os = plat.os)) + elif match_os: + _add(deps, deps_select, req.name, platform(os = plat.os)) + elif match_version and default_abi: + _add(deps, deps_select, req.name, platform(abi = plat.abi)) + if plat.abi == default_abi: + _add(deps, deps_select, req.name, platform()) + elif match_version: + _add(deps, deps_select, req.name, None) + else: + fail("BUG: {} support is not implemented".format(req.marker)) + + _maybe_add_common_dep(deps, deps_select, platforms, req.name) diff --git a/python/private/pypi/pep508_evaluate.bzl b/python/private/pypi/pep508_evaluate.bzl index f45eb75cdb..f8ef553034 100644 --- a/python/private/pypi/pep508_evaluate.bzl +++ b/python/private/pypi/pep508_evaluate.bzl @@ -138,7 +138,7 @@ def evaluate(marker, *, env, strict = True, **kwargs): """ tokens = tokenize(marker) - ast = _new_expr(**kwargs) + ast = _new_expr(marker = marker, **kwargs) for _ in range(len(tokens) * 2): if not tokens: break @@ -219,17 +219,20 @@ def _not_fn(x): return not x def _new_expr( + *, + marker, and_fn = _and_fn, or_fn = _or_fn, not_fn = _not_fn): # buildifier: disable=uninitialized self = struct( + marker = marker, tree = [], parse = lambda **kwargs: _parse(self, **kwargs), value = lambda: _value(self), # This is a way for us to have a handle to the currently constructed # expression tree branch. - current = lambda: self._current[0] if self._current else None, + current = lambda: self._current[-1] if self._current else None, _current = [], _and = and_fn, _or = or_fn, @@ -313,6 +316,7 @@ def marker_expr(left, op, right, *, env, strict = True): # # The following normalizes the values left = env.get(_ENV_ALIASES, {}).get(var_name, {}).get(left, left) + else: var_name = left left = env[left] @@ -392,12 +396,15 @@ def _append(self, value): current.tree.append(value) elif hasattr(current.tree[-1], "append"): current.tree[-1].append(value) - else: + elif hasattr(current.tree, "_append"): current.tree._append(value) + else: + fail("Cannot evaluate '{}' in '{}', current: {}".format(value, self.marker, current)) def _open_parenthesis(self): """Add an extra node into the tree to perform evaluate inside parenthesis.""" self._current.append(_new_expr( + marker = self.marker, and_fn = self._and, or_fn = self._or, not_fn = self._not, diff --git a/python/private/pypi/whl_installer/BUILD.bazel b/python/private/pypi/whl_installer/BUILD.bazel index 5fb617004d..49f1a119c1 100644 --- a/python/private/pypi/whl_installer/BUILD.bazel +++ b/python/private/pypi/whl_installer/BUILD.bazel @@ -6,7 +6,6 @@ py_library( srcs = [ "arguments.py", "namespace_pkgs.py", - "platform.py", "wheel.py", "wheel_installer.py", ], diff --git a/python/private/pypi/whl_installer/arguments.py b/python/private/pypi/whl_installer/arguments.py index 29bea8026e..bb841ea9ab 100644 --- a/python/private/pypi/whl_installer/arguments.py +++ b/python/private/pypi/whl_installer/arguments.py @@ -17,8 +17,6 @@ import pathlib from typing import Any, Dict, Set -from python.private.pypi.whl_installer.platform import Platform - def parser(**kwargs: Any) -> argparse.ArgumentParser: """Create a parser for the wheel_installer tool.""" @@ -41,12 +39,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser: action="store", help="Extra arguments to pass down to pip.", ) - parser.add_argument( - "--platform", - action="extend", - type=Platform.from_string, - help="Platforms to target dependencies. Can be used multiple times.", - ) parser.add_argument( "--pip_data_exclude", action="store", diff --git a/python/private/pypi/whl_installer/platform.py b/python/private/pypi/whl_installer/platform.py deleted file mode 100644 index 11dd6e37ab..0000000000 --- a/python/private/pypi/whl_installer/platform.py +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility class to inspect an extracted wheel directory""" - -import platform -import sys -from dataclasses import dataclass -from enum import Enum -from typing import Any, Dict, Iterator, List, Optional, Union - - -class OS(Enum): - linux = 1 - osx = 2 - windows = 3 - darwin = osx - win32 = windows - - @classmethod - def interpreter(cls) -> "OS": - "Return the interpreter operating system." - return cls[sys.platform.lower()] - - def __str__(self) -> str: - return self.name.lower() - - -class Arch(Enum): - x86_64 = 1 - x86_32 = 2 - aarch64 = 3 - ppc = 4 - ppc64le = 5 - s390x = 6 - arm = 7 - amd64 = x86_64 - arm64 = aarch64 - i386 = x86_32 - i686 = x86_32 - x86 = x86_32 - - @classmethod - def interpreter(cls) -> "Arch": - "Return the currently running interpreter architecture." - # FIXME @aignas 2023-12-13: Hermetic toolchain on Windows 3.11.6 - # is returning an empty string here, so lets default to x86_64 - return cls[platform.machine().lower() or "x86_64"] - - def __str__(self) -> str: - return self.name.lower() - - -def _as_int(value: Optional[Union[OS, Arch]]) -> int: - """Convert one of the enums above to an int for easier sorting algorithms. - - Args: - value: The value of an enum or None. - - Returns: - -1 if we get None, otherwise, the numeric value of the given enum. - """ - if value is None: - return -1 - - return int(value.value) - - -def host_interpreter_minor_version() -> int: - return sys.version_info.minor - - -@dataclass(frozen=True) -class Platform: - os: Optional[OS] = None - arch: Optional[Arch] = None - minor_version: Optional[int] = None - - @classmethod - def all( - cls, - want_os: Optional[OS] = None, - minor_version: Optional[int] = None, - ) -> List["Platform"]: - return sorted( - [ - cls(os=os, arch=arch, minor_version=minor_version) - for os in OS - for arch in Arch - if not want_os or want_os == os - ] - ) - - @classmethod - def host(cls) -> List["Platform"]: - """Use the Python interpreter to detect the platform. - - We extract `os` from sys.platform and `arch` from platform.machine - - Returns: - A list of parsed values which makes the signature the same as - `Platform.all` and `Platform.from_string`. - """ - return [ - Platform( - os=OS.interpreter(), - arch=Arch.interpreter(), - minor_version=host_interpreter_minor_version(), - ) - ] - - def all_specializations(self) -> Iterator["Platform"]: - """Return the platform itself and all its unambiguous specializations. - - For more info about specializations see - https://bazel.build/docs/configurable-attributes - """ - yield self - if self.arch is None: - for arch in Arch: - yield Platform(os=self.os, arch=arch, minor_version=self.minor_version) - if self.os is None: - for os in OS: - yield Platform(os=os, arch=self.arch, minor_version=self.minor_version) - if self.arch is None and self.os is None: - for os in OS: - for arch in Arch: - yield Platform(os=os, arch=arch, minor_version=self.minor_version) - - def __lt__(self, other: Any) -> bool: - """Add a comparison method, so that `sorted` returns the most specialized platforms first.""" - if not isinstance(other, Platform) or other is None: - raise ValueError(f"cannot compare {other} with Platform") - - self_arch, self_os = _as_int(self.arch), _as_int(self.os) - other_arch, other_os = _as_int(other.arch), _as_int(other.os) - - if self_os == other_os: - return self_arch < other_arch - else: - return self_os < other_os - - def __str__(self) -> str: - if self.minor_version is None: - if self.os is None and self.arch is None: - return "//conditions:default" - - if self.arch is None: - return f"@platforms//os:{self.os}" - else: - return f"{self.os}_{self.arch}" - - if self.arch is None and self.os is None: - return f"@//python/config_settings:is_python_3.{self.minor_version}" - - if self.arch is None: - return f"cp3{self.minor_version}_{self.os}_anyarch" - - if self.os is None: - return f"cp3{self.minor_version}_anyos_{self.arch}" - - return f"cp3{self.minor_version}_{self.os}_{self.arch}" - - @classmethod - def from_string(cls, platform: Union[str, List[str]]) -> List["Platform"]: - """Parse a string and return a list of platforms""" - platform = [platform] if isinstance(platform, str) else list(platform) - ret = set() - for p in platform: - if p == "host": - ret.update(cls.host()) - continue - - abi, _, tail = p.partition("_") - if not abi.startswith("cp"): - # The first item is not an abi - tail = p - abi = "" - os, _, arch = tail.partition("_") - arch = arch or "*" - - minor_version = int(abi[len("cp3") :]) if abi else None - - if arch != "*": - ret.add( - cls( - os=OS[os] if os != "*" else None, - arch=Arch[arch], - minor_version=minor_version, - ) - ) - - else: - ret.update( - cls.all( - want_os=OS[os] if os != "*" else None, - minor_version=minor_version, - ) - ) - - return sorted(ret) - - # NOTE @aignas 2023-12-05: below is the minimum number of accessors that are defined in - # https://peps.python.org/pep-0496/ to make rules_python generate dependencies. - # - # WARNING: It may not work in cases where the python implementation is different between - # different platforms. - - # derived from OS - @property - def os_name(self) -> str: - if self.os == OS.linux or self.os == OS.osx: - return "posix" - elif self.os == OS.windows: - return "nt" - else: - return "" - - @property - def sys_platform(self) -> str: - if self.os == OS.linux: - return "linux" - elif self.os == OS.osx: - return "darwin" - elif self.os == OS.windows: - return "win32" - else: - return "" - - @property - def platform_system(self) -> str: - if self.os == OS.linux: - return "Linux" - elif self.os == OS.osx: - return "Darwin" - elif self.os == OS.windows: - return "Windows" - else: - return "" - - # derived from OS and Arch - @property - def platform_machine(self) -> str: - """Guess the target 'platform_machine' marker. - - NOTE @aignas 2023-12-05: this may not work on really new systems, like - Windows if they define the platform markers in a different way. - """ - if self.arch == Arch.x86_64: - return "x86_64" - elif self.arch == Arch.x86_32 and self.os != OS.osx: - return "i386" - elif self.arch == Arch.x86_32: - return "" - elif self.arch == Arch.aarch64 and self.os == OS.linux: - return "aarch64" - elif self.arch == Arch.aarch64: - # Assuming that OSX and Windows use this one since the precedent is set here: - # https://github.com/cgohlke/win_arm64-wheels - return "arm64" - elif self.os != OS.linux: - return "" - elif self.arch == Arch.ppc: - return "ppc" - elif self.arch == Arch.ppc64le: - return "ppc64le" - elif self.arch == Arch.s390x: - return "s390x" - else: - return "" - - def env_markers(self, extra: str) -> Dict[str, str]: - # If it is None, use the host version - minor_version = self.minor_version or host_interpreter_minor_version() - - return { - "extra": extra, - "os_name": self.os_name, - "sys_platform": self.sys_platform, - "platform_machine": self.platform_machine, - "platform_system": self.platform_system, - "platform_release": "", # unset - "platform_version": "", # unset - "python_version": f"3.{minor_version}", - # FIXME @aignas 2024-01-14: is putting zero last a good idea? Maybe we should - # use `20` or something else to avoid having weird issues where the full version is used for - # matching and the author decides to only support 3.y.5 upwards. - "implementation_version": f"3.{minor_version}.0", - "python_full_version": f"3.{minor_version}.0", - # we assume that the following are the same as the interpreter used to setup the deps: - # "implementation_name": "cpython" - # "platform_python_implementation: "CPython", - } diff --git a/python/private/pypi/whl_installer/wheel.py b/python/private/pypi/whl_installer/wheel.py index d95b33a194..da81b5ea9f 100644 --- a/python/private/pypi/whl_installer/wheel.py +++ b/python/private/pypi/whl_installer/wheel.py @@ -25,275 +25,6 @@ from packaging.requirements import Requirement from pip._vendor.packaging.utils import canonicalize_name -from python.private.pypi.whl_installer.platform import ( - Platform, - host_interpreter_minor_version, -) - - -@dataclass(frozen=True) -class FrozenDeps: - deps: List[str] - deps_select: Dict[str, List[str]] - - -class Deps: - """Deps is a dependency builder that has a build() method to return FrozenDeps.""" - - def __init__( - self, - name: str, - requires_dist: List[str], - *, - extras: Optional[Set[str]] = None, - platforms: Optional[Set[Platform]] = None, - ): - """Create a new instance and parse the requires_dist - - Args: - name (str): The name of the whl distribution - requires_dist (list[Str]): The Requires-Dist from the METADATA of the whl - distribution. - extras (set[str], optional): The list of requested extras, defaults to None. - platforms (set[Platform], optional): The list of target platforms, defaults to - None. If the list of platforms has multiple `minor_version` values, it - will change the code to generate the select statements using - `@rules_python//python/config_settings:is_python_3.y` conditions. - """ - self.name: str = Deps._normalize(name) - self._platforms: Set[Platform] = platforms or set() - self._target_versions = {p.minor_version for p in platforms or {}} - self._default_minor_version = None - if platforms and len(self._target_versions) > 2: - # TODO @aignas 2024-06-23: enable this to be set via a CLI arg - # for being more explicit. - self._default_minor_version = host_interpreter_minor_version() - - if None in self._target_versions and len(self._target_versions) > 2: - raise ValueError( - f"all python versions need to be specified explicitly, got: {platforms}" - ) - - # Sort so that the dictionary order in the FrozenDeps is deterministic - # without the final sort because Python retains insertion order. That way - # the sorting by platform is limited within the Platform class itself and - # the unit-tests for the Deps can be simpler. - reqs = sorted( - (Requirement(wheel_req) for wheel_req in requires_dist), - key=lambda x: f"{x.name}:{sorted(x.extras)}", - ) - - want_extras = self._resolve_extras(reqs, extras) - - # Then add all of the requirements in order - self._deps: Set[str] = set() - self._select: Dict[Platform, Set[str]] = defaultdict(set) - for req in reqs: - self._add_req(req, want_extras) - - def _add(self, dep: str, platform: Optional[Platform]): - dep = Deps._normalize(dep) - - # Self-edges are processed in _resolve_extras - if dep == self.name: - return - - if not platform: - self._deps.add(dep) - - # If the dep is in the platform-specific list, remove it from the select. - pop_keys = [] - for p, deps in self._select.items(): - if dep not in deps: - continue - - deps.remove(dep) - if not deps: - pop_keys.append(p) - - for p in pop_keys: - self._select.pop(p) - return - - if dep in self._deps: - # If the dep is already in the main dependency list, no need to add it in the - # platform-specific dependency list. - return - - # Add the platform-specific dep - self._select[platform].add(dep) - - # Add the dep to specializations of the given platform if they - # exist in the select statement. - for p in platform.all_specializations(): - if p not in self._select: - continue - - self._select[p].add(dep) - - if len(self._select[platform]) == 1: - # We are adding a new item to the select and we need to ensure that - # existing dependencies from less specialized platforms are propagated - # to the newly added dependency set. - for p, deps in self._select.items(): - # Check if the existing platform overlaps with the given platform - if p == platform or platform not in p.all_specializations(): - continue - - self._select[platform].update(self._select[p]) - - def _maybe_add_common_dep(self, dep): - if len(self._target_versions) < 2: - return - - platforms = [Platform()] + [ - Platform(minor_version=v) for v in self._target_versions - ] - - # If the dep is targeting all target python versions, lets add it to - # the common dependency list to simplify the select statements. - for p in platforms: - if p not in self._select: - return - - if dep not in self._select[p]: - return - - # All of the python version-specific branches have the dep, so lets add - # it to the common deps. - self._deps.add(dep) - for p in platforms: - self._select[p].remove(dep) - if not self._select[p]: - self._select.pop(p) - - @staticmethod - def _normalize(name: str) -> str: - return re.sub(r"[-_.]+", "_", name).lower() - - def _resolve_extras( - self, reqs: List[Requirement], extras: Optional[Set[str]] - ) -> Set[str]: - """Resolve extras which are due to depending on self[some_other_extra]. - - Some packages may have cyclic dependencies resulting from extras being used, one example is - `etils`, where we have one set of extras as aliases for other extras - and we have an extra called 'all' that includes all other extras. - - Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32. - - When the `requirements.txt` is generated by `pip-tools`, then it is likely that - this step is not needed, but for other `requirements.txt` files this may be useful. - - NOTE @aignas 2023-12-08: the extra resolution is not platform dependent, - but in order for it to become platform dependent we would have to have - separate targets for each extra in extras. - """ - - # Resolve any extra extras due to self-edges, empty string means no - # extras The empty string in the set is just a way to make the handling - # of no extras and a single extra easier and having a set of {"", "foo"} - # is equivalent to having {"foo"}. - extras = extras or {""} - - self_reqs = [] - for req in reqs: - if Deps._normalize(req.name) != self.name: - continue - - if req.marker is None: - # I am pretty sure we cannot reach this code as it does not - # make sense to specify packages in this way, but since it is - # easy to handle, lets do it. - # - # TODO @aignas 2023-12-08: add a test - extras = extras | req.extras - else: - # process these in a separate loop - self_reqs.append(req) - - # A double loop is not strictly optimal, but always correct without recursion - for req in self_reqs: - if any(req.marker.evaluate({"extra": extra}) for extra in extras): - extras = extras | req.extras - else: - continue - - # Iterate through all packages to ensure that we include all of the extras from previously - # visited packages. - for req_ in self_reqs: - if any(req_.marker.evaluate({"extra": extra}) for extra in extras): - extras = extras | req_.extras - - return extras - - def _add_req(self, req: Requirement, extras: Set[str]) -> None: - if req.marker is None: - self._add(req.name, None) - return - - marker_str = str(req.marker) - - if not self._platforms: - if any(req.marker.evaluate({"extra": extra}) for extra in extras): - self._add(req.name, None) - return - - # NOTE @aignas 2023-12-08: in order to have reasonable select statements - # we do have to have some parsing of the markers, so it begs the question - # if packaging should be reimplemented in Starlark to have the best solution - # for now we will implement it in Python and see what the best parsing result - # can be before making this decision. - match_os = any( - tag in marker_str - for tag in [ - "os_name", - "sys_platform", - "platform_system", - ] - ) - match_arch = "platform_machine" in marker_str - match_version = "version" in marker_str - - if not (match_os or match_arch or match_version): - if any(req.marker.evaluate({"extra": extra}) for extra in extras): - self._add(req.name, None) - return - - for plat in self._platforms: - if not any( - req.marker.evaluate(plat.env_markers(extra)) for extra in extras - ): - continue - - if match_arch and self._default_minor_version: - self._add(req.name, plat) - if plat.minor_version == self._default_minor_version: - self._add(req.name, Platform(plat.os, plat.arch)) - elif match_arch: - self._add(req.name, Platform(plat.os, plat.arch)) - elif match_os and self._default_minor_version: - self._add(req.name, Platform(plat.os, minor_version=plat.minor_version)) - if plat.minor_version == self._default_minor_version: - self._add(req.name, Platform(plat.os)) - elif match_os: - self._add(req.name, Platform(plat.os)) - elif match_version and self._default_minor_version: - self._add(req.name, Platform(minor_version=plat.minor_version)) - if plat.minor_version == self._default_minor_version: - self._add(req.name, Platform()) - elif match_version: - self._add(req.name, None) - - # Merge to common if possible after processing all platforms - self._maybe_add_common_dep(req.name) - - def build(self) -> FrozenDeps: - return FrozenDeps( - deps=sorted(self._deps), - deps_select={str(p): sorted(deps) for p, deps in self._select.items()}, - ) - class Wheel: """Representation of the compressed .whl file""" @@ -344,18 +75,6 @@ def entry_points(self) -> Dict[str, Tuple[str, str]]: return entry_points_mapping - def dependencies( - self, - extras_requested: Set[str] = None, - platforms: Optional[Set[Platform]] = None, - ) -> FrozenDeps: - return Deps( - self.name, - extras=extras_requested, - platforms=platforms, - requires_dist=self.metadata.get_all("Requires-Dist", []), - ).build() - def unzip(self, directory: str) -> None: installation_schemes = { "purelib": "/site-packages", diff --git a/python/private/pypi/whl_installer/wheel_installer.py b/python/private/pypi/whl_installer/wheel_installer.py index ef8181c30d..c7695d92e8 100644 --- a/python/private/pypi/whl_installer/wheel_installer.py +++ b/python/private/pypi/whl_installer/wheel_installer.py @@ -23,7 +23,7 @@ import sys from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, Optional, Set, Tuple from pip._vendor.packaging.utils import canonicalize_name @@ -103,9 +103,7 @@ def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None: def _extract_wheel( wheel_file: str, - extras: Dict[str, Set[str]], enable_implicit_namespace_pkgs: bool, - platforms: List[wheel.Platform], installation_dir: Path = Path("."), ) -> None: """Extracts wheel into given directory and creates py_library and filegroup targets. @@ -113,7 +111,6 @@ def _extract_wheel( Args: wheel_file: the filepath of the .whl installation_dir: the destination directory for installation of the wheel. - extras: a list of extras to add as dependencies for the installed wheel enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is """ @@ -123,25 +120,19 @@ def _extract_wheel( if not enable_implicit_namespace_pkgs: _setup_namespace_pkg_compatibility(installation_dir) - extras_requested = extras[whl.name] if whl.name in extras else set() - - dependencies = whl.dependencies(extras_requested, platforms) + metadata = { + "python_version": sys.version.partition(" ")[0], + "entry_points": [ + { + "name": name, + "module": module, + "attribute": attribute, + } + for name, (module, attribute) in sorted(whl.entry_points().items()) + ], + } with open(os.path.join(installation_dir, "metadata.json"), "w") as f: - metadata = { - "name": whl.name, - "version": whl.version, - "deps": dependencies.deps, - "deps_by_platform": dependencies.deps_select, - "entry_points": [ - { - "name": name, - "module": module, - "attribute": attribute, - } - for name, (module, attribute) in sorted(whl.entry_points().items()) - ], - } json.dump(metadata, f) @@ -155,13 +146,9 @@ def main() -> None: if args.whl_file: whl = Path(args.whl_file) - name, extras_for_pkg = _parse_requirement_for_extra(args.requirement) - extras = {name: extras_for_pkg} if extras_for_pkg and name else dict() _extract_wheel( wheel_file=whl, - extras=extras, enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, - platforms=arguments.get_platforms(args), ) return diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 493f11353e..54f9ff3909 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -21,9 +21,13 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS", "use_isolated") load(":deps.bzl", "all_repo_names", "record_files") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":parse_requirements.bzl", "host_platform") load(":parse_whl_name.bzl", "parse_whl_name") load(":patch_whl.bzl", "patch_whl") +load(":pep508_deps.bzl", "deps") +load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") +load(":whl_metadata.bzl", "whl_metadata") load(":whl_target_platforms.bzl", "whl_target_platforms") _CPPFLAGS = "CPPFLAGS" @@ -361,7 +365,7 @@ def _whl_library_impl(rctx): arguments = args + [ "--whl-file", whl_path, - ] + ["--platform={}".format(p) for p in target_platforms], + ], srcs = rctx.attr._python_srcs, environment = environment, quiet = rctx.attr.quiet, @@ -396,17 +400,60 @@ def _whl_library_impl(rctx): ) entry_points[entry_point_without_py] = entry_point_script_name + # TODO @aignas 2025-04-04: move this to whl_library_targets.bzl to have + # this in the analysis phase. + # + # This means that whl_library_targets will have to accept the following args: + # * name - the name of the package in the METADATA. + # * requires_dist - the list of METADATA Requires-Dist. + # * platforms - the list of target platforms. The target_platforms + # should come from the hub repo via a 'load' statement so that they don't + # need to be passed as an argument to `whl_library`. + # * extras - the list of required extras. This comes from the + # `rctx.attr.requirement` for now. In the future the required extras could + # stay in the hub repo, where we calculate the extra aliases that we need + # to create automatically and this way expose the targets for the specific + # extras. The first step will be to generate a target per extra for the + # `py_library` and `filegroup`. Maybe we need to have a special provider + # or an output group so that we can return the `whl` file from the + # `py_library` target? filegroup can use output groups to expose files. + # * host_python_version/versons - the list of python versions to support + # should come from the hub, similar to how the target platforms are specified. + # + # Extra things that we should move at the same time: + # * group_name, group_deps - this info can stay in the hub repository so that + # it is piped at the analysis time and changing the requirement groups does + # cause to re-fetch the deps. + python_version = metadata["python_version"] + metadata = whl_metadata( + install_dir = rctx.path("site-packages"), + read_fn = rctx.read, + logger = logger, + ) + + # TODO @aignas 2025-04-09: this will later be removed when loaded through the hub + major_minor, _, _ = python_version.rpartition(".") + package_deps = deps( + name = metadata.name, + requires_dist = metadata.requires_dist, + platforms = target_platforms or [ + "cp{}_{}".format(major_minor.replace(".", ""), host_platform(rctx)), + ], + extras = requirement(rctx.attr.requirement).extras, + host_python_version = python_version, + ) + build_file_contents = generate_whl_library_build_bazel( name = whl_path.basename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), - dependencies = metadata["deps"], - dependencies_by_platform = metadata["deps_by_platform"], + dependencies = package_deps.deps, + dependencies_by_platform = package_deps.deps_select, group_name = rctx.attr.group_name, group_deps = rctx.attr.group_deps, data_exclude = rctx.attr.pip_data_exclude, tags = [ - "pypi_name=" + metadata["name"], - "pypi_version=" + metadata["version"], + "pypi_name=" + metadata.name, + "pypi_version=" + metadata.version, ], entry_points = entry_points, annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 95031e6181..d32746b604 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -90,8 +90,6 @@ def whl_library_targets( native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. """ - _ = name # buildifier: @unused - dependencies = sorted([normalize_name(d) for d in dependencies]) dependencies_by_platform = { platform: sorted([normalize_name(d) for d in deps]) diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl new file mode 100644 index 0000000000..8a86ffbff1 --- /dev/null +++ b/python/private/pypi/whl_metadata.bzl @@ -0,0 +1,108 @@ +"""A simple function to find the METADATA file and parse it""" + +_NAME = "Name: " +_PROVIDES_EXTRA = "Provides-Extra: " +_REQUIRES_DIST = "Requires-Dist: " +_VERSION = "Version: " + +def whl_metadata(*, install_dir, read_fn, logger): + """Find and parse the METADATA file in the extracted whl contents dir. + + Args: + install_dir: {type}`path` location where the wheel has been extracted. + read_fn: the function used to read files. + logger: the function used to log failures. + + Returns: + A struct with parsed values: + * `name`: {type}`str` the name of the wheel. + * `version`: {type}`str` the version of the wheel. + * `requires_dist`: {type}`list[str]` the list of requirements. + * `provides_extra`: {type}`list[str]` the list of extras that this package + provides. + """ + metadata_file = find_whl_metadata(install_dir = install_dir, logger = logger) + contents = read_fn(metadata_file) + result = parse_whl_metadata(contents) + + if not (result.name and result.version): + logger.fail("Failed to parsed the wheel METADATA file:\n{}".format(contents)) + return None + + return result + +def parse_whl_metadata(contents): + """Parse .whl METADATA file + + Args: + contents: {type}`str` the contents of the file. + + Returns: + A struct with parsed values: + * `name`: {type}`str` the name of the wheel. + * `version`: {type}`str` the version of the wheel. + * `requires_dist`: {type}`list[str]` the list of requirements. + * `provides_extra`: {type}`list[str]` the list of extras that this package + provides. + """ + parsed = { + "name": "", + "provides_extra": [], + "requires_dist": [], + "version": "", + } + for line in contents.strip().split("\n"): + if not line.strip(): + # Stop parsing on first empty line, which marks the end of the + # headers containing the metadata. + break + + if line.startswith(_NAME): + _, _, value = line.partition(_NAME) + parsed["name"] = value.strip() + elif line.startswith(_VERSION): + _, _, value = line.partition(_VERSION) + parsed["version"] = value.strip() + elif line.startswith(_REQUIRES_DIST): + _, _, value = line.partition(_REQUIRES_DIST) + parsed["requires_dist"].append(value.strip(" ")) + elif line.startswith(_PROVIDES_EXTRA): + _, _, value = line.partition(_PROVIDES_EXTRA) + parsed["provides_extra"].append(value.strip(" ")) + + return struct( + name = parsed["name"], + provides_extra = parsed["provides_extra"], + requires_dist = parsed["requires_dist"], + version = parsed["version"], + ) + +def find_whl_metadata(*, install_dir, logger): + """Find the whl METADATA file in the install_dir. + + Args: + install_dir: {type}`path` location where the wheel has been extracted. + logger: the function used to log failures. + + Returns: + {type}`path` The path to the METADATA file. + """ + dist_info = None + for maybe_dist_info in install_dir.readdir(): + # first find the ".dist-info" folder + if not (maybe_dist_info.is_dir and maybe_dist_info.basename.endswith(".dist-info")): + continue + + dist_info = maybe_dist_info + metadata_file = dist_info.get_child("METADATA") + + if metadata_file.exists: + return metadata_file + + break + + if dist_info: + logger.fail("The METADATA file for the wheel could not be found in '{}/{}'".format(install_dir.basename, dist_info.basename)) + else: + logger.fail("The '*.dist-info' directory could not be found in '{}'".format(install_dir.basename)) + return None diff --git a/tests/pypi/pep508/BUILD.bazel b/tests/pypi/pep508/BUILD.bazel index 575f28ada6..7eab2e096a 100644 --- a/tests/pypi/pep508/BUILD.bazel +++ b/tests/pypi/pep508/BUILD.bazel @@ -1,6 +1,11 @@ +load(":deps_tests.bzl", "deps_test_suite") load(":evaluate_tests.bzl", "evaluate_test_suite") load(":requirement_tests.bzl", "requirement_test_suite") +deps_test_suite( + name = "deps_tests", +) + evaluate_test_suite( name = "evaluate_tests", ) diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl new file mode 100644 index 0000000000..44031ab6a5 --- /dev/null +++ b/tests/pypi/pep508/deps_tests.bzl @@ -0,0 +1,385 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for construction of Python version matching config settings.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:pep508_deps.bzl", "deps") # buildifier: disable=bzl-visibility + +_tests = [] + +def test_simple_deps(env): + got = deps( + "foo", + requires_dist = ["bar-Bar"], + ) + env.expect.that_collection(got.deps).contains_exactly(["bar_bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({}) + +_tests.append(test_simple_deps) + +def test_can_add_os_specific_deps(env): + got = deps( + "foo", + requires_dist = [ + "bar", + "an_osx_dep; sys_platform=='darwin'", + "posix_dep; os_name=='posix'", + "win_dep; os_name=='nt'", + ], + platforms = [ + "linux_x86_64", + "osx_x86_64", + "osx_aarch64", + "windows_x86_64", + ], + host_python_version = "3.3.1", + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "@platforms//os:linux": ["posix_dep"], + "@platforms//os:osx": ["an_osx_dep", "posix_dep"], + "@platforms//os:windows": ["win_dep"], + }) + +_tests.append(test_can_add_os_specific_deps) + +def test_can_add_os_specific_deps_with_python_version(env): + got = deps( + "foo", + requires_dist = [ + "bar", + "an_osx_dep; sys_platform=='darwin'", + "posix_dep; os_name=='posix'", + "win_dep; os_name=='nt'", + ], + platforms = [ + "cp33_linux_x86_64", + "cp33_osx_x86_64", + "cp33_osx_aarch64", + "cp33_windows_x86_64", + ], + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "@platforms//os:linux": ["posix_dep"], + "@platforms//os:osx": ["an_osx_dep", "posix_dep"], + "@platforms//os:windows": ["win_dep"], + }) + +_tests.append(test_can_add_os_specific_deps_with_python_version) + +def test_deps_are_added_to_more_specialized_platforms(env): + got = deps( + "foo", + requires_dist = [ + "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", + "mac_dep; sys_platform=='darwin'", + ], + platforms = [ + "osx_x86_64", + "osx_aarch64", + ], + host_python_version = "3.8.4", + ) + + env.expect.that_collection(got.deps).contains_exactly([]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "@platforms//os:osx": ["mac_dep"], + "osx_aarch64": ["m1_dep", "mac_dep"], + }) + +_tests.append(test_deps_are_added_to_more_specialized_platforms) + +def test_deps_from_more_specialized_platforms_are_propagated(env): + got = deps( + "foo", + requires_dist = [ + "a_mac_dep; sys_platform=='darwin'", + "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", + ], + platforms = [ + "osx_x86_64", + "osx_aarch64", + ], + host_python_version = "3.8.4", + ) + + env.expect.that_collection(got.deps).contains_exactly([]) + env.expect.that_dict(got.deps_select).contains_exactly( + { + "@platforms//os:osx": ["a_mac_dep"], + "osx_aarch64": ["a_mac_dep", "m1_dep"], + }, + ) + +_tests.append(test_deps_from_more_specialized_platforms_are_propagated) + +def test_non_platform_markers_are_added_to_common_deps(env): + got = deps( + "foo", + requires_dist = [ + "bar", + "baz; implementation_name=='cpython'", + "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", + ], + platforms = [ + "linux_x86_64", + "osx_x86_64", + "osx_aarch64", + "windows_x86_64", + ], + host_python_version = "3.8.4", + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "osx_aarch64": ["m1_dep"], + }) + +_tests.append(test_non_platform_markers_are_added_to_common_deps) + +def test_self_is_ignored(env): + got = deps( + "foo", + requires_dist = [ + "bar", + "req_dep; extra == 'requests'", + "foo[requests]; extra == 'ssl'", + "ssl_lib; extra == 'ssl'", + ], + extras = ["ssl"], + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar", "req_dep", "ssl_lib"]) + env.expect.that_dict(got.deps_select).contains_exactly({}) + +_tests.append(test_self_is_ignored) + +def test_self_dependencies_can_come_in_any_order(env): + got = deps( + "foo", + requires_dist = [ + "bar", + "baz; extra == 'feat'", + "foo[feat2]; extra == 'all'", + "foo[feat]; extra == 'feat2'", + "zdep; extra == 'all'", + ], + extras = ["all"], + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar", "baz", "zdep"]) + env.expect.that_dict(got.deps_select).contains_exactly({}) + +_tests.append(test_self_dependencies_can_come_in_any_order) + +def _test_can_get_deps_based_on_specific_python_version(env): + requires_dist = [ + "bar", + "baz; python_version < '3.8'", + "posix_dep; os_name=='posix' and python_version >= '3.8'", + ] + + py38 = deps( + "foo", + requires_dist = requires_dist, + platforms = ["cp38_linux_x86_64"], + ) + py37 = deps( + "foo", + requires_dist = requires_dist, + platforms = ["cp37_linux_x86_64"], + ) + + env.expect.that_collection(py37.deps).contains_exactly(["bar", "baz"]) + env.expect.that_dict(py37.deps_select).contains_exactly({}) + env.expect.that_collection(py38.deps).contains_exactly(["bar"]) + env.expect.that_dict(py38.deps_select).contains_exactly({"@platforms//os:linux": ["posix_dep"]}) + +_tests.append(_test_can_get_deps_based_on_specific_python_version) + +def _test_no_version_select_when_single_version(env): + requires_dist = [ + "bar", + "baz; python_version >= '3.8'", + "posix_dep; os_name=='posix'", + "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", + "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'", + ] + host_python_version = "3.7.5" + + got = deps( + "foo", + requires_dist = requires_dist, + platforms = [ + "cp38_linux_x86_64", + "cp38_windows_x86_64", + ], + host_python_version = host_python_version, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "@platforms//os:linux": ["posix_dep", "posix_dep_with_version"], + "linux_x86_64": ["arch_dep", "posix_dep", "posix_dep_with_version"], + "windows_x86_64": ["arch_dep"], + }) + +_tests.append(_test_no_version_select_when_single_version) + +def _test_can_get_version_select(env): + requires_dist = [ + "bar", + "baz; python_version < '3.8'", + "baz_new; python_version >= '3.8'", + "posix_dep; os_name=='posix'", + "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", + "arch_dep; platform_machine=='x86_64' and python_version < '3.8'", + ] + host_python_version = "3.7.4" + + got = deps( + "foo", + requires_dist = requires_dist, + platforms = [ + "cp3{}_{}_x86_64".format(minor, os) + for minor in [7, 8, 9] + for os in ["linux", "windows"] + ], + host_python_version = host_python_version, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + str(Label("//python/config_settings:is_python_3.7")): ["baz"], + str(Label("//python/config_settings:is_python_3.8")): ["baz_new"], + str(Label("//python/config_settings:is_python_3.9")): ["baz_new"], + "@platforms//os:linux": ["baz", "posix_dep"], + "cp37_linux_anyarch": ["baz", "posix_dep"], + "cp37_linux_x86_64": ["arch_dep", "baz", "posix_dep"], + "cp37_windows_x86_64": ["arch_dep", "baz"], + "cp38_linux_anyarch": [ + "baz_new", + "posix_dep", + "posix_dep_with_version", + ], + "cp39_linux_anyarch": [ + "baz_new", + "posix_dep", + "posix_dep_with_version", + ], + "linux_x86_64": ["arch_dep", "baz", "posix_dep"], + "windows_x86_64": ["arch_dep", "baz"], + "//conditions:default": ["baz"], + }) + +_tests.append(_test_can_get_version_select) + +def _test_deps_spanning_all_target_py_versions_are_added_to_common(env): + requires_dist = [ + "bar", + "baz (<2,>=1.11) ; python_version < '3.8'", + "baz (<2,>=1.14) ; python_version >= '3.8'", + ] + host_python_version = "3.8.4" + + got = deps( + "foo", + requires_dist = requires_dist, + platforms = [ + "cp3{}_linux_x86_64".format(minor) + for minor in [7, 8, 9] + ], + host_python_version = host_python_version, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"]) + env.expect.that_dict(got.deps_select).contains_exactly({}) + +_tests.append(_test_deps_spanning_all_target_py_versions_are_added_to_common) + +def _test_deps_are_not_duplicated(env): + host_python_version = "3.7.4" + + # See an example in + # https://files.pythonhosted.org/packages/76/9e/db1c2d56c04b97981c06663384f45f28950a73d9acf840c4006d60d0a1ff/opencv_python-4.9.0.80-cp37-abi3-win32.whl.metadata + requires_dist = [ + "bar >=0.1.0 ; python_version < '3.7'", + "bar >=0.2.0 ; python_version >= '3.7'", + "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'", + "bar >=0.4.0 ; python_version >= '3.9'", + "bar >=0.5.0 ; python_version <= '3.9' and platform_system == 'Darwin' and platform_machine == 'arm64'", + "bar >=0.5.0 ; python_version >= '3.10' and platform_system == 'Darwin'", + "bar >=0.5.0 ; python_version >= '3.10'", + "bar >=0.6.0 ; python_version >= '3.11'", + ] + + got = deps( + "foo", + requires_dist = requires_dist, + platforms = [ + "cp3{}_{}_{}".format(minor, os, arch) + for minor in [7, 10] + for os in ["linux", "osx", "windows"] + for arch in ["x86_64", "aarch64"] + ], + host_python_version = host_python_version, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({}) + +_tests.append(_test_deps_are_not_duplicated) + +def _test_deps_are_not_duplicated_when_encountering_platform_dep_first(env): + host_python_version = "3.7.1" + + # Note, that we are sorting the incoming `requires_dist` and we need to ensure that we are not getting any + # issues even if the platform-specific line comes first. + requires_dist = [ + "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'", + "bar >=0.5.0 ; python_version >= '3.9'", + ] + + got = deps( + "foo", + requires_dist = requires_dist, + platforms = [ + "cp37_linux_aarch64", + "cp37_linux_x86_64", + "cp310_linux_aarch64", + "cp310_linux_x86_64", + ], + host_python_version = host_python_version, + ) + + # TODO @aignas 2025-02-24: this test case in the python version is passing but + # I am not sure why. The starlark version behaviour looks more correct. + env.expect.that_collection(got.deps).contains_exactly([]) + env.expect.that_dict(got.deps_select).contains_exactly({ + str(Label("//python/config_settings:is_python_3.10")): ["bar"], + "cp310_linux_aarch64": ["bar"], + "cp37_linux_aarch64": ["bar"], + "linux_aarch64": ["bar"], + }) + +_tests.append(_test_deps_are_not_duplicated_when_encountering_platform_dep_first) + +def deps_test_suite(name): # buildifier: disable=function-docstring + test_suite( + name = name, + basic_tests = _tests, + ) diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index 80b70f4dad..14e5e40b43 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -148,6 +148,8 @@ def _logical_expression_tests(env): # expr "os_name == 'fo'": False, "(os_name == 'fo')": False, + "((os_name == 'fo'))": False, + "((os_name == 'foo'))": True, "not (os_name == 'fo')": True, # and diff --git a/tests/pypi/whl_installer/BUILD.bazel b/tests/pypi/whl_installer/BUILD.bazel index 040e4d765f..fea6a46d01 100644 --- a/tests/pypi/whl_installer/BUILD.bazel +++ b/tests/pypi/whl_installer/BUILD.bazel @@ -27,18 +27,6 @@ py_test( ], ) -py_test( - name = "platform_test", - size = "small", - srcs = [ - "platform_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", - ], -) - py_test( name = "wheel_installer_test", size = "small", @@ -50,15 +38,3 @@ py_test( ":lib", ], ) - -py_test( - name = "wheel_test", - size = "small", - srcs = [ - "wheel_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", - ], -) diff --git a/tests/pypi/whl_installer/arguments_test.py b/tests/pypi/whl_installer/arguments_test.py index 5538054a59..9f73ae96a9 100644 --- a/tests/pypi/whl_installer/arguments_test.py +++ b/tests/pypi/whl_installer/arguments_test.py @@ -15,7 +15,7 @@ import json import unittest -from python.private.pypi.whl_installer import arguments, wheel +from python.private.pypi.whl_installer import arguments class ArgumentsTestCase(unittest.TestCase): @@ -49,18 +49,6 @@ def test_deserialize_structured_args(self) -> None: self.assertEqual(args["environment"], {"PIP_DO_SOMETHING": "True"}) self.assertEqual(args["extra_pip_args"], []) - def test_platform_aggregation(self) -> None: - parser = arguments.parser() - args = parser.parse_args( - args=[ - "--platform=linux_*", - "--platform=osx_*", - "--platform=windows_*", - "--requirement=foo", - ] - ) - self.assertEqual(set(wheel.Platform.all()), arguments.get_platforms(args)) - if __name__ == "__main__": unittest.main() diff --git a/tests/pypi/whl_installer/platform_test.py b/tests/pypi/whl_installer/platform_test.py deleted file mode 100644 index 2aeb4caa69..0000000000 --- a/tests/pypi/whl_installer/platform_test.py +++ /dev/null @@ -1,154 +0,0 @@ -import unittest -from random import shuffle - -from python.private.pypi.whl_installer.platform import ( - OS, - Arch, - Platform, - host_interpreter_minor_version, -) - - -class MinorVersionTest(unittest.TestCase): - def test_host(self): - host = host_interpreter_minor_version() - self.assertIsNotNone(host) - - -class PlatformTest(unittest.TestCase): - def test_can_get_host(self): - host = Platform.host() - self.assertIsNotNone(host) - self.assertEqual(1, len(Platform.from_string("host"))) - self.assertEqual(host, Platform.from_string("host")) - - def test_can_get_linux_x86_64_without_py_version(self): - got = Platform.from_string("linux_x86_64") - want = Platform(os=OS.linux, arch=Arch.x86_64) - self.assertEqual(want, got[0]) - - def test_can_get_specific_from_string(self): - got = Platform.from_string("cp33_linux_x86_64") - want = Platform(os=OS.linux, arch=Arch.x86_64, minor_version=3) - self.assertEqual(want, got[0]) - - def test_can_get_all_for_py_version(self): - cp39 = Platform.all(minor_version=9) - self.assertEqual(21, len(cp39), f"Got {cp39}") - self.assertEqual(cp39, Platform.from_string("cp39_*")) - - def test_can_get_all_for_os(self): - linuxes = Platform.all(OS.linux, minor_version=9) - self.assertEqual(7, len(linuxes)) - self.assertEqual(linuxes, Platform.from_string("cp39_linux_*")) - - def test_can_get_all_for_os_for_host_python(self): - linuxes = Platform.all(OS.linux) - self.assertEqual(7, len(linuxes)) - self.assertEqual(linuxes, Platform.from_string("linux_*")) - - def test_specific_version_specializations(self): - any_py33 = Platform(minor_version=3) - - # When - all_specializations = list(any_py33.all_specializations()) - - want = ( - [any_py33] - + [ - Platform(arch=arch, minor_version=any_py33.minor_version) - for arch in Arch - ] - + [Platform(os=os, minor_version=any_py33.minor_version) for os in OS] - + Platform.all(minor_version=any_py33.minor_version) - ) - self.assertEqual(want, all_specializations) - - def test_aarch64_specializations(self): - any_aarch64 = Platform(arch=Arch.aarch64) - all_specializations = list(any_aarch64.all_specializations()) - want = [ - Platform(os=None, arch=Arch.aarch64), - Platform(os=OS.linux, arch=Arch.aarch64), - Platform(os=OS.osx, arch=Arch.aarch64), - Platform(os=OS.windows, arch=Arch.aarch64), - ] - self.assertEqual(want, all_specializations) - - def test_linux_specializations(self): - any_linux = Platform(os=OS.linux) - all_specializations = list(any_linux.all_specializations()) - want = [ - Platform(os=OS.linux, arch=None), - Platform(os=OS.linux, arch=Arch.x86_64), - Platform(os=OS.linux, arch=Arch.x86_32), - Platform(os=OS.linux, arch=Arch.aarch64), - Platform(os=OS.linux, arch=Arch.ppc), - Platform(os=OS.linux, arch=Arch.ppc64le), - Platform(os=OS.linux, arch=Arch.s390x), - Platform(os=OS.linux, arch=Arch.arm), - ] - self.assertEqual(want, all_specializations) - - def test_osx_specializations(self): - any_osx = Platform(os=OS.osx) - all_specializations = list(any_osx.all_specializations()) - # NOTE @aignas 2024-01-14: even though in practice we would only have - # Python on osx aarch64 and osx x86_64, we return all arch posibilities - # to make the code simpler. - want = [ - Platform(os=OS.osx, arch=None), - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.x86_32), - Platform(os=OS.osx, arch=Arch.aarch64), - Platform(os=OS.osx, arch=Arch.ppc), - Platform(os=OS.osx, arch=Arch.ppc64le), - Platform(os=OS.osx, arch=Arch.s390x), - Platform(os=OS.osx, arch=Arch.arm), - ] - self.assertEqual(want, all_specializations) - - def test_platform_sort(self): - platforms = [ - Platform(os=OS.linux, arch=None), - Platform(os=OS.linux, arch=Arch.x86_64), - Platform(os=OS.osx, arch=None), - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.aarch64), - ] - shuffle(platforms) - platforms.sort() - want = [ - Platform(os=OS.linux, arch=None), - Platform(os=OS.linux, arch=Arch.x86_64), - Platform(os=OS.osx, arch=None), - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.aarch64), - ] - - self.assertEqual(want, platforms) - - def test_wheel_os_alias(self): - self.assertEqual("osx", str(OS.osx)) - self.assertEqual(str(OS.darwin), str(OS.osx)) - - def test_wheel_arch_alias(self): - self.assertEqual("x86_64", str(Arch.x86_64)) - self.assertEqual(str(Arch.amd64), str(Arch.x86_64)) - - def test_wheel_platform_alias(self): - give = Platform( - os=OS.darwin, - arch=Arch.amd64, - ) - alias = Platform( - os=OS.osx, - arch=Arch.x86_64, - ) - - self.assertEqual("osx_x86_64", str(give)) - self.assertEqual(str(alias), str(give)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/pypi/whl_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py index 7139779c3e..3c118af3c4 100644 --- a/tests/pypi/whl_installer/wheel_installer_test.py +++ b/tests/pypi/whl_installer/wheel_installer_test.py @@ -22,39 +22,6 @@ from python.private.pypi.whl_installer import wheel_installer -class TestRequirementExtrasParsing(unittest.TestCase): - def test_parses_requirement_for_extra(self) -> None: - cases = [ - ("name[foo]", ("name", frozenset(["foo"]))), - ("name[ Foo123 ]", ("name", frozenset(["Foo123"]))), - (" name1[ foo ] ", ("name1", frozenset(["foo"]))), - ("Name[foo]", ("name", frozenset(["foo"]))), - ("name_foo[bar]", ("name-foo", frozenset(["bar"]))), - ( - "name [fred,bar] @ http://foo.com ; python_version=='2.7'", - ("name", frozenset(["fred", "bar"])), - ), - ( - "name[quux, strange];python_version<'2.7' and platform_version=='2'", - ("name", frozenset(["quux", "strange"])), - ), - ( - "name; (os_name=='a' or os_name=='b') and os_name=='c'", - (None, None), - ), - ( - "name@http://foo.com", - (None, None), - ), - ] - - for case, expected in cases: - with self.subTest(): - self.assertTupleEqual( - wheel_installer._parse_requirement_for_extra(case), expected - ) - - class TestWhlFilegroup(unittest.TestCase): def setUp(self) -> None: self.wheel_name = "example_minimal_package-0.0.1-py3-none-any.whl" @@ -68,10 +35,8 @@ def tearDown(self): def test_wheel_exists(self) -> None: wheel_installer._extract_wheel( Path(self.wheel_path), - installation_dir=Path(self.wheel_dir), - extras={}, enable_implicit_namespace_pkgs=False, - platforms=[], + installation_dir=Path(self.wheel_dir), ) want_files = [ @@ -92,11 +57,8 @@ def test_wheel_exists(self) -> None: metadata_file_content = json.load(metadata_file) want = dict( - version="0.0.1", - name="example-minimal-package", - deps=[], - deps_by_platform={}, entry_points=[], + python_version="3.11.11", ) self.assertEqual(want, metadata_file_content) diff --git a/tests/pypi/whl_installer/wheel_test.py b/tests/pypi/whl_installer/wheel_test.py deleted file mode 100644 index 404218e12b..0000000000 --- a/tests/pypi/whl_installer/wheel_test.py +++ /dev/null @@ -1,371 +0,0 @@ -import unittest -from unittest import mock - -from python.private.pypi.whl_installer import wheel -from python.private.pypi.whl_installer.platform import OS, Arch, Platform - -_HOST_INTERPRETER_FN = ( - "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" -) - - -class DepsTest(unittest.TestCase): - def test_simple(self): - deps = wheel.Deps("foo", requires_dist=["bar"]) - - got = deps.build() - - self.assertIsInstance(got, wheel.FrozenDeps) - self.assertEqual(["bar"], got.deps) - self.assertEqual({}, got.deps_select) - - def test_can_add_os_specific_deps(self): - deps = wheel.Deps( - "foo", - requires_dist=[ - "bar", - "an_osx_dep; sys_platform=='darwin'", - "posix_dep; os_name=='posix'", - "win_dep; os_name=='nt'", - ], - platforms={ - Platform(os=OS.linux, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.aarch64), - Platform(os=OS.windows, arch=Arch.x86_64), - }, - ) - - got = deps.build() - - self.assertEqual(["bar"], got.deps) - self.assertEqual( - { - "@platforms//os:linux": ["posix_dep"], - "@platforms//os:osx": ["an_osx_dep", "posix_dep"], - "@platforms//os:windows": ["win_dep"], - }, - got.deps_select, - ) - - def test_can_add_os_specific_deps_with_specific_python_version(self): - deps = wheel.Deps( - "foo", - requires_dist=[ - "bar", - "an_osx_dep; sys_platform=='darwin'", - "posix_dep; os_name=='posix'", - "win_dep; os_name=='nt'", - ], - platforms={ - Platform(os=OS.linux, arch=Arch.x86_64, minor_version=8), - Platform(os=OS.osx, arch=Arch.x86_64, minor_version=8), - Platform(os=OS.osx, arch=Arch.aarch64, minor_version=8), - Platform(os=OS.windows, arch=Arch.x86_64, minor_version=8), - }, - ) - - got = deps.build() - - self.assertEqual(["bar"], got.deps) - self.assertEqual( - { - "@platforms//os:linux": ["posix_dep"], - "@platforms//os:osx": ["an_osx_dep", "posix_dep"], - "@platforms//os:windows": ["win_dep"], - }, - got.deps_select, - ) - - def test_deps_are_added_to_more_specialized_platforms(self): - got = wheel.Deps( - "foo", - requires_dist=[ - "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", - "mac_dep; sys_platform=='darwin'", - ], - platforms={ - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.aarch64), - }, - ).build() - - self.assertEqual( - wheel.FrozenDeps( - deps=[], - deps_select={ - "osx_aarch64": ["m1_dep", "mac_dep"], - "@platforms//os:osx": ["mac_dep"], - }, - ), - got, - ) - - def test_deps_from_more_specialized_platforms_are_propagated(self): - got = wheel.Deps( - "foo", - requires_dist=[ - "a_mac_dep; sys_platform=='darwin'", - "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", - ], - platforms={ - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.aarch64), - }, - ).build() - - self.assertEqual([], got.deps) - self.assertEqual( - { - "osx_aarch64": ["a_mac_dep", "m1_dep"], - "@platforms//os:osx": ["a_mac_dep"], - }, - got.deps_select, - ) - - def test_non_platform_markers_are_added_to_common_deps(self): - got = wheel.Deps( - "foo", - requires_dist=[ - "bar", - "baz; implementation_name=='cpython'", - "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", - ], - platforms={ - Platform(os=OS.linux, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.x86_64), - Platform(os=OS.osx, arch=Arch.aarch64), - Platform(os=OS.windows, arch=Arch.x86_64), - }, - ).build() - - self.assertEqual(["bar", "baz"], got.deps) - self.assertEqual( - { - "osx_aarch64": ["m1_dep"], - }, - got.deps_select, - ) - - def test_self_is_ignored(self): - deps = wheel.Deps( - "foo", - requires_dist=[ - "bar", - "req_dep; extra == 'requests'", - "foo[requests]; extra == 'ssl'", - "ssl_lib; extra == 'ssl'", - ], - extras={"ssl"}, - ) - - got = deps.build() - - self.assertEqual(["bar", "req_dep", "ssl_lib"], got.deps) - self.assertEqual({}, got.deps_select) - - def test_self_dependencies_can_come_in_any_order(self): - deps = wheel.Deps( - "foo", - requires_dist=[ - "bar", - "baz; extra == 'feat'", - "foo[feat2]; extra == 'all'", - "foo[feat]; extra == 'feat2'", - "zdep; extra == 'all'", - ], - extras={"all"}, - ) - - got = deps.build() - - self.assertEqual(["bar", "baz", "zdep"], got.deps) - self.assertEqual({}, got.deps_select) - - def test_can_get_deps_based_on_specific_python_version(self): - requires_dist = [ - "bar", - "baz; python_version < '3.8'", - "posix_dep; os_name=='posix' and python_version >= '3.8'", - ] - - py38_deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=[ - Platform(os=OS.linux, arch=Arch.x86_64, minor_version=8), - ], - ).build() - py37_deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=[ - Platform(os=OS.linux, arch=Arch.x86_64, minor_version=7), - ], - ).build() - - self.assertEqual(["bar", "baz"], py37_deps.deps) - self.assertEqual({}, py37_deps.deps_select) - self.assertEqual(["bar"], py38_deps.deps) - self.assertEqual({"@platforms//os:linux": ["posix_dep"]}, py38_deps.deps_select) - - @mock.patch(_HOST_INTERPRETER_FN) - def test_no_version_select_when_single_version(self, mock_host_interpreter_version): - requires_dist = [ - "bar", - "baz; python_version >= '3.8'", - "posix_dep; os_name=='posix'", - "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", - "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'", - ] - mock_host_interpreter_version.return_value = 7 - - self.maxDiff = None - - deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=[ - Platform(os=os, arch=Arch.x86_64, minor_version=minor) - for minor in [8] - for os in [OS.linux, OS.windows] - ], - ) - got = deps.build() - - self.assertEqual(["bar", "baz"], got.deps) - self.assertEqual( - { - "@platforms//os:linux": ["posix_dep", "posix_dep_with_version"], - "linux_x86_64": ["arch_dep", "posix_dep", "posix_dep_with_version"], - "windows_x86_64": ["arch_dep"], - }, - got.deps_select, - ) - - @mock.patch(_HOST_INTERPRETER_FN) - def test_can_get_version_select(self, mock_host_interpreter_version): - requires_dist = [ - "bar", - "baz; python_version < '3.8'", - "baz_new; python_version >= '3.8'", - "posix_dep; os_name=='posix'", - "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", - "arch_dep; platform_machine=='x86_64' and python_version < '3.8'", - ] - mock_host_interpreter_version.return_value = 7 - - self.maxDiff = None - - deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=[ - Platform(os=os, arch=Arch.x86_64, minor_version=minor) - for minor in [7, 8, 9] - for os in [OS.linux, OS.windows] - ], - ) - got = deps.build() - - self.assertEqual(["bar"], got.deps) - self.assertEqual( - { - "//conditions:default": ["baz"], - "@//python/config_settings:is_python_3.7": ["baz"], - "@//python/config_settings:is_python_3.8": ["baz_new"], - "@//python/config_settings:is_python_3.9": ["baz_new"], - "@platforms//os:linux": ["baz", "posix_dep"], - "cp37_linux_x86_64": ["arch_dep", "baz", "posix_dep"], - "cp37_windows_x86_64": ["arch_dep", "baz"], - "cp37_linux_anyarch": ["baz", "posix_dep"], - "cp38_linux_anyarch": [ - "baz_new", - "posix_dep", - "posix_dep_with_version", - ], - "cp39_linux_anyarch": [ - "baz_new", - "posix_dep", - "posix_dep_with_version", - ], - "linux_x86_64": ["arch_dep", "baz", "posix_dep"], - "windows_x86_64": ["arch_dep", "baz"], - }, - got.deps_select, - ) - - @mock.patch(_HOST_INTERPRETER_FN) - def test_deps_spanning_all_target_py_versions_are_added_to_common( - self, mock_host_version - ): - requires_dist = [ - "bar", - "baz (<2,>=1.11) ; python_version < '3.8'", - "baz (<2,>=1.14) ; python_version >= '3.8'", - ] - mock_host_version.return_value = 8 - - deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=Platform.from_string(["cp37_*", "cp38_*", "cp39_*"]), - ) - got = deps.build() - - self.assertEqual(["bar", "baz"], got.deps) - self.assertEqual({}, got.deps_select) - - @mock.patch(_HOST_INTERPRETER_FN) - def test_deps_are_not_duplicated(self, mock_host_version): - mock_host_version.return_value = 7 - - # See an example in - # https://files.pythonhosted.org/packages/76/9e/db1c2d56c04b97981c06663384f45f28950a73d9acf840c4006d60d0a1ff/opencv_python-4.9.0.80-cp37-abi3-win32.whl.metadata - requires_dist = [ - "bar >=0.1.0 ; python_version < '3.7'", - "bar >=0.2.0 ; python_version >= '3.7'", - "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'", - "bar >=0.4.0 ; python_version >= '3.9'", - "bar >=0.5.0 ; python_version <= '3.9' and platform_system == 'Darwin' and platform_machine == 'arm64'", - "bar >=0.5.0 ; python_version >= '3.10' and platform_system == 'Darwin'", - "bar >=0.5.0 ; python_version >= '3.10'", - "bar >=0.6.0 ; python_version >= '3.11'", - ] - - deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=Platform.from_string(["cp37_*", "cp310_*"]), - ) - got = deps.build() - - self.assertEqual(["bar"], got.deps) - self.assertEqual({}, got.deps_select) - - @mock.patch(_HOST_INTERPRETER_FN) - def test_deps_are_not_duplicated_when_encountering_platform_dep_first( - self, mock_host_version - ): - mock_host_version.return_value = 7 - - # Note, that we are sorting the incoming `requires_dist` and we need to ensure that we are not getting any - # issues even if the platform-specific line comes first. - requires_dist = [ - "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'", - "bar >=0.5.0 ; python_version >= '3.9'", - ] - - deps = wheel.Deps( - "foo", - requires_dist=requires_dist, - platforms=Platform.from_string(["cp37_*", "cp310_*"]), - ) - got = deps.build() - - self.assertEqual(["bar"], got.deps) - self.assertEqual({}, got.deps_select) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/pypi/whl_metadata/BUILD.bazel b/tests/pypi/whl_metadata/BUILD.bazel new file mode 100644 index 0000000000..3f1d665dd2 --- /dev/null +++ b/tests/pypi/whl_metadata/BUILD.bazel @@ -0,0 +1,5 @@ +load(":whl_metadata_tests.bzl", "whl_metadata_test_suite") + +whl_metadata_test_suite( + name = "whl_metadata_tests", +) diff --git a/tests/pypi/whl_metadata/whl_metadata_tests.bzl b/tests/pypi/whl_metadata/whl_metadata_tests.bzl new file mode 100644 index 0000000000..4acbc9213d --- /dev/null +++ b/tests/pypi/whl_metadata/whl_metadata_tests.bzl @@ -0,0 +1,147 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "subjects") +load( + "//python/private/pypi:whl_metadata.bzl", + "find_whl_metadata", + "parse_whl_metadata", +) # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_empty(env): + fake_path = struct( + basename = "site-packages", + readdir = lambda watch = None: [], + ) + fail_messages = [] + find_whl_metadata(install_dir = fake_path, logger = struct( + fail = fail_messages.append, + )) + env.expect.that_collection(fail_messages).contains_exactly([ + "The '*.dist-info' directory could not be found in 'site-packages'", + ]) + +_tests.append(_test_empty) + +def _test_contains_dist_info_but_no_metadata(env): + fake_path = struct( + basename = "site-packages", + readdir = lambda watch = None: [ + struct( + basename = "something.dist-info", + is_dir = True, + get_child = lambda basename: struct( + basename = basename, + exists = False, + ), + ), + ], + ) + fail_messages = [] + find_whl_metadata(install_dir = fake_path, logger = struct( + fail = fail_messages.append, + )) + env.expect.that_collection(fail_messages).contains_exactly([ + "The METADATA file for the wheel could not be found in 'site-packages/something.dist-info'", + ]) + +_tests.append(_test_contains_dist_info_but_no_metadata) + +def _test_contains_metadata(env): + fake_path = struct( + basename = "site-packages", + readdir = lambda watch = None: [ + struct( + basename = "something.dist-info", + is_dir = True, + get_child = lambda basename: struct( + basename = basename, + exists = True, + ), + ), + ], + ) + fail_messages = [] + got = find_whl_metadata(install_dir = fake_path, logger = struct( + fail = fail_messages.append, + )) + env.expect.that_collection(fail_messages).contains_exactly([]) + env.expect.that_str(got.basename).equals("METADATA") + +_tests.append(_test_contains_metadata) + +def _parse_whl_metadata(env, **kwargs): + result = parse_whl_metadata(**kwargs) + + return env.expect.that_struct( + struct( + name = result.name, + version = result.version, + requires_dist = result.requires_dist, + provides_extra = result.provides_extra, + ), + attrs = dict( + name = subjects.str, + version = subjects.str, + requires_dist = subjects.collection, + provides_extra = subjects.collection, + ), + ) + +def _test_parse_metadata_invalid(env): + got = _parse_whl_metadata( + env, + contents = "", + ) + got.name().equals("") + got.version().equals("") + got.requires_dist().contains_exactly([]) + got.provides_extra().contains_exactly([]) + +_tests.append(_test_parse_metadata_invalid) + +def _test_parse_metadata_basic(env): + got = _parse_whl_metadata( + env, + contents = """\ +Name: foo +Version: 0.0.1 +""", + ) + got.name().equals("foo") + got.version().equals("0.0.1") + got.requires_dist().contains_exactly([]) + got.provides_extra().contains_exactly([]) + +_tests.append(_test_parse_metadata_basic) + +def _test_parse_metadata_all(env): + got = _parse_whl_metadata( + env, + contents = """\ +Name: foo +Version: 0.0.1 +Requires-Dist: bar; extra == "all" +Provides-Extra: all + +Requires-Dist: this will be ignored +""", + ) + got.name().equals("foo") + got.version().equals("0.0.1") + got.requires_dist().contains_exactly([ + "bar; extra == \"all\"", + ]) + got.provides_extra().contains_exactly([ + "all", + ]) + +_tests.append(_test_parse_metadata_all) + +def whl_metadata_test_suite(name): # buildifier: disable=function-docstring + test_suite( + name = name, + basic_tests = _tests, + )