diff --git a/CHANGELOG.md b/CHANGELOG.md index d5b757e02e..8eb269c39c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,12 @@ A brief description of the categories of changes: [`pip.parse#extra_pip_args`](https://rules-python.readthedocs.io/en/latest/api/rules_python/python/extensions/pip.html#pip.parse.extra_pip_args) * (pip.parse) {attr}`pip.parse.whl_modifications` now normalizes the given whl names and now `pyyaml` and `PyYAML` will both work. +* (bzlmod) `pip.parse` spoke repository naming will be changed in an upcoming + release in places where the users specify different package versions per + platform in the same hub repository. The naming of the spoke repos is considered + an implementation detail and we advise the users to use the `hub` repository + directly to avoid such breakage in the future. If `rules_python` is missing + features to allow one to do that, please raise tickets. {#v0-0-0-fixed} ### Fixed @@ -51,6 +57,12 @@ A brief description of the categories of changes: pass the `extra_pip_args` value when building an `sdist`. * (pypi) The patched wheel filenames from now on are using local version specifiers which fixes usage of the said wheels using standard package managers. +* (bzlmod) The extension evaluation has been adjusted to always generate the + same lock file irrespective if `experimental_index_url` is set by any module + or not. Fixes + [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known + issue is that it may break `bazel query` and in these use cases it is + advisable to use `cquery` or switch to `download_only = True` {#v0-0-0-added} ### Added @@ -63,6 +75,11 @@ A brief description of the categories of changes: * (pip.parse) {attr}`pip.parse.extra_hub_aliases` can now be used to expose extra targets created by annotations in whl repositories. Fixes [#2187](https://github.com/bazelbuild/rules_python/issues/2187). +* (bzlmod) `pip.parse` now supports `whl-only` setup using + `download_only = True` where users can specify multiple requirements files + and use the `pip` backend to do the downloading. This was only available for + users setting {bzl:obj}`pip.parse.experimental_index_url`, but now users have + more options whilst we continue to work on stabilizing the experimental feature. {#v0-0-0-removed} ### Removed diff --git a/MODULE.bazel b/MODULE.bazel index de14b86f1b..50f137690c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -56,6 +56,10 @@ register_toolchains("@pythons_hub//:all") pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal") pip.parse( + # NOTE @aignas 2024-10-26: We have an integration test that depends on us + # being able to build sdists for this hub, so explicitly set this to False. + download_only = False, + experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", python_version = "3.11", requirements_by_platform = { @@ -90,17 +94,20 @@ dev_python.override( ) dev_pip = use_extension( - "//python/private/pypi:pip.bzl", - "pip_internal", + "//python/extensions:pip.bzl", + "pip", dev_dependency = True, ) dev_pip.parse( - download_only = True, # this will not add the `sdist` values to the transitive closures at all. + download_only = True, + experimental_index_url = "https://pypi.org/simple", hub_name = "dev_pip", python_version = "3.11", requirements_lock = "//docs:requirements.txt", ) dev_pip.parse( + download_only = True, + experimental_index_url = "https://pypi.org/simple", hub_name = "pypiserver", python_version = "3.11", requirements_lock = "//examples/wheel:requirements_server.txt", diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md index 636fefb33d..28e630c61d 100644 --- a/docs/pypi-dependencies.md +++ b/docs/pypi-dependencies.md @@ -307,6 +307,59 @@ leg of the dependency manually. For instance by making perhaps `apache-airflow-providers-common-sql`. +(bazel-downloader)= +### Multi-platform support + +Multi-platform support of cross-building the wheels can be done in two ways - either +using {bzl:attr}`experimental_index_url` for the {bzl:obj}`pip.parse` bzlmod tag class +or by using the {bzl:attr}`pip.parse.download_only` setting. In this section we +are going to outline quickly how one can use the latter option. + +Let's say you have 2 requirements files: +``` +# requirements.linux_x86_64.txt +--platform=manylinux_2_17_x86_64 +--python-version=39 +--implementation=cp +--abi=cp39 + +foo==0.0.1 --hash=sha256:deadbeef +bar==0.0.1 --hash=sha256:deadb00f +``` + +``` +# requirements.osx_aarch64.txt contents +--platform=macosx_10_9_arm64 +--python-version=39 +--implementation=cp +--abi=cp39 + +foo==0.0.3 --hash=sha256:deadbaaf +``` + +With these 2 files your {bzl:obj}`pip.parse` could look like: +``` +pip.parse( + hub_name = "pip", + python_version = "3.9", + # Tell `pip` to ignore sdists + download_only = True, + requirements_by_platform = { + "requirements.linux_x86_64.txt": "linux_x86_64", + "requirements.osx_aarch64.txt": "osx_aarch64", + }, +) +``` + +With this, the `pip.parse` will create a hub repository that is going to +support only two platforms - `cp39_osx_aarch64` and `cp39_linux_x86_64` and it +will only use `wheels` and ignore any sdists that it may find on the PyPI +compatible indexes. + +```{note} +This is only supported on `bzlmd`. +``` + (bazel-downloader)= ### Bazel downloader and multi-platform wheel hub repository. diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 4381cb0d70..1b8bbbf5e3 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -221,6 +221,9 @@ pip.parse( "host", ], hub_name = "pip", + # Parse all requirements files for the same lock file on all OSes, this will + # become the default with 1.0 release + parse_all_requirements_files = True, python_version = "3.10", # The requirements files for each platform that we want to support. requirements_by_platform = { diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index c115ef974f..eb578f681d 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1392,8 +1392,8 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "E5Yr6AjquyIy5ae3c7URmvtPPOm2j+7XOr58GOHp8vw=", - "usagesDigest": "iVxh/vcpGrSKpO8rafQwAe7uq+pHhasSXC7Pg4o/1dw=", + "bzlTransitiveDigest": "0Qn7Q9FuTxYCxMKm2DsW7mbXYcxL71sS/l1baXvY1vA=", + "usagesDigest": "GGeczTmsfE4YHAy32dV/jfOfbYmpyu/QGe35drFuZ5E=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", "@@rules_python~//python/private/pypi/whl_installer/platform.py": "b944b908b25a2f97d6d9f491504ad5d2507402d7e37c802ee878783f87f2aa11", @@ -6587,8 +6587,8 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "wz5L+/+R6gOtD681pNVgPUUipqqPH0bP/b0e22JbSOI=", - "usagesDigest": "LYtSAPzhPjmfD9vF39mCED1UQSvHEo2Hv+aK5Z4ZWWc=", + "bzlTransitiveDigest": "SnuwsgZv1SGZz4jVPvwaEUwPTnea18fXIueD9vSR3sQ=", + "usagesDigest": "O2O2oBIbKEglN2K3FECsRxUKVS/zg/6a86F3MO1ZtmY=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements_linux.txt": "8175b4c8df50ae2f22d1706961884beeb54e7da27bd2447018314a175981997d", "@@rules_python~//tools/publish/requirements_windows.txt": "7673adc71dc1a81d3661b90924d7a7c0fc998cd508b3cb4174337cef3f2de556", diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index 492acf1895..9f3f4d4e48 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -148,6 +148,13 @@ def config_settings( ) def _dist_config_settings(*, suffix, plat_flag_values, **kwargs): + if kwargs.get("constraint_values"): + # Add python version + platform config settings + _dist_config_setting( + name = suffix.strip("_"), + **kwargs + ) + flag_values = {_flags.dist: ""} # First create an sdist, we will be building upon the flag values, which @@ -277,7 +284,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): return ret -def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native = native, **kwargs): +def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs): """A macro to create a target that matches is_pip_whl_auto and one more value. Args: @@ -310,6 +317,10 @@ def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native # `python_version` setting. return + if not is_pip_whl: + native.config_setting(name = _name, **kwargs) + return + config_setting_name = _name + "_setting" native.config_setting(name = config_setting_name, **kwargs) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index c5660272ec..7b31d0d50c 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,7 +31,7 @@ load(":render_pkg_aliases.bzl", "whl_alias") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_library.bzl", "whl_library") -load(":whl_repo_name.bzl", "whl_repo_name") +load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name") def _major_minor_version(version): version = semver(version) @@ -260,10 +260,10 @@ def _create_whl_repos( if v != default }) + is_exposed = False if get_index_urls: # TODO @aignas 2024-05-26: move to a separate function found_something = False - is_exposed = False for requirement in requirements: is_exposed = is_exposed or requirement.is_exposed dists = requirement.whls @@ -319,35 +319,69 @@ def _create_whl_repos( exposed_packages[whl_name] = None continue - requirement = select_requirement( - requirements, - platform = None if pip_attr.download_only else repository_platform, - ) - if not requirement: - # Sometimes the package is not present for host platform if there - # are whls specified only in particular requirements files, in that - # case just continue, however, if the download_only flag is set up, - # then the user can also specify the target platform of the wheel - # packages they want to download, in that case there will be always - # a requirement here, so we will not be in this code branch. + if not pip_attr.parse_all_requirements_files: + requirement = select_requirement( + requirements, + platform = None if pip_attr.download_only else repository_platform, + ) + if not requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + continue + elif get_index_urls: + logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + + whl_library_args["requirement"] = requirement.requirement_line + if requirement.extra_pip_args: + whl_library_args["extra_pip_args"] = requirement.extra_pip_args + + # We sort so that the lock-file remains the same no matter the order of how the + # args are manipulated in the code going before. + repo_name = "{}_{}".format(pip_name, whl_name) + whl_libraries[repo_name] = dict(whl_library_args.items()) + whl_map.setdefault(whl_name, []).append( + whl_alias( + repo = repo_name, + version = major_minor, + ), + ) continue - elif get_index_urls: - logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) - whl_library_args["requirement"] = requirement.requirement_line - if requirement.extra_pip_args: - whl_library_args["extra_pip_args"] = requirement.extra_pip_args + is_exposed = False + for requirement in requirements: + is_exposed = is_exposed or requirement.is_exposed + if get_index_urls: + logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + + args = dict(whl_library_args) # make a copy + args["requirement"] = requirement.requirement_line + if requirement.extra_pip_args: + args["extra_pip_args"] = requirement.extra_pip_args + + if pip_attr.download_only: + args.setdefault("experimental_target_platforms", requirement.target_platforms) + + target_platforms = requirement.target_platforms if len(requirements) > 1 else [] + repo_name = pypi_repo_name( + pip_name, + whl_name, + *target_platforms + ) + whl_libraries[repo_name] = args + whl_map.setdefault(whl_name, []).append( + whl_alias( + repo = repo_name, + version = major_minor, + target_platforms = target_platforms or None, + ), + ) - # We sort so that the lock-file remains the same no matter the order of how the - # args are manipulated in the code going before. - repo_name = "{}_{}".format(pip_name, whl_name) - whl_libraries[repo_name] = dict(whl_library_args.items()) - whl_map.setdefault(whl_name, []).append( - whl_alias( - repo = repo_name, - version = major_minor, - ), - ) + if is_exposed: + exposed_packages[whl_name] = None return struct( is_reproducible = is_reproducible, @@ -502,7 +536,8 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map[hub_name] = pip_attr.experimental_requirement_cycles return struct( - # We sort the output here so that the lock file is sorted + # We sort so that the lock-file remains the same no matter the order of how the + # args are manipulated in the code going before. whl_mods = dict(sorted(whl_mods.items())), hub_whl_map = { hub_name: { @@ -518,7 +553,10 @@ You cannot use both the additive_build_content and additive_build_content_file a } for hub_name, group_map in sorted(hub_group_map.items()) }, - exposed_packages = {k: sorted(v) for k, v in sorted(exposed_packages.items())}, + exposed_packages = { + k: sorted(v) + for k, v in sorted(exposed_packages.items()) + }, extra_aliases = { hub_name: { whl_name: sorted(aliases) @@ -526,7 +564,10 @@ You cannot use both the additive_build_content and additive_build_content_file a } for hub_name, extra_whl_aliases in extra_aliases.items() }, - whl_libraries = dict(sorted(whl_libraries.items())), + whl_libraries = { + k: dict(sorted(args.items())) + for k, args in sorted(whl_libraries.items()) + }, is_reproducible = is_reproducible, ) @@ -601,10 +642,8 @@ def _pip_impl(module_ctx): # Build all of the wheel modifications if the tag class is called. _whl_mods_impl(mods.whl_mods) - for name, args in sorted(mods.whl_libraries.items()): - # We sort so that the lock-file remains the same no matter the order of how the - # args are manipulated in the code going before. - whl_library(name = name, **dict(sorted(args.items()))) + for name, args in mods.whl_libraries.items(): + whl_library(name = name, **args) for hub_name, whl_map in mods.hub_whl_map.items(): hub_repository( @@ -746,6 +785,20 @@ find in case extra indexes are specified. """, default = True, ), + "parse_all_requirements_files": attr.bool( + default = False, + doc = """\ +A temporary flag to enable users to make `pip` extension result always +the same independent of the whether transitive dependencies use {bzl:attr}`experimental_index_url` or not. + +This enables users to migrate to a solution that fixes +[#2268](https://github.com/bazelbuild/rules_python/issues/2268). + +::::{deprecated} 0.38.0 +This is a transition flag and will be removed in a subsequent release. +:::: +""", + ), "python_version": attr.string( mandatory = True, doc = """ @@ -907,24 +960,22 @@ extension. pypi_internal = module_extension( doc = """\ This extension is used to make dependencies from pypi available. - For now this is intended to be used internally so that usage of the `pip` extension in `rules_python` does not affect the evaluations of the extension for the consumers. - pip.parse: To use, call `pip.parse()` and specify `hub_name` and your requirements file. Dependencies will be downloaded and made available in a repo named after the `hub_name` argument. - Each `pip.parse()` call configures a particular Python version. Multiple calls can be made to configure different Python versions, and will be grouped by the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy` to automatically resolve to different, Python version-specific, libraries. - pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. + +TODO: will be removed before 1.0. """, implementation = _pip_non_reproducible, tag_classes = { diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index aacc8bdbc0..a43217dbc2 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -168,7 +168,7 @@ def parse_requirements( ) ret = {} - for whl_name, reqs in requirements_by_platform.items(): + for whl_name, reqs in sorted(requirements_by_platform.items()): requirement_target_platforms = {} for r in reqs.values(): target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) @@ -212,6 +212,8 @@ def parse_requirements( def select_requirement(requirements, *, platform): """A simple function to get a requirement for a particular platform. + Only used in WORKSPACE. + Args: requirements (list[struct]): The list of requirements as returned by the `parse_requirements` function above. @@ -243,6 +245,8 @@ def select_requirement(requirements, *, platform): def host_platform(ctx): """Return a string representation of the repository OS. + Only used in WORKSPACE. + Args: ctx (struct): The `module_ctx` or `repository_ctx` attribute. diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 60f4b54306..7a759799dd 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -326,16 +326,19 @@ def multiplatform_whl_aliases(*, aliases, **kwargs): ret = [] versioned_additions = {} for alias in aliases: - if not alias.filename: + if not alias.filename and not alias.target_platforms: ret.append(alias) continue config_settings, all_versioned_settings = get_filename_config_settings( # TODO @aignas 2024-05-27: pass the parsed whl to reduce the # number of duplicate operations. - filename = alias.filename, + filename = alias.filename or "", target_platforms = alias.target_platforms, python_version = alias.version, + # If we have multiple platforms but no wheel filename, lets use different + # config settings. + non_whl_prefix = "sdist" if alias.filename else "", **kwargs ) @@ -437,10 +440,7 @@ def get_whl_flag_versions(aliases): if a.version: python_versions[a.version] = None - if not a.filename: - continue - - if a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"): + if a.filename and a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"): parsed = parse_whl_name(a.filename) else: for plat in a.target_platforms or []: @@ -499,10 +499,11 @@ def get_filename_config_settings( *, filename, target_platforms, - glibc_versions, - muslc_versions, - osx_versions, - python_version): + python_version, + glibc_versions = None, + muslc_versions = None, + osx_versions = None, + non_whl_prefix = "sdist"): """Get the filename config settings. Args: @@ -512,6 +513,8 @@ def get_filename_config_settings( muslc_versions: list[tuple[int, int]], list of versions. osx_versions: list[tuple[int, int]], list of versions. python_version: the python version to generate the config_settings for. + non_whl_prefix: the prefix of the config setting when the whl we don't have + a filename ending with ".whl". Returns: A tuple: @@ -520,19 +523,20 @@ def get_filename_config_settings( """ prefixes = [] suffixes = [] - if (0, 0) in glibc_versions: - fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value") - if (0, 0) in muslc_versions: - fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value") - if (0, 0) in osx_versions: - fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value") - - glibc_versions = sorted(glibc_versions) - muslc_versions = sorted(muslc_versions) - osx_versions = sorted(osx_versions) setting_supported_versions = {} if filename.endswith(".whl"): + if (0, 0) in glibc_versions: + fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value") + if (0, 0) in muslc_versions: + fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value") + if (0, 0) in osx_versions: + fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value") + + glibc_versions = sorted(glibc_versions) + muslc_versions = sorted(muslc_versions) + osx_versions = sorted(osx_versions) + parsed = parse_whl_name(filename) if parsed.python_tag == "py2.py3": py = "py" @@ -547,10 +551,10 @@ def get_filename_config_settings( abi = parsed.abi_tag if parsed.platform_tag == "any": - prefixes = ["{}_{}_any".format(py, abi)] + prefixes = ["_{}_{}_any".format(py, abi)] suffixes = [_non_versioned_platform(p) for p in target_platforms or []] else: - prefixes = ["{}_{}".format(py, abi)] + prefixes = ["_{}_{}".format(py, abi)] suffixes = _whl_config_setting_suffixes( platform_tag = parsed.platform_tag, glibc_versions = glibc_versions, @@ -559,12 +563,12 @@ def get_filename_config_settings( setting_supported_versions = setting_supported_versions, ) else: - prefixes = ["sdist"] + prefixes = [""] if not non_whl_prefix else ["_" + non_whl_prefix] suffixes = [_non_versioned_platform(p) for p in target_platforms or []] versioned = { - ":is_cp{}_{}_{}".format(python_version, p, suffix): { - version: ":is_cp{}_{}_{}".format(python_version, p, setting) + ":is_cp{}{}_{}".format(python_version, p, suffix): { + version: ":is_cp{}{}_{}".format(python_version, p, setting) for version, setting in versions.items() } for p in prefixes @@ -572,9 +576,9 @@ def get_filename_config_settings( } if suffixes or versioned: - return [":is_cp{}_{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned + return [":is_cp{}{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned else: - return [":is_cp{}_{}".format(python_version, p) for p in prefixes], setting_supported_versions + return [":is_cp{}{}".format(python_version, p) for p in prefixes], setting_supported_versions def _whl_config_setting_suffixes( platform_tag, diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl index 295f5a45c4..38ed600cd1 100644 --- a/python/private/pypi/whl_repo_name.bzl +++ b/python/private/pypi/whl_repo_name.bzl @@ -22,12 +22,12 @@ def whl_repo_name(prefix, filename, sha256): """Return a valid whl_library repo name given a distribution filename. Args: - prefix: str, the prefix of the whl_library. - filename: str, the filename of the distribution. - sha256: str, the sha256 of the distribution. + prefix: {type}`str` the prefix of the whl_library. + filename: {type}`str` the filename of the distribution. + sha256: {type}`str` the sha256 of the distribution. Returns: - a string that can be used in `whl_library`. + a string that can be used in {obj}`whl_library`. """ parts = [prefix] @@ -50,3 +50,22 @@ def whl_repo_name(prefix, filename, sha256): parts.append(sha256[:8]) return "_".join(parts) + +def pypi_repo_name(prefix, whl_name, *target_platforms): + """Return a valid whl_library given a requirement line. + + Args: + prefix: {type}`str` the prefix of the whl_library. + whl_name: {type}`str` the whl_name to use. + *target_platforms: {type}`list[str]` the target platforms to use in the name. + + Returns: + {type}`str` that can be used in {obj}`whl_library`. + """ + parts = [ + prefix, + normalize_name(whl_name), + ] + parts.extend([p.partition("_")[-1] for p in target_platforms]) + + return "_".join(parts) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index aa120af83d..0405bad4d8 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -20,14 +20,14 @@ load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disa _tests = [] -def _mock_mctx(*modules, environ = {}, read = None, os_name = "unittest", os_arch = "exotic"): +def _mock_mctx(*modules, environ = {}, read = None): return struct( os = struct( environ = environ, - name = os_name, - arch = os_arch, + name = "unittest", + arch = "exotic", ), - read = read or (lambda _: "simple==0.0.1 --hash=sha256:deadbeef"), + read = read or (lambda _: "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf"), modules = [ struct( name = modules[0].name, @@ -61,7 +61,6 @@ def _parse_modules(env, **kwargs): attrs = dict( is_reproducible = subjects.bool, exposed_packages = subjects.dict, - extra_aliases = subjects.dict, hub_group_map = subjects.dict, hub_whl_map = subjects.dict, whl_libraries = subjects.dict, @@ -69,29 +68,6 @@ def _parse_modules(env, **kwargs): ), ) -def _whl_mods( - *, - whl_name, - hub_name, - additive_build_content = None, - additive_build_content_file = None, - copy_executables = {}, - copy_files = {}, - data = [], - data_exclude_glob = [], - srcs_exclude_glob = []): - return struct( - additive_build_content = additive_build_content, - additive_build_content_file = additive_build_content_file, - copy_executables = copy_executables, - copy_files = copy_files, - data = data, - data_exclude_glob = data_exclude_glob, - hub_name = hub_name, - srcs_exclude_glob = srcs_exclude_glob, - whl_name = whl_name, - ) - def _parse( *, hub_name, @@ -109,6 +85,7 @@ def _parse( extra_pip_args = [], isolated = True, netrc = None, + parse_all_requirements_files = True, pip_data_exclude = None, python_interpreter = None, python_interpreter_target = None, @@ -136,6 +113,7 @@ def _parse( hub_name = hub_name, isolated = isolated, netrc = netrc, + parse_all_requirements_files = parse_all_requirements_files, pip_data_exclude = pip_data_exclude, python_interpreter = python_interpreter, python_interpreter_target = python_interpreter_target, @@ -176,47 +154,137 @@ def _test_simple(env): ) pypi.is_reproducible().equals(True) - pypi.exposed_packages().contains_exactly({"pypi": []}) + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) - pypi.hub_whl_map().contains_exactly({"pypi": {}}) - pypi.whl_libraries().contains_exactly({}) + pypi.hub_whl_map().contains_exactly({"pypi": { + "simple": [ + struct( + config_setting = "//_config:is_python_3.15", + filename = None, + repo = "pypi_315_simple", + target_platforms = None, + version = "3.15", + ), + ], + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + }, + }) pypi.whl_mods().contains_exactly({}) _tests.append(_test_simple) -def _test_simple_with_whl_mods(env): +def _test_simple_multiple_requirements(env): pypi = _parse_modules( env, module_ctx = _mock_mctx( _mod( name = "rules_python", - whl_mods = [ - _whl_mods( - additive_build_content = """\ -filegroup( - name = "foo", - srcs = ["all"], -)""", - hub_name = "whl_mods_hub", - whl_name = "simple", + parse = [ + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_darwin = "darwin.txt", + requirements_windows = "win.txt", ), ], + ), + read = lambda x: { + "darwin.txt": "simple==0.0.2 --hash=sha256:deadb00f", + "win.txt": "simple==0.0.1 --hash=sha256:deadbeef", + }[x], + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + ) + + pypi.is_reproducible().equals(True) + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) + pypi.hub_group_map().contains_exactly({"pypi": {}}) + pypi.hub_whl_map().contains_exactly({"pypi": { + "simple": [ + struct( + config_setting = "//_config:is_python_3.15", + filename = None, + repo = "pypi_315_simple_windows_x86_64", + target_platforms = [ + "cp315_windows_x86_64", + ], + version = "3.15", + ), + struct( + config_setting = "//_config:is_python_3.15", + filename = None, + repo = "pypi_315_simple_osx_aarch64_osx_x86_64", + target_platforms = [ + "cp315_osx_aarch64", + "cp315_osx_x86_64", + ], + version = "3.15", + ), + ], + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple_osx_aarch64_osx_x86_64": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "simple==0.0.2 --hash=sha256:deadb00f", + }, + "pypi_315_simple_windows_x86_64": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_simple_multiple_requirements) + +def _test_download_only_multiple(env): + pypi = _parse_modules( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", parse = [ _parse( hub_name = "pypi", python_version = "3.15", - requirements_lock = "requirements.txt", - extra_hub_aliases = { - "simple": ["foo"], - }, - whl_modifications = { - "@whl_mods_hub//:simple.json": "simple", + download_only = True, + requirements_by_platform = { + "requirements.linux_x86_64.txt": "linux_x86_64", + "requirements.osx_aarch64.txt": "osx_aarch64", }, ), ], ), - os_name = "linux", - os_arch = "aarch64", + read = lambda x: { + "requirements.linux_x86_64.txt": """\ +--platform=manylinux_2_17_x86_64 +--python-version=315 +--implementation=cp +--abi=cp315 + +simple==0.0.1 --hash=sha256:deadbeef +extra==0.0.1 --hash=sha256:deadb00f +""", + "requirements.osx_aarch64.txt": """\ +--platform=macosx_10_9_arm64 +--python-version=315 +--implementation=cp +--abi=cp315 + +simple==0.0.3 --hash=sha256:deadbaaf +""", + }[x], ), available_interpreters = { "python_3_15_host": "unit_test_interpreter_target", @@ -224,45 +292,67 @@ filegroup( ) pypi.is_reproducible().equals(True) - pypi.exposed_packages().contains_exactly({"pypi": []}) - pypi.extra_aliases().contains_exactly({ - "pypi": {"simple": ["foo"]}, - }) + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { - "simple": [ + "extra": [ struct( config_setting = "//_config:is_python_3.15", filename = None, - repo = "pypi_315_simple", + repo = "pypi_315_extra", target_platforms = None, version = "3.15", ), ], + "simple": [ + struct( + config_setting = "//_config:is_python_3.15", + filename = None, + repo = "pypi_315_simple_linux_x86_64", + target_platforms = ["cp315_linux_x86_64"], + version = "3.15", + ), + struct( + config_setting = "//_config:is_python_3.15", + filename = None, + repo = "pypi_315_simple_osx_aarch64", + target_platforms = ["cp315_osx_aarch64"], + version = "3.15", + ), + ], }}) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple": { - "annotation": "@whl_mods_hub//:simple.json", + "pypi_315_extra": { + "dep_template": "@pypi//{name}:{target}", + "download_only": True, + "experimental_target_platforms": ["cp315_linux_x86_64"], + "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "extra==0.0.1 --hash=sha256:deadb00f", + }, + "pypi_315_simple_linux_x86_64": { "dep_template": "@pypi//{name}:{target}", + "download_only": True, + "experimental_target_platforms": ["cp315_linux_x86_64"], + "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", "repo": "pypi_315", "requirement": "simple==0.0.1 --hash=sha256:deadbeef", }, - }) - pypi.whl_mods().contains_exactly({ - "whl_mods_hub": { - "simple": struct( - build_content = "filegroup(\n name = \"foo\",\n srcs = [\"all\"],\n)", - copy_executables = {}, - copy_files = {}, - data = [], - data_exclude_glob = [], - srcs_exclude_glob = [], - ), + "pypi_315_simple_osx_aarch64": { + "dep_template": "@pypi//{name}:{target}", + "download_only": True, + "experimental_target_platforms": ["cp315_osx_aarch64"], + "extra_pip_args": ["--platform=macosx_10_9_arm64", "--python-version=315", "--implementation=cp", "--abi=cp315"], + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "simple==0.0.3 --hash=sha256:deadbaaf", }, }) + pypi.whl_mods().contains_exactly({}) -_tests.append(_test_simple_with_whl_mods) +_tests.append(_test_download_only_multiple) def _test_simple_get_index(env): got_simpleapi_download_args = [] @@ -310,7 +400,10 @@ def _test_simple_get_index(env): ], ), read = lambda x: { - "requirements.txt": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadb00f", + "requirements.txt": """ +simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadb00f +some_pkg==0.0.1 +""", }[x], ), available_interpreters = { @@ -320,26 +413,37 @@ def _test_simple_get_index(env): ) pypi.is_reproducible().equals(False) - pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) + pypi.exposed_packages().contains_exactly({"pypi": ["simple", "some_pkg"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) - pypi.hub_whl_map().contains_exactly({"pypi": { - "simple": [ - struct( - config_setting = "//_config:is_python_3.15", - filename = "simple-0.0.1-py3-none-any.whl", - repo = "pypi_315_simple_py3_none_any_deadb00f", - target_platforms = None, - version = "3.15", - ), - struct( - config_setting = "//_config:is_python_3.15", - filename = "simple-0.0.1.tar.gz", - repo = "pypi_315_simple_sdist_deadbeef", - target_platforms = None, - version = "3.15", - ), - ], - }}) + pypi.hub_whl_map().contains_exactly({ + "pypi": { + "simple": [ + struct( + config_setting = "//_config:is_python_3.15", + filename = "simple-0.0.1-py3-none-any.whl", + repo = "pypi_315_simple_py3_none_any_deadb00f", + target_platforms = None, + version = "3.15", + ), + struct( + config_setting = "//_config:is_python_3.15", + filename = "simple-0.0.1.tar.gz", + repo = "pypi_315_simple_sdist_deadbeef", + target_platforms = None, + version = "3.15", + ), + ], + "some_pkg": [ + struct( + config_setting = "//_config:is_python_3.15", + filename = None, + repo = "pypi_315_some_pkg", + target_platforms = None, + version = "3.15", + ), + ], + }, + }) pypi.whl_libraries().contains_exactly({ "pypi_315_simple_py3_none_any_deadb00f": { "dep_template": "@pypi//{name}:{target}", @@ -372,9 +476,7 @@ def _test_simple_get_index(env): "cp315_osx_x86_64", "cp315_windows_x86_64", ], - "extra_pip_args": [ - "--extra-args-for-sdist-building", - ], + "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "simple-0.0.1.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", "repo": "pypi_315", @@ -382,8 +484,29 @@ def _test_simple_get_index(env): "sha256": "deadbeef", "urls": ["example.org"], }, + # We are falling back to regular `pip` + "pypi_315_some_pkg": { + "dep_template": "@pypi//{name}:{target}", + "extra_pip_args": ["--extra-args-for-sdist-building"], + "python_interpreter_target": "unit_test_interpreter_target", + "repo": "pypi_315", + "requirement": "some_pkg==0.0.1", + }, }) pypi.whl_mods().contains_exactly({}) + env.expect.that_dict(got_simpleapi_download_kwargs).contains_exactly({ + "attr": struct( + auth_patterns = {}, + envsubst = {}, + extra_index_urls = [], + index_url = "pypi.org", + index_url_overrides = {}, + netrc = None, + sources = ["simple", "some_pkg"], + ), + "cache": {}, + "parallel_download": False, + }) _tests.append(_test_simple_get_index) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index c719ad6972..a6e17bebec 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -29,6 +29,16 @@ foo[extra]==0.0.1 --hash=sha256:deadbeef """, "requirements_linux": """\ foo==0.0.3 --hash=sha256:deadbaaf +""", + # download_only = True + "requirements_linux_download_only": """\ +--platform=manylinux_2_17_x86_64 +--python-version=39 +--implementation=cp +--abi=cp39 + +foo==0.0.1 --hash=sha256:deadbeef +bar==0.0.1 --hash=sha256:deadb00f """, "requirements_lock": """\ foo[extra]==0.0.1 --hash=sha256:deadbeef @@ -43,6 +53,14 @@ foo[extra]==0.0.1 ;marker --hash=sha256:deadbeef bar==0.0.1 --hash=sha256:deadbeef """, "requirements_osx": """\ +foo==0.0.3 --hash=sha256:deadbaaf +""", + "requirements_osx_download_only": """\ +--platform=macosx_10_9_arm64 +--python-version=39 +--implementation=cp +--abi=cp39 + foo==0.0.3 --hash=sha256:deadbaaf """, "requirements_windows": """\ @@ -229,6 +247,66 @@ def _test_multi_os(env): _tests.append(_test_multi_os) +def _test_multi_os_legacy(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_linux_download_only": ["cp39_linux_x86_64"], + "requirements_osx_download_only": ["cp39_osx_aarch64"], + }, + ) + + env.expect.that_dict(got).contains_exactly({ + "bar": [ + struct( + distribution = "bar", + extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"], + is_exposed = False, + requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", + sdist = None, + srcs = struct( + requirement = "bar==0.0.1", + shas = ["deadb00f"], + version = "0.0.1", + ), + target_platforms = ["cp39_linux_x86_64"], + whls = [], + ), + ], + "foo": [ + struct( + distribution = "foo", + extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"], + is_exposed = True, + requirement_line = "foo==0.0.1 --hash=sha256:deadbeef", + sdist = None, + srcs = struct( + requirement = "foo==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = ["cp39_linux_x86_64"], + whls = [], + ), + struct( + distribution = "foo", + extra_pip_args = ["--platform=macosx_10_9_arm64", "--python-version=39", "--implementation=cp", "--abi=cp39"], + is_exposed = True, + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + sdist = None, + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = ["cp39_osx_aarch64"], + whls = [], + ), + ], + }) + +_tests.append(_test_multi_os_legacy) + def _test_select_requirement_none_platform(env): got = select_requirement( [ diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9de309b295..f5187788ea 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -387,6 +387,24 @@ def _test_get_python_versions(env): _tests.append(_test_get_python_versions) +def _test_get_python_versions_with_target_platforms(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias(repo = "foo", version = "3.3", target_platforms = ["cp33_linux_x86_64"]), + whl_alias(repo = "foo", version = "3.2", target_platforms = ["cp32_linux_x86_64", "cp32_osx_aarch64"]), + ], + ) + want = { + "python_versions": ["3.2", "3.3"], + "target_platforms": [ + "linux_x86_64", + "osx_aarch64", + ], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_get_python_versions_with_target_platforms) + def _test_get_python_versions_from_filenames(env): got = get_whl_flag_versions( aliases = [ @@ -660,6 +678,29 @@ def _test_multiplatform_whl_aliases_nofilename(env): _tests.append(_test_multiplatform_whl_aliases_nofilename) +def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): + aliases = [ + whl_alias( + repo = "foo", + config_setting = "//:ignored", + version = "3.1", + target_platforms = [ + "cp31_linux_x86_64", + "cp31_linux_aarch64", + ], + ), + ] + + got = multiplatform_whl_aliases(aliases = aliases) + + want = [ + whl_alias(config_setting = "//_config:is_cp3.1_linux_x86_64", repo = "foo", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_linux_aarch64", repo = "foo", version = "3.1"), + ] + env.expect.that_collection(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) + def _test_multiplatform_whl_aliases_filename(env): aliases = [ whl_alias( @@ -734,6 +775,52 @@ def _test_multiplatform_whl_aliases_filename_versioned(env): _tests.append(_test_multiplatform_whl_aliases_filename_versioned) +def _mock_alias(container): + return lambda name, **kwargs: container.append(name) + +def _mock_config_setting(container): + def _inner(name, flag_values = None, constraint_values = None, **_): + if flag_values or constraint_values: + container.append(name) + return + + fail("At least one of 'flag_values' or 'constraint_values' needs to be set") + + return _inner + +def _test_config_settings_exist_legacy(env): + aliases = [ + whl_alias( + repo = "repo", + version = "3.11", + target_platforms = [ + "cp311_linux_aarch64", + "cp311_linux_x86_64", + ], + ), + ] + available_config_settings = [] + config_settings( + python_versions = ["3.11"], + native = struct( + alias = _mock_alias(available_config_settings), + config_setting = _mock_config_setting(available_config_settings), + ), + target_platforms = [ + "linux_aarch64", + "linux_x86_64", + ], + ) + + got_aliases = multiplatform_whl_aliases( + aliases = aliases, + ) + got = [a.config_setting.partition(":")[-1] for a in got_aliases] + + env.expect.that_collection(available_config_settings).contains_at_least(got) + +_tests.append(_test_config_settings_exist_legacy) + def _test_config_settings_exist(env): for py_tag in ["py2.py3", "py3", "py311", "cp311"]: if py_tag == "py2.py3": @@ -771,12 +858,11 @@ def _test_config_settings_exist(env): ), ] available_config_settings = [] - mock_rule = lambda name, **kwargs: available_config_settings.append(name) config_settings( python_versions = ["3.11"], native = struct( - alias = mock_rule, - config_setting = mock_rule, + alias = _mock_alias(available_config_settings), + config_setting = _mock_config_setting(available_config_settings), ), **kwargs )