diff --git a/CHANGELOG.md b/CHANGELOG.md index f69e94ec65..1ce59e4935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) +* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`, + `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to + use the `pip.default` to configure it yourself. If you are interested in graduating the + platform, consider helping set us up CI for them and update the documentation. * (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations. * (core) #!/usr/bin/env bash is now used as a shebang in the stage1 bootstrap template. @@ -88,6 +92,10 @@ END_UNRELEASED_TEMPLATE ([#3043](https://github.com/bazel-contrib/rules_python/issues/3043)). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* (pypi) The selection of the whls has been changed and should no longer result + in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient + when running `bazel query` due to fewer repositories being included + ({gh-issue}`2849`). * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8 ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)). diff --git a/MODULE.bazel b/MODULE.bazel index 9db287dc28..1ce95e9f23 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -70,20 +70,30 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") config_settings = [ "@platforms//cpu:{}".format(cpu), "@platforms//os:linux", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), ], env = {"platform_version": "0"}, + marker = "python_version >= '3.13'" if freethreaded else "", os_name = "linux", - platform = "linux_{}".format(cpu), + platform = "linux_{}{}".format(cpu, freethreaded), + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "linux_{}".format(cpu), + "manylinux_*_{}".format(cpu), + ], ) for cpu in [ "x86_64", "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", + ] + for freethreaded in [ + "", + "_freethreaded", ] ] @@ -93,32 +103,75 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") config_settings = [ "@platforms//cpu:{}".format(cpu), "@platforms//os:osx", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), ], # We choose the oldest non-EOL version at the time when we release `rules_python`. # See https://endoflife.date/macos env = {"platform_version": "14.0"}, + marker = "python_version >= '3.13'" if freethreaded else "", os_name = "osx", - platform = "osx_{}".format(cpu), + platform = "osx_{}{}".format(cpu, freethreaded), + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "macosx_*_{}".format(suffix) + for suffix in platform_tag_cpus + ], ) - for cpu in [ - "aarch64", - "x86_64", + for cpu, platform_tag_cpus in { + "aarch64": [ + "universal2", + "arm64", + ], + "x86_64": [ + "universal2", + "x86_64", + ], + }.items() + for freethreaded in [ + "", + "_freethreaded", + ] +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:windows", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), + ], + env = {"platform_version": "0"}, + marker = "python_version >= '3.13'" if freethreaded else "", + os_name = "windows", + platform = "windows_{}{}".format(cpu, freethreaded), + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = whl_platform_tags, + ) + for cpu, whl_platform_tags in { + "x86_64": ["win_amd64"], + }.items() + for freethreaded in [ + "", + "_freethreaded", ] ] -pip.default( - arch_name = "x86_64", - config_settings = [ - "@platforms//cpu:x86_64", - "@platforms//os:windows", - ], - env = {"platform_version": "0"}, - os_name = "windows", - platform = "windows_x86_64", -) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. + # + # how do we test sdists? Maybe just worth adding a single sdist somewhere? download_only = False, experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 841c096dcf..95e1090f53 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -158,6 +158,23 @@ pip.whl_mods( ) use_repo(pip, "whl_mods_hub") +# Because below we are using `windows_aarch64` platform, we have to define various +# properties for it. +pip.default( + arch_name = "aarch64", + config_settings = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], + env = { + "platform_version": "0", + }, + os_name = "windows", + platform = "windows_aarch64", + whl_abi_tags = [], # default to all ABIs + whl_platform_tags = ["win_amd64"], +) + # To fetch pip dependencies, use pip.parse. We can pass in various options, # but typically we pass requirements and the Python version. The Python # version must have been configured by a corresponding `python.toolchain()` diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b098f29e94..847c85d634 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -116,11 +116,11 @@ bzl_library( ":parse_whl_name_bzl", ":pep508_env_bzl", ":pip_repository_attrs_bzl", + ":python_tag_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", ":whl_library_bzl", ":whl_repo_name_bzl", - ":whl_target_platforms_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_bzl", @@ -209,7 +209,7 @@ bzl_library( ":parse_requirements_txt_bzl", ":pypi_repo_utils_bzl", ":requirements_files_by_platform_bzl", - ":whl_target_platforms_bzl", + ":select_whl_bzl", "//python/private:normalize_name_bzl", "//python/private:repo_utils_bzl", ], @@ -252,6 +252,9 @@ bzl_library( bzl_library( name = "pep508_env_bzl", srcs = ["pep508_env.bzl"], + deps = [ + "//python/private:version_bzl", + ], ) bzl_library( @@ -263,11 +266,6 @@ bzl_library( ], ) -bzl_library( - name = "pep508_platform_bzl", - srcs = ["pep508_platform.bzl"], -) - bzl_library( name = "pep508_requirement_bzl", srcs = ["pep508_requirement.bzl"], @@ -338,6 +336,14 @@ bzl_library( ], ) +bzl_library( + name = "python_tag_bzl", + srcs = ["python_tag.bzl"], + deps = [ + "//python/private:version_bzl", + ], +) + bzl_library( name = "render_pkg_aliases_bzl", srcs = ["render_pkg_aliases.bzl"], @@ -359,6 +365,16 @@ bzl_library( ], ) +bzl_library( + name = "select_whl_bzl", + srcs = ["select_whl.bzl"], + deps = [ + ":parse_whl_name_bzl", + ":python_tag_bzl", + "//python/private:version_bzl", + ], +) + bzl_library( name = "simpleapi_download_bzl", srcs = ["simpleapi_download.bzl"], @@ -422,5 +438,4 @@ bzl_library( bzl_library( name = "whl_target_platforms_bzl", srcs = ["whl_target_platforms.bzl"], - deps = [":parse_whl_name_bzl"], ) diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 6167cdbc96..4d6a39a1df 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -43,11 +43,11 @@ def evaluate_markers(*, requirements, platforms): for req_string, platform_strings in requirements.items(): req = requirement(req_string) for platform_str in platform_strings: - env = platforms.get(platform_str) - if not env: + plat = platforms.get(platform_str) + if not plat: fail("Please define platform: '{}'".format(platform_str)) - if evaluate(req.marker, env = env): + if evaluate(req.marker, env = plat.env): ret.setdefault(req_string, []).append(platform_str) return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 096256e4be..fbcc63befb 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -30,7 +30,9 @@ load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") +load(":pep508_evaluate.bzl", "evaluate") load(":pip_repository_attrs.bzl", "ATTRS") +load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_config_setting.bzl", "whl_config_setting") @@ -68,19 +70,47 @@ def _whl_mods_impl(whl_mods_dict): def _platforms(*, python_version, minor_mapping, config): platforms = {} - python_version = full_version( - version = python_version, - minor_mapping = minor_mapping, + python_version = version.parse( + full_version( + version = python_version, + minor_mapping = minor_mapping, + ), + strict = True, ) - abi = "cp3{}".format(python_version[2:]) for platform, values in config.platforms.items(): + # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too + # many times. + abi = "{}{}{}.{}".format( + python_tag(values.env["implementation_name"]), + python_version.release[0], + python_version.release[1], + python_version.release[2], + ) key = "{}_{}".format(abi, platform) - platforms[key] = env(struct( - abi = abi, + + env_ = env( + env = values.env, os = values.os_name, arch = values.arch_name, - )) | values.env + python_version = python_version.string, + ) + + if values.marker and not evaluate(values.marker, env = env_): + continue + + platforms[key] = struct( + env = env_, + triple = "{}_{}_{}".format(abi, values.os_name, values.arch_name), + whl_abi_tags = [ + v.format( + major = python_version.release[0], + minor = python_version.release[1], + ) + for v in values.whl_abi_tags + ], + whl_platform_tags = values.whl_platform_tags, + ) return platforms def _create_whl_repos( @@ -152,6 +182,8 @@ def _create_whl_repos( )) python_interpreter_target = available_interpreters[python_name] + # TODO @aignas 2025-06-29: we should not need the version in the pip_name if + # we are using pipstar and we are downloading the wheel using the downloader pip_name = "{}_{}".format( hub_name, version_label(pip_attr.python_version), @@ -178,17 +210,19 @@ def _create_whl_repos( whl_group_mapping = {} requirement_cycles = {} + platforms = _platforms( + python_version = pip_attr.python_version, + minor_mapping = minor_mapping, + config = config, + ) + if evaluate_markers: # This is most likely unit tests pass elif config.enable_pipstar: evaluate_markers = lambda _, requirements: evaluate_markers_star( requirements = requirements, - platforms = _platforms( - python_version = pip_attr.python_version, - minor_mapping = minor_mapping, - config = config, - ), + platforms = platforms, ) else: # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either @@ -207,7 +241,13 @@ def _create_whl_repos( # spin up a Python interpreter. evaluate_markers = lambda module_ctx, requirements: evaluate_markers_py( module_ctx, - requirements = requirements, + requirements = { + k: { + p: platforms[p].triple + for p in plats + } + for k, plats in requirements.items() + }, python_interpreter = pip_attr.python_interpreter, python_interpreter_target = python_interpreter_target, srcs = pip_attr._evaluate_markers_srcs, @@ -223,19 +263,24 @@ def _create_whl_repos( requirements_osx = pip_attr.requirements_darwin, requirements_windows = pip_attr.requirements_windows, extra_pip_args = pip_attr.extra_pip_args, - platforms = sorted(config.platforms), # here we only need keys + platforms = sorted(platforms), # here we only need keys python_version = full_version( version = pip_attr.python_version, minor_mapping = minor_mapping, ), logger = logger, ), + platforms = platforms, extra_pip_args = pip_attr.extra_pip_args, get_index_urls = get_index_urls, evaluate_markers = evaluate_markers, logger = logger, ) + use_downloader = { + normalize_name(s): False + for s in pip_attr.simpleapi_skip + } exposed_packages = {} for whl in requirements_by_platform: if whl.is_exposed: @@ -289,11 +334,20 @@ def _create_whl_repos( whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, + use_downloader = use_downloader.get( + whl.name, + get_index_urls != None, # defaults to True if the get_index_urls is defined + ), auth_patterns = pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, enable_pipstar = config.enable_pipstar, ) + if repo == None: + # NOTE @aignas 2025-07-07: we guard against an edge-case where there + # are more platforms defined than there are wheels for and users + # disallow building from sdist. + continue repo_name = "{}_{}".format(pip_name, repo.repo_name) if repo_name in whl_libraries: @@ -303,6 +357,16 @@ def _create_whl_repos( )) whl_libraries[repo_name] = repo.args + if not config.enable_pipstar and "experimental_target_platforms" in repo.args: + whl_libraries[repo_name] |= { + "experimental_target_platforms": sorted({ + # TODO @aignas 2025-07-07: this should be solved in a better way + platforms[candidate].triple.partition("_")[-1]: None + for p in repo.args["experimental_target_platforms"] + for candidate in platforms + if candidate.endswith(p) + }), + } whl_map.setdefault(whl.name, {})[repo.config_setting] = repo_name return struct( @@ -312,7 +376,17 @@ def _create_whl_repos( whl_libraries = whl_libraries, ) -def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, netrc, auth_patterns, python_version, enable_pipstar = False): +def _whl_repo( + *, + src, + whl_library_args, + is_multiple_versions, + download_only, + netrc, + auth_patterns, + python_version, + use_downloader, + enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -325,19 +399,24 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net args["extra_pip_args"] = src.extra_pip_args if not src.url or (not is_whl and download_only): - # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if is_multiple_versions else [] - return struct( - repo_name = pypi_repo_name( - normalize_name(src.distribution), - *target_platforms - ), - args = args, - config_setting = whl_config_setting( - version = python_version, - target_platforms = target_platforms or None, - ), - ) + if download_only and use_downloader: + # If the user did not allow using sdists and we are using the downloader + # and we are not using simpleapi_skip for this + return None + else: + # Fallback to a pip-installed wheel + target_platforms = src.target_platforms if is_multiple_versions else [] + return struct( + repo_name = pypi_repo_name( + normalize_name(src.distribution), + *target_platforms + ), + args = args, + config_setting = whl_config_setting( + version = python_version, + target_platforms = target_platforms or None, + ), + ) # This is no-op because pip is not used to download the wheel. args.pop("download_only", None) @@ -359,44 +438,136 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net for p in src.target_platforms ] - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if is_whl and not src.filename.endswith("-any.whl"): - pass - elif is_multiple_versions: - target_platforms = src.target_platforms - return struct( repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, - filename = src.filename, - target_platforms = target_platforms, + target_platforms = src.target_platforms, ), ) -def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False): +def _configure( + config, + *, + platform, + os_name, + arch_name, + config_settings, + env = {}, + marker, + whl_abi_tags, + whl_platform_tags, + override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform: - if not override and config.get("platforms", {}).get(platform): + + if platform and ( + os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env or marker + ): + if not override and config["platforms"].get(platform): return for key in env: if key not in _SUPPORTED_PEP508_KEYS: fail("Unsupported key in the PEP508 environment: {}".format(key)) - config["platforms"][platform] = struct( - name = platform.replace("-", "_").lower(), - os_name = os_name, - arch_name = arch_name, - config_settings = config_settings, - env = env, - ) + config["platforms"].setdefault(platform, {}) + for key, value in { + "arch_name": arch_name, + "config_settings": config_settings, + "env": env, + "marker": marker, + "name": platform.replace("-", "_").lower(), + "os_name": os_name, + "whl_abi_tags": whl_abi_tags, + "whl_platform_tags": whl_platform_tags, + }.items(): + if not value: + continue + + if not override and config.get(key): + continue + + config["platforms"][platform][key] = value else: config["platforms"].pop(platform) +def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, marker = "", whl_abi_tags = [], whl_platform_tags = []): + # NOTE @aignas 2025-07-08: the least preferred is the first item in the list + if "any" not in whl_platform_tags: + # the lowest priority one needs to be the first one + whl_platform_tags = ["any"] + whl_platform_tags + + whl_abi_tags = whl_abi_tags or ["abi3", "cp{major}{minor}"] + if "none" not in whl_abi_tags: + # the lowest priority one needs to be the first one + whl_abi_tags = ["none"] + whl_abi_tags + + return struct( + name = name, + arch_name = arch_name, + os_name = os_name, + config_settings = config_settings, + env = { + # defaults for env + "implementation_name": "cpython", + } | env, + marker = marker, + whl_abi_tags = whl_abi_tags, + whl_platform_tags = whl_platform_tags, + ) + +def build_config( + *, + module_ctx, + enable_pipstar): + """Parse 'configure' and 'default' extension tags + + Args: + module_ctx: {type}`module_ctx` module context. + enable_pipstar: {type}`bool` a flag to enable dropping Python dependency for + evaluation of the extension. + + Returns: + A struct with the configuration. + """ + defaults = { + "platforms": {}, + } + for mod in module_ctx.modules: + if not (mod.is_root or mod.name == "rules_python"): + continue + + platform = None + for tag in mod.tags.default: + platform = tag.platform or platform + _configure( + defaults, + arch_name = tag.arch_name, + config_settings = tag.config_settings, + env = tag.env, + os_name = tag.os_name, + marker = tag.marker, + platform = platform, + override = mod.is_root, + whl_abi_tags = tag.whl_abi_tags, + whl_platform_tags = tag.whl_platform_tags, + # TODO @aignas 2025-05-19: add more attr groups: + # * for AUTH - the default `netrc` usage could be configured through a common + # attribute. + # * for index/downloader config. This includes all of those attributes for + # overrides, etc. Index overrides per platform could be also used here. + ) + + return struct( + platforms = { + name: _plat(**values) + for name, values in defaults["platforms"].items() + }, + enable_pipstar = enable_pipstar, + ) + def parse_modules( module_ctx, _fail = fail, @@ -447,33 +618,7 @@ You cannot use both the additive_build_content and additive_build_content_file a srcs_exclude_glob = whl_mod.srcs_exclude_glob, ) - defaults = { - "enable_pipstar": enable_pipstar, - "platforms": {}, - } - for mod in module_ctx.modules: - if not (mod.is_root or mod.name == "rules_python"): - continue - - for tag in mod.tags.default: - _configure( - defaults, - arch_name = tag.arch_name, - config_settings = tag.config_settings, - env = tag.env, - os_name = tag.os_name, - platform = tag.platform, - override = mod.is_root, - # TODO @aignas 2025-05-19: add more attr groups: - # * for AUTH - the default `netrc` usage could be configured through a common - # attribute. - # * for index/downloader config. This includes all of those attributes for - # overrides, etc. Index overrides per platform could be also used here. - # * for whl selection - selecting preferences of which `platform_tag`s we should use - # for what. We could also model the `cp313t` freethreaded as separate platforms. - ) - - config = struct(**defaults) + config = build_config(module_ctx = module_ctx, enable_pipstar = enable_pipstar) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} @@ -601,6 +746,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases.setdefault(hub_name, {}) for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) + if hub_name not in exposed_packages: exposed_packages[hub_name] = out.exposed_packages else: @@ -611,6 +757,14 @@ You cannot use both the additive_build_content and additive_build_content_file a intersection[pkg] = None exposed_packages[hub_name] = intersection whl_libraries.update(out.whl_libraries) + for whl_name, lib in out.whl_libraries.items(): + if enable_pipstar: + whl_libraries.setdefault(whl_name, lib) + elif whl_name in lib: + fail("'{}' already in created".format(whl_name)) + else: + # replicate whl_libraries.update(out.whl_libraries) + whl_libraries[whl_name] = lib # TODO @aignas 2024-04-05: how do we support different requirement # cycles for different abis/oses? For now we will need the users to @@ -658,6 +812,7 @@ You cannot use both the additive_build_content and additive_build_content_file a k: dict(sorted(args.items())) for k, args in sorted(whl_libraries.items()) }, + config = config, ) def _pip_impl(module_ctx): @@ -760,6 +915,7 @@ _default_attrs = { "arch_name": attr.string( doc = """\ The CPU architecture name to be used. +You can use any cpu name from the `@platforms//cpu:` package. :::{note} Either this or {attr}`env` `platform_machine` key should be specified. @@ -773,25 +929,6 @@ The list of labels to `config_setting` targets that need to be matched for the p selected. """, ), - "os_name": attr.string( - doc = """\ -The OS name to be used. - -:::{note} -Either this or the appropriate `env` keys should be specified. -::: -""", - ), - "platform": attr.string( - doc = """\ -A platform identifier which will be used as the unique identifier within the extension evaluation. -If you are defining custom platforms in your project and don't want things to clash, use extension -[isolation] feature. - -[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate -""", - ), -} | { "env": attr.string_dict( doc = """\ The values to use for environment markers when evaluating an expression. @@ -814,9 +951,88 @@ Supported keys: ::::{note} This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. :::: +""", + ), + "marker": attr.string( + doc = """\ +A marker which will be evaluated to disable the target platform for certain python versions. This +is especially useful when defining freethreaded platform variants. """, ), # The values for PEP508 env marker evaluation during the lock file parsing + "os_name": attr.string( + doc = """\ +The OS name to be used. +You can use any OS name from the `@platforms//os:` package. + +:::{note} +Either this or the appropriate `env` keys should be specified. +::: +""", + ), + "platform": attr.string( + doc = """\ +A platform identifier which will be used as the unique identifier within the extension evaluation. +If you are defining custom platforms in your project and don't want things to clash, use extension +[isolation] feature. + +[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), + "whl_abi_tags": attr.string_list( + doc = """\ +A list of ABIs to select wheels for. The values can be either strings or include template +parameters like `{major}` and `{minor}` which will be replaced with python version parts. e.g. +`cp{major}{minor}` will result in `cp313` given the full python version is `3.13.5`. +Will always include `"none"` even if it is not specified. + +:::{note} +We select a single wheel and the last match will take precedence. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#abi-tag) for more information. +::: +""", + ), + "whl_platform_tags": attr.string_list( + doc = """\ +A list of `platform_tag` matchers so that we can select the best wheel based on the user +preference. +Will always include `"any"` even if it is not specified. + +The items in this list can contain a single `*` character that is equivalent to matching the +latest available version component in the platform_tag. Note, if the wheel platform tag does not +have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular +character. + +We will always select the highest available `platform_tag` version that is compatible with the +target platform. + +:::{note} +We select a single wheel and the last match will take precedence, if the platform_tag that we +match has a version component (e.g. `android_x_arch`, then the version `x` will be used in the +matching algorithm). + +If the matcher you provide has `*`, then we will match a wheel with the highest available target platform, i.e. if `musllinux_1_1_arch` and `musllinux_1_2_arch` are both present, then we will select `musllinux_1_2_arch`. +Otherwise we will select the highest available version that is equal or lower to the specifier, i.e. if `manylinux_2_12` and `manylinux_2_17` wheels are present and the matcher is `manylinux_2_15`, then we will match `manylinux_2_12` but not `manylinux_2_17`. +::: + +:::{note} +The following tag prefixes should be used instead of the legacy equivalents: +* `manylinux_2_5` instead of `manylinux1` +* `manylinux_2_12` instead of `manylinux2010` +* `manylinux_2_17` instead of `manylinux2014` + +When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the +latest format. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. +::: +""", + ), } _SUPPORTED_PEP508_KEYS = [ diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 9c610f11d3..ebd447d95d 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -31,13 +31,14 @@ load("//python/private:repo_utils.bzl", "repo_utils") load(":index_sources.bzl", "index_sources") load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":pep508_requirement.bzl", "requirement") -load(":whl_target_platforms.bzl", "select_whls") +load(":select_whl.bzl", "select_whl") def parse_requirements( ctx, *, requirements_by_platform = {}, extra_pip_args = [], + platforms = {}, get_index_urls = None, evaluate_markers = None, extract_url_srcs = True, @@ -46,6 +47,7 @@ def parse_requirements( Args: ctx: A context that has .read function that would read contents from a label. + platforms: The target platform descriptions. requirements_by_platform (label_keyed_string_dict): a way to have different package versions (or different packages) for different os, arch combinations. @@ -88,7 +90,7 @@ def parse_requirements( requirements = {} for file, plats in requirements_by_platform.items(): if logger: - logger.debug(lambda: "Using {} for {}".format(file, plats)) + logger.trace(lambda: "Using {} for {}".format(file, plats)) contents = ctx.read(file) # Parse the requirements file directly in starlark to get the information @@ -161,7 +163,7 @@ def parse_requirements( # VCS package references. env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers) if logger: - logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( + logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( reqs_with_env_markers, env_marker_target_platforms, )) @@ -196,6 +198,7 @@ def parse_requirements( name = name, reqs = reqs, index_urls = index_urls, + platforms = platforms, env_marker_target_platforms = env_marker_target_platforms, extract_url_srcs = extract_url_srcs, logger = logger, @@ -203,7 +206,7 @@ def parse_requirements( ) ret.append(item) if not item.is_exposed and logger: - logger.debug(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( + logger.trace(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( name, sorted(requirement_target_platforms), sorted(requirements), @@ -219,38 +222,43 @@ def _package_srcs( name, reqs, index_urls, + platforms, logger, env_marker_target_platforms, extract_url_srcs): """A function to return sources for a particular package.""" srcs = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(name), - logger = logger, - ) - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - target_platforms = sorted(target_platforms) + extra_pip_args = tuple(r.extra_pip_args) - all_dists = [] + whls - if sdist: - all_dists.append(sdist) + for target_platform in target_platforms: + if platforms and target_platform not in platforms: + fail("The target platform '{}' could not be found in {}".format( + target_platform, + platforms.keys(), + )) - if extract_url_srcs and all_dists: - req_line = r.srcs.requirement - else: - all_dists = [struct( - url = "", - filename = "", - sha256 = "", - yanked = False, - )] - req_line = r.srcs.requirement_line + dist = _add_dists( + requirement = r, + target_platform = platforms.get(target_platform), + index_urls = index_urls.get(name), + logger = logger, + ) + if logger: + logger.debug(lambda: "The whl dist is: {}".format(dist.filename if dist else dist)) + + if extract_url_srcs and dist: + req_line = r.srcs.requirement + else: + dist = struct( + url = "", + filename = "", + sha256 = "", + yanked = False, + ) + req_line = r.srcs.requirement_line - extra_pip_args = tuple(r.extra_pip_args) - for dist in all_dists: key = ( dist.filename, req_line, @@ -269,9 +277,9 @@ def _package_srcs( yanked = dist.yanked, ), ) - for p in target_platforms: - if p not in entry.target_platforms: - entry.target_platforms.append(p) + + if target_platform not in entry.target_platforms: + entry.target_platforms.append(target_platform) return srcs.values() @@ -325,7 +333,7 @@ def host_platform(ctx): repo_utils.get_platforms_cpu_name(ctx), ) -def _add_dists(*, requirement, index_urls, logger = None): +def _add_dists(*, requirement, index_urls, target_platform, logger = None): """Populate dists based on the information from the PyPI index. This function will modify the given requirements_by_platform data structure. @@ -333,6 +341,7 @@ def _add_dists(*, requirement, index_urls, logger = None): Args: requirement: The result of parse_requirements function. index_urls: The result of simpleapi_download. + target_platform: The target_platform information. logger: A logger for printing diagnostic info. """ @@ -342,7 +351,7 @@ def _add_dists(*, requirement, index_urls, logger = None): logger.debug(lambda: "Could not detect the filename from the URL, falling back to pip: {}".format( requirement.srcs.url, )) - return [], None + return None # Handle direct URLs in requirements dist = struct( @@ -353,12 +362,12 @@ def _add_dists(*, requirement, index_urls, logger = None): ) if dist.filename.endswith(".whl"): - return [dist], None + return dist else: - return [], dist + return dist if not index_urls: - return [], None + return None whls = [] sdist = None @@ -401,11 +410,16 @@ def _add_dists(*, requirement, index_urls, logger = None): for reason, dists in yanked.items() ])) - # Filter out the wheels that are incompatible with the target_platforms. - whls = select_whls( + if not target_platform: + # The pipstar platforms are undefined here, so we cannot do any matching + return sdist + + # Select a single wheel that can work on the target_platform + return select_whl( whls = whls, - want_platforms = requirement.target_platforms, + python_version = target_platform.env["python_full_version"], + implementation_name = target_platform.env["implementation_name"], + whl_abi_tags = target_platform.whl_abi_tags, + whl_platform_tags = target_platform.whl_platform_tags, logger = logger, - ) - - return whls, sdist + ) or sdist diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index c2d404bc3e..5031ebae12 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -15,6 +15,20 @@ """This module is for implementing PEP508 environment definition. """ +load("//python/private:version.bzl", "version") + +_DEFAULT = "//conditions:default" + +# Here we store the aliases in the platform so that the users can specify any valid target in +# there. +_cpu_aliases = { + "arm": "aarch32", + "arm64": "aarch64", +} +_os_aliases = { + "macos": "osx", +} + # See https://stackoverflow.com/a/45125525 platform_machine_aliases = { # These pairs mean the same hardware, but different values may be used @@ -59,7 +73,7 @@ platform_machine_select_map = { "@platforms//cpu:x86_64": "x86_64", # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # Platform system returns results from the `uname` call. @@ -73,7 +87,7 @@ _platform_system_values = { "linux": "Linux", "netbsd": "NetBSD", "openbsd": "OpenBSD", - "osx": "Darwin", + "osx": "Darwin", # NOTE: macos is an alias to osx, we handle it through _os_aliases "windows": "Windows", } @@ -83,7 +97,7 @@ platform_system_select_map = { } | { # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # The copy of SO [answer](https://stackoverflow.com/a/13874620) containing @@ -123,18 +137,19 @@ _sys_platform_values = { "ios": "ios", "linux": "linux", "openbsd": "openbsd", - "osx": "darwin", + "osx": "darwin", # NOTE: macos is an alias to osx, we handle it through _os_aliases "wasi": "wasi", "windows": "win32", } sys_platform_select_map = { + # These values are decided by the sys.platform docs. "@platforms//os:{}".format(bazel_os): py_platform for bazel_os, py_platform in _sys_platform_values.items() } | { # For lack of a better option, use empty string. No standard doc/spec # about sys_platform value. - "//conditions:default": "", + _DEFAULT: "", } # The "java" value is documented, but with Jython defunct, @@ -142,53 +157,58 @@ sys_platform_select_map = { # The os.name value is technically a property of the runtime, not the # targetted runtime OS, but the distinction shouldn't matter if # things are properly configured. -_os_name_values = { - "linux": "posix", - "osx": "posix", - "windows": "nt", -} - os_name_select_map = { - "@platforms//os:{}".format(bazel_os): py_os - for bazel_os, py_os in _os_name_values.items() -} | { - "//conditions:default": "posix", + "@platforms//os:windows": "nt", + _DEFAULT: "posix", } -def env(target_platform, *, extra = None): +def _set_default(env, env_key, m, key): + """Set the default value in the env if it is not already set.""" + default = m.get(key, m[_DEFAULT]) + env.setdefault(env_key, default) + +def env(*, env = None, os, arch, python_version = "", extra = None): """Return an env target platform NOTE: This is for use during the loading phase. For the analysis phase, `env_marker_setting()` constructs the env dict. Args: - target_platform: {type}`str` the target platform identifier, e.g. - `cp33_linux_aarch64` + env: {type}`str` the environment. + os: {type}`str` the OS name. + arch: {type}`str` the CPU name. + python_version: {type}`str` the full python version. extra: {type}`str` the extra value to be added into the env. Returns: A dict that can be used as `env` in the marker evaluation. """ - env = create_env() + env = env or {} + env = env | create_env() if extra != None: env["extra"] = extra - if target_platform.abi: - minor_version, _, micro_version = target_platform.abi[3:].partition(".") - micro_version = micro_version or "0" - env = env | { - "implementation_version": "3.{}.{}".format(minor_version, micro_version), - "python_full_version": "3.{}.{}".format(minor_version, micro_version), - "python_version": "3.{}".format(minor_version), - } - if target_platform.os and target_platform.arch: - os = target_platform.os + if python_version: + v = version.parse(python_version) + major = v.release[0] + minor = v.release[1] + micro = v.release[2] if len(v.release) > 2 else 0 env = env | { - "os_name": _os_name_values.get(os, ""), - "platform_machine": target_platform.arch, - "platform_system": _platform_system_values.get(os, ""), - "sys_platform": _sys_platform_values.get(os, ""), + "implementation_version": "{}.{}.{}".format(major, minor, micro), + "python_full_version": "{}.{}.{}".format(major, minor, micro), + "python_version": "{}.{}".format(major, minor), } + + if os: + os = "@platforms//os:{}".format(_os_aliases.get(os, os)) + _set_default(env, "os_name", os_name_select_map, os) + _set_default(env, "platform_system", platform_system_select_map, os) + _set_default(env, "sys_platform", sys_platform_select_map, os) + + if arch: + arch = "@platforms//cpu:{}".format(_cpu_aliases.get(arch, arch)) + _set_default(env, "platform_machine", platform_machine_select_map, arch) + set_missing_env_defaults(env) return env diff --git a/python/private/pypi/pep508_platform.bzl b/python/private/pypi/pep508_platform.bzl deleted file mode 100644 index 381a8d7a08..0000000000 --- a/python/private/pypi/pep508_platform.bzl +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""The platform abstraction -""" - -def platform(*, abi = None, os = None, arch = None): - """platform returns a struct for the platform. - - Args: - abi: {type}`str | None` the target ABI, e.g. `"cp39"`. - os: {type}`str | None` the target os, e.g. `"linux"`. - arch: {type}`str | None` the target CPU, e.g. `"aarch64"`. - - Returns: - A struct. - """ - - # Note, this is used a lot as a key in dictionaries, so it cannot contain - # methods. - return struct( - abi = abi, - os = os, - arch = arch, - ) - -def platform_from_str(p, python_version): - """Return a platform from a string. - - Args: - p: {type}`str` the actual string. - python_version: {type}`str` the python version to add to platform if needed. - - Returns: - A struct that is returned by the `_platform` function. - """ - if p.startswith("cp"): - abi, _, p = p.partition("_") - elif python_version: - major, _, tail = python_version.partition(".") - abi = "cp{}{}".format(major, tail) - else: - abi = None - - os, _, arch = p.partition("_") - return platform(abi = abi, os = os or None, arch = arch or None) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index e63bd6c3d1..3df56f24ff 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -94,7 +94,12 @@ def _pip_repository_impl(rctx): extra_pip_args = rctx.attr.extra_pip_args, evaluate_markers = lambda rctx, requirements: evaluate_markers_py( rctx, - requirements = requirements, + requirements = { + # NOTE @aignas 2025-07-07: because we don't distinguish between + # freethreaded and non-freethreaded, it is a 1:1 mapping. + req: {p: p for p in plats} + for req, plats in requirements.items() + }, python_interpreter = rctx.attr.python_interpreter, python_interpreter_target = rctx.attr.python_interpreter_target, srcs = rctx.attr._evaluate_markers_srcs, diff --git a/python/private/pypi/python_tag.bzl b/python/private/pypi/python_tag.bzl new file mode 100644 index 0000000000..224c5f96f0 --- /dev/null +++ b/python/private/pypi/python_tag.bzl @@ -0,0 +1,41 @@ +"A simple utility function to get the python_tag from the implementation name" + +load("//python/private:version.bzl", "version") + +# Taken from +# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag +_PY_TAGS = { + # "py": Generic Python (does not require implementation-specific features) + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", + "python": "py", +} +PY_TAG_GENERIC = "py" + +def python_tag(implementation_name, python_version = ""): + """Get the python_tag from the implementation_name. + + Args: + implementation_name: {type}`str` the implementation name, e.g. "cpython" + python_version: {type}`str` a version who can be parsed using PEP440 compliant + parser. + + Returns: + A {type}`str` that represents the python_tag with a version if the + python_version is given. + """ + if python_version: + v = version.parse(python_version, strict = True) + suffix = "{}{}".format( + v.release[0], + v.release[1] if len(v.release) > 1 else "", + ) + else: + suffix = "" + + return "{}{}".format( + _PY_TAGS.get(implementation_name, implementation_name), + suffix, + ) diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl index d8d3651461..356bd4416e 100644 --- a/python/private/pypi/requirements_files_by_platform.bzl +++ b/python/private/pypi/requirements_files_by_platform.bzl @@ -37,7 +37,9 @@ def _default_platforms(*, filter, platforms): if not prefix: return platforms - match = [p for p in platforms if p.startswith(prefix)] + match = [p for p in platforms if p.startswith(prefix) or ( + p.startswith("cp") and p.partition("_")[-1].startswith(prefix) + )] else: match = [p for p in platforms if filter in p] @@ -140,7 +142,7 @@ def requirements_files_by_platform( if logger: logger.debug(lambda: "Platforms from pip args: {}".format(platforms_from_args)) - default_platforms = [_platform(p, python_version) for p in platforms] + default_platforms = platforms if platforms_from_args: lock_files = [ @@ -252,6 +254,6 @@ def requirements_files_by_platform( ret = {} for plat, file in requirements.items(): - ret.setdefault(file, []).append(plat) + ret.setdefault(file, []).append(_platform(plat, python_version = python_version)) return ret diff --git a/python/private/pypi/requirements_parser/resolve_target_platforms.py b/python/private/pypi/requirements_parser/resolve_target_platforms.py index c899a943cc..accacf5bfa 100755 --- a/python/private/pypi/requirements_parser/resolve_target_platforms.py +++ b/python/private/pypi/requirements_parser/resolve_target_platforms.py @@ -50,8 +50,8 @@ def main(): hashes = prefix + hashes req = Requirement(entry) - for p in target_platforms: - (platform,) = Platform.from_string(p) + for p, triple in target_platforms.items(): + (platform,) = Platform.from_string(triple) if not req.marker or req.marker.evaluate(platform.env_markers("")): response.setdefault(requirement_line, []).append(p) diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl new file mode 100644 index 0000000000..91268d6735 --- /dev/null +++ b/python/private/pypi/select_whl.bzl @@ -0,0 +1,230 @@ +"Select a single wheel that fits the parameters of a target platform." + +load("//python/private:version.bzl", "version") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") + +_ANDROID = "android" +_IOS = "ios" +_MANYLINUX = "manylinux" +_MACOSX = "macosx" +_MUSLLINUX = "musllinux" + +def _value_priority(*, tag, values): + keys = [] + for priority, wp in enumerate(values): + if tag == wp: + keys.append(priority) + + return max(keys) if keys else None + +def _platform_tag_priority(*, tag, values): + # Implements matching platform tag + # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ + + if not ( + tag.startswith(_ANDROID) or + tag.startswith(_IOS) or + tag.startswith(_MACOSX) or + tag.startswith(_MANYLINUX) or + tag.startswith(_MUSLLINUX) + ): + res = _value_priority(tag = tag, values = values) + if res == None: + return res + + return (res, (0, 0)) + + plat, _, tail = tag.partition("_") + major, _, tail = tail.partition("_") + if not plat.startswith(_ANDROID): + minor, _, arch = tail.partition("_") + else: + minor = "0" + arch = tail + version = (int(major), int(minor)) + + keys = [] + for priority, wp in enumerate(values): + want_plat, sep, tail = wp.partition("_") + if not sep: + continue + + if want_plat != plat: + continue + + want_major, _, tail = tail.partition("_") + if want_major == "*": + want_major = "" + want_minor = "" + want_arch = tail + elif plat.startswith(_ANDROID): + want_minor = "0" + want_arch = tail + else: + want_minor, _, want_arch = tail.partition("_") + + if want_arch != arch: + continue + + want_version = (int(want_major), int(want_minor)) if want_major else None + if not want_version or version <= want_version: + keys.append((priority, version)) + + return max(keys) if keys else None + +def _python_tag_priority(*, tag, implementation, py_version): + if tag.startswith(PY_TAG_GENERIC): + ver_str = tag[len(PY_TAG_GENERIC):] + elif tag.startswith(implementation): + ver_str = tag[len(implementation):] + else: + return None + + # Add a 0 at the end in case it is a single digit + ver_str = "{}.{}".format(ver_str[0], ver_str[1:] or "0") + + ver = version.parse(ver_str) + if not version.is_compatible(py_version, ver): + return None + + return ( + tag.startswith(implementation), + version.key(ver), + ) + +def _candidates_by_priority( + *, + whls, + implementation_name, + python_version, + whl_abi_tags, + whl_platform_tags, + logger): + """Calculate the priority of each wheel + + Returns: + A dictionary where keys are priority tuples which allows us to sort and pick the + last item. + """ + py_version = version.parse(python_version, strict = True) + implementation = python_tag(implementation_name) + + ret = {} + for whl in whls: + parsed = parse_whl_name(whl.filename) + priority = None + + # See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#compressed-tag-sets + for platform in parsed.platform_tag.split("."): + platform = _platform_tag_priority(tag = platform, values = whl_platform_tags) + if platform == None: + if logger: + logger.debug(lambda: "The platform_tag in '{}' does not match given list: {}".format( + whl.filename, + whl_platform_tags, + )) + continue + + for py in parsed.python_tag.split("."): + py = _python_tag_priority( + tag = py, + implementation = implementation, + py_version = py_version, + ) + if py == None: + if logger: + logger.debug(lambda: "The python_tag in '{}' does not match implementation or version: {} {}".format( + whl.filename, + implementation, + py_version.string, + )) + continue + + for abi in parsed.abi_tag.split("."): + abi = _value_priority( + tag = abi, + values = whl_abi_tags, + ) + if abi == None: + if logger: + logger.debug(lambda: "The abi_tag in '{}' does not match given list: {}".format( + whl.filename, + whl_abi_tags, + )) + continue + + # 1. Prefer platform wheels + # 2. Then prefer implementation/python version + # 3. Then prefer more specific ABI wheels + candidate = (platform, py, abi) + priority = priority or candidate + if candidate > priority: + priority = candidate + + if priority == None: + if logger: + logger.debug(lambda: "The whl '{}' is incompatible".format( + whl.filename, + )) + continue + + ret[priority] = whl + + return ret + +def select_whl( + *, + whls, + python_version, + whl_platform_tags, + whl_abi_tags, + implementation_name = "cpython", + limit = 1, + logger = None): + """Select a whl that is the most suitable for the given platform. + + Args: + whls: {type}`list[struct]` a list of candidates which have a `filename` + attribute containing the `whl` filename. + python_version: {type}`str` the target python version. + implementation_name: {type}`str` the `implementation_name` from the target_platform env. + whl_abi_tags: {type}`list[str]` The whl abi tags to select from. The preference is + for wheels that have ABI values appearing later in the `whl_abi_tags` list. + whl_platform_tags: {type}`list[str]` The whl platform tags to select from. + The platform tag may contain `*` and this means that if the platform tag is + versioned (e.g. `manylinux`), then we will select the highest available + platform version, e.g. if `manylinux_2_17` and `manylinux_2_5` wheels are both + compatible, we will select `manylinux_2_17`. Otherwise for versioned platform + tags we select the highest *compatible* version, e.g. if `manylinux_2_6` + support is requested, then we would select `manylinux_2_5` in the previous + example. This allows us to pass the same filtering parameters when selecting + all of the whl dependencies irrespective of what actual platform tags they + contain. + limit: {type}`int` number of wheels to return. Defaults to 1. + logger: {type}`struct` the logger instance. + + Returns: + {type}`list[struct] | struct | None`, a single struct from the `whls` input + argument or `None` if a match is not found. If the `limit` is greater than + one, then we will return a list. + """ + candidates = _candidates_by_priority( + whls = whls, + implementation_name = implementation_name, + python_version = python_version, + whl_abi_tags = whl_abi_tags, + whl_platform_tags = whl_platform_tags, + logger = logger, + ) + + if not candidates: + return None + + res = [i[1] for i in sorted(candidates.items())] + if logger: + logger.debug(lambda: "Sorted candidates:\n{}".format( + "\n".join([c.filename for c in res]), + )) + + return res[-1] if limit == 1 else res[-limit:] diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl index 6ea3f120c3..6c3dd5da83 100644 --- a/python/private/pypi/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -16,8 +16,6 @@ A starlark implementation of the wheel platform tag parsing to get the target platform. """ -load(":parse_whl_name.bzl", "parse_whl_name") - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -46,136 +44,6 @@ _OS_PREFIXES = { "win": "windows", } # buildifier: disable=unsorted-dict-items -def select_whls(*, whls, want_platforms = [], logger = None): - """Select a subset of wheels suitable for target platforms from a list. - - Args: - whls(list[struct]): A list of candidates which have a `filename` - attribute containing the `whl` filename. - want_platforms(str): The platforms in "{abi}_{os}_{cpu}" or "{os}_{cpu}" format. - logger: A logger for printing diagnostic messages. - - Returns: - A filtered list of items from the `whls` arg where `filename` matches - the selected criteria. If no match is found, an empty list is returned. - """ - if not whls: - return [] - - want_abis = { - "abi3": None, - "none": None, - } - - _want_platforms = {} - version_limit = None - - for p in want_platforms: - if not p.startswith("cp3"): - fail("expected all platforms to start with ABI, but got: {}".format(p)) - - abi, _, os_cpu = p.partition("_") - abi, _, _ = abi.partition(".") - _want_platforms[os_cpu] = None - - # TODO @aignas 2025-04-20: add a test - _want_platforms["{}_{}".format(abi, os_cpu)] = None - - version_limit_candidate = int(abi[3:]) - if not version_limit: - version_limit = version_limit_candidate - if version_limit and version_limit != version_limit_candidate: - fail("Only a single python version is supported for now") - - # For some legacy implementations the wheels may target the `cp3xm` ABI - _want_platforms["{}m_{}".format(abi, os_cpu)] = None - want_abis[abi] = None - want_abis[abi + "m"] = None - - # Also add freethreaded wheels if we find them since we started supporting them - _want_platforms["{}t_{}".format(abi, os_cpu)] = None - want_abis[abi + "t"] = None - - want_platforms = sorted(_want_platforms) - - candidates = {} - for whl in whls: - parsed = parse_whl_name(whl.filename) - - if logger: - logger.trace(lambda: "Deciding whether to use '{}'".format(whl.filename)) - - supported_implementations = {} - whl_version_min = 0 - for tag in parsed.python_tag.split("."): - supported_implementations[tag[:2]] = None - - if tag.startswith("cp3") or tag.startswith("py3"): - version = int(tag[len("..3"):] or 0) - else: - # In this case it should be eithor "cp2" or "py2" and we will default - # to `whl_version_min` = 0 - continue - - if whl_version_min == 0 or version < whl_version_min: - whl_version_min = version - - if not ("cp" in supported_implementations or "py" in supported_implementations): - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations)) - continue - - if want_abis and parsed.abi_tag not in want_abis: - # Filter out incompatible ABIs - if logger: - logger.trace(lambda: "Discarding the whl because the whl abi did not match") - continue - - if whl_version_min > version_limit: - if logger: - logger.trace(lambda: "Discarding the whl because the whl supported python version is too high") - continue - - compatible = False - if parsed.platform_tag == "any": - compatible = True - else: - for p in whl_target_platforms(parsed.platform_tag, abi_tag = parsed.abi_tag.strip("m") if parsed.abi_tag.startswith("cp") else None): - if p.target_platform in want_platforms: - compatible = True - break - - if not compatible: - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms)) - continue - - for implementation in supported_implementations: - candidates.setdefault( - ( - parsed.abi_tag, - parsed.platform_tag, - ), - {}, - ).setdefault( - ( - # prefer cp implementation - implementation == "cp", - # prefer higher versions - whl_version_min, - # prefer abi3 over none - parsed.abi_tag != "none", - # prefer cpx abi over abi3 - parsed.abi_tag != "abi3", - ), - [], - ).append(whl) - - return [ - candidates[key][sorted(v)[-1]][-1] - for key, v in candidates.items() - ] - def whl_target_platforms(platform_tag, abi_tag = ""): """Parse the wheel abi and platform tags and return (os, cpu) tuples. diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 52e0e29cb0..be4bcb1e68 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -16,7 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # buildifier: disable=bzl-visibility load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility @@ -58,20 +58,23 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo whl_mods = whl_mods, default = default or [ _default( - platform = "{}_{}".format(os, cpu), + platform = "{}_{}{}".format(os, cpu, freethreaded), os_name = os, arch_name = cpu, config_settings = [ "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else ["abi3", "cp{major}{minor}"], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "x86_64"), - ("linux", "aarch64"), - ("osx", "aarch64"), - ("windows", "aarch64"), - ] + for (os, cpu, freethreaded), whl_platform_tags in { + ("linux", "x86_64", ""): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "x86_64", "_freethreaded"): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64", ""): ["linux_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64", ""): ["macosx_*_arm64"], + ("windows", "aarch64", ""): ["win_arm64"], + }.items() ], ), is_root = is_root, @@ -92,22 +95,37 @@ def _parse_modules(env, enable_pipstar = 0, **kwargs): ), ) +def _build_config(env, enable_pipstar = 0, **kwargs): + return env.expect.that_struct( + build_config( + enable_pipstar = enable_pipstar, + **kwargs + ), + attrs = dict( + platforms = subjects.dict, + enable_pipstar = subjects.bool, + ), + ) + def _default( + *, arch_name = None, config_settings = None, os_name = None, platform = None, + whl_platform_tags = None, env = None, - whl_limit = None, - whl_platforms = None): + marker = None, + whl_abi_tags = None): return struct( arch_name = arch_name, os_name = os_name, platform = platform, + whl_platform_tags = whl_platform_tags or [], config_settings = config_settings, env = env or {}, - whl_platforms = whl_platforms, - whl_limit = whl_limit, + marker = marker or "", + whl_abi_tags = whl_abi_tags or [], ) def _parse( @@ -383,6 +401,96 @@ new-package==0.0.1 --hash=sha256:deadb00f2 _tests.append(_test_simple_multiple_python_versions) +def _test_simple_multiple_platforms_with_extras(env): + """TODO(hartikainen): Test that reproduces a multi-platform-with-extras issue.""" + # This test case is based on my issue where different requirement strings for the same package + # (`jax` vs `jax[cuda12]`) for multiple platforms caused a "duplicate library" error (for details, + # see https://github.com/bazel-contrib/rules_python/issues/2797#issuecomment-3143914644). + pypi = _parse_modules( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", + parse = [ + _parse( + hub_name = "pypi", + python_version = "3.12", + download_only = True, + requirements_by_platform = { + "requirements.linux_arm64.txt": "linux_aarch64", + "requirements.linux_x86_64.txt": "linux_x86_64", + "requirements.macos_arm64.txt": "osx_aarch64", + }, + experimental_index_url = "pypi.org", + ), + ], + ), + read = lambda x: { + "requirements.linux_arm64.txt": """\ +jax==0.7.0 \ + --hash=sha256:4dd8924f171ed73a4f1a6191e2f800ae1745069989b69fabc45593d6b6504003 \ + --hash=sha256:62833036cbaf4641d66ae94c61c0446890a91b2c0d153946583a0ebe04877a76 +""", + "requirements.linux_x86_64.txt": """\ +jax[cuda12]==0.7.0 \ + --hash=sha256:62833036cbaf4641d66ae94c61c0446890a91b2c0d153946583a0ebe04877a76 +""", + "requirements.macos_arm64.txt": """\ +jax==0.7.0 \ + --hash=sha256:4dd8924f171ed73a4f1a6191e2f800ae1745069989b69fabc45593d6b6504003 \ + --hash=sha256:62833036cbaf4641d66ae94c61c0446890a91b2c0d153946583a0ebe04877a76 +""", + }[x], + ), + available_interpreters = { + "python_3_12_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.12": "3.12.11"}, + simpleapi_download = lambda *_, **__: { + "jax": parse_simpleapi_html( + url = "https://example.com/jax", + content = """ +jax-0.7.0.tar.gz +jax-0.7.0-py3-none-any.whl +""", + ), + }, + ) + + pypi.exposed_packages().contains_exactly({"pypi": ["jax"]}) + # TODO(hartikainen): Check these expectations. + pypi.hub_whl_map().contains_exactly({"pypi": { + "jax": { + "pypi_312_jax_py3_none_any_62833036": [ + whl_config_setting( + # TODO(hartikainen): I think all these platforms use the same `.whl` + # and thus all three platforms should be included in the same + # `target_platforms` here? + target_platforms = ["cp312_linux_arm64", "cp312_linux_x86_64", "cp312_osx_aarch64"], + version = "3.12", + ), + ], + }, + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_312_jax_py3_none_any_62833036": { + "dep_template": "@pypi//{name}:{target}", + "download_only": True, + "experimental_target_platforms": ["linux_arm64", "linux_x86_64", "osx_aarch64"], + "filename": "jax-0.7.0-py3-none-any.whl", + "python_interpreter_target": "unit_test_interpreter_target", + # NOTE(hartikainen): Perhaps this is part of the problem? + # This should say `jax[cuda12]==0.7.0` for `linux_x86_64` platform and + # `jax==0.7.0` for `linux_arm64` and `osx_aarch64`. + "requirement": "jax[cuda12]==0.7.0", + "sha256": "62833036cbaf4641d66ae94c61c0446890a91b2c0d153946583a0ebe04877a76", + "urls": ["https://example.com/jax-0.7.0-py3-none-any.whl"], + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_simple_multiple_platforms_with_extras) + def _test_simple_with_markers(env): pypi = _parse_modules( env, @@ -433,10 +541,11 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ version = "3.15", ), ], - "pypi_315_torch_linux_x86_64": [ + "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": [ whl_config_setting( target_platforms = [ "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", ], version = "3.15", ), @@ -449,7 +558,7 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1 --hash=sha256:deadbeef", }, - "pypi_315_torch_linux_x86_64": { + "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -503,18 +612,21 @@ def _test_torch_experimental_index_url(env): "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "aarch64"), - ("linux", "x86_64"), - ("osx", "aarch64"), - ("windows", "x86_64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "x86_64"): ["win_amd64"], + ("windows", "aarch64"): ["win_arm64"], # this should be ignored + }.items() ], parse = [ _parse( hub_name = "pypi", python_version = "3.12", + download_only = True, experimental_index_url = "https://torch.index", requirements_lock = "universal.txt", ), @@ -571,25 +683,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", + target_platforms = ["cp312_linux_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( - filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + target_platforms = ["cp312_linux_aarch64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", + target_platforms = ["cp312_windows_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( - filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", + target_platforms = ["cp312_osx_aarch64"], version = "3.12", ), ], @@ -598,10 +710,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -610,10 +719,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -622,10 +728,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -634,10 +737,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -844,78 +944,84 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "pypi": { "direct_sdist_without_sha": { "pypi_315_any_name": [ - struct( - config_setting = None, - filename = "any-name.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "direct_without_sha": { "pypi_315_direct_without_sha_0_0_1_py3_none_any": [ - struct( - config_setting = None, - filename = "direct_without_sha-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "git_dep": { "pypi_315_git_dep": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "pip_fallback": { "pypi_315_pip_fallback": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "simple": { "pypi_315_simple_py3_none_any_deadb00f": [ - struct( - config_setting = None, - filename = "simple-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", - ), - ], - "pypi_315_simple_sdist_deadbeef": [ - struct( - config_setting = None, - filename = "simple-0.0.1.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_other_pkg": { "pypi_315_some_py3_none_any_deadb33f": [ - struct( - config_setting = None, - filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_pkg": { "pypi_315_some_pkg_py3_none_any_deadbaaf": [ - struct( - config_setting = None, - filename = "some_pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], @@ -978,21 +1084,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "sha256": "deadb00f", "urls": ["example2.org"], }, - "pypi_315_simple_sdist_deadbeef": { - "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], - "extra_pip_args": ["--extra-args-for-sdist-building"], - "filename": "simple-0.0.1.tar.gz", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1", - "sha256": "deadbeef", - "urls": ["example.org"], - }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ @@ -1081,12 +1172,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_linux_aarch64_linux_x86_64": [ + "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", ], config_setting = None, filename = None, @@ -1107,7 +1199,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_linux_aarch64_linux_x86_64": { + "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", @@ -1133,6 +1225,7 @@ def _test_pipstar_platforms(env): platform = "my{}_{}".format(os, cpu), os_name = os, arch_name = cpu, + marker = "python_version ~= \"3.13\"", config_settings = [ "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), @@ -1206,6 +1299,48 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' _tests.append(_test_pipstar_platforms) +def _test_build_pipstar_platform(env): + config = _build_config( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", + default = [ + _default( + platform = "myplat", + os_name = "linux", + arch_name = "x86_64", + ), + _default( + config_settings = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + ], + ), + ), + enable_pipstar = True, + ) + config.enable_pipstar().equals(True) + config.platforms().contains_exactly({ + "myplat": struct( + name = "myplat", + os_name = "linux", + arch_name = "x86_64", + config_settings = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + env = {"implementation_name": "cpython"}, + marker = "", + whl_abi_tags = ["none", "abi3", "cp{major}{minor}"], + whl_platform_tags = ["any"], + ), + }) + +_tests.append(_test_build_pipstar_platform) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 82fdd0a051..08bf3d3787 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -16,6 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -522,6 +523,26 @@ def _test_overlapping_shas_with_index_results(env): "requirements_linux": ["cp39_linux_x86_64"], "requirements_osx": ["cp39_osx_x86_64"], }, + platforms = { + "cp39_linux_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["any"], + ), + "cp39_osx_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "osx", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["macosx_*_x86_64"], + ), + }, get_index_urls = lambda _, __: { "foo": struct( sdists = { @@ -563,20 +584,10 @@ def _test_overlapping_shas_with_index_results(env): filename = "foo-0.0.1-py3-none-any.whl", requirement_line = "foo==0.0.3", sha256 = "deadbaaf", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + target_platforms = ["cp39_linux_x86_64"], url = "super2", yanked = False, ), - struct( - distribution = "foo", - extra_pip_args = [], - filename = "foo-0.0.1.tar.gz", - requirement_line = "foo==0.0.3", - sha256 = "5d15t", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], - url = "sdist", - yanked = False, - ), struct( distribution = "foo", extra_pip_args = [], diff --git a/tests/pypi/pep508/BUILD.bazel b/tests/pypi/pep508/BUILD.bazel index 7eab2e096a..36fce0fa89 100644 --- a/tests/pypi/pep508/BUILD.bazel +++ b/tests/pypi/pep508/BUILD.bazel @@ -1,4 +1,5 @@ load(":deps_tests.bzl", "deps_test_suite") +load(":env_tests.bzl", "env_test_suite") load(":evaluate_tests.bzl", "evaluate_test_suite") load(":requirement_tests.bzl", "requirement_test_suite") @@ -6,6 +7,10 @@ deps_test_suite( name = "deps_tests", ) +env_test_suite( + name = "env_tests", +) + evaluate_test_suite( name = "evaluate_tests", ) diff --git a/tests/pypi/pep508/env_tests.bzl b/tests/pypi/pep508/env_tests.bzl new file mode 100644 index 0000000000..cfd94a1b01 --- /dev/null +++ b/tests/pypi/pep508/env_tests.bzl @@ -0,0 +1,69 @@ +"""Tests to check for env construction.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_env_defaults(env): + got = pep508_env(os = "exotic", arch = "exotic", python_version = "3.1.1") + got.pop("_aliases") + env.expect.that_dict(got).contains_exactly({ + "implementation_name": "cpython", + "implementation_version": "3.1.1", + "os_name": "posix", + "platform_machine": "", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": "", + "platform_version": "0", + "python_full_version": "3.1.1", + "python_version": "3.1", + "sys_platform": "", + }) + +_tests.append(_test_env_defaults) + +def _test_env_freebsd(env): + got = pep508_env(os = "freebsd", arch = "arm64", python_version = "3.1.1") + got.pop("_aliases") + env.expect.that_dict(got).contains_exactly({ + "implementation_name": "cpython", + "implementation_version": "3.1.1", + "os_name": "posix", + "platform_machine": "aarch64", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": "FreeBSD", + "platform_version": "0", + "python_full_version": "3.1.1", + "python_version": "3.1", + "sys_platform": "freebsd", + }) + +_tests.append(_test_env_freebsd) + +def _test_env_macos(env): + got = pep508_env(os = "macos", arch = "arm64", python_version = "3.1.1") + got.pop("_aliases") + env.expect.that_dict(got).contains_exactly({ + "implementation_name": "cpython", + "implementation_version": "3.1.1", + "os_name": "posix", + "platform_machine": "aarch64", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": "Darwin", + "platform_version": "0", + "python_full_version": "3.1.1", + "python_version": "3.1", + "sys_platform": "darwin", + }) + +_tests.append(_test_env_macos) + +def env_test_suite(name): # buildifier: disable=function-docstring + test_suite( + name = name, + basic_tests = _tests, + ) diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index cc867f346c..7843f88e89 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -16,7 +16,6 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # buildifier: disable=bzl-visibility -load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility _tests = [] @@ -244,26 +243,37 @@ _tests.append(_evaluate_partial_only_extra) def _evaluate_with_aliases(env): # When - for target_platform, tests in { + for (os, cpu), tests in { # buildifier: @unsorted-dict-items - "osx_aarch64": { + ("osx", "aarch64"): { "platform_system == 'Darwin' and platform_machine == 'arm64'": True, "platform_system == 'Darwin' and platform_machine == 'aarch64'": True, "platform_system == 'Darwin' and platform_machine == 'amd64'": False, }, - "osx_x86_64": { + ("osx", "x86_64"): { "platform_system == 'Darwin' and platform_machine == 'amd64'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": True, }, - "osx_x86_32": { + ("osx", "x86_32"): { "platform_system == 'Darwin' and platform_machine == 'i386'": True, "platform_system == 'Darwin' and platform_machine == 'i686'": True, "platform_system == 'Darwin' and platform_machine == 'x86_32'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": False, }, + ("freebsd", "x86_32"): { + "platform_system == 'FreeBSD' and platform_machine == 'i386'": True, + "platform_system == 'FreeBSD' and platform_machine == 'i686'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_32'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_64'": False, + "platform_system == 'FreeBSD' and os_name == 'posix'": True, + }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): - _check_evaluate(env, input, want, pep508_env(platform_from_str(target_platform, ""))) + _check_evaluate(env, input, want, pep508_env( + os = os, + arch = cpu, + python_version = "3.2", + )) _tests.append(_evaluate_with_aliases) diff --git a/tests/pypi/python_tag/BUILD.bazel b/tests/pypi/python_tag/BUILD.bazel new file mode 100644 index 0000000000..d4b37cea16 --- /dev/null +++ b/tests/pypi/python_tag/BUILD.bazel @@ -0,0 +1,3 @@ +load(":python_tag_tests.bzl", "python_tag_test_suite") + +python_tag_test_suite(name = "python_tag_tests") diff --git a/tests/pypi/python_tag/python_tag_tests.bzl b/tests/pypi/python_tag/python_tag_tests.bzl new file mode 100644 index 0000000000..ca86575e5b --- /dev/null +++ b/tests/pypi/python_tag/python_tag_tests.bzl @@ -0,0 +1,34 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:python_tag.bzl", "python_tag") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_without_version(env): + for give, expect in { + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", + "python": "py", + "something_else": "something_else", + }.items(): + got = python_tag(give) + env.expect.that_str(got).equals(expect) + +_tests.append(_test_without_version) + +def _test_with_version(env): + got = python_tag("cpython", "3.1.15") + env.expect.that_str(got).equals("cp31") + +_tests.append(_test_with_version) + +def python_tag_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/select_whl/BUILD.bazel b/tests/pypi/select_whl/BUILD.bazel new file mode 100644 index 0000000000..0ad8cba0cd --- /dev/null +++ b/tests/pypi/select_whl/BUILD.bazel @@ -0,0 +1,3 @@ +load(":select_whl_tests.bzl", "select_whl_test_suite") + +select_whl_test_suite(name = "select_whl_tests") diff --git a/tests/pypi/select_whl/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl new file mode 100644 index 0000000000..28e17ba3b3 --- /dev/null +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -0,0 +1,463 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility +load("//python/private/pypi:select_whl.bzl", "select_whl") # buildifier: disable=bzl-visibility + +WHL_LIST = [ + "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", + "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-cp311-win32.whl", + "pkg-0.0.1-cp311-cp311-win_amd64.whl", + "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-win32.whl", + "pkg-0.0.1-cp37-cp37m-win_amd64.whl", + "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", + "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp39-cp39-win32.whl", + "pkg-0.0.1-cp39-cp39-win_amd64.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + # Extra examples that should be discarded + "pkg-0.0.1-py27-cp27mu-win_amd64.whl", + "pkg-0.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", +] + +def _match(env, got, *want_filenames): + if not want_filenames: + env.expect.that_collection(got).has_size(len(want_filenames)) + return + + got = [g for g in got if g] + got_filenames = [g.filename for g in got] + env.expect.that_collection(got_filenames).contains_exactly(want_filenames).in_order() + + if got: + # Check that we pass the original structs + env.expect.that_str(got[0].other).equals("dummy") + +def _select_whl(whls, debug = False, **kwargs): + return select_whl( + whls = [ + struct( + filename = f, + other = "dummy", + ) + for f in whls + ], + logger = repo_utils.logger(struct( + os = struct( + environ = { + REPO_DEBUG_ENV_VAR: "1", + REPO_VERBOSITY_ENV_VAR: "TRACE" if debug else "INFO", + }, + ), + ), "unit-test"), + **kwargs + ) + +_tests = [] + +def _test_not_select_py2(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ], + whl_platform_tags = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ) + +_tests.append(_test_not_select_py2) + +def _test_not_select_abi3(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-any.whl", + # the following should be ignored + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-p1.p2.p2.whl", + ], + whl_platform_tags = ["any", "p1"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + debug = True, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + ) + +_tests.append(_test_not_select_abi3) + +def _test_select_cp312(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + "pkg-0.0.1-cp314-none-any.whl", + ], + whl_platform_tags = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 5, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + ) + +_tests.append(_test_select_cp312) + +def _test_simplest(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + ] + + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.0", + ) + _match( + env, + [got], + "pkg-0.0.1-py3-abi3-any.whl", + ) + +_tests.append(_test_simplest) + +def _test_select_by_supported_py_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + ] + + for minor_version, match in { + 8: "pkg-0.0.1-py3-abi3-any.whl", + 11: "pkg-0.0.1-py311-abi3-any.whl", + }.items(): + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.{}".format(minor_version), + ) + _match(env, [got], match) + +_tests.append(_test_select_by_supported_py_version) + +def _test_select_by_supported_cp_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + "pkg-0.0.1-cp311-abi3-any.whl", + ] + + for minor_version, match in { + 11: "pkg-0.0.1-cp311-abi3-any.whl", + 8: "pkg-0.0.1-py3-abi3-any.whl", + }.items(): + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.{}".format(minor_version), + ) + _match(env, [got], match) + +_tests.append(_test_select_by_supported_cp_version) + +def _test_supported_cp_version_manylinux(env): + whls = [ + "pkg-0.0.1-py2.py3-none-manylinux_1_1_x86_64.whl", + "pkg-0.0.1-py3-none-manylinux_1_1_x86_64.whl", + "pkg-0.0.1-py311-none-manylinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-none-manylinux_1_1_x86_64.whl", + ] + + for minor_version, match in { + 8: "pkg-0.0.1-py3-none-manylinux_1_1_x86_64.whl", + 11: "pkg-0.0.1-cp311-none-manylinux_1_1_x86_64.whl", + }.items(): + got = _select_whl( + whls = whls, + whl_platform_tags = ["manylinux_1_1_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.{}".format(minor_version), + ) + _match(env, [got], match) + +_tests.append(_test_supported_cp_version_manylinux) + +def _test_ignore_unsupported(env): + whls = ["pkg-0.0.1-xx3-abi3-any.whl"] + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["none"], + python_version = "3.0", + ) + if got: + _match(env, [got], None) + +_tests.append(_test_ignore_unsupported) + +def _test_match_abi_and_not_py_version(env): + # Check we match the ABI and not the py version + whls = WHL_LIST + whl_platform_tags = [ + "musllinux_*_x86_64", + "manylinux_*_x86_64", + ] + got = _select_whl( + whls = whls, + whl_platform_tags = whl_platform_tags, + whl_abi_tags = ["abi3", "cp37m"], + python_version = "3.7", + ) + _match( + env, + [got], + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + ) + + got = _select_whl( + whls = whls, + whl_platform_tags = whl_platform_tags[::-1], + whl_abi_tags = ["abi3", "cp37m"], + python_version = "3.7", + ) + _match( + env, + [got], + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_match_abi_and_not_py_version) + +def _test_select_filename_with_many_tags(env): + # Check we can select a filename with many platform tags + got = _select_whl( + whls = WHL_LIST, + whl_platform_tags = [ + "any", + "musllinux_*_i686", + "manylinux_*_i686", + ], + whl_abi_tags = ["none", "abi3", "cp39"], + python_version = "3.9", + limit = 5, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + ) + +_tests.append(_test_select_filename_with_many_tags) + +def _test_freethreaded_wheels(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = WHL_LIST, + whl_platform_tags = [ + "any", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none", "abi3", "cp313", "cp313t"], + python_version = "3.13", + limit = 8, + ) + _match( + env, + got, + # The last item has the most priority + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_freethreaded_wheels) + +def _test_pytags_all_possible(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ], + whl_platform_tags = ["win_amd64"], + whl_abi_tags = ["none"], + python_version = "3.12", + ) + _match( + env, + [got], + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ) + +_tests.append(_test_pytags_all_possible) + +def _test_manylinx_musllinux_pref(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = [ + "manylinux_*_x86_64", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # there is only one wheel, just select that + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_manylinx_musllinux_pref) + +def _test_multiple_musllinux(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = ["musllinux_*_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the highest version that is matching + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + ) + +_tests.append(_test_multiple_musllinux) + +def _test_multiple_musllinux_exact_params(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = ["musllinux_1_2_x86_64", "musllinux_1_1_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the lowest version, because of the input to the function + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_multiple_musllinux_exact_params) + +def _test_android(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-android_4_x86_64.whl", + "pkg-0.0.1-py3-none-android_8_x86_64.whl", + ], + whl_platform_tags = ["android_5_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the highest version that is matching + "pkg-0.0.1-py3-none-android_4_x86_64.whl", + ) + +_tests.append(_test_android) + +def select_whl_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel index 6c35b08d32..fec25af033 100644 --- a/tests/pypi/whl_target_platforms/BUILD.bazel +++ b/tests/pypi/whl_target_platforms/BUILD.bazel @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":select_whl_tests.bzl", "select_whl_test_suite") load(":whl_target_platforms_tests.bzl", "whl_target_platforms_test_suite") -select_whl_test_suite(name = "select_whl_tests") - whl_target_platforms_test_suite(name = "whl_target_platforms_tests") diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl deleted file mode 100644 index 1674ac5ef2..0000000000 --- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"" - -load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility -load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility - -WHL_LIST = [ - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-win32.whl", - "pkg-0.0.1-cp311-cp311-win_amd64.whl", - "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-win32.whl", - "pkg-0.0.1-cp37-cp37m-win_amd64.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-cp39-win32.whl", - "pkg-0.0.1-cp39-cp39-win_amd64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py310-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", -] - -def _match(env, got, *want_filenames): - if not want_filenames: - env.expect.that_collection(got).has_size(len(want_filenames)) - return - - got_filenames = [g.filename for g in got] - env.expect.that_collection(got_filenames).contains_exactly(want_filenames) - - if got: - # Check that we pass the original structs - env.expect.that_str(got[0].other).equals("dummy") - -def _select_whls(whls, debug = False, **kwargs): - return select_whls( - whls = [ - struct( - filename = f, - other = "dummy", - ) - for f in whls - ], - logger = repo_utils.logger(struct( - os = struct( - environ = { - REPO_DEBUG_ENV_VAR: "1", - REPO_VERBOSITY_ENV_VAR: "TRACE" if debug else "INFO", - }, - ), - ), "unit-test"), - **kwargs - ) - -_tests = [] - -def _test_simplest(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ], - want_platforms = ["cp30_ignored"], - ) - _match( - env, - got, - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_simplest) - -def _test_select_by_supported_py_version(env): - for minor_version, match in { - 8: "pkg-0.0.1-py3-abi3-any.whl", - 11: "pkg-0.0.1-py311-abi3-any.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_select_by_supported_py_version) - -def _test_select_by_supported_cp_version(env): - for minor_version, match in { - 11: "pkg-0.0.1-cp311-abi3-any.whl", - 8: "pkg-0.0.1-py3-abi3-any.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - "pkg-0.0.1-cp311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_select_by_supported_cp_version) - -def _test_supported_cp_version_manylinux(env): - for minor_version, match in { - 8: "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - 11: "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py311-none-manylinux_x86_64.whl", - "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - ], - want_platforms = ["cp3{}_linux_x86_64".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_supported_cp_version_manylinux) - -def _test_ignore_unsupported(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-xx3-abi3-any.whl", - ], - want_platforms = ["cp30_ignored"], - ) - _match(env, got) - -_tests.append(_test_ignore_unsupported) - -def _test_match_abi_and_not_py_version(env): - # Check we match the ABI and not the py version - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp37_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_match_abi_and_not_py_version) - -def _test_select_filename_with_many_tags(env): - # Check we can select a filename with many platform tags - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_32"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_select_filename_with_many_tags) - -def _test_osx_prefer_arch_specific(env): - # Check that we prefer the specific wheel - got = _select_whls( - whls = WHL_LIST, - want_platforms = ["cp311_osx_x86_64", "cp311_osx_x86_32"], - ) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp311_osx_aarch64"]) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_prefer_arch_specific) - -def _test_osx_fallback_to_universal2(env): - # Check that we can use the universal2 if the arm wheel is not available - got = _select_whls( - whls = [w for w in WHL_LIST if "arm64" not in w], - want_platforms = ["cp311_osx_aarch64"], - ) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_fallback_to_universal2) - -def _test_prefer_manylinux_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_prefer_manylinux_wheels) - -def _test_freethreaded_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_freethreaded_wheels) - -def _test_micro_version_freethreaded(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313.3_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_micro_version_freethreaded) - -def select_whl_test_suite(name): - """Create the test suite. - - Args: - name: the name of the test suite - """ - test_suite(name = name, basic_tests = _tests)