Merge branch 'main' into chore/pip-parse-rm-deprecated

aignas · aignas · commit d7e14316fb07 · 2024-11-19T10:31:41.000+09:00
diff --git a/.bazelrc b/.bazelrc
@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -73,10 +73,16 @@ Unreleased changes template.
 
 Other changes:
 * (python_repository) Start honoring the `strip_prefix` field for `zstd` archives.
+* (pypi) {bzl:obj}`pip_parse.extra_hub_aliases` now works in WORKSPACE files.
 
 {#v0-0-0-fixed}
 ### Fixed
 * (toolchains) stop depending on `uname` to get the value of the host platform.
+* (pypi): Correctly handle multiple versions of the same package in the requirements
+  files which is useful when including different PyTorch builds (e.g. <pytorch+cpu> vs <pytorch+cu118> ) for different target platforms.
+  Fixes ([2337](https://github.com/bazelbuild/rules_python/issues/2337)).
+* (uv): Correct the sha256sum for the `uv` binary for aarch64-apple-darwin.
+  Fixes ([2411](https://github.com/bazelbuild/rules_python/issues/2411)).
 
 {#v0-0-0-added}
 ### Added
@@ -87,10 +93,17 @@ Other changes:
   `RULES_PYTHON_REPO_TOOLCHAIN_{VERSION}_{OS}_{ARCH}` env variable setting. For
   example, this allows one to use `freethreaded` python interpreter in the
   `repository_rule` to build a wheel from `sdist`.
+* (toolchain) The python interpreters targeting `muslc` libc have been added
+  for the latest toolchain versions for each minor Python version. You can control
+  the toolchain selection by using the
+  {bzl:obj}`//python/config_settings:py_linux_libc` build flag.
 
 {#v0-0-0-removed}
 ### Removed
-* Nothing removed.
+* (pypi): Remove `pypi_install_dependencies` macro that has been included in
+  {bzl:obj}`py_repositories` for a long time.
+* (bzlmod): Remove `DEFAULT_PYTHON_VERSION` from `interpreters.bzl` file. If
+  you need the version, please use it from the `versions.bzl` file instead.
 
 {#v0-40-0}
 ## [0.40.0] - 2024-11-17
diff --git a/docs/toolchains.md b/docs/toolchains.md
@@ -451,3 +451,10 @@ The toolchain() calls should be in a separate BUILD file from everything else.
 This avoids Bazel having to perform unnecessary work when it discovers the list
 of available toolchains.
 :::
+
+## Toolchain selection flags
+
+Currently the following flags are used to influence toolchain selection:
+* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant.
+* {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting
+  the freethreaded experimental Python builds available from `3.13.0` onwards.
diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
@@ -6,14 +6,14 @@ load(
     "BootstrapImplFlag",
     "ExecToolsToolchainFlag",
     "FreeThreadedFlag",
+    "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
 )
 load(
     "//python/private/pypi:flags.bzl",
     "UniversalWhlFlag",
     "UseWhlFlag",
-    "WhlLibcFlag",
     "define_pypi_internal_flags",
 )
 load(":config_settings.bzl", "construct_config_settings")
@@ -87,8 +87,8 @@ string_flag(
 # This is used for pip and hermetic toolchain resolution.
 string_flag(
     name = "py_linux_libc",
-    build_setting_default = WhlLibcFlag.GLIBC,
-    values = sorted(WhlLibcFlag.__members__.values()),
+    build_setting_default = LibcFlag.GLIBC,
+    values = LibcFlag.flag_values(),
     # NOTE: Only public because it is used in pip hub and toolchain repos.
     visibility = ["//visibility:public"],
 )
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
@@ -35,14 +35,6 @@ bzl_library(
     deps = ["//python/private/pypi:pip_compile_bzl"],
 )
 
-bzl_library(
-    name = "repositories_bzl",
-    srcs = ["repositories.bzl"],
-    deps = [
-        "//python/private/pypi:deps_bzl",
-    ],
-)
-
 filegroup(
     name = "distribution",
     srcs = glob(["**"]),
diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl
diff --git a/python/private/flags.bzl b/python/private/flags.bzl
@@ -132,3 +132,14 @@ FreeThreadedFlag = enum(
     # Do not use freethreaded python toolchain and wheels.
     NO = "no",
 )
+
+# Determines which libc flavor is preferred when selecting the toolchain and
+# linux whl distributions.
+#
+# buildifier: disable=name-conventions
+LibcFlag = FlagEnum(
+    # Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64)
+    GLIBC = "glibc",
+    # Prefer musl wheels (e.g. musllinux_2_17_x86_64)
+    MUSL = "musl",
+)
diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl
@@ -141,6 +141,16 @@ Special values: `host` (for generating deps for the host platform only) and
 NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
 """,
     ),
+    "extra_hub_aliases": attr.string_list_dict(
+        doc = """\
+Extra aliases to make for specific wheels in the hub repo. This is useful when
+paired with the {attr}`whl_modifications`.
+
+:::{versionadded} 0.38.0
+:::
+""",
+        mandatory = False,
+    ),
     "extra_pip_args": attr.string_list(
         doc = """Extra arguments to pass on to pip. Must not contain spaces.
 
diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl
@@ -39,7 +39,8 @@ Note, that here the specialization of musl vs manylinux wheels is the same in
 order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
 """
 
-load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag")
+load("//python/private:flags.bzl", "LibcFlag")
+load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag")
 
 FLAGS = struct(
     **{
@@ -251,14 +252,14 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):
 
     elif os == "linux":
         for os_prefix, linux_libc in {
-            os: WhlLibcFlag.GLIBC,
-            "many" + os: WhlLibcFlag.GLIBC,
-            "musl" + os: WhlLibcFlag.MUSL,
+            os: LibcFlag.GLIBC,
+            "many" + os: LibcFlag.GLIBC,
+            "musl" + os: LibcFlag.MUSL,
         }.items():
-            if linux_libc == WhlLibcFlag.GLIBC:
+            if linux_libc == LibcFlag.GLIBC:
                 libc_versions = glibc_versions
                 libc_flag = FLAGS.pip_whl_glibc_version
-            elif linux_libc == WhlLibcFlag.MUSL:
+            elif linux_libc == LibcFlag.MUSL:
                 libc_versions = muslc_versions
                 libc_flag = FLAGS.pip_whl_muslc_version
             else:
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
@@ -710,16 +710,6 @@ The indexes must support Simple API as described here:
 https://packaging.python.org/en/latest/specifications/simple-repository-api/
 """,
         ),
-        "extra_hub_aliases": attr.string_list_dict(
-            doc = """\
-Extra aliases to make for specific wheels in the hub repo. This is useful when
-paired with the {attr}`whl_modifications`.
-
-:::{versionadded} 0.38.0
-:::
-""",
-            mandatory = False,
-        ),
         "hub_name": attr.string(
             mandatory = True,
             doc = """
diff --git a/python/private/pypi/flags.bzl b/python/private/pypi/flags.bzl
@@ -44,16 +44,6 @@ UniversalWhlFlag = enum(
     UNIVERSAL = "universal",
 )
 
-# Determines which libc flavor is preferred when selecting the linux whl distributions.
-#
-# buildifier: disable=name-conventions
-WhlLibcFlag = enum(
-    # Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64)
-    GLIBC = "glibc",
-    # Prefer musl wheels (e.g. musllinux_2_17_x86_64)
-    MUSL = "musl",
-)
-
 INTERNAL_FLAGS = [
     "dist",
     "whl_plat",
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
@@ -32,6 +32,20 @@ load(":index_sources.bzl", "index_sources")
 load(":parse_requirements_txt.bzl", "parse_requirements_txt")
 load(":whl_target_platforms.bzl", "select_whls")
 
+def _extract_version(entry):
+    """Extract the version part from the requirement string.
+
+
+    Args:
+        entry: {type}`str` The requirement string.
+    """
+    version_start = entry.find("==")
+    if version_start != -1:
+        # Extract everything after '==' until the next space or end of the string
+        version, _, _ = entry[version_start + 2:].partition(" ")
+        return version
+    return None
+
 def parse_requirements(
         ctx,
         *,
@@ -92,7 +106,7 @@ def parse_requirements(
         # are returned as just the base package name. e.g., `foo[bar]` results
         # in an entry like `("foo", "foo[bar] == 1.0 ...")`.
         requirements_dict = {
-            normalize_name(entry[0]): entry
+            (normalize_name(entry[0]), _extract_version(entry[1])): entry
             for entry in sorted(
                 parse_result.requirements,
                 # Get the longest match and fallback to original WORKSPACE sorting,
diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl
@@ -177,6 +177,7 @@ def _pip_repository_impl(rctx):
             pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)]
             for pkg in bzl_packages or []
         },
+        extra_hub_aliases = rctx.attr.extra_hub_aliases,
     )
     for path, contents in aliases.items():
         rctx.file(path, contents)
diff --git a/python/private/pythons_hub.bzl b/python/private/pythons_hub.bzl
@@ -86,7 +86,6 @@ _interpreters_bzl_template = """
 INTERPRETER_LABELS = {{
 {interpreter_labels}
 }}
-DEFAULT_PYTHON_VERSION = "{default_python_version}"
 """
 
 _line_for_hub_template = """\
@@ -125,8 +124,6 @@ def _hub_repo_impl(rctx):
     rctx.file(
         "interpreters.bzl",
         _interpreters_bzl_template.format(
-            # TODO @aignas 2024-09-28: before 1.0 remove the value from here
-            default_python_version = rctx.attr.default_python_version,
             interpreter_labels = interpreter_labels,
         ),
         executable = False,
diff --git a/python/uv/private/versions.bzl b/python/uv/private/versions.bzl
@@ -70,7 +70,7 @@ UV_PLATFORMS = {
 UV_TOOL_VERSIONS = {
     "0.4.25": {
         "aarch64-apple-darwin": struct(
-            sha256 = "35786030f926e3d34d186edc0ea3989698e57755852af9ae4b39da5109abcbfa",
+            sha256 = "bb2ff4348114ef220ca52e44d5086640c4a1a18f797a5f1ab6f8559fc37b1230",
         ),
         "aarch64-unknown-linux-gnu": struct(
             sha256 = "4485852eb8013530c4275cd222c0056ce123f92742321f012610f1b241463f39",
diff --git a/python/versions.bzl b/python/versions.bzl
diff --git a/tests/integration/pip_parse/WORKSPACE b/tests/integration/pip_parse/WORKSPACE
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
