Merge branch 'main' into pip_compile_test_with_proxy

Author: herewasmike

CHANGELOG.md

@@ -96,6 +96,8 @@ END_UNRELEASED_TEMPLATE
 * (pypi) When running under `bazel test`, be sure that temporary `requirements` file
   remains writable.
 * (py_test, py_binary) Allow external files to be used for main
+* (pypi) Correctly aggregate the sources when the hashes specified in the lockfile differ
+  by platform even though the same version is used. Fixes [#2648](https://github.com/bazel-contrib/rules_python/issues/2648).
 * (pypi) `compile_pip_requirements` test rule works behind the proxy
 
 {#v0-0-0-added}
@@ -110,6 +112,8 @@ END_UNRELEASED_TEMPLATE
   Set the `RULES_PYTHON_ENABLE_PIPSTAR=1` environment variable to enable it.
 * (utils) Add a way to run a REPL for any `rules_python` target that returns
   a `PyInfo` provider.
+* (uv) Handle `.netrc` and `auth_patterns` auth when downloading `uv`. Work towards
+  [#1975](https://github.com/bazel-contrib/rules_python/issues/1975).
 * (toolchains) Arbitrary python-build-standalone runtimes can be registered
   and activated with custom flags. See the [Registering custom runtimes]
   docs and {obj}`single_version_platform_override()` API docs for more

python/private/pypi/parse_requirements.bzl

@@ -223,7 +223,7 @@ def _package_srcs(
         env_marker_target_platforms,
         extract_url_srcs):
     """A function to return sources for a particular package."""
-    srcs = []
+    srcs = {}
     for r in sorted(reqs.values(), key = lambda r: r.requirement_line):
         whls, sdist = _add_dists(
             requirement = r,
@@ -249,21 +249,31 @@ def _package_srcs(
             )]
             req_line = r.srcs.requirement_line
 
+        extra_pip_args = tuple(r.extra_pip_args)
         for dist in all_dists:
-            srcs.append(
+            key = (
+                dist.filename,
+                req_line,
+                extra_pip_args,
+            )
+            entry = srcs.setdefault(
+                key,
                 struct(
                     distribution = name,
                     extra_pip_args = r.extra_pip_args,
                     requirement_line = req_line,
-                    target_platforms = target_platforms,
+                    target_platforms = [],
                     filename = dist.filename,
                     sha256 = dist.sha256,
                     url = dist.url,
                     yanked = dist.yanked,
                 ),
             )
+            for p in target_platforms:
+                if p not in entry.target_platforms:
+                    entry.target_platforms.append(p)
 
-    return srcs
+    return srcs.values()
 
 def select_requirement(requirements, *, platform):
     """A simple function to get a requirement for a particular platform.

python/uv/private/BUILD.bazel

@@ -62,13 +62,15 @@ bzl_library(
         ":toolchain_types_bzl",
         ":uv_repository_bzl",
         ":uv_toolchains_repo_bzl",
+        "//python/private:auth_bzl",
     ],
 )
 
 bzl_library(
     name = "uv_repository_bzl",
     srcs = ["uv_repository.bzl"],
     visibility = ["//python/uv:__subpackages__"],
+    deps = ["//python/private:auth_bzl"],
 )
 
 bzl_library(

python/uv/private/uv.bzl

@@ -18,6 +18,7 @@ EXPERIMENTAL: This is experimental and may be removed without notice
 A module extension for working with uv.
 """
 
+load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
 load(":toolchain_types.bzl", "UV_TOOLCHAIN_TYPE")
 load(":uv_repository.bzl", "uv_repository")
 load(":uv_toolchains_repo.bzl", "uv_toolchains_repo")
@@ -77,7 +78,7 @@ The version of uv to configure the sources for. If this is not specified it will
 last version used in the module or the default version set by `rules_python`.
 """,
     ),
-}
+} | AUTH_ATTRS
 
 default = tag_class(
     doc = """\
@@ -133,7 +134,7 @@ for a particular version.
     },
 )
 
-def _configure(config, *, platform, compatible_with, target_settings, urls = [], sha256 = "", override = False, **values):
+def _configure(config, *, platform, compatible_with, target_settings, auth_patterns, urls = [], sha256 = "", override = False, **values):
     """Set the value in the config if the value is provided"""
     for key, value in values.items():
         if not value:
@@ -144,6 +145,7 @@ def _configure(config, *, platform, compatible_with, target_settings, urls = [],
 
         config[key] = value
 
+    config.setdefault("auth_patterns", {}).update(auth_patterns)
     config.setdefault("platforms", {})
     if not platform:
         if compatible_with or target_settings or urls:
@@ -173,7 +175,8 @@ def process_modules(
         hub_name = "uv",
         uv_repository = uv_repository,
         toolchain_type = str(UV_TOOLCHAIN_TYPE),
-        hub_repo = uv_toolchains_repo):
+        hub_repo = uv_toolchains_repo,
+        get_auth = get_auth):
     """Parse the modules to get the config for 'uv' toolchains.
 
     Args:
@@ -182,6 +185,7 @@ def process_modules(
         uv_repository: the rule to create a uv_repository override.
         toolchain_type: the toolchain type to use here.
         hub_repo: the hub repo factory function to use.
+        get_auth: the auth function to use.
 
     Returns:
         the result of the hub_repo. Mainly used for tests.
@@ -216,6 +220,8 @@ def process_modules(
                 compatible_with = tag.compatible_with,
                 target_settings = tag.target_settings,
                 override = mod.is_root,
+                netrc = tag.netrc,
+                auth_patterns = tag.auth_patterns,
             )
 
     for key in [
@@ -271,6 +277,8 @@ def process_modules(
                 sha256 = tag.sha256,
                 urls = tag.urls,
                 override = mod.is_root,
+                netrc = tag.netrc,
+                auth_patterns = tag.auth_patterns,
             )
 
     if not versions:
@@ -301,6 +309,11 @@ def process_modules(
             for platform, src in config.get("urls", {}).items()
             if src.urls
         }
+        auth = {
+            "auth_patterns": config.get("auth_patterns"),
+            "netrc": config.get("netrc"),
+        }
+        auth = {k: v for k, v in auth.items() if v}
 
         # Or fallback to fetching them from GH manifest file
         # Example file: https://github.com/astral-sh/uv/releases/download/0.6.3/dist-manifest.json
@@ -313,6 +326,8 @@ def process_modules(
                 ),
                 manifest_filename = config["manifest_filename"],
                 platforms = sorted(platforms),
+                get_auth = get_auth,
+                **auth
             )
 
         for platform_name, platform in platforms.items():
@@ -327,6 +342,7 @@ def process_modules(
                 platform = platform_name,
                 urls = urls[platform_name].urls,
                 sha256 = urls[platform_name].sha256,
+                **auth
             )
 
             toolchain_names.append(toolchain_name)
@@ -363,7 +379,7 @@ def _overlap(first_collection, second_collection):
 
     return False
 
-def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename, platforms):
+def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename, platforms, get_auth = get_auth, **auth_attrs):
     """Download the results about remote tool sources.
 
     This relies on the tools using the cargo packaging to infer the actual
@@ -431,10 +447,13 @@ def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename
                     "aarch64-apple-darwin"
                 ]
     """
+    auth_attr = struct(**auth_attrs)
     dist_manifest = module_ctx.path(manifest_filename)
+    urls = [base_url + "/" + manifest_filename]
     result = module_ctx.download(
-        base_url + "/" + manifest_filename,
+        url = urls,
         output = dist_manifest,
+        auth = get_auth(module_ctx, urls, ctx_attr = auth_attr),
     )
     if not result.success:
         fail(result)
@@ -454,11 +473,13 @@ def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename
 
         checksum_fname = checksum["name"]
         checksum_path = module_ctx.path(checksum_fname)
+        urls = ["{}/{}".format(base_url, checksum_fname)]
         downloads[checksum_path] = struct(
             download = module_ctx.download(
-                "{}/{}".format(base_url, checksum_fname),
+                url = urls,
                 output = checksum_path,
                 block = False,
+                auth = get_auth(module_ctx, urls, ctx_attr = auth_attr),
             ),
             archive_fname = fname,
             platforms = checksum["target_triples"],
@@ -473,7 +494,7 @@ def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename
 
         sha256, _, checksummed_fname = module_ctx.read(checksum_path).partition(" ")
         checksummed_fname = checksummed_fname.strip(" *\n")
-        if archive_fname != checksummed_fname:
+        if checksummed_fname and archive_fname != checksummed_fname:
             fail("The checksum is for a different file, expected '{}' but got '{}'".format(
                 archive_fname,
                 checksummed_fname,

python/uv/private/uv_repository.bzl

@@ -18,6 +18,8 @@ EXPERIMENTAL: This is experimental and may be removed without notice
 Create repositories for uv toolchain dependencies
 """
 
+load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
+
 UV_BUILD_TMPL = """\
 # Generated by repositories.bzl
 load("@rules_python//python/uv:uv_toolchain.bzl", "uv_toolchain")
@@ -43,6 +45,7 @@ def _uv_repo_impl(repository_ctx):
         url = repository_ctx.attr.urls,
         sha256 = repository_ctx.attr.sha256,
         stripPrefix = strip_prefix,
+        auth = get_auth(repository_ctx, repository_ctx.attr.urls),
     )
 
     binary = "uv.exe" if is_windows else "uv"
@@ -70,5 +73,5 @@ uv_repository = repository_rule(
         "sha256": attr.string(mandatory = False),
         "urls": attr.string_list(mandatory = True),
         "version": attr.string(mandatory = True),
-    },
+    } | AUTH_ATTRS,
 )

tests/pypi/parse_requirements/parse_requirements_tests.bzl

@@ -38,7 +38,7 @@ foo[extra]==0.0.1 \
 foo @ git+https://github.com/org/foo.git@deadbeef
 """,
         "requirements_linux": """\
-foo==0.0.3 --hash=sha256:deadbaaf
+foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:5d15t
 """,
         # download_only = True
         "requirements_linux_download_only": """\
@@ -67,7 +67,7 @@ foo==0.0.4 @ https://example.org/foo-0.0.4.whl
 foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef
 """,
         "requirements_osx": """\
-foo==0.0.3 --hash=sha256:deadbaaf
+foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:deadb11f --hash=sha256:5d15t
 """,
         "requirements_osx_download_only": """\
 --platform=macosx_10_9_arm64
@@ -251,7 +251,7 @@ def _test_multi_os(env):
                 struct(
                     distribution = "foo",
                     extra_pip_args = [],
-                    requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf",
+                    requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:5d15t",
                     target_platforms = ["linux_x86_64"],
                     url = "",
                     filename = "",
@@ -515,6 +515,84 @@ def _test_git_sources(env):
 
 _tests.append(_test_git_sources)
 
+def _test_overlapping_shas_with_index_results(env):
+    got = parse_requirements(
+        ctx = _mock_ctx(),
+        requirements_by_platform = {
+            "requirements_linux": ["cp39_linux_x86_64"],
+            "requirements_osx": ["cp39_osx_x86_64"],
+        },
+        get_index_urls = lambda _, __: {
+            "foo": struct(
+                sdists = {
+                    "5d15t": struct(
+                        url = "sdist",
+                        sha256 = "5d15t",
+                        filename = "foo-0.0.1.tar.gz",
+                        yanked = False,
+                    ),
+                },
+                whls = {
+                    "deadb11f": struct(
+                        url = "super2",
+                        sha256 = "deadb11f",
+                        filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl",
+                        yanked = False,
+                    ),
+                    "deadbaaf": struct(
+                        url = "super2",
+                        sha256 = "deadbaaf",
+                        filename = "foo-0.0.1-py3-none-any.whl",
+                        yanked = False,
+                    ),
+                },
+            ),
+        },
+    )
+
+    env.expect.that_collection(got).contains_exactly([
+        struct(
+            name = "foo",
+            is_exposed = True,
+            # TODO @aignas 2025-05-25: how do we rename this?
+            is_multiple_versions = True,
+            srcs = [
+                struct(
+                    distribution = "foo",
+                    extra_pip_args = [],
+                    filename = "foo-0.0.1-py3-none-any.whl",
+                    requirement_line = "foo==0.0.3",
+                    sha256 = "deadbaaf",
+                    target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"],
+                    url = "super2",
+                    yanked = False,
+                ),
+                struct(
+                    distribution = "foo",
+                    extra_pip_args = [],
+                    filename = "foo-0.0.1.tar.gz",
+                    requirement_line = "foo==0.0.3",
+                    sha256 = "5d15t",
+                    target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"],
+                    url = "sdist",
+                    yanked = False,
+                ),
+                struct(
+                    distribution = "foo",
+                    extra_pip_args = [],
+                    filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl",
+                    requirement_line = "foo==0.0.3",
+                    sha256 = "deadb11f",
+                    target_platforms = ["cp39_osx_x86_64"],
+                    url = "super2",
+                    yanked = False,
+                ),
+            ],
+        ),
+    ])
+
+_tests.append(_test_overlapping_shas_with_index_results)
+
 def parse_requirements_test_suite(name):
     """Create the test suite.