Merge branch 'main' into fix/only-call-simple-api-for-distros-with-hashes

Author: aignas

python/private/pypi/extension.bzl

@@ -105,7 +105,6 @@ def _create_whl_repos(
 
     # containers to aggregate outputs from this function
     whl_map = {}
-    exposed_packages = {}
     extra_aliases = {
         whl_name: {alias: True for alias in aliases}
         for whl_name, aliases in pip_attr.extra_hub_aliases.items()
@@ -219,8 +218,6 @@ def _create_whl_repos(
     )
 
     for whl_name, requirements in requirements_by_platform.items():
-        whl_name = normalize_name(whl_name)
-
         group_name = whl_group_mapping.get(whl_name)
         group_deps = requirement_cycles.get(group_name, [])
 
@@ -261,68 +258,55 @@ def _create_whl_repos(
             if v != default
         })
 
-        is_exposed = False
-        if get_index_urls:
-            # TODO @aignas 2024-05-26: move to a separate function
-            found_something = False
-            for requirement in requirements:
-                is_exposed = is_exposed or requirement.is_exposed
-                dists = requirement.whls
-                if not pip_attr.download_only and requirement.sdist:
-                    dists = dists + [requirement.sdist]
-
-                for distribution in dists:
-                    found_something = True
-                    is_reproducible = False
-
-                    args = dict(whl_library_args)
-                    if pip_attr.netrc:
-                        args["netrc"] = pip_attr.netrc
-                    if pip_attr.auth_patterns:
-                        args["auth_patterns"] = pip_attr.auth_patterns
-
-                    if not distribution.filename.endswith(".whl"):
-                        # pip is not used to download wheels and the python
-                        # `whl_library` helpers are only extracting things, however
-                        # for sdists, they will be built by `pip`, so we still
-                        # need to pass the extra args there.
-                        args["extra_pip_args"] = requirement.extra_pip_args
-
-                    # This is no-op because pip is not used to download the wheel.
-                    args.pop("download_only", None)
-
-                    repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
-                    args["requirement"] = requirement.srcs.requirement
-                    args["urls"] = [distribution.url]
-                    args["sha256"] = distribution.sha256
-                    args["filename"] = distribution.filename
-                    args["experimental_target_platforms"] = requirement.target_platforms
-
-                    # Pure python wheels or sdists may need to have a platform here
-                    target_platforms = None
-                    if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
-                        if len(requirements) > 1:
-                            target_platforms = requirement.target_platforms
-
-                    whl_libraries[repo_name] = args
-
-                    whl_map.setdefault(whl_name, {})[whl_config_setting(
-                        version = major_minor,
-                        filename = distribution.filename,
-                        target_platforms = target_platforms,
-                    )] = repo_name
-
-            if found_something:
-                if is_exposed:
-                    exposed_packages[whl_name] = None
-                continue
-
-        is_exposed = False
+        # TODO @aignas 2024-05-26: move to a separate function
         for requirement in requirements:
-            is_exposed = is_exposed or requirement.is_exposed
-            if get_index_urls:
-                logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.srcs.requirement_line))
+            dists = requirement.whls
+            if not pip_attr.download_only and requirement.sdist:
+                dists = dists + [requirement.sdist]
+
+            for distribution in dists:
+                args = dict(whl_library_args)
+                if pip_attr.netrc:
+                    args["netrc"] = pip_attr.netrc
+                if pip_attr.auth_patterns:
+                    args["auth_patterns"] = pip_attr.auth_patterns
+
+                if not distribution.filename.endswith(".whl"):
+                    # pip is not used to download wheels and the python
+                    # `whl_library` helpers are only extracting things, however
+                    # for sdists, they will be built by `pip`, so we still
+                    # need to pass the extra args there.
+                    args["extra_pip_args"] = requirement.extra_pip_args
+
+                # This is no-op because pip is not used to download the wheel.
+                args.pop("download_only", None)
+
+                repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
+                args["requirement"] = requirement.srcs.requirement
+                args["urls"] = [distribution.url]
+                args["sha256"] = distribution.sha256
+                args["filename"] = distribution.filename
+                args["experimental_target_platforms"] = requirement.target_platforms
+
+                # Pure python wheels or sdists may need to have a platform here
+                target_platforms = None
+                if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
+                    if len(requirements) > 1:
+                        target_platforms = requirement.target_platforms
+
+                whl_libraries[repo_name] = args
+
+                whl_map.setdefault(whl_name, {})[whl_config_setting(
+                    version = major_minor,
+                    filename = distribution.filename,
+                    target_platforms = target_platforms,
+                )] = repo_name
+
+            if dists:
+                is_reproducible = False
+                continue
 
+            # Fallback to a pip-installed wheel
             args = dict(whl_library_args)  # make a copy
             args["requirement"] = requirement.srcs.requirement_line
             if requirement.extra_pip_args:
@@ -343,13 +327,14 @@ def _create_whl_repos(
                 target_platforms = target_platforms or None,
             )] = repo_name
 
-        if is_exposed:
-            exposed_packages[whl_name] = None
-
     return struct(
         is_reproducible = is_reproducible,
         whl_map = whl_map,
-        exposed_packages = exposed_packages,
+        exposed_packages = {
+            whl_name: None
+            for whl_name, requirements in requirements_by_platform.items()
+            if len([r for r in requirements if r.is_exposed]) > 0
+        },
         extra_aliases = extra_aliases,
         whl_libraries = whl_libraries,
     )

python/private/pypi/parse_requirements.bzl

@@ -204,6 +204,9 @@ def parse_requirements(
                 sorted(requirements),
             ))
 
+        # Return normalized names
+        ret_requirements = ret.setdefault(normalize_name(whl_name), [])
+
         for r in sorted(reqs.values(), key = lambda r: r.requirement_line):
             whls, sdist = _add_dists(
                 requirement = r,
@@ -212,7 +215,7 @@ def parse_requirements(
             )
 
             target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms)
-            ret.setdefault(whl_name, []).append(
+            ret_requirements.append(
                 struct(
                     distribution = r.distribution,
                     srcs = r.srcs,