feat(bzlmod): cross-platform builds without experimental_index_url

CHANGELOG.md

@@ -31,7 +31,11 @@ A brief description of the categories of changes:
 
 {#v0-0-0-fixed}
 ### Fixed
-- Nothing yet
+- The extension evaluation has been adjusted to always generate the same lock
+  file irrespective if `experimental_index_url` is set by any module or not.
+  Fixes [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known
+  issue is that it may break `bazel query` and in these usecases it is advisable
+  to use `cquery` until we have `sdist` cross-building from source fully working.
 
 {#v0-0-0-added}
 ### Added

MODULE.bazel

@@ -54,8 +54,10 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal")
+pip = use_extension("//python/extensions:pip.bzl", "pip")
 pip.parse(
+    download_only = True,
+    experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
@@ -89,17 +91,20 @@ dev_python.override(
 )
 
 dev_pip = use_extension(
-    "//python/private/pypi:pip.bzl",
-    "pip_internal",
+    "//python/extensions:pip.bzl",
+    "pip",
     dev_dependency = True,
 )
 dev_pip.parse(
-    download_only = True,  # this will not add the `sdist` values to the transitive closures at all.
+    download_only = True,
+    experimental_index_url = "https://pypi.org/simple",
     hub_name = "dev_pip",
     python_version = "3.11",
     requirements_lock = "//docs:requirements.txt",
 )
 dev_pip.parse(
+    download_only = True,
+    experimental_index_url = "https://pypi.org/simple",
     hub_name = "pypiserver",
     python_version = "3.11",
     requirements_lock = "//examples/wheel:requirements_server.txt",

python/private/pypi/config_settings.bzl

@@ -148,6 +148,13 @@ def config_settings(
             )
 
 def _dist_config_settings(*, suffix, plat_flag_values, **kwargs):
+    if kwargs.get("constraint_values"):
+        # Add python version + platform config settings
+        _dist_config_setting(
+            name = suffix.strip("_"),
+            **kwargs
+        )
+
     flag_values = {_flags.dist: ""}
 
     # First create an sdist, we will be building upon the flag values, which
@@ -277,7 +284,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):
 
     return ret
 
-def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native = native, **kwargs):
+def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs):
     """A macro to create a target that matches is_pip_whl_auto and one more value.
 
     Args:
@@ -310,6 +317,10 @@ def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native
         # `python_version` setting.
         return
 
+    if not is_pip_whl:
+        native.config_setting(name = _name, **kwargs)
+        return
+
     config_setting_name = _name + "_setting"
     native.config_setting(name = config_setting_name, **kwargs)
 

python/private/pypi/extension.bzl

@@ -24,7 +24,7 @@ load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
 load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS")
 load(":hub_repository.bzl", "hub_repository")
-load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
+load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":render_pkg_aliases.bzl", "whl_alias")
@@ -212,7 +212,6 @@ def _create_whl_repos(
         logger = logger,
     )
 
-    repository_platform = host_platform(module_ctx)
     for whl_name, requirements in requirements_by_platform.items():
         # We are not using the "sanitized name" because the user
         # would need to guess what name we modified the whl name
@@ -260,10 +259,10 @@ def _create_whl_repos(
             if v != default
         })
 
+        is_exposed = False
         if get_index_urls:
             # TODO @aignas 2024-05-26: move to a separate function
             found_something = False
-            is_exposed = False
             for requirement in requirements:
                 is_exposed = is_exposed or requirement.is_exposed
                 dists = requirement.whls
@@ -319,35 +318,30 @@ def _create_whl_repos(
                     exposed_packages[whl_name] = None
                 continue
 
-        requirement = select_requirement(
-            requirements,
-            platform = None if pip_attr.download_only else repository_platform,
-        )
-        if not requirement:
-            # Sometimes the package is not present for host platform if there
-            # are whls specified only in particular requirements files, in that
-            # case just continue, however, if the download_only flag is set up,
-            # then the user can also specify the target platform of the wheel
-            # packages they want to download, in that case there will be always
-            # a requirement here, so we will not be in this code branch.
-            continue
-        elif get_index_urls:
-            logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
-
-        whl_library_args["requirement"] = requirement.requirement_line
-        if requirement.extra_pip_args:
-            whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+        for i, requirement in enumerate(requirements):
+            is_exposed = is_exposed or requirement.is_exposed
+            if get_index_urls:
+                logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
+
+            whl_library_args["requirement"] = requirement.requirement_line
+            if requirement.extra_pip_args:
+                whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+
+            repo_name = "{}_{}".format(pip_name, whl_name)
+            if len(requirements) > 1:
+                repo_name = "{}__{}".format(repo_name, i)
+
+            whl_libraries[repo_name] = dict(whl_library_args.items())
+            whl_map.setdefault(whl_name, []).append(
+                whl_alias(
+                    repo = repo_name,
+                    version = major_minor,
+                    target_platforms = requirement.target_platforms if len(requirements) > 1 else None,
+                ),
+            )
 
-        # We sort so that the lock-file remains the same no matter the order of how the
-        # args are manipulated in the code going before.
-        repo_name = "{}_{}".format(pip_name, whl_name)
-        whl_libraries[repo_name] = dict(whl_library_args.items())
-        whl_map.setdefault(whl_name, []).append(
-            whl_alias(
-                repo = repo_name,
-                version = major_minor,
-            ),
-        )
+        if is_exposed:
+            exposed_packages[whl_name] = None
 
     return struct(
         is_reproducible = is_reproducible,
@@ -497,7 +491,8 @@ You cannot use both the additive_build_content and additive_build_content_file a
             hub_group_map[hub_name] = pip_attr.experimental_requirement_cycles
 
     return struct(
-        # We sort the output here so that the lock file is sorted
+        # We sort so that the lock-file remains the same no matter the order of how the
+        # args are manipulated in the code going before.
         whl_mods = dict(sorted(whl_mods.items())),
         hub_whl_map = {
             hub_name: {
@@ -513,8 +508,14 @@ You cannot use both the additive_build_content and additive_build_content_file a
             }
             for hub_name, group_map in sorted(hub_group_map.items())
         },
-        exposed_packages = {k: sorted(v) for k, v in sorted(exposed_packages.items())},
-        whl_libraries = dict(sorted(whl_libraries.items())),
+        exposed_packages = {
+            k: sorted(v)
+            for k, v in sorted(exposed_packages.items())
+        },
+        whl_libraries = {
+            k: dict(sorted(args.items()))
+            for k, args in sorted(whl_libraries.items())
+        },
         is_reproducible = is_reproducible,
     )
 
@@ -589,10 +590,8 @@ def _pip_impl(module_ctx):
     # Build all of the wheel modifications if the tag class is called.
     _whl_mods_impl(mods.whl_mods)
 
-    for name, args in sorted(mods.whl_libraries.items()):
-        # We sort so that the lock-file remains the same no matter the order of how the
-        # args are manipulated in the code going before.
-        whl_library(name = name, **dict(sorted(args.items())))
+    for name, args in mods.whl_libraries.items():
+        whl_library(name = name, **args)
 
     for hub_name, whl_map in mods.hub_whl_map.items():
         hub_repository(
@@ -617,13 +616,6 @@ def _pip_impl(module_ctx):
     else:
         return None
 
-def _pip_non_reproducible(module_ctx):
-    _pip_impl(module_ctx)
-
-    # We default to calling the PyPI index and that will go into the
-    # MODULE.bazel.lock file, hence return nothing here.
-    return None
-
 def _pip_parse_ext_attrs(**kwargs):
     """Get the attributes for the pip extension.
 
@@ -881,55 +873,6 @@ extension.
     },
 )
 
-pypi_internal = module_extension(
-    doc = """\
-This extension is used to make dependencies from pypi available.
-
-For now this is intended to be used internally so that usage of the `pip`
-extension in `rules_python` does not affect the evaluations of the extension
-for the consumers.
-
-pip.parse:
-To use, call `pip.parse()` and specify `hub_name` and your requirements file.
-Dependencies will be downloaded and made available in a repo named after the
-`hub_name` argument.
-
-Each `pip.parse()` call configures a particular Python version. Multiple calls
-can be made to configure different Python versions, and will be grouped by
-the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
-to automatically resolve to different, Python version-specific, libraries.
-
-pip.whl_mods:
-This tag class is used to help create JSON files to describe modifications to
-the BUILD files for wheels.
-""",
-    implementation = _pip_non_reproducible,
-    tag_classes = {
-        "override": _override_tag,
-        "parse": tag_class(
-            attrs = _pip_parse_ext_attrs(
-                experimental_index_url = "https://pypi.org/simple",
-            ),
-            doc = """\
-This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
-This tag class reuses most of the attributes found in {bzl:obj}`pip_parse`.
-The exception is it does not use the arg 'repo_prefix'.  We set the repository
-prefix for the user and the alias arg is always True in bzlmod.
-""",
-        ),
-        "whl_mods": tag_class(
-            attrs = _whl_mod_attrs(),
-            doc = """\
-This tag class is used to create JSON file that are used when calling wheel_builder.py.  These
-JSON files contain instructions on how to modify a wheel's project.  Each of the attributes
-create different modifications based on the type of attribute. Previously to bzlmod these
-JSON files where referred to as annotations, and were renamed to whl_modifications in this
-extension.
-""",
-        ),
-    },
-)
-
 def _whl_mods_repo_impl(rctx):
     rctx.file("BUILD.bazel", "")
     for whl_name, mods in rctx.attr.whl_mods.items():

python/private/pypi/parse_requirements.bzl

@@ -168,7 +168,7 @@ def parse_requirements(
         )
 
     ret = {}
-    for whl_name, reqs in requirements_by_platform.items():
+    for whl_name, reqs in sorted(requirements_by_platform.items()):
         requirement_target_platforms = {}
         for r in reqs.values():
             target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms)
@@ -212,6 +212,8 @@ def parse_requirements(
 def select_requirement(requirements, *, platform):
     """A simple function to get a requirement for a particular platform.
 
+    Only used in WORKSPACE.
+
     Args:
         requirements (list[struct]): The list of requirements as returned by
             the `parse_requirements` function above.
@@ -243,6 +245,8 @@ def select_requirement(requirements, *, platform):
 def host_platform(ctx):
     """Return a string representation of the repository OS.
 
+    Only used in WORKSPACE.
+
     Args:
         ctx (struct): The `module_ctx` or `repository_ctx` attribute.
 

python/private/pypi/pip.bzl

@@ -14,7 +14,6 @@
 
 "pip module extensions for use with bzlmod."
 
-load("//python/private/pypi:extension.bzl", "pypi", "pypi_internal")
+load("//python/private/pypi:extension.bzl", "pypi")
 
 pip = pypi
-pip_internal = pypi_internal