feat(pypi): enable pipstar by default (#3225)

Before this PR we were using our python helpers to parse the whl
METADATA for a
particular host (or experimentally - target) platform and the fixed
dependency
list would be written to the `BUILD.bazel` files materialized by the
`whl_library` repository rule.

This PR adds extra plumbing to leverage the Starlark implementation of
the
METADATA file parsing (a.k.a. pipstar) which moves the evaluation to
analysis
phase as we will get the target platform details via the
`env_marker_config_setting` from the `toolchain` used for the
dependencies.

Since users are normally working with a subset of platforms that the
METADATA
would pull in, we are writing the list of packages from the requirements
file to
a `bzl` file so that `pipstar` knows which packages to consider when
parsing the
METADATA. This will ensure that `bazel query` continues to work.

Since just passing the list of packages to the `whl_library` would cause
refetches of all of the `whl_library` dependencies when we add or remove
a
package, we pass a path where to load the symbol called `packages` from,
which
means that only the analysis phase will be affected because the macros
will be
re-evaluated if one adds or removes a package.

We still need to download the correct platform-specific wheels for the
right
platform in order to fully resolve the referenced tickets. With this PR
we are
deprecating the `experimental_target_platforms` attribute since it has
no longer
any effect.

Fixes #2949
Work towards #260
Work towards #2241

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: aignas

CHANGELOG.md

@@ -74,6 +74,14 @@ END_UNRELEASED_TEMPLATE
 * (toolchains) `py_runtime` and `PyRuntimeInfo` reject Python 2 settings.
   Setting `py_runtime.python_version = "PY2"` or non-None
   `PyRuntimeInfo.py2_runtime` is an error.
+* (pypi) `pipstar` flag has been flipped to be enabled by default, to turn it
+  off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please
+  add a comment to
+  [#2949](https://github.com/bazel-contrib/rules_python/issues/2949).
+  With this release we are deprecating {obj}`pip.parse.experimental_target_platforms` and
+  {obj}`pip_repository.experimental_target_platforms`. For users using `WORKSPACE` and
+  vendoring the `requirements.bzl` file, please re-vendor so that downstream is unaffected
+  when the APIs get removed.
 
 {#v0-0-0-fixed}
 ### Fixed

examples/pip_parse_vendored/requirements.bzl

@@ -4,7 +4,7 @@
 """
 
 load("@rules_python//python:pip.bzl", "pip_utils")
-load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")
+load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library")
 
 all_requirements = [
     "@my_project_pip_deps_vendored_certifi//:pkg",
@@ -91,12 +91,17 @@ def install_deps(**whl_library_kwargs):
         for requirement in group_requirements
     }
 
-    group_repo = "my_project_pip_deps_vendored__groups"
-    group_library(
-        name = group_repo,
+    config_repo = "my_project_pip_deps_vendored__config"
+    whl_config_repo(
+        name = "my_project_pip_deps_vendored__config",
         repo_prefix = "my_project_pip_deps_vendored_",
         groups = all_requirement_groups,
+        whl_map = {
+            p: ""
+            for p in all_whl_requirements_by_package
+        },
     )
+    config_load = "@{}//:config.bzl".format(config_repo)
 
     # Install wheels which may be participants in a group
     whl_config = dict(_config)
@@ -112,5 +117,6 @@ def install_deps(**whl_library_kwargs):
             group_name = group_name,
             group_deps = group_deps,
             annotation = _get_annotation(requirement),
+            config_load = config_load,
             **whl_config
         )

python/pip_install/BUILD.bazel

@@ -22,9 +22,9 @@ bzl_library(
     name = "pip_repository_bzl",
     srcs = ["pip_repository.bzl"],
     deps = [
-        "//python/private/pypi:group_library_bzl",
         "//python/private/pypi:package_annotation_bzl",
         "//python/private/pypi:pip_repository_bzl",
+        "//python/private/pypi:whl_config_repo_bzl",
         "//python/private/pypi:whl_library_bzl",
     ],
 )

python/pip_install/pip_repository.bzl

@@ -14,13 +14,14 @@
 
 ""
 
-load("//python/private/pypi:group_library.bzl", _group_library = "group_library")
 load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation")
 load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository")
+load("//python/private/pypi:whl_config_repo.bzl", _whl_config_repo = "whl_config_repo")
 load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library")
 
 # Re-exports for backwards compatibility
-group_library = _group_library
+group_library = _whl_config_repo
 pip_repository = _pip_repository
 whl_library = _whl_library
+whl_config_repo = _whl_config_repo
 package_annotation = _package_annotation

python/private/internal_config_repo.bzl

@@ -22,7 +22,7 @@ load("//python/private:text_util.bzl", "render")
 load(":repo_utils.bzl", "repo_utils")
 
 _ENABLE_PIPSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PIPSTAR"
-_ENABLE_PIPSTAR_DEFAULT = "0"
+_ENABLE_PIPSTAR_DEFAULT = "1"
 _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME = "RULES_PYTHON_DEPRECATION_WARNINGS"
 _ENABLE_DEPRECATION_WARNINGS_DEFAULT = "0"
 

python/private/pypi/BUILD.bazel

@@ -157,14 +157,6 @@ bzl_library(
     ],
 )
 
-bzl_library(
-    name = "group_library_bzl",
-    srcs = ["group_library.bzl"],
-    deps = [
-        ":generate_group_library_build_bazel_bzl",
-    ],
-)
-
 bzl_library(
     name = "hub_builder_bzl",
     srcs = ["hub_builder.bzl"],
@@ -408,6 +400,15 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "whl_config_repo_bzl",
+    srcs = ["whl_config_repo.bzl"],
+    deps = [
+        ":generate_group_library_build_bazel_bzl",
+        "//python/private:text_util_bzl",
+    ],
+)
+
 bzl_library(
     name = "whl_config_setting_bzl",
     srcs = ["whl_config_setting.bzl"],

python/private/pypi/config.bzl.tmpl.bzlmod renamed to python/private/pypi/config.bzl.tmpl

@@ -2,8 +2,6 @@
 
 NOTE: This is internal `rules_python` API and if you would like to depend on it, please raise an issue
 with your usecase. This may change in between rules_python versions without any notice.
-
-@generated by rules_python pip.parse bzlmod extension.
 """
 
-whl_map = %%WHL_MAP%%
+packages = %%PACKAGES%%

python/private/pypi/generate_whl_library_build_bazel.bzl

@@ -72,6 +72,7 @@ def generate_whl_library_build_bazel(
             "requires",
             "metadata_name",
             "metadata_version",
+            "packages",
             "include",
         ]
     else:
@@ -82,17 +83,13 @@ def generate_whl_library_build_bazel(
             "target_platforms",
             "default_python_version",
         ]
-        dep_template = kwargs.get("dep_template")
-        loads.append(
-            """load("{}", "{}")""".format(
-                dep_template.format(
-                    name = "",
-                    target = "config.bzl",
-                ),
-                "whl_map",
-            ),
-        )
-        kwargs["include"] = "whl_map"
+        packages_load = kwargs.pop("config_load")
+        if not kwargs.get("requires_dist"):
+            # no deps, we can leave the extra loads out
+            pass
+        else:
+            loads.append("""load("{}", "{}")""".format(packages_load, "packages"))
+            kwargs["include"] = "packages"
 
     for arg in unsupported_args:
         if kwargs.get(arg):

python/private/pypi/group_library.bzl

@@ -451,6 +451,7 @@ def _common_args(self, module_ctx, *, pip_attr):
     # attrs.
     whl_library_args = dict(
         dep_template = "@{}//{{name}}:{{target}}".format(self.name),
+        config_load = "@{}//:config.bzl".format(self.name),
     )
     maybe_args = dict(
         # The following values are safe to omit if they have false like values