refactor: starlark reimplementation of pip_repository (#1043)

Previously we were using pip_parse python scripts. This has a few drawbacks:

* Requires system python to be present.
* Usage of a Python script makes it harder to reason as there is an extra layer of abstraction.
* Extending/reusing code between multi_pip_parse and pip_parse is hard.

Now we use Starlark to parse the requirements.txt into requirements.bzl.

Author: aignas

docs/pip.md

@@ -20,7 +20,6 @@ genrule(
         # Replace the bazel 6.0.0 specific comment with something that bazel 5.4.0 would produce.
         # This enables this example to be run as a test under bazel 5.4.0.
         """sed -e 's#@//#//#'""",
-        """tr "'" '"' """,
         """sed 's#"@python39_.*//:bin/python3"#interpreter#' >$@""",
     ]),
 )

docs/pip_repository.md

@@ -14,29 +14,20 @@ all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "
 _packages = [("pip_certifi", "certifi==2022.12.7     --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3     --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"), ("pip_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
 _config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
 _annotations = {}
-_bzlmod = False
 
 def _clean_name(name):
     return name.replace("-", "_").replace(".", "_").lower()
 
 def requirement(name):
-    if _bzlmod:
-        return "@@pip//:" + _clean_name(name) + "_pkg"
     return "@pip_" + _clean_name(name) + "//:pkg"
 
 def whl_requirement(name):
-    if _bzlmod:
-        return "@@pip//:" + _clean_name(name) + "_whl"
     return "@pip_" + _clean_name(name) + "//:whl"
 
 def data_requirement(name):
-    if _bzlmod:
-        return "@@pip//:" + _clean_name(name) + "_data"
     return "@pip_" + _clean_name(name) + "//:data"
 
 def dist_info_requirement(name):
-    if _bzlmod:
-        return "@@pip//:" + _clean_name(name) + "_dist_info"
     return "@pip_" + _clean_name(name) + "//:dist_info"
 
 def entry_point(pkg, script = None):
@@ -46,7 +37,7 @@ def entry_point(pkg, script = None):
 
 def _get_annotation(requirement):
     # This expects to parse `setuptools==58.2.0     --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11`
-    # down wo `setuptools`.
+    # down to `setuptools`.
     name = requirement.split(" ")[0].split("=")[0].split("[")[0]
     return _annotations.get(name)
 

examples/pip_parse_vendored/BUILD.bazel

@@ -14,9 +14,8 @@
 
 "Module extensions for use with bzlmod"
 
-load("@rules_python//python:pip.bzl", "pip_parse")
 load("@rules_python//python:repositories.bzl", "python_register_toolchains")
-load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "use_isolated", "whl_library")
+load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
 load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
@@ -68,22 +67,17 @@ def _pip_impl(module_ctx):
 
             # Parse the requirements file directly in starlark to get the information
             # needed for the whl_libary declarations below. This is needed to contain
-            # the pip_parse logic to a single module extension.
+            # the pip_repository logic to a single module extension.
             requirements_lock_content = module_ctx.read(requrements_lock)
             parse_result = parse_requirements(requirements_lock_content)
             requirements = parse_result.requirements
             extra_pip_args = attr.extra_pip_args + parse_result.options
 
             # Create the repository where users load the `requirement` macro. Under bzlmod
             # this does not create the install_deps() macro.
-            pip_parse(
+            pip_repository_bzlmod(
                 name = attr.name,
                 requirements_lock = attr.requirements_lock,
-                bzlmod = True,
-                timeout = attr.timeout,
-                python_interpreter = attr.python_interpreter,
-                python_interpreter_target = attr.python_interpreter_target,
-                quiet = attr.quiet,
             )
 
             for name, requirement_line in requirements:
@@ -114,7 +108,7 @@ def _pip_parse_ext_attrs():
         "name": attr.string(mandatory = True),
     }, **pip_repository_attrs)
 
-    # Like the pip_parse macro, we end up setting this manually so
+    # Like the pip_repository rule, we end up setting this manually so
     # don't allow users to override it.
     attrs.pop("repo_prefix")
 

examples/pip_parse_vendored/requirements.bzl

@@ -47,7 +47,7 @@ def pip_install(requirements = None, name = "pip", **kwargs):
     print("pip_install is deprecated. Please switch to pip_parse. pip_install will be removed in a future release.")
     pip_parse(requirements = requirements, name = name, **kwargs)
 
-def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_deps", bzlmod = False, **kwargs):
+def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_deps", **kwargs):
     """Accepts a locked/compiled requirements file and installs the dependencies listed within.
 
     Those dependencies become available in a generated `requirements.bzl` file.
@@ -143,14 +143,9 @@ def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_
         requirements (Label): Deprecated. See requirements_lock.
         name (str, optional): The name of the generated repository. The generated repositories
             containing each requirement will be of the form `<name>_<requirement-name>`.
-        bzlmod (bool, optional): Whether this rule is being run under a bzlmod module extension.
         **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule.
     """
-
-    # Don't try to fetch dependencies under bzlmod because they are already fetched via the internal_deps
-    # module extention, and because the maybe-install pattern doesn't work under bzlmod.
-    if not bzlmod:
-        pip_install_dependencies()
+    pip_install_dependencies()
 
     # Temporary compatibility shim.
     # pip_install was previously document to use requirements while pip_parse was using requirements_lock.
@@ -160,8 +155,6 @@ def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_
     pip_repository(
         name = name,
         requirements_lock = reqs_to_use,
-        repo_prefix = "{}_".format(name),
-        bzlmod = bzlmod,
         **kwargs
     )
 

python/extensions.bzl

@@ -4,7 +4,6 @@ filegroup(
         "BUILD.bazel",
         "//python/pip_install/tools/dependency_resolver:distribution",
         "//python/pip_install/tools/lib:distribution",
-        "//python/pip_install/tools/lock_file_generator:distribution",
         "//python/pip_install/tools/wheel_installer:distribution",
         "//python/pip_install/private:distribution",
     ],
@@ -24,7 +23,6 @@ filegroup(
     srcs = [
         "//python/pip_install/tools/dependency_resolver:py_srcs",
         "//python/pip_install/tools/lib:py_srcs",
-        "//python/pip_install/tools/lock_file_generator:py_srcs",
         "//python/pip_install/tools/wheel_installer:py_srcs",
     ],
     visibility = ["//python/pip_install/private:__pkg__"],