Refactor and separate concerns of external python package handling code (#953)

Author: groodt

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/keith/pre-commit-buildifier
-    rev: 5.1.0.1
+    rev: 6.0.0
     hooks:
       - id: buildifier
         args: &args 

MODULE.bazel

@@ -1,15 +1,13 @@
 module(
     name = "rules_python",
-    compatibility_level = 1,
     version = "0.0.0",
+    compatibility_level = 1,
 )
 
 bazel_dep(name = "platforms", version = "0.0.4")
 
 internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps")
-
 internal_deps.install()
-
 use_repo(
     internal_deps,
     "pypi__build",

examples/bzlmod/MODULE.bazel

@@ -1,35 +1,30 @@
 module(
     name = "example_bzlmod",
-    compatibility_level = 1,
     version = "0.0.0",
+    compatibility_level = 1,
 )
 
 bazel_dep(name = "rules_python", version = "0.0.0")
-
 local_path_override(
     module_name = "rules_python",
     path = "../..",
 )
 
 python = use_extension("@rules_python//python:extensions.bzl", "python")
-
 python.toolchain(
     name = "python3_9",
     python_version = "3.9",
 )
-
 use_repo(python, "python3_9_toolchains")
 
 register_toolchains(
     "@python3_9_toolchains//:all",
 )
 
 pip = use_extension("@rules_python//python:extensions.bzl", "pip")
-
 pip.parse(
     name = "pip",
     requirements_lock = "//:requirements_lock.txt",
     requirements_windows = "//:requirements_windows.txt",
 )
-
 use_repo(pip, "pip")

python/extensions.bzl

@@ -78,7 +78,7 @@ def _pip_impl(module_ctx):
                     environment = attr.environment,
                 )
 
-# Keep in sync with python/pip_install/extract_wheels/bazel.py
+# Keep in sync with python/pip_install/tools/bazel.py
 def _sanitize_name(name):
     return name.replace("-", "_").replace(".", "_").lower()
 

python/pip_install/BUILD.bazel

@@ -1,11 +1,11 @@
-exports_files(["pip_compile.py"])
-
 filegroup(
     name = "distribution",
     srcs = glob(["*.bzl"]) + [
         "BUILD.bazel",
-        "pip_compile.py",
-        "//python/pip_install/extract_wheels:distribution",
+        "//python/pip_install/tools/dependency_resolver:distribution",
+        "//python/pip_install/tools/lib:distribution",
+        "//python/pip_install/tools/lock_file_generator:distribution",
+        "//python/pip_install/tools/wheel_installer:distribution",
         "//python/pip_install/private:distribution",
     ],
     visibility = ["//:__pkg__"],
@@ -22,7 +22,10 @@ filegroup(
 filegroup(
     name = "py_srcs",
     srcs = [
-        "//python/pip_install/extract_wheels:py_srcs",
+        "//python/pip_install/tools/dependency_resolver:py_srcs",
+        "//python/pip_install/tools/lib:py_srcs",
+        "//python/pip_install/tools/lock_file_generator:py_srcs",
+        "//python/pip_install/tools/wheel_installer:py_srcs",
     ],
     visibility = ["//python/pip_install/private:__pkg__"],
 )

python/pip_install/pip_repository.bzl

@@ -307,7 +307,7 @@ def _pip_repository_impl(rctx):
     args = [
         python_interpreter,
         "-m",
-        "python.pip_install.extract_wheels.parse_requirements_to_bzl",
+        "python.pip_install.tools.lock_file_generator.lock_file_generator",
         "--requirements_lock",
         rctx.path(requirements_txt),
         "--requirements_lock_label",
@@ -524,7 +524,7 @@ def _whl_library_impl(rctx):
     args = [
         python_interpreter,
         "-m",
-        "python.pip_install.extract_wheels.wheel_installer",
+        "python.pip_install.tools.wheel_installer.wheel_installer",
         "--requirement",
         rctx.attr.requirement,
         "--repo",

python/pip_install/private/pip_install_utils.bzl

@@ -1,9 +1,9 @@
 """Utilities for `rules_python` pip rules"""
 
 _SRCS_TEMPLATE = """\
-\"\"\"A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
 
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
 `bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
 "\"\"
 

python/pip_install/private/srcs.bzl

@@ -1,18 +1,21 @@
-"""A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
 
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
 `bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
 """
 
 # Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
 # sources changed.
 PIP_INSTALL_PY_SRCS = [
-    "@rules_python//python/pip_install/extract_wheels:__init__.py",
-    "@rules_python//python/pip_install/extract_wheels:annotation.py",
-    "@rules_python//python/pip_install/extract_wheels:arguments.py",
-    "@rules_python//python/pip_install/extract_wheels:bazel.py",
-    "@rules_python//python/pip_install/extract_wheels:namespace_pkgs.py",
-    "@rules_python//python/pip_install/extract_wheels:parse_requirements_to_bzl.py",
-    "@rules_python//python/pip_install/extract_wheels:wheel.py",
-    "@rules_python//python/pip_install/extract_wheels:wheel_installer.py",
+    "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py",
+    "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py",
+    "@rules_python//python/pip_install/tools/lib:__init__.py",
+    "@rules_python//python/pip_install/tools/lib:annotation.py",
+    "@rules_python//python/pip_install/tools/lib:arguments.py",
+    "@rules_python//python/pip_install/tools/lib:bazel.py",
+    "@rules_python//python/pip_install/tools/lock_file_generator:__init__.py",
+    "@rules_python//python/pip_install/tools/lock_file_generator:lock_file_generator.py",
+    "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py",
+    "@rules_python//python/pip_install/tools/wheel_installer:wheel.py",
+    "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py",
 ]

python/pip_install/repositories.bzl

@@ -89,7 +89,7 @@ py_library(
     srcs = glob(["**/*.py"]),
     data = glob(["**/*"], exclude=[
         # These entries include those put into user-installed dependencies by
-        # data_exclude in /python/pip_install/extract_wheels/bazel.py
+        # data_exclude in /python/pip_install/tools/bazel.py
         # to avoid non-determinism following pip install's behavior.
         "**/*.py",
         "**/*.pyc",

python/pip_install/requirements.bzl

@@ -57,7 +57,7 @@ def compile_pip_requirements(
 
     # Use the Label constructor so this is expanded in the context of the file
     # where it appears, which is to say, in @rules_python
-    pip_compile = Label("//python/pip_install:pip_compile.py")
+    pip_compile = Label("//python/pip_install/tools/dependency_resolver:dependency_resolver.py")
 
     loc = "$(rootpath {})"