feat(pip): Add a clean-sheet pip implementation

.pre-commit-config.yaml

@@ -4,7 +4,7 @@
 
 # Commitizen runs in commit-msg stage
 # but we don't want to run the other hooks on commit messages
-default_stages: [commit]
+default_stages: [pre-commit]
 
 repos:
   # Check formatting and lint for starlark code

MODULE.bazel

@@ -14,8 +14,9 @@ module(
 bazel_dep(name = "aspect_bazel_lib", version = "2.16.0")
 bazel_dep(name = "aspect_tools_telemetry", version = "0.2.6")
 bazel_dep(name = "bazel_skylib", version = "1.4.2")
-bazel_dep(name = "rules_python", version = "0.29.0")
-bazel_dep(name = "platforms", version = "0.0.7")
+bazel_dep(name = "rules_python", version = "1.6.3")
+bazel_dep(name = "rules_python_gazelle_plugin", version = "1.6.3")
+bazel_dep(name = "platforms", version = "1.0.0")
 
 bazel_lib = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
 bazel_lib.expand_template()
@@ -27,7 +28,7 @@ use_repo(tel, "aspect_tools_telemetry_report")
 python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True)
 python.toolchain(
     is_default = False,
-    python_version = "3.9",
+    python_version = "3.13",
 )
 
 tools = use_extension("//py:extensions.bzl", "py_tools")
@@ -114,3 +115,60 @@ oci.pull(
     tag = "latest",
 )
 use_repo(oci, "ubuntu", "ubuntu_linux_amd64", "ubuntu_linux_arm64_v8")
+
+# This is what `rules_uv` really boils down to, so vendoring that
+bazel_dep(name = "rules_multitool", version = "0.11.0")
+
+multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool")
+multitool.hub(lockfile = "//pip/private/uv:uv.lock.json")
+use_repo(multitool, "multitool")
+
+# Used as part of the sdist flow
+bazel_dep(name = "tar.bzl", version = "0.6.0")
+
+# The toml tool that pip2 uses
+tomltool = use_extension("//pip/private/tomltool:extension.bzl", "tomltool")
+use_repo(tomltool, "tomltool")
+
+# Mandatory pip2 host configuration
+host = use_extension("//pip/private/host:extension.bzl", "host_platform")
+use_repo(host, "aspect_rules_py_pip_host")
+
+# Testing
+pip2 = use_extension("//pip:extension.bzl", "pip")
+pip2.declare_hub(hub_name = "pip2")
+pip2.declare_venv(
+    hub_name = "pip2",
+    venv_name = "default",
+)
+pip2.lockfile(
+    hub_name = "pip2",
+    lockfile = "//:uv-default.lock",
+    venv_name = "default",
+)
+pip2.declare_venv(
+    hub_name = "pip2",
+    venv_name = "airflow",
+)
+pip2.lockfile(
+    hub_name = "pip2",
+    lockfile = "//:uv-airflow.lock",
+    venv_name = "airflow",
+)
+
+# pip2.add_sdist_deps(
+#     hub_name =
+#     venv_name =
+#     name = "annoy",
+#     deps = [
+#         "",
+#     ],
+# )
+
+# pip2.use_src_build(
+#     hub_name =
+#     venv_name =
+#     src = "", # firstparty
+# )
+
+use_repo(pip2, "pip2")

WORKSPACE

@@ -23,7 +23,7 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist
 
 python_register_toolchains(
     name = "python_toolchain_3_8",
-    python_version = "3.8.12",
+    python_version = "3.8",
     # Setting `set_python_version_constraint` will set special constraints on the registered toolchain.
     # This means that this toolchain registration will only be selected for `py_binary` / `py_test` targets
     # that have the `python_version = "3.8.12"` attribute set. Targets that have no `python_attribute` will use
@@ -35,7 +35,7 @@ python_register_toolchains(
 # py_test/py_binary target even if it has python_version attribute set.
 python_register_toolchains(
     name = "python_toolchain",
-    python_version = "3.9",
+    python_version = "3.12",
 )
 
 py_repositories()
@@ -317,3 +317,10 @@ oci_pull(
 load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain")
 
 container_structure_test_register_toolchain(name = "cst")
+
+# Forwards-compat to the new pip machinery mostly so docgen works
+load("//pip/private/host:repository.bzl", "host_platform_repo")
+
+host_platform_repo(
+    name = "aspect_rules_py_pip_host",
+)

docs/BUILD.bazel

@@ -37,8 +37,13 @@ stardoc_with_diff_test(
 )
 
 stardoc_with_diff_test(
-    name = "venv",
-    bzl_library_target = "//py/private/py_venv:py_venv",
+    name = "unstable",
+    bzl_library_target = "//py/unstable:defs",
+)
+
+stardoc_with_diff_test(
+    name = "pip",
+    bzl_library_target = "//pip/private:extension",
 )
 
 update_docs(name = "update")

examples/pip2/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template")
+load("//py/unstable:defs.bzl", "py_venv_binary")
+
+expand_template(
+    name = "stamped",
+    stamp_substitutions = {
+        "<BUILD_TIMESTAMP>": "{{BUILD_TIMESTAMP}}",
+    },
+    template = "say.py",
+)
+
+py_venv_binary(
+    name = "say",
+    srcs = [
+        ":stamped",
+    ],
+    imports = [
+        ".",
+    ],
+    main = ":stamped",
+    venv = "default",
+    deps = [
+        "@pip2//click",
+        "@pip2//cowsay",
+    ],
+)

examples/pip2/say.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import sys
+
+print("---")
+import _virtualenv
+
+output_base = _virtualenv.__file__.split("/execroot/")[0]
+execroot = f"{output_base}/execroot"
+external = f"{output_base}/external"
+runfiles = _virtualenv.__file__.split(".runfiles/")[0] + ".runfiles"
+
+def _simplify(s):
+    if isinstance(s, str):
+        return s \
+            .replace(sys.prefix, "${PYTHONHOME}") \
+            .replace(runfiles, "${RUNFILES}") \
+            .replace(execroot, "${BAZEL_EXECROOT}") \
+            .replace(external, "${BAZEL_EXTERNAL}") \
+            .replace(output_base, "${BAZEL_BASE}")
+
+    elif isinstance(s, list):
+        return [_simplify(it) for it in s]
+
+print("virtualenv:", _simplify(_virtualenv.__file__))
+import sys
+print("sys.prefix:", _simplify(sys.prefix))
+print("sys.path:")
+for it in _simplify(sys.path):
+    print(" -", it)
+import site
+print("site.PREFIXES:")
+for it in _simplify(site.PREFIXES):
+    print(" -", it)
+
+import cowsay
+import click
+
+@click.command()
+@click.argument("who", envvar="USER")
+def main(who):
+    cowsay.cow(f'hello {who}! (built at <BUILD_TIMESTAMP>)')
+
+main()

internal_deps.bzl

@@ -134,3 +134,12 @@ def rules_py_internal_deps():
         strip_prefix = "container-structure-test-56c7201716d770c0f820a9c19207ba2ea77c34f8",
         sha256 = "29632b3226bb9c7dcde340b6efcc48cc5f32e60ec47c86a144922e234e4d9ce7",
     )
+
+    _http_archive(
+        name = "yq.bzl",
+        urls = [
+            "https://github.com/bazel-contrib/yq.bzl/archive/refs/tags/v0.2.0.tar.gz",
+        ],
+        strip_prefix = "yq.bzl-0.2.0",
+        sha256 = "FIXME",
+    )

pip/extension.bzl

@@ -0,0 +1,5 @@
+""""""
+
+load("//pip/private:extension.bzl", _pip = "pip")
+
+pip = _pip

pip/private/BUILD.bazel

@@ -0,0 +1,51 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+package(default_visibility = [
+    "//docs:__pkg__",
+    "//pip/private:__subpackages__",
+])
+
+bzl_library(
+    name = "extension",
+    srcs = ["extension.bzl"],
+    deps = [
+        ":normalize_name",
+        ":parse_whl_name",
+        ":sccs",
+        ":sha1",
+        "//pip/private/constraints:repository",
+        "//pip/private/constraints/platform:defs",
+        "//pip/private/pip_hub:repository",
+        "//pip/private/sdist_build:repository",
+        "//pip/private/tomltool:toml",
+        "//pip/private/venv_hub:repository",
+        "//pip/private/whl_install:repository",
+        "@bazel_tools//tools/build_defs/repo:cache.bzl",
+        "@bazel_tools//tools/build_defs/repo:http.bzl",
+        "@bazel_tools//tools/build_defs/repo:utils.bzl",
+    ],
+)
+
+bzl_library(
+    name = "sccs",
+    srcs = ["sccs.bzl"],
+    deps = [],
+)
+
+bzl_library(
+    name = "sha1",
+    srcs = ["sha1.bzl"],
+    deps = ["@aspect_bazel_lib//lib:strings"],
+)
+
+bzl_library(
+    name = "normalize_name",
+    srcs = ["normalize_name.bzl"],
+    deps = [],
+)
+
+bzl_library(
+    name = "parse_whl_name",
+    srcs = ["parse_whl_name.bzl"],
+    deps = ["@bazel_skylib//lib:new_sets"],
+)

pip/private/constraints/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+bzl_library(
+    name = "defs",
+    srcs = ["defs.bzl"],
+    visibility = ["//:__subpackages__"],
+    deps = ["@bazel_skylib//lib:selects"],
+)
+
+bzl_library(
+    name = "repository",
+    srcs = ["repository.bzl"],
+    visibility = ["//:__subpackages__"],
+    deps = [],
+)

pip/private/constraints/abi/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load(":macro.bzl", "generate")
+
+package(default_visibility = ["//visibility:public"])
+
+generate()
+
+bzl_library(
+    name = "macro",
+    srcs = ["macro.bzl"],
+    deps = [
+        "//pip/private/constraints:defs",
+        "@bazel_skylib//lib:selects",
+    ],
+)