feat(pip): Add a clean-sheet pip implementation

MODULE.bazel

@@ -14,8 +14,8 @@ module(
 bazel_dep(name = "aspect_bazel_lib", version = "2.16.0")
 bazel_dep(name = "aspect_tools_telemetry", version = "0.2.6")
 bazel_dep(name = "bazel_skylib", version = "1.4.2")
-bazel_dep(name = "rules_python", version = "0.29.0")
-bazel_dep(name = "platforms", version = "0.0.7")
+bazel_dep(name = "rules_python", version = "1.6.3")
+bazel_dep(name = "platforms", version = "1.0.0")
 
 bazel_lib = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
 bazel_lib.expand_template()
@@ -27,7 +27,7 @@ use_repo(tel, "aspect_tools_telemetry_report")
 python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True)
 python.toolchain(
     is_default = False,
-    python_version = "3.9",
+    python_version = "3.13",
 )
 
 tools = use_extension("//py:extensions.bzl", "py_tools")
@@ -114,3 +114,21 @@ oci.pull(
     tag = "latest",
 )
 use_repo(oci, "ubuntu", "ubuntu_linux_amd64", "ubuntu_linux_arm64_v8")
+
+
+# Mandatory pip2 host configuration
+host = use_extension("//pip/private/host:extension.bzl", "host_platform")
+use_repo(host, "aspect_rules_py_pip_host")
+
+# Testing
+pip2 = use_extension("//pip:extension.bzl", "pip")
+
+pip2.declare_hub(hub_name = "pip2")
+
+pip2.declare_venv(hub_name = "pip2", venv_name = "default")
+pip2.lockfile(hub_name = "pip2", venv_name = "default", lockfile = "//:uv-default.lock")
+
+pip2.declare_venv(hub_name = "pip2", venv_name = "airflow")
+pip2.lockfile(hub_name = "pip2", venv_name = "airflow", lockfile = "//:uv-airflow.lock")
+
+use_repo(pip2, "pip2")

WORKSPACE

@@ -23,7 +23,7 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist
 
 python_register_toolchains(
     name = "python_toolchain_3_8",
-    python_version = "3.8.12",
+    python_version = "3.8",
     # Setting `set_python_version_constraint` will set special constraints on the registered toolchain.
     # This means that this toolchain registration will only be selected for `py_binary` / `py_test` targets
     # that have the `python_version = "3.8.12"` attribute set. Targets that have no `python_attribute` will use
@@ -35,7 +35,7 @@ python_register_toolchains(
 # py_test/py_binary target even if it has python_version attribute set.
 python_register_toolchains(
     name = "python_toolchain",
-    python_version = "3.9",
+    python_version = "3.13",
 )
 
 py_repositories()

examples/pip2/BUILD.bazel

@@ -0,0 +1,25 @@
+load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template")
+load("//py/unstable:defs.bzl", "py_venv", "py_venv_binary")
+
+expand_template(
+    name = "stamped",
+    stamp_substitutions = {
+        "<BUILD_TIMESTAMP>": "{{BUILD_TIMESTAMP}}",
+    },
+    template = "say.py",
+)
+
+py_venv_binary(
+    name = "say",
+    srcs = [
+        ":stamped",
+    ],
+    imports = [
+        ".",
+    ],
+    main = ":stamped",
+    deps = [
+        "@pip2//package:cowsay",
+        "@pip2//package:pandas",
+    ],
+)

examples/pip2/say.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import sys
+
+print("---")
+import _virtualenv
+
+output_base = _virtualenv.__file__.split("/execroot/")[0]
+execroot = f"{output_base}/execroot"
+external = f"{output_base}/external"
+runfiles = _virtualenv.__file__.split(".runfiles/")[0] + ".runfiles"
+
+def _simplify(s):
+    if isinstance(s, str):
+        return s \
+            .replace(sys.prefix, "${PYTHONHOME}") \
+            .replace(runfiles, "${RUNFILES}") \
+            .replace(execroot, "${BAZEL_EXECROOT}") \
+            .replace(external, "${BAZEL_EXTERNAL}") \
+            .replace(output_base, "${BAZEL_BASE}")
+
+    elif isinstance(s, list):
+        return [_simplify(it) for it in s]
+
+print("virtualenv:", _simplify(_virtualenv.__file__))
+import sys
+print("sys.prefix:", _simplify(sys.prefix))
+print("sys.path:")
+for it in _simplify(sys.path):
+    print(" -", it)
+import site
+print("site.PREFIXES:")
+for it in _simplify(site.PREFIXES):
+    print(" -", it)
+
+import cowsay
+
+cowsay.cow('hello py_venv! (built at <BUILD_TIMESTAMP>)')
+
+
+import pandas
+print("pandas", _simplify(pandas.__file__))
+
+import numpy  # pandas transitive
+print("numpy", _simplify(numpy.__file__))

internal_deps.bzl

@@ -134,3 +134,12 @@ def rules_py_internal_deps():
         strip_prefix = "container-structure-test-56c7201716d770c0f820a9c19207ba2ea77c34f8",
         sha256 = "29632b3226bb9c7dcde340b6efcc48cc5f32e60ec47c86a144922e234e4d9ce7",
     )
+
+    _http_archive(
+        name = "yq.bzl",
+        urls = [
+            "https://github.com/bazel-contrib/yq.bzl/archive/refs/tags/v0.2.0.tar.gz",
+        ],
+        strip_prefix = "yq.bzl-0.2.0",
+        sha256 = "FIXME",
+    )

pip/extension.bzl

@@ -0,0 +1,3 @@
+load("//pip/private:extension.bzl", _pip = "pip")
+
+pip = _pip

pip/private/constraints/BUILD.bazel

@@ -0,0 +1,3 @@
+load(":macros.bzl", "generate")
+
+generate()

pip/private/constraints/abi/BUILD.bazel

@@ -0,0 +1,5 @@
+load(":macro.bzl", "generate")
+
+package(default_visibility = ["//visibility:public"])
+
+generate()

pip/private/constraints/abi/macro.bzl

@@ -0,0 +1,122 @@
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("//pip/private/constraints:defs.bzl", "MAJORS", "MINORS", "INTERPRETERS", "FLAGS")
+
+# FIXME: Where does abi 2/3/4 fit in here?
+# FIXME: Where do ABI feature flags fit in here?
+def generate():
+    """
+    Lay down `py3`, `py312`, `cp3`, `cp312` etc and critically `any`.
+
+    The interpretation is a bit tricky because `cp`
+    """
+
+    # FIXME: Is there a better/worse way to do this?
+    selects.config_setting_group(
+        name = "none",
+        match_all = [
+            "//conditions:default",
+        ]
+    )
+
+    native.constraint_setting(
+        name = "feature_pydebug",
+        default_constraint_value = ":pydebug_disabled",
+    )
+    native.constraint_value(
+        name = "pydebug_enabled",
+        constraint_setting = ":feature_pydebug",
+    )
+    native.constraint_value(
+        name = "pydebug_disabled",
+        constraint_setting = ":feature_pydebug",
+    )
+
+    native.constraint_setting(
+        name = "feature_pymalloc",
+        default_constraint_value = ":pymalloc_disabled",
+    )
+    native.constraint_value(
+        name = "pymalloc_enabled",
+        constraint_setting = ":feature_pymalloc",
+    )
+    native.constraint_value(
+        name = "pymalloc_disabled",
+        constraint_setting = ":feature_pymalloc",
+    )
+
+    native.constraint_setting(
+        name = "feature_freethreading",
+        default_constraint_value = ":freethreading_disabled",
+    )
+    native.constraint_value(
+        name = "freethreading_enabled",
+        constraint_setting = ":feature_freethreading",
+    )
+    native.constraint_value(
+        name = "freethreading_disabled",
+        constraint_setting = ":feature_freethreading",
+    )
+
+    native.constraint_setting(
+        name = "feature_wide_unicode",
+        default_constraint_value = ":wide_unicode_disabled",
+    )
+    native.constraint_value(
+        name = "wide_unicode_enabled",
+        constraint_setting = ":feature_wide_unicode",
+    )
+    native.constraint_value(
+        name = "wide_unicode_disabled",
+        constraint_setting = ":feature_wide_unicode",
+    )
+
+    native.alias(
+        name = "abi3",
+        actual = "is_py33"
+    )
+
+    for interpreter in INTERPRETERS:
+        for major in MAJORS:
+            # selects.config_setting_group(
+            #     name = "{}{}".format(interpreter, major),
+            #     match_all = [
+            #         "//pip/private/constraints/python/interpreter:{}".format(interpreter),
+            #         "//pip/private/constraints/python/major:{}".format(major),
+            #     ]
+            # )
+
+            for minor in MINORS:
+                selects.config_setting_group(
+                    name = "is_{}{}{}".format(interpreter, major, minor),
+                    match_all = [
+                        # "//pip/private/constraints/python/interpreter:{}".format(interpreter),
+                        "//pip/private/constraints/python/major:{}".format(major),
+                        "//pip/private/constraints/python/minor:{}".format(minor),
+                    ]
+                )
+
+                for d in [False, True]:
+                    for m in [False, True]:
+                        for t in [False, True]:
+                            for u in [False, True]:
+                                selects.config_setting_group(
+                                    # This is a bit out of hand I admit
+                                    name = "{0}{1}{2}{3}{4}{5}{6}".format(
+                                        interpreter,
+                                        major,
+                                        minor,
+                                        "d" if d else "",
+                                        "m" if m else "",
+                                        "t" if t else "",
+                                        "u" if u else "",
+                                    ),
+                                    match_all = (
+                                        [
+                                            ":is_{}{}{}".format(interpreter, major, minor),
+                                        ]
+                                        + ([":pydebug_enabled"] if d else [])
+                                        + ([":pymalloc_enabled"] if m else [])
+                                        + ([":freethreading_enabled"] if t else [])
+                                        + ([":wide_unicode_enabled"] if u else [])
+                                    ),
+                                )

pip/private/constraints/defs.bzl

@@ -0,0 +1,63 @@
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+MAJORS = [2, 3]        # There is no 4
+MINORS = range(0, 21)
+PATCHES = range(0, 31)
+INTERPRETERS = [
+    "py", # Generic
+    "cp", # CPython
+    # "jy", # Jython
+    # "ip", # IronPython
+    # "pp", # PyPy, has its own ABI scheme :|
+]
+# FIXME: We're ignoring these for now which isn't ideal
+FLAGS = {
+    "d": "pydebug",
+    "m": "pymalloc",
+    "t": "freethreading",
+    "u": "wide-unicode",  # Deprecated in 3.13
+}
+
+def generate_gte_ladder(stages):
+    """
+    Accept a list of names of individual conditions representing specific versions.
+    These names happen to be in ascending order and are generated by some other codepath.
+
+    Given these conditon lable names generate a sequence ("ladder") of equality
+    comparison operations + or with the "greater" comparison.
+
+    So for [macos_10_0, macos_10_1, macos_11_0, macos_11_1, ...]
+    We want to get
+
+        selects.config_setting_group(
+            name = "gte_macos_11_0",
+            match_any = [
+               ":macos_11_0",
+               ":gte_macos_10_1",
+            ],
+        )
+        selects.config_setting_group(
+            name = "gte_macos_10_1",
+            match_any = [
+               ":macos_10_1",
+               ":gte_macos_10_0",
+            ],
+        )
+
+    As an added refinement the stages are a struct
+
+        struct(
+           name = <name of the ladder step>,
+           condition = <condition for this ladder step>,
+        )
+
+    """
+
+    # We loop up to the second-to-last item to ensure we always have a 'next' stage.
+    for i, current_stage in enumerate(stages):
+        selects.config_setting_group(
+            name = "{}".format(current_stage.name),
+            match_any = [
+                ":{}".format(current_stage.condition),
+            ] + ([":{}".format(stages[i+1].name)] if i+1 < len(stages) else []),
+        )

pip/private/constraints/platform/BUILD.bazel

@@ -0,0 +1,5 @@
+load(":macro.bzl", "generate")
+
+package(default_visibility = ["//visibility:public"])
+
+generate()