feat: don't require users to build tools from source (#287)

alexeagle · web-flow · commit d83ef8bc5d2f · 2024-03-01T16:15:42.000-08:00
Wire up the toolchains along two different codepaths, based on whether
our version was stamped as a release, or otherwise a pre-release.

When a release:
- fetch tools from GitHub releases using `rctx.download`
- each fetched tool goes into a separate external repo with its own
`toolchain` declaration, so Bazel fetches only the needed ones
- the resulting repos are returned to repositories.bzl which registers
them

When a pre-release:
- continue to build the tools from source, using existing toolchains

---

### Type of change

- New feature or functionality (change which adds functionality)

### Test plan

- Manual testing; please provide instructions so we can reproduce:
Use my personal fork to test against a release I performed there.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -29,3 +29,9 @@ jobs:
           {"folder": ".", "bazelversion": "6.4.0"},
           {"folder": ".", "bzlmodEnabled": true}
         ]
+
+  verify-bcr-patches:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: patch --dry-run -p1 < .bcr/patches/*.patch
diff --git a/e2e/smoke/WORKSPACE.bazel b/e2e/smoke/WORKSPACE.bazel
@@ -53,6 +53,7 @@ crate_repositories()
 # Alternatively, you can skip calling this function, so long as you've
 # already fetched all the dependencies.
 load("@aspect_rules_py//py:repositories.bzl", "rules_py_dependencies")
+
 rules_py_dependencies()
 
 # "Installation" for rules_python
diff --git a/py/private/toolchain/tools.bzl b/py/private/toolchain/tools.bzl
@@ -1,31 +1,41 @@
-"""Toolchain for making toolchains"""
+"""Utilities for making toolchains"""
+load("//tools:integrity.bzl", "RELEASED_BINARY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
-load("//tools:version.bzl", "IS_PRERELEASE")
+# The expected config for each tool, whether it runs in an action or at runtime
+RUST_BIN_CFG = {
+    "unpack": "exec",
+    "venv": "target",
+}
 
 TOOLCHAIN_PLATFORMS = {
     "darwin_amd64": struct(
-        release_platform = "macos-amd64",
+        arch = "x86_64",
+        vendor_os_abi = "apple-darwin",
         compatible_with = [
             "@platforms//os:macos",
             "@platforms//cpu:x86_64",
         ],
     ),
     "darwin_arm64": struct(
-        release_platform = "macos-arm64",
+        arch = "aarch64",
+        vendor_os_abi = "apple-darwin",
         compatible_with = [
             "@platforms//os:macos",
             "@platforms//cpu:aarch64",
         ],
     ),
     "linux_amd64": struct(
-        release_platform = "linux-amd64",
+        arch = "x86_64",
+        vendor_os_abi = "unknown-linux-gnu",
         compatible_with = [
             "@platforms//os:linux",
             "@platforms//cpu:x86_64",
         ],
     ),
     "linux_arm64": struct(
-        release_platform = "linux-arm64",
+        arch = "aarch64",
+        vendor_os_abi = "unknown-linux-gnu",
         compatible_with = [
             "@platforms//os:linux",
             "@platforms//cpu:aarch64",
@@ -56,7 +66,7 @@ def _toolchain_impl(ctx):
 
     return [toolchain_info, default_info, template_variables]
 
-_toolchain = rule(
+py_tool_toolchain = rule(
     implementation = _toolchain_impl,
     attrs = {
         "bin": attr.label(
@@ -69,49 +79,97 @@ _toolchain = rule(
     },
 )
 
-def _make_toolchain_name(name, platform):
-    return "{}_{}_toolchain".format(name, platform)
-
-def make_toolchain(name, toolchain_type, tools, cfg = "exec"):
+def source_toolchain(name, toolchain_type, bin):
     """Makes vtool toolchain and repositories
 
     Args:
         name: Override the prefix for the generated toolchain repositories.
         toolchain_type: Toolchain type reference.
-        tools: Mapping of tool binary to platform.
-        cfg: Generate a toolchain for the target or exec config.
+        bin: the rust_binary target
     """
 
-    if IS_PRERELEASE:
-        toolchain_rule = "{}_toolchain_source".format(name)
-        _toolchain(
-            name = toolchain_rule,
-            bin = tools["from-source"],
-            template_var = "{}_BIN".format(name.upper()),
-        )
-        native.toolchain(
-            name = _make_toolchain_name(name, "source"),
-            toolchain = toolchain_rule,
-            toolchain_type = toolchain_type,
+    toolchain_rule = "{}_toolchain_source".format(name)
+    py_tool_toolchain(
+        name = toolchain_rule,
+        bin = bin,
+        template_var = "{}_BIN".format(name.upper()),
+    )
+    native.toolchain(
+        name = "{}_source_toolchain".format(name),
+        toolchain = toolchain_rule,
+        toolchain_type = toolchain_type,
+    )
+
+def _tool_repo_impl(rctx):
+    build_content = """\
+# Generated by @aspect_rules_py//py/private/toolchain:tools.bzl
+load("@aspect_rules_py//py/private/toolchain:tools.bzl", "py_tool_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+"""
+    # For manual testing, override these environment variables
+    # TODO: use rctx.getenv when available, see https://github.com/bazelbuild/bazel/pull/20944
+    release_fork = "aspect-build"
+    release_version = VERSION
+    if "RULES_PY_RELEASE_FORK" in rctx.os.environ:
+        release_fork = rctx.os.environ["RULES_PY_RELEASE_FORK"]
+    if "RULES_PY_RELEASE_VERSION" in rctx.os.environ:
+        release_version = rctx.os.environ["RULES_PY_RELEASE_VERSION"]
+
+    for tool, cfg in RUST_BIN_CFG.items():
+        filename = "-".join([
+            tool,
+            TOOLCHAIN_PLATFORMS[rctx.attr.platform].arch,
+            TOOLCHAIN_PLATFORMS[rctx.attr.platform].vendor_os_abi,
+        ])
+        url = "https://github.com/{}/rules_py/releases/download/v{}/{}".format(
+            release_fork,
+            release_version,
+            filename
         )
-        return
-
-    for [platform, meta] in TOOLCHAIN_PLATFORMS.items():
-        toolchain_rule = "{}_toolchain_{}".format(name, platform)
-        _toolchain(
-            name = toolchain_rule,
-            bin = tools[meta.release_platform],
-            template_var = "{}_BIN".format(name.upper()),
+        rctx.download(
+            url = url,
+            sha256 = RELEASED_BINARY_INTEGRITY[filename],
+            executable = True,
+            output = tool,
         )
+        build_content += """\
+py_tool_toolchain(name = "concrete_{tool}_toolchain", bin = "{tool}", template_var = "{tool_upper}_BIN")
+
+toolchain(
+    name = "{tool}_toolchain",
+    {cfg}_compatible_with = {compatible_with},
+    toolchain = "concrete_{tool}_toolchain",
+    toolchain_type = "@aspect_rules_py//py/private/toolchain/{tool}:toolchain_type",
+)
+""".format(
+    cfg = cfg,
+    compatible_with = TOOLCHAIN_PLATFORMS[rctx.attr.platform].compatible_with,
+    tool = tool,
+    tool_upper = tool.upper(),
+)
+
+    rctx.file("BUILD.bazel", build_content)
 
-        args = dict({
-            "name": _make_toolchain_name(name, platform),
-            "toolchain": toolchain_rule,
-            "toolchain_type": toolchain_type,
-        })
-        if cfg == "exec":
-            args.update({"exec_compatible_with": meta.compatible_with})
-        else:
-            args.update({"target_compatible_with": meta.compatible_with})
-
-        native.toolchain(**args)
+_tool_repo = repository_rule(
+    doc = "Download pre-built binary tools and create toolchains for them",
+    implementation = _tool_repo_impl,
+    attrs = {
+        "platform": attr.string(mandatory = True, values = TOOLCHAIN_PLATFORMS.keys()),
+    }
+)
+
+def binary_tool_repos(name):
+    """Create a downloaded toolchain for every tool under every supported platform.
+
+    Args:
+        name: prefix used in created repositories
+    Returns:
+        list of toolchain targets to register
+    """
+    result = []
+    for platform in TOOLCHAIN_PLATFORMS.keys():
+        plat_repo_name = ".".join([name, platform])
+        result.append("@{}//:all".format(plat_repo_name))
+        _tool_repo(name = plat_repo_name, platform = platform)
+    return result
diff --git a/py/private/toolchain/unpack/BUILD.bazel b/py/private/toolchain/unpack/BUILD.bazel
@@ -1,17 +1,12 @@
-load("//py/private/toolchain:tools.bzl", "make_toolchain")
+load("//py/private/toolchain:tools.bzl", "source_toolchain")
 
 toolchain_type(
     name = "toolchain_type",
+    visibility = ["//visibility:public"],
 )
 
-make_toolchain(
+source_toolchain(
     name = "unpack",
     toolchain_type = ":toolchain_type",
-    tools = {
-        "from-source": "//py/tools/unpack_bin",
-        "macos-amd64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-x86_64-apple-darwin",
-        "macos-arm64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-aarch64-apple-darwin",
-        "linux-amd64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-x86_64-unknown-linux-musl",
-        "linux-arm64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-aarch64-unknown-linux-musl",
-    },
+    bin = "//py/tools/unpack_bin",
 )
diff --git a/py/private/toolchain/venv/BUILD.bazel b/py/private/toolchain/venv/BUILD.bazel
@@ -1,18 +1,16 @@
-load("//py/private/toolchain:tools.bzl", "make_toolchain")
+load("//py/private/toolchain:tools.bzl", "source_toolchain")
 
 toolchain_type(
     name = "toolchain_type",
+    visibility = ["//visibility:public"],
 )
 
-make_toolchain(
+source_toolchain(
     name = "venv",
-    cfg = "target",
+    # When running from source, we assume the target is the exec platform.
+    # TODO: when we have cross-compile Mac -> Linux, pre-release users should be able to
+    # build a py_image on their Mac and have the linux venv rust_binary end up in the container.
+    # cfg = "target",
     toolchain_type = ":toolchain_type",
-    tools = {
-        "from-source": "//py/tools/venv_bin",
-        "macos-amd64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-x86_64-apple-darwin",
-        "macos-arm64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-aarch64-apple-darwin",
-        "linux-amd64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-x86_64-unknown-linux-musl",
-        "linux-arm64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-aarch64-unknown-linux-musl",
-    },
+    bin = "//py/tools/venv_bin",
 )
diff --git a/py/repositories.bzl b/py/repositories.bzl
@@ -7,6 +7,8 @@ See https://docs.bazel.build/versions/main/skylark/deploying.html#dependencies
 load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 load("//py/private/toolchain:autodetecting.bzl", _register_autodetecting_python_toolchain = "register_autodetecting_python_toolchain")
+load("//py/private/toolchain:tools.bzl", "binary_tool_repos")
+load("//tools:version.bzl", "IS_PRERELEASE")
 
 def http_archive(name, **kwargs):
     maybe(_http_archive, name = name, **kwargs)
@@ -21,11 +23,13 @@ register_autodetecting_python_toolchain = _register_autodetecting_python_toolcha
 # and released only in semver majors.
 
 # buildifier: disable=unnamed-macro
-def rules_py_dependencies(register_py_toolchains = True):
-    """Fetch rules_py's dependencies, and optionally register toolchains.
+def rules_py_dependencies(name = "rules_py_tools", register = True, prerelease = IS_PRERELEASE):
+    """Fetch rules_py's dependencies
 
     Args:
-        register_py_toolchains: When true, rules_py's toolchains are automatically registered.
+        name: prefix for generated repositories
+        register: whether to register the toolchains created, should be false under bzlmod
+        prerelease: whether to build rust tools from source, rather than download pre-built binaries
     """
 
     # The minimal version of bazel_skylib we require
@@ -49,9 +53,18 @@ def rules_py_dependencies(register_py_toolchains = True):
         strip_prefix = "rules_python-0.31.0",
         url = "https://github.com/bazelbuild/rules_python/releases/download/0.31.0/rules_python-0.31.0.tar.gz",
     )
+    
+    # When running from a release version:
+    # Fetch remote tools from the release and create toolchain for them
+    if prerelease:
+        if register:
+            native.register_toolchains(
+                "@aspect_rules_py//py/private/toolchain/venv/...",
+                "@aspect_rules_py//py/private/toolchain/unpack/...",
+            )
+        return
 
-    if register_py_toolchains:
-        native.register_toolchains(
-            "@aspect_rules_py//py/private/toolchain/venv/...",
-            "@aspect_rules_py//py/private/toolchain/unpack/...",
-        )
+    toolchains = binary_tool_repos(name)
+    if register:
+        native.register_toolchains(*toolchains)
+            
diff --git a/tools/integrity.bzl b/tools/integrity.bzl
@@ -4,4 +4,14 @@ This file contents are entirely replaced during release publishing.
 The checked in content is only here to allow load() statements in the sources to resolve.
 """
 
-RELEASED_BINARY_INTEGRITY = {}
+# FROM https://github.com/alexeagle/rules_py/releases/tag/v0.101.0
+RELEASED_BINARY_INTEGRITY = {
+  "unpack-aarch64-apple-darwin": "5199cb98d59535ba8609c4a538be99d8759e971ab9d1e680d932091529860868",
+  "unpack-x86_64-apple-darwin": "23d0eb9ec09609c00d65779234f1210506eba1bf64090a01cef71d5175e3e6b6",
+  "venv-aarch64-apple-darwin": "e02b478358641abf72ca5192948101e18a3dabccb48e7ee704759ad39c2f718a",
+  "venv-x86_64-apple-darwin": "13d0c08a23b1064489df522d2327cfbaf0058b9364ca2ffee51b5b6f06a6930e",
+  "unpack-aarch64-unknown-linux-gnu": "48552e399a1f2ab97e62ca7fce5783b6214e284330c7555383f43acf82446636",
+  "unpack-x86_64-unknown-linux-gnu": "fd265552bfd236efef519f81ce783322a50d8d7ab5af5d08a713e519cedff87f",
+  "venv-aarch64-unknown-linux-gnu": "b05d6cbe485e8ed1872f330012929126778744208e47485d7e2be285a027c7ca",
+  "venv-x86_64-unknown-linux-gnu": "61f53fe45da5f2f160907c87c1a2d663bc57eee667520a46490a837825be61c9"
+}
diff --git a/tools/version.bzl b/tools/version.bzl
@@ -1,6 +1,6 @@
 "version information. replaced with stamped info with each release"
 
-# This is automagically replace by git during git archive using `git export-subst`
+# Automagically "stamped" by git during `git archive` thanks to `export-subst` line in .gitattributes.
 # See https://git-scm.com/docs/git-archive#Documentation/git-archive.txt-export-subst
 _VERSION_PRIVATE = "$Format:%(describe:tags=true)$"
 
