Update distros without download (#614)

Instead of computing the SHA256s ourselves we get them from github
directly.
Also add additional 21.1.5 distros.

Author: helly25

toolchain/internal/llvm_distributions.bzl

@@ -706,12 +706,14 @@ _llvm_distributions = {
     "clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz": "511e4e7e0a43156cb1410578285f1db246ebb400db0018cd304c84a369562b6d",
 
     # 21.1.5
-    "LLVM-21.1.5-Linux-ARM64.tar.xz": "c9a1ee5d1a1698a8eb0abda1c1e44c812378aec32f89cc4fbbb41865237359a9",
-    "LLVM-21.1.5-Linux-X64.tar.xz": "6279d78feeeb8e839a397f0bca7b1c0594972224d59525496416653d9b9c077f",
-    "clang+llvm-21.1.5-x86_64-pc-windows-msvc.tar.xz": "eba824f1379fdb1a385f6dff8d19275a57348f621c752ce93b6d11256741e349",
+    "clang+llvm-21.1.5-aarch64-pc-windows-msvc.tar.xz": "sha256:dcc7a6f9e3ff02f5b49b23e6f91abe2f9431972d72ab59f7b7d9f8b436ea1ca3",
+    "clang+llvm-21.1.5-armv7a-linux-gnueabihf.tar.gz": "sha256:42a964c0ea68764ef8e222f5f979a400a803a912e5df273358652d4017ca3411",
+    "clang+llvm-21.1.5-x86_64-pc-windows-msvc.tar.xz": "sha256:eba824f1379fdb1a385f6dff8d19275a57348f621c752ce93b6d11256741e349",
+    "LLVM-21.1.5-Linux-ARM64.tar.xz": "sha256:c9a1ee5d1a1698a8eb0abda1c1e44c812378aec32f89cc4fbbb41865237359a9",
+    "LLVM-21.1.5-Linux-X64.tar.xz": "sha256:6279d78feeeb8e839a397f0bca7b1c0594972224d59525496416653d9b9c077f",
 
     # Refer to variable declaration on how to update!
-    # Example update (without deleting): utils/llvm_checksums.sh -g -t /tmp/llvm -v 21.1.5
+    # Example update (without download): utils/llvm_checksums.sh -D -g -t /tmp/llvm -v 21.1.5
 }
 
 # Note: Unlike the user-specified llvm_mirror attribute, the URL prefixes in
@@ -779,6 +781,13 @@ def _full_url(url):
         return "file://" + url
     return url
 
+def _normalize_and_check_sha256(sha256):
+    if sha256:
+        sha256 = sha256.removeprefix("sha256:")
+        if len(sha256) != 64:
+            return None, "Attribute sha256 needs exactly 64 hex characters."
+    return sha256, None
+
 def download_llvm(rctx):
     """Download the LLVM distribution for the given context."""
     urls = []
@@ -793,6 +802,10 @@ def download_llvm(rctx):
     if not urls:
         urls, sha256, strip_prefix = _distribution_urls(rctx)
 
+    sha256, shaerr = _normalize_and_check_sha256(sha256)
+    if shaerr:
+        fail("ERROR: " + shaerr)
+
     res = rctx.download_and_extract(
         [_full_url(url) for url in urls],
         sha256 = sha256,
@@ -1443,6 +1456,13 @@ def _distributions_test_writer_impl(ctx):
             extra_llvm_distributions = extra_llvm_distributions,
             parsed_llvm_version = version,
         )
+        for basename, distribution in all_llvm_distributions.items():
+            _, shaerr = _normalize_and_check_sha256(distribution.sha256)
+            if shaerr:
+                output.append("err: {basename}: bad sha256: {shaerr}".format(
+                    basename = basename,
+                    shaerr = shaerr,
+                ))
         for arch in arch_list:
             for os in os_list:
                 if version < MIN_VERSION:

utils/llvm_checksums.sh

@@ -17,8 +17,9 @@ set -euo pipefail
 
 use_github_host=0
 tmp_dir=
+download=1
 
-while getopts "t:v:gh" opt; do
+while getopts "t:v:ghD" opt; do
   case "${opt}" in
   "t") tmp_dir="${OPTARG}" ;;
   "v") llvm_version="${OPTARG}" ;;
@@ -30,6 +31,7 @@ while getopts "t:v:gh" opt; do
     echo "-g           - Use github to download releases."
     exit 2
     ;;
+  "D") download=0 ;;
   *)
     echo "invalid option: -${OPTARG}"
     exit 1
@@ -67,11 +69,21 @@ llvm_host() {
 github_host() {
   output_dir="${tmp_dir}/${llvm_version}"
   mkdir -p "${output_dir}"
+  if ((download)); then
+    echo ""
+    echo "===="
+    echo "Checksums for clang+llvm distributions are (${output_dir}):"
+    echo "    # ${llvm_version}"
+    curl -s "https://api.github.com/repos/llvm/llvm-project/releases/tags/llvmorg-${llvm_version}" |
+      tee ./releases.json |
+      jq -r '.assets[]|select(any(.name; test("^(clang[+]llvm|LLVM)-.*tar.(xz|gz)$")))|"    \""+(.browser_download_url|split("/")|.[-1]|sub("%2B";"+"))+"\": \""+.digest+"\","'
+    exit 0
+  fi
   (
     cd "${output_dir}"
     curl -s "https://api.github.com/repos/llvm/llvm-project/releases/tags/llvmorg-${llvm_version}" |
       tee ./releases.json |
-      jq '.assets[]|select(any(.name; test("^(clang[+]llvm|LLVM)-.*tar.(xz|gz)$")))|.browser_download_url' |
+      jq '.assets[]|select(any(.name .digest; test("^(clang[+]llvm|LLVM)-.*tar.(xz|gz)$")))|.browser_download_url' |
       tee ./filtered_urls.txt |
       xargs -n1 curl -L -O -C -
   )