Reland "go_sdk: store SDK filenames and hashes in lockfile facts" (#4493)

**What type of PR is this?**

Feature

**What does this PR do? Why is it needed?**

This moves the download of the "all versions" JSON, which can't hit the
repository cache, from each individual `go_download_sdk` into the module
extension. If the current version of Bazel supports facts, this
information will also be persisted in the lockfile, allowing for truly
airgapped builds assuming an up-to-date download (formerly repository)
cache.

This is a reland of #4393 with the following improvements:
* Adapted to the more limited facts API that actually got merged.
* Prefetching of SDK hashes is now performed on a best-effort basis so
that otherwise airgapped builds that provide SDK hashes do not result in
failures.

See bazelbuild/bazel#26198 for the PR that added
facts support to Bazel.

**Which issues(s) does this PR fix?**

Fixes #3945 

**Other notes for review**

You can verify that this works by running `bazel mod show_repo
@go_default_sdk` in the BCR test repo.

Author: fmeum

go/private/extensions.bzl

@@ -15,7 +15,7 @@
 load("@io_bazel_rules_go_bazel_features//:features.bzl", "bazel_features")
 load("//go/private:go_mod.bzl", "version_from_go_mod")
 load("//go/private:nogo.bzl", "DEFAULT_NOGO", "NOGO_DEFAULT_EXCLUDES", "NOGO_DEFAULT_INCLUDES", "go_register_nogo")
-load("//go/private:sdk.bzl", "detect_host_platform", "go_download_sdk_rule", "go_host_sdk_rule", "go_multiple_toolchains", "go_wrap_sdk_rule")
+load("//go/private:sdk.bzl", "detect_host_platform", "fetch_sdks_by_version", "go_download_sdk_rule", "go_host_sdk_rule", "go_multiple_toolchains", "go_wrap_sdk_rule")
 
 def host_compatible_toolchain_impl(ctx):
     ctx.file("BUILD.bazel")
@@ -197,6 +197,38 @@ def _go_sdk_impl(ctx):
     first_host_compatible_toolchain = None
     host_detected_goos, host_detected_goarch = detect_host_platform(ctx)
     toolchains = []
+
+    all_sdks_by_version = {}
+    used_sdks_by_version = {}
+    facts = getattr(ctx, "facts", {})
+
+    def get_sdks_by_version_cached(version):
+        # Avoid a download without a known digest in the SDK repo rule by fetching the SDKs filename
+        # and digest here. When using a version of Bazel that supports module extension facts, this
+        # info will be persisted in the lockfile, allowing for truly airgapped builds with an
+        # up-to-date lockfile and download (formerly repository) cache.
+        sdks = facts.get(version)
+        if sdks == None:
+            # Lazily fetch the information about all SDKs so that we avoid the download if the facts
+            # already contain all the versions we care about. We take care to only do this once and
+            # also accept failures to support airgapped builds: the user may have set sdk hashes on
+            # all SDK repos they actually intend to use, but others (e.g., the default SDK added by
+            # rules_go) trigger this path even if they would never be selected by toolchain
+            # resolution. We must not break those builds.
+            if not all_sdks_by_version:
+                all_sdks_by_version.clear()
+                all_sdks_by_version.update(fetch_sdks_by_version(ctx, allow_fail = True) or {
+                    "fetch_failed_but_should_not_fetch_again_sentinel": [],
+                })
+            sdks = all_sdks_by_version.get(version)
+        if sdks == None:
+            # This is either caused by an invalid version or because we are in an airgapped build
+            # and the version wasn't present in facts. Since we don't want to fail in the latter
+            # case, we leave it to the repository rule to report a useful error message.
+            return None
+        used_sdks_by_version[version] = sdks
+        return sdks
+
     for module in ctx.modules:
         # Apply wrapped toolchains first to override specific platforms from the
         # default toolchain or any downloads.
@@ -261,18 +293,12 @@ def _go_sdk_impl(ctx):
                 index = index,
             )
 
-            # Keep in sync with the other calls to `go_download_sdk_rule` above and below.
-            go_download_sdk_rule(
+            _download_sdk(
+                get_sdks_by_version = get_sdks_by_version_cached,
                 name = name,
                 goos = download_tag.goos,
                 goarch = download_tag.goarch,
-                sdks = download_tag.sdks,
-                experiments = download_tag.experiments,
-                patches = download_tag.patches,
-                patch_strip = download_tag.patch_strip,
-                urls = download_tag.urls,
-                version = download_tag.version,
-                strip_prefix = download_tag.strip_prefix,
+                download_tag = download_tag,
             )
 
             if (not download_tag.goos or download_tag.goos == host_detected_goos) and (not download_tag.goarch or download_tag.goarch == host_detected_goarch):
@@ -306,18 +332,12 @@ def _go_sdk_impl(ctx):
                         suffix = "_{}_{}".format(goos, goarch),
                     )
 
-                    # Keep in sync with the other calls to `go_download_sdk_rule` above.
-                    go_download_sdk_rule(
+                    _download_sdk(
+                        get_sdks_by_version = get_sdks_by_version_cached,
                         name = default_name,
                         goos = goos,
                         goarch = goarch,
-                        sdks = download_tag.sdks,
-                        experiments = download_tag.experiments,
-                        patches = download_tag.patches,
-                        patch_strip = download_tag.patch_strip,
-                        urls = download_tag.urls,
-                        version = download_tag.version,
-                        strip_prefix = download_tag.strip_prefix,
+                        download_tag = download_tag,
                     )
 
                     toolchains.append(struct(
@@ -379,7 +399,14 @@ def _go_sdk_impl(ctx):
     )
 
     if bazel_features.external_deps.extension_metadata_has_reproducible:
-        return ctx.extension_metadata(reproducible = True)
+        kwargs = {
+            "reproducible": True,
+        }
+
+        # See get_sdks_by_version_cached above for details on these facts.
+        if hasattr(ctx, "facts"):
+            kwargs["facts"] = used_sdks_by_version
+        return ctx.extension_metadata(**kwargs)
     else:
         return None
 
@@ -409,6 +436,25 @@ def _left_pad_zero(index, length):
         fail("index must be non-negative")
     return ("0" * length + str(index))[-length:]
 
+def _download_sdk(*, get_sdks_by_version, name, goos, goarch, download_tag):
+    version = download_tag.version
+    sdks = download_tag.sdks
+    if version and not sdks:
+        sdks = get_sdks_by_version(version)
+
+    go_download_sdk_rule(
+        name = name,
+        goos = goos,
+        goarch = goarch,
+        sdks = sdks,
+        experiments = download_tag.experiments,
+        patches = download_tag.patches,
+        patch_strip = download_tag.patch_strip,
+        urls = download_tag.urls,
+        version = download_tag.version,
+        strip_prefix = download_tag.strip_prefix,
+    )
+
 go_sdk_extra_kwargs = {
     # The choice of a host-compatible SDK is expressed in repository rule attribute values and
     # depends on host OS and architecture.

go/private/sdk.bzl

@@ -81,17 +81,7 @@ def _go_download_sdk_impl(ctx):
             ctx.report_progress("Finding latest Go version")
         else:
             ctx.report_progress("Finding Go SHA-256 sums")
-        ctx.download(
-            url = [
-                "https://go.dev/dl/?mode=json&include=all",
-                "https://golang.google.cn/dl/?mode=json&include=all",
-            ],
-            output = "versions.json",
-        )
-
-        data = ctx.read("versions.json")
-        ctx.delete("versions.json")
-        sdks_by_version = _parse_versions_json(data)
+        sdks_by_version = fetch_sdks_by_version(ctx)
 
         if not version:
             highest_version = None
@@ -580,6 +570,31 @@ def _parse_versions_json(data):
         for sdk in sdks
     }
 
+def fetch_sdks_by_version(ctx, allow_fail = False):
+    result = ctx.download(
+        url = [
+            "https://go.dev/dl/?mode=json&include=all",
+            "https://golang.google.cn/dl/?mode=json&include=all",
+        ],
+        output = "versions.json",
+        allow_fail = allow_fail,
+    )
+    if not result.success:
+        return None
+    data = ctx.read("versions.json")
+
+    # If the download is redirected through a proxy such as Artifactory, it may
+    # drop the query parameters and return an HTML page instead. In that case,
+    # just return an empty map if allow_fail is set. It is unfortunately not
+    # possible to attempt parsing as JSON and catch the error.
+    if (not data or data[0] != "[") and allow_fail:
+        return None
+
+    # module_ctx doesn't have delete, but its files are temporary anyway.
+    if hasattr(ctx, "delete"):
+        ctx.delete("versions.json")
+    return _parse_versions_json(data)
+
 def parse_version(version):
     """Parses a version string like "1.15.5" and returns a tuple of numbers or None"""
     l, r = 0, 0

tests/bcr/.bazelversion

@@ -1 +1,3 @@
-7.4.1
+1079973bd513b69fb56becc7fe678c03449b30c3
+# A commit on the 8.5.0 release branch with support for facts.
+# TODO: Update to 8.5.0 when released.