go_sdk: store SDK filenames and hashes in lockfile facts (#4393)

**What type of PR is this?**

Feature

**What does this PR do? Why is it needed?**

This moves the download of the "all versions" JSON, which can't hit the
repository cache, from each individual `go_download_sdk` into the module
extension. If the current version of Bazel supports facts, this
information will also be persisted in the lockfile, allowing for truly
airgapped builds assuming an up-to-date download (formerly repository)
cache.

See bazelbuild/bazel#26198 for the PR that adds
facts support to Bazel.

**Which issues(s) does this PR fix?**

Fixes #3945 

**Other notes for review**

Author: fmeum

go/private/extensions.bzl

@@ -15,7 +15,7 @@
 load("@io_bazel_rules_go_bazel_features//:features.bzl", "bazel_features")
 load("//go/private:go_mod.bzl", "version_from_go_mod")
 load("//go/private:nogo.bzl", "DEFAULT_NOGO", "NOGO_DEFAULT_EXCLUDES", "NOGO_DEFAULT_INCLUDES", "go_register_nogo")
-load("//go/private:sdk.bzl", "detect_host_platform", "go_download_sdk_rule", "go_host_sdk_rule", "go_multiple_toolchains", "go_wrap_sdk_rule")
+load("//go/private:sdk.bzl", "detect_host_platform", "fetch_sdks_by_version", "go_download_sdk_rule", "go_host_sdk_rule", "go_multiple_toolchains", "go_wrap_sdk_rule")
 
 def host_compatible_toolchain_impl(ctx):
     ctx.file("BUILD.bazel")
@@ -190,6 +190,10 @@ def _go_sdk_impl(ctx):
     first_host_compatible_toolchain = None
     host_detected_goos, host_detected_goarch = detect_host_platform(ctx)
     toolchains = []
+
+    sdks_by_version = getattr(ctx, "facts", None) or {}
+    used_versions = {}
+
     for module in ctx.modules:
         # Apply wrapped toolchains first to override specific platforms from the
         # default toolchain or any downloads.
@@ -252,18 +256,14 @@ def _go_sdk_impl(ctx):
                 index = index,
             )
 
-            # Keep in sync with the other calls to `go_download_sdk_rule` above and below.
-            go_download_sdk_rule(
+            _download_sdk(
+                module_ctx = ctx,
+                sdks_by_version = sdks_by_version,
+                used_versions = used_versions,
                 name = name,
                 goos = download_tag.goos,
                 goarch = download_tag.goarch,
-                sdks = download_tag.sdks,
-                experiments = download_tag.experiments,
-                patches = download_tag.patches,
-                patch_strip = download_tag.patch_strip,
-                urls = download_tag.urls,
-                version = download_tag.version,
-                strip_prefix = download_tag.strip_prefix,
+                download_tag = download_tag,
             )
 
             if (not download_tag.goos or download_tag.goos == host_detected_goos) and (not download_tag.goarch or download_tag.goarch == host_detected_goarch):
@@ -297,18 +297,14 @@ def _go_sdk_impl(ctx):
                         suffix = "_{}_{}".format(goos, goarch),
                     )
 
-                    # Keep in sync with the other calls to `go_download_sdk_rule` above.
-                    go_download_sdk_rule(
+                    _download_sdk(
+                        module_ctx = ctx,
+                        sdks_by_version = sdks_by_version,
+                        used_versions = used_versions,
                         name = default_name,
                         goos = goos,
                         goarch = goarch,
-                        sdks = download_tag.sdks,
-                        experiments = download_tag.experiments,
-                        patches = download_tag.patches,
-                        patch_strip = download_tag.patch_strip,
-                        urls = download_tag.urls,
-                        version = download_tag.version,
-                        strip_prefix = download_tag.strip_prefix,
+                        download_tag = download_tag,
                     )
 
                     toolchains.append(struct(
@@ -370,7 +366,18 @@ def _go_sdk_impl(ctx):
     )
 
     if bazel_features.external_deps.extension_metadata_has_reproducible:
-        return ctx.extension_metadata(reproducible = True)
+        kwargs = {
+            "reproducible": True,
+        }
+
+        # See _download_sdk below for details on these facts.
+        if hasattr(ctx, "facts"):
+            kwargs["facts"] = {
+                version: sdk_info
+                for version, sdk_info in sdks_by_version.items()
+                if version in used_versions
+            }
+        return ctx.extension_metadata(**kwargs)
     else:
         return None
 
@@ -400,6 +407,37 @@ def _left_pad_zero(index, length):
         fail("index must be non-negative")
     return ("0" * length + str(index))[-length:]
 
+def _download_sdk(*, module_ctx, sdks_by_version, used_versions, name, goos, goarch, download_tag):
+    version = download_tag.version
+    sdks = download_tag.sdks
+    if version and not sdks:
+        # Avoid a download without a known digest in the SDK repo rule by fetching the SDKs filename
+        # and digest here. When using a version of Bazel that supports module extension facts, this
+        # info will be persisted in the lockfile, allowing for truly airgapped builds with an
+        # up-to-date lockfile and download (formerly repository) cache.
+        if version not in sdks_by_version:
+            # Lazily fetch the information about all SDKs so that we avoid the download if the facts
+            # already contain all the versions we care about.
+            sdks_by_version.clear()
+            sdks_by_version.update(fetch_sdks_by_version(module_ctx))
+        if version not in sdks_by_version:
+            fail("go_sdk: no SDKs found for version {} requested by".format(version), download_tag)
+        used_versions[version] = True
+        sdks = sdks_by_version[version]
+
+    go_download_sdk_rule(
+        name = name,
+        goos = goos,
+        goarch = goarch,
+        sdks = sdks,
+        experiments = download_tag.experiments,
+        patches = download_tag.patches,
+        patch_strip = download_tag.patch_strip,
+        urls = download_tag.urls,
+        version = download_tag.version,
+        strip_prefix = download_tag.strip_prefix,
+    )
+
 go_sdk_extra_kwargs = {
     # The choice of a host-compatible SDK is expressed in repository rule attribute values and
     # depends on host OS and architecture.

go/private/sdk.bzl

@@ -81,17 +81,7 @@ def _go_download_sdk_impl(ctx):
             ctx.report_progress("Finding latest Go version")
         else:
             ctx.report_progress("Finding Go SHA-256 sums")
-        ctx.download(
-            url = [
-                "https://go.dev/dl/?mode=json&include=all",
-                "https://golang.google.cn/dl/?mode=json&include=all",
-            ],
-            output = "versions.json",
-        )
-
-        data = ctx.read("versions.json")
-        ctx.delete("versions.json")
-        sdks_by_version = _parse_versions_json(data)
+        sdks_by_version = fetch_sdks_by_version(ctx)
 
         if not version:
             highest_version = None
@@ -582,6 +572,21 @@ def _parse_versions_json(data):
         for sdk in sdks
     }
 
+def fetch_sdks_by_version(ctx):
+    ctx.download(
+        url = [
+            "https://go.dev/dl/?mode=json&include=all",
+            "https://golang.google.cn/dl/?mode=json&include=all",
+        ],
+        output = "versions.json",
+    )
+    data = ctx.read("versions.json")
+
+    # module_ctx doesn't have delete, but its files are temporary anyway.
+    if hasattr(ctx, "delete"):
+        ctx.delete("versions.json")
+    return _parse_versions_json(data)
+
 def parse_version(version):
     """Parses a version string like "1.15.5" and returns a tuple of numbers or None"""
     l, r = 0, 0

tests/bcr/MODULE.bazel

@@ -19,6 +19,7 @@ local_path_override(
 )
 
 bazel_dep(name = "gazelle", version = "0.33.0")
+bazel_dep(name = "platforms", version = "1.0.0")
 bazel_dep(name = "protobuf", version = "3.19.6")
 bazel_dep(name = "rules_shell", version = "0.4.1")
 

tests/bcr/WORKSPACE.bzlmod

@@ -3,7 +3,3 @@ module(name = "other_module")
 bazel_dep(name = "rules_go", version = "")
 
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
-
-# Request an invalid SDK to verify that it isn't fetched since the test module registers a toolchain
-# that takes precedence.
-go_sdk.download(version = "3.0.0")