Compile cmd/pack from source

Author: fmeum

go.mod

@@ -1,6 +1,6 @@
 module github.com/bazelbuild/rules_go
 
-toolchain go1.23.6
+toolchain go1.25.0
 
 go 1.22.0
 

go/private/BUILD.sdk.bazel

@@ -80,13 +80,21 @@ go_tool_binary(
     exec_compatible_with = {exec_compatible_with},
     ldflags = "-X main.rulesGoStdlibPrefix={}".format(RULES_GO_STDLIB_PREFIX),
     sdk = ":go_sdk",
+    # The .exe suffix is required on Windows and harmless on other platforms.
+    # Output attributes are not configurable, so we use it everywhere.
+    out_pack = "pack.exe",
 )
 
 non_go_reset_target(
     name = "builder_reset",
     dep = ":builder",
 )
 
+non_go_reset_target(
+    name = "pack_reset",
+    dep = ":pack.exe",
+)
+
 # TODO(jayconrod): Gazelle depends on this file directly. This dependency
 # should be broken, and this rule should be folded into go_sdk.
 package_list(
@@ -98,6 +106,7 @@ package_list(
 
 declare_go_toolchains(
     builder = ":builder_reset",
+    pack = ":pack_reset",
     host_goos = "{goos}",
     sdk = ":go_sdk",
 )

go/private/actions/compilepkg.bzl

@@ -94,7 +94,7 @@ def emit_compilepkg(
         archives = archives + [go.coverdata]
 
     sdk = go.sdk
-    inputs_direct = (sources + embedsrcs + [sdk.package_list] +
+    inputs_direct = (sources + embedsrcs + [sdk.package_list, go.toolchain._pack] +
                      [archive.data.export_file for archive in archives])
     inputs_transitive = [sdk.headers, sdk.tools, go.stdlib.libs, headers]
     outputs = [out_lib, out_export]
@@ -103,6 +103,7 @@ def emit_compilepkg(
     shared_args.add_all(sources, before_each = "-src")
 
     compile_args = go.tool_args(go)
+    compile_args.add("-pack", go.toolchain._pack)
     compile_args.add_all(embedsrcs, before_each = "-embedsrc", expand_directories = False)
     compile_args.add_all(
         sources + [out_lib] + embedsrcs,
@@ -200,7 +201,6 @@ def emit_compilepkg(
     go.actions.run(
         inputs = depset(inputs_direct, transitive = inputs_transitive),
         outputs = outputs,
-        tools = [go.toolchain.sdk.go],
         mnemonic = "GoCompilePkgExternal" if is_external_pkg else "GoCompilePkg",
         executable = go.toolchain._builder,
         arguments = ["compilepkg", shared_args, compile_args],

go/private/common.bzl

@@ -211,15 +211,11 @@ COVERAGE_OPTIONS_DENYLIST = {
     "-fcoverage-mapping": None,
 }
 
-_RULES_GO_RAW_REPO_NAME = str(Label("//:unused"))[:-len("//:unused")]
-
-# When rules_go is the main repository and Bazel < 6 is used, the repo name does
-# not start with a "@", so we need to add it.
-RULES_GO_REPO_NAME = _RULES_GO_RAW_REPO_NAME if _RULES_GO_RAW_REPO_NAME.startswith("@") else "@" + _RULES_GO_RAW_REPO_NAME
+RULES_GO_REPO_NAME = str(Label("//:unused"))[:-len("//:unused")]
 RULES_GO_STDLIB_PREFIX = RULES_GO_REPO_NAME + "//stdlib:"
 
 # TODO: Remove the "and" once the rules_go repo itself uses Bzlmod.
-RULES_GO_IS_BZLMOD_REPO = _RULES_GO_RAW_REPO_NAME.lstrip("@") != "io_bazel_rules_go" and _RULES_GO_RAW_REPO_NAME.lstrip("@")
+RULES_GO_IS_BZLMOD_REPO = RULES_GO_REPO_NAME.lstrip("@") != "io_bazel_rules_go" and RULES_GO_REPO_NAME.lstrip("@")
 
 # Marks an action as supporting path mapping (--experimental_output_paths=strip).
 # See https://www.youtube.com/watch?v=Et1rjb7ixUU for more details.

go/private/go_toolchain.bzl

@@ -50,6 +50,7 @@ def _go_toolchain_impl(ctx):
 
             # Internal fields -- may be read by emit functions.
             _builder = ctx.executable.builder,
+            _pack = ctx.executable.pack,
         ),
     ]
 
@@ -63,6 +64,12 @@ go_toolchain = rule(
             executable = True,
             doc = "Tool used to execute most Go actions",
         ),
+        "pack": attr.label(
+            mandatory = True,
+            cfg = "exec",
+            executable = True,
+            doc = "Tool used to pack object files into archives",
+        ),
         "goos": attr.string(
             mandatory = True,
             doc = "Default target OS",
@@ -89,7 +96,7 @@ go_toolchain = rule(
     provides = [platform_common.ToolchainInfo],
 )
 
-def declare_go_toolchains(host_goos, sdk, builder):
+def declare_go_toolchains(host_goos, sdk, builder, pack):
     """Declares go_toolchain targets for each platform."""
     for p in PLATFORMS:
         if p.cgo:
@@ -111,6 +118,7 @@ def declare_go_toolchains(host_goos, sdk, builder):
             goarch = p.goarch,
             sdk = sdk,
             builder = builder,
+            pack = pack,
             link_flags = link_flags,
             cgo_link_flags = cgo_link_flags,
             tags = ["manual"],

go/private/rules/binary.bzl

@@ -497,7 +497,10 @@ set GOTOOLCHAIN=local
 set GO111MODULE=off
 set GOTELEMETRY=off
 set GOENV=off
-{go} build -o {out} -trimpath -ldflags \"-buildid='' {ldflags}\" {srcs}
+{go} build -o {out_pack} -trimpath -ldflags \"-buildid='' {ldflags}\" cmd/pack
+if %ERRORLEVEL% EQU 0 (
+  {go} build -o {out} -trimpath -ldflags \"-buildid='' {ldflags}\" {srcs}
+)
 set GO_EXIT_CODE=%ERRORLEVEL%
 RMDIR /S /Q "{gotmp}"
 MKDIR "{gotmp}"
@@ -506,6 +509,7 @@ exit /b %GO_EXIT_CODE%
             gotmp = gotmp.path.replace("/", "\\"),
             go = sdk.go.path.replace("/", "\\"),
             out = out.path,
+            out_pack = ctx.outputs.out_pack.path,
             srcs = " ".join([f.path for f in ctx.files.srcs]),
             ldflags = ctx.attr.ldflags,
         )
@@ -521,32 +525,35 @@ exit /b %GO_EXIT_CODE%
                 transitive = [sdk.headers, sdk.srcs, sdk.tools],
             ),
             toolchain = None,
-            outputs = [out, gotmp],
+            outputs = [out, ctx.outputs.out_pack, gotmp],
             mnemonic = "GoToolchainBinaryBuild",
         )
     else:
-        # -a flag instructs the compiler to not read from GOCACHE and force a rebuild.
-        # This provides extra safety in cases of unsandboxed execution.
+        # We do not use -a here as the cache drastically reduces the time spent
+        # on the second go build (roughly 50% faster).
         args = ctx.actions.args()
-        args.add("build")
-        args.add("-a")
-        args.add("-o", out)
-        args.add("-trimpath")
-        args.add("-ldflags", ctx.attr.ldflags, format = '-buildid="" %s')
+        args.add(ctx.outputs.out_pack)
+        args.add(out)
         args.add_all(ctx.files.srcs)
 
         ctx.actions.run_shell(
             command = """
-trap "HOME={HOME} GOROOT={GOROOT} {go} clean -cache" EXIT;
-HOME={HOME} {go} "$@" """.format(
+set -eu
+export HOME={HOME}
+trap "{go} clean -cache" EXIT;
+{go} build -trimpath -ldflags='-buildid="" {ldflags}' -o "$1" cmd/pack
+shift
+{go} build -trimpath -ldflags='-buildid="" {ldflags}' -o "$@"
+""".format(
                 go = sdk.go.path,
-                GOROOT = sdk.root_file.dirname,
+                ldflags = ctx.attr.ldflags,
                 # The value of GOCACHE/GOPATH are determined from HOME.
                 # We place them in the execroot to avoid dependency on `mktemp` and because we don't know
                 # a safe scratch space on all systems. Note that HOME must be an absolute path, otherwise the
                 # Go toolchain will write some outputs to the wrong place and the result will be uncacheable.
-                # We use a hardcoded UUID to prevent collisions with anything else under unsandboxed strategy.
-                HOME = "$(pwd)/_go_tool_binary-fake-home-85e96dea-541b-4188-8d13-5c2c42bdbd06",
+                # We include the output path of this action to prevent collisions with anything else,
+                # including differently configured versions of the same target, under an unsandboxed strategy.
+                HOME = "$(pwd)/_go_tool_binary-fake-home-" + out.path.replace("/", "_"),
             ),
             arguments = [args],
             tools = [sdk.go],
@@ -562,7 +569,7 @@ HOME={HOME} {go} "$@" """.format(
                 transitive = [sdk.headers, sdk.srcs, sdk.libs, sdk.tools],
             ),
             toolchain = None,
-            outputs = [out],
+            outputs = [out, ctx.outputs.out_pack],
             mnemonic = "GoToolchainBinaryBuild",
         )
 
@@ -586,6 +593,7 @@ go_tool_binary = rule(
         "ldflags": attr.string(
             doc = "Raw value to pass to go build via -ldflags without tokenization",
         ),
+        "out_pack": attr.output(),
     },
     executable = True,
     doc = """Used instead of go_binary for executables used in the toolchain.
@@ -594,6 +602,11 @@ go_tool_binary depends on tools and libraries that are part of the Go SDK.
 It does not depend on other toolchains. It can only compile binaries that
 just have a main package and only depend on the standard library and don't
 require build constraints.
+
+It is currently only used to build the `builder` tool maintained as part of
+rules_go as well as the `pack` tool provided by the Go SDK in source form
+only as of Go 1.25. Combining both builds into a single action drastically
+reduces the overall build time due to Go's own caching mechanism.
 """,
 )
 

go/private/rules/transition.bzl

@@ -344,6 +344,7 @@ non_go_reset_target = rule(
         "dep": attr.label(
             mandatory = True,
             cfg = non_go_tool_transition,
+            allow_files = True,
         ),
         "_allowlist_function_transition": attr.label(
             default = "@bazel_tools//tools/allowlists/function_transition_allowlist",

go/tools/builders/compilepkg.go

@@ -37,6 +37,7 @@ func compilePkg(args []string) error {
 
 	fs := flag.NewFlagSet("GoCompilePkg", flag.ExitOnError)
 	goenv := envFlags(fs)
+	var pack string
 	var unfilteredSrcs, coverSrcs, embedSrcs, embedLookupDirs, embedRoots, recompileInternalDeps multiFlag
 	var deps archiveMultiFlag
 	var importPath, packagePath, packageListPath, coverMode string
@@ -45,6 +46,7 @@ func compilePkg(args []string) error {
 	var gcFlags, asmFlags, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags quoteMultiFlag
 	var coverFormat string
 	var pgoprofile string
+	fs.StringVar(&pack, "pack", "", "Path of the pack tool.")
 	fs.Var(&unfilteredSrcs, "src", ".go, .c, .cc, .m, .mm, .s, or .S file to be filtered and compiled")
 	fs.Var(&coverSrcs, "cover", ".go file that should be instrumented for coverage (must also be a -src)")
 	fs.Var(&embedSrcs, "embedsrc", "file that may be compiled into the package with a //go:embed directive")
@@ -106,6 +108,7 @@ func compilePkg(args []string) error {
 
 	return compileArchive(
 		goenv,
+		pack,
 		importPath,
 		packagePath,
 		srcs,
@@ -137,6 +140,7 @@ func compilePkg(args []string) error {
 
 func compileArchive(
 	goenv *env,
+	pack string,
 	importPath string,
 	packagePath string,
 	srcs archiveSrcs,
@@ -456,7 +460,7 @@ func compileArchive(
 	// Pack .o and .syso files into the archive. These may come from cgo generated code,
 	// cgo dependencies (cdeps), windows resource file generation, or assembly.
 	if len(objFiles) > 0 {
-		if err := appendToArchive(goenv, outLinkObj, objFiles); err != nil {
+		if err := appendToArchive(goenv, pack, outLinkObj, objFiles); err != nil {
 			return err
 		}
 	}
@@ -531,9 +535,9 @@ func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPa
 	return goenv.runCommand(args)
 }
 
-func appendToArchive(goenv *env, outPath string, objFiles []string) error {
+func appendToArchive(goenv *env, pack, outPath string, objFiles []string) error {
 	// Use abs to work around long path issues on Windows.
-	args := goenv.goCmd("tool", "pack", "r", abs(outPath))
+	args := []string{pack, "r", abs(outPath)}
 	args = append(args, objFiles...)
 	return goenv.runCommand(args)
 }

go/tools/builders/stdlib.go

@@ -18,7 +18,6 @@ import (
 	"flag"
 	"fmt"
 	"go/build"
-	"runtime"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -171,21 +170,5 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`)
 	if err := goenv.runCommand(installArgs); err != nil {
 		return err
 	}
-
-	// Install the "cmd/pack" command from source if it's missing as it's no longer shipped with Go distributions
-	// after go1.25 https://github.com/golang/go/issues/74080
-	if _, err := os.Stat(filepath.Join(output, "pkg/tool", runtime.GOOS + "_" + runtime.GOARCH, "pack")); err != nil && os.IsNotExist(err) {
-		originalOS, originalARCH := os.Getenv("GOOS"), os.Getenv("GOARCH")
-		os.Setenv("GOOS", runtime.GOOS)
-		os.Setenv("GOARCH", runtime.GOARCH)
-		defer func() {
-			os.Setenv("GOOS", originalOS)
-			os.Setenv("GOARCH", originalARCH)
-		}()
-		toolArgs := goenv.goCmd("install", "-toolexec", abs(os.Args[0])+" filterbuildid", "cmd/pack")
-		if err := goenv.runCommand(toolArgs); err != nil {
-			return err
-		}
-	}
 	return nil
 }

tests/bcr/BUILD.bazel

@@ -81,7 +81,7 @@ go_test(
 sdk_transition_test(
     name = "wrap_sdk_test",
     binary = ":wrap_test",
-    sdk_version = "1.23.6",
+    sdk_version = "1.25.0",
 )
 
 go_library(