Skip to content

Commit aeaede0

Browse files
alexeaglekarenwuz
authored andcommitted
feat(bazel): wire up prebuilt protoc toolchain (#24115)
Upstreamed from my https://github.com/aspect-build/toolchains_protoc which provided this feature outside the protobuf repo. Adds: - private API: module_extension + tag `protoc.prebuilt_toolchain` to provide and register toolchain for bzlmod users - WORKSPACE users can live with the inconvenience of having to create toolchains themselves - example of proto_library rule with intentionally non-functional cc toolchain, exercising the feature This needs work with @thesayyn to: - constrain users to choosing a protoc version agreeing with design doc https://docs.google.com/document/d/16N-eU-0zHbWmxEuaUAIFTJwQLekIXZ8zIC3K4XDr20E/edit - also register lang toolchains when the flag is enabled Work towards #19558 Closes #24115 COPYBARA_INTEGRATE_REVIEW=#24115 from protocolbuffers:alexeagle/use_prebuilt_toolchain b18667d PiperOrigin-RevId: 844859103
1 parent 35d91d3 commit aeaede0

File tree

17 files changed

+357
-19
lines changed

17 files changed

+357
-19
lines changed

.github/workflows/release_prep.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ mkdir -p ${PREFIX}/bazel/private
3737
cat >${INTEGRITY_FILE} <<EOF
3838
"Generated during release by release_prep.sh"
3939
40+
RELEASE_VERSION="${TAG}"
4041
RELEASED_BINARY_INTEGRITY = $(
4142
curl -s https://api.github.com/repos/protocolbuffers/protobuf/releases/tags/${TAG} \
4243
| jq -f <(echo "$filter_releases")

.github/workflows/test_bazel.yml

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,12 @@ jobs:
2929
runner: [ ubuntu, windows, macos ]
3030
bazelversion: [ '7.6.1', '8.0.0' ]
3131
bzlmod: [ true, false ]
32-
toolchain_resolution: [ "", "--incompatible_enable_proto_toolchain_resolution=true" ]
32+
toolchain_resolution:
33+
# Default flags, uses from-source protoc
34+
- ""
35+
# still uses from-source protoc unless
36+
# --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc is set
37+
- "--incompatible_enable_proto_toolchain_resolution=true"
3338
runs-on: ${{ matrix.runner }}-latest
3439
name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Examples ${{ matrix.runner }} ${{ matrix.bazelversion }}${{ matrix.bzlmod && ' (bzlmod)' || '' }} ${{ matrix.toolchain_resolution && ' (toolchain resolution)' || '' }}
3540
steps:
@@ -72,3 +77,14 @@ jobs:
7277
bash: >
7378
cd examples;
7479
bazel build //... @com_google_protobuf-examples-with-hyphen//... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} --enable_workspace=${{ !matrix.bzlmod }} ${{ matrix.toolchain_resolution }};
80+
81+
- name: Prebuilt test
82+
if: ${{ matrix.bzlmod && (!matrix.continuous-only || inputs.continuous-run) }}
83+
uses: protocolbuffers/protobuf-ci/bazel@v5
84+
with:
85+
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
86+
bazel-cache: examples-prebuilt-${{ matrix.bazelversion }}-${{ matrix.toolchain_resolution }}
87+
version: ${{ matrix.bazelversion }}
88+
bash: >
89+
cd examples/example_without_cc_toolchain;
90+
bazel build //... $BAZEL_FLAGS --enable_bzlmod=true ${{ matrix.toolchain_resolution }};

MODULE.bazel

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,27 @@ register_toolchains(
115115
dev_dependency = True,
116116
)
117117

118-
# Proto toolchains
118+
# Define toolchains that use pre-built protoc binaries.
119+
prebuilt_protoc = use_extension("//bazel/private:prebuilt_protoc_extension.bzl", "protoc")
120+
use_repo(
121+
prebuilt_protoc,
122+
"prebuilt_protoc.linux_aarch_64",
123+
"prebuilt_protoc.osx_aarch_64",
124+
"prebuilt_protoc.linux_ppcle_64",
125+
"prebuilt_protoc.linux_s390_64",
126+
"prebuilt_protoc.linux_x86_32",
127+
"prebuilt_protoc.linux_x86_64",
128+
"prebuilt_protoc.osx_x86_64",
129+
"prebuilt_protoc.win32",
130+
"prebuilt_protoc.win64",
131+
)
132+
133+
# However this registration only matters if the config_setting for prefer_prebuilt_protoc is true,
134+
# using --@protobuf//bazel/toolchains:prefer_prebuilt_protoc
135+
register_toolchains("//bazel/private/toolchains/prebuilt:all")
136+
137+
# From-source protobuf toolchains
138+
# Fallback if nothing is already registered
119139
register_toolchains("//bazel/private/toolchains:all")
120140

121141
SUPPORTED_PYTHON_VERSIONS = [
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
"Module extensions for use under bzlmod"
2+
3+
load("@bazel_skylib//lib:modules.bzl", "modules")
4+
load("//bazel/private:prebuilt_protoc_toolchain.bzl", "prebuilt_protoc_repo")
5+
load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
6+
7+
def create_all_toolchain_repos(name = "prebuilt_protoc"):
8+
for platform in PROTOBUF_PLATFORMS.keys():
9+
prebuilt_protoc_repo(
10+
# We must replace hyphen with underscore to workaround rules_python py_proto_library constraint
11+
name = ".".join([name, platform.replace("-", "_")]),
12+
platform = platform,
13+
)
14+
15+
protoc = modules.as_extension(create_all_toolchain_repos)
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
"Repository rule that downloads a pre-compiled protoc from our official release for a single platform."
2+
3+
load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
4+
load(":prebuilt_tool_integrity.bzl", "RELEASED_BINARY_INTEGRITY", "RELEASE_VERSION")
5+
6+
def release_version_to_artifact_name(release_version, platform):
7+
# versions have a "v" prefix like "v28.0"
8+
stripped_version = release_version.removeprefix("v")
9+
10+
# release candidate versions like "v29.0-rc3" have artifact names
11+
# like "protoc-29.0-rc-3-osx-x86_64.zip"
12+
artifact_version = stripped_version.replace("rc", "rc-")
13+
14+
return "{}-{}-{}.zip".format(
15+
"protoc",
16+
artifact_version,
17+
platform,
18+
)
19+
20+
def _prebuilt_protoc_repo_impl(rctx):
21+
filename = release_version_to_artifact_name(
22+
RELEASE_VERSION,
23+
rctx.attr.platform,
24+
)
25+
rctx.download_and_extract(
26+
url = "https://github.com/protocolbuffers/protobuf/releases/download/{}/{}".format(
27+
RELEASE_VERSION,
28+
filename,
29+
),
30+
sha256 = RELEASED_BINARY_INTEGRITY[filename],
31+
)
32+
33+
rctx.file("BUILD.bazel", """\
34+
# Generated by @protobuf//bazel/private:prebuilt_protoc_toolchain.bzl
35+
load("@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", "proto_toolchain")
36+
37+
package(default_visibility = ["//visibility:public"])
38+
39+
proto_toolchain(
40+
name = "prebuilt_protoc_toolchain",
41+
proto_compiler = "{protoc_label}",
42+
)
43+
""".format(
44+
protoc_label = "bin/protoc.exe" if rctx.attr.platform.startswith("win") else "bin/protoc",
45+
))
46+
47+
prebuilt_protoc_repo = repository_rule(
48+
doc = "Download a pre-built protoc and create a concrete toolchains for it",
49+
implementation = _prebuilt_protoc_repo_impl,
50+
attrs = {
51+
"platform": attr.string(
52+
doc = "A platform that protobuf ships a release for",
53+
mandatory = True,
54+
values = PROTOBUF_PLATFORMS.keys(),
55+
),
56+
},
57+
)

bazel/private/prebuilt_tool_integrity.bzl

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,19 @@
33
This file contents are entirely replaced during release publishing, by .github/workflows/release_prep.sh
44
so that the integrity of the prebuilt tools is included in the release artifact.
55
6-
The checked in content is only here to allow load() statements in the sources to resolve.
6+
The checked in content is only here to allow load() statements in the sources to resolve, and permit local testing.
77
"""
88

9-
# Create a mapping for every tool name to the hash of /dev/null
10-
NULLSHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
11-
RELEASED_BINARY_INTEGRITY = {
12-
"-".join([
13-
"protoc",
14-
os,
15-
arch,
16-
]): NULLSHA
17-
for [os, arch] in {
18-
"linux": ["aarch_64", "x86_64"],
19-
}
20-
}
9+
# An arbitrary version of protobuf that includes pre-built binaries.
10+
# See /examples/example_without_cc_toolchain which uses this for testing.
11+
# TODO: add some automation to update this version occasionally.
12+
_TEST_VERSION = "v33.0"
13+
_TEST_SHAS = dict()
14+
15+
# Add a couple platforms which are commonly used for testing.
16+
_TEST_SHAS["protoc-33.0-linux-x86_64.zip"] = "d99c011b799e9e412064244f0be417e5d76c9b6ace13a2ac735330fa7d57ad8f"
17+
_TEST_SHAS["protoc-33.0-osx-aarch_64.zip"] = "3cf55dd47118bd2efda9cd26b74f8bbbfcf5beb1bf606bc56ad4c001b543f6d3"
18+
_TEST_SHAS["protoc-33.0-win64.zip"] = "3742cd49c8b6bd78b6760540367eb0ff62fa70a1032e15dafe131bfaf296986a"
19+
20+
RELEASE_VERSION = _TEST_VERSION
21+
RELEASED_BINARY_INTEGRITY = _TEST_SHAS

bazel/private/proto_library_rule.bzl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ def _proto_library_impl(ctx):
113113
default_runfiles = ctx.runfiles(), # empty
114114
data_runfiles = data_runfiles,
115115
),
116+
OutputGroupInfo(_validation = ctx.attr._authenticity_validation[OutputGroupInfo]._validation),
116117
]
117118

118119
def _process_srcs(ctx, srcs, import_prefix, strip_import_prefix):
@@ -375,6 +376,10 @@ List of files containing extension declarations. This attribute is only allowed
375376
for use with MessageSet.
376377
""",
377378
),
379+
"_authenticity_validation": attr.label(
380+
default = "//bazel/private/toolchains/prebuilt:authenticity_validation",
381+
doc = "Validate that the binary registered on the toolchain is produced by protobuf team",
382+
),
378383
# buildifier: disable=attr-license (calling attr.license())
379384
"licenses": attr.license() if hasattr(attr, "license") else attr.string_list(),
380385
"_experimental_proto_descriptor_sets_include_source_info": attr.label(
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
"""Create lazy definitions to reference the pre-built protoc toolchains.
2+
3+
Ensures that Bazel only downloads required binaries for selected toolchains.
4+
In particular, see comment below on the toolchain#toolchain attribute.
5+
"""
6+
7+
load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
8+
load(":protoc_authenticity.bzl", "protoc_authenticity")
9+
10+
[
11+
toolchain(
12+
name = "{}_toolchain".format(platform.replace("-", "_")),
13+
exec_compatible_with = meta["compatible_with"],
14+
# Toolchain resolution will only permit this toolchain if the config_setting for prefer_prebuilt_protoc is true,
15+
target_settings = ["@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc.flag_set"],
16+
# Bazel does not follow this attribute during analysis, so the referenced repo
17+
# will only be fetched if this toolchain is selected.
18+
toolchain = "@prebuilt_protoc.{}//:prebuilt_protoc_toolchain".format(platform.replace("-", "_")),
19+
toolchain_type = "@com_google_protobuf//bazel/private:proto_toolchain_type",
20+
)
21+
for platform, meta in PROTOBUF_PLATFORMS.items()
22+
]
23+
24+
# Support verification of user-registered toolchains
25+
protoc_authenticity(
26+
name = "authenticity_validation",
27+
fail_on_mismatch = select({
28+
"//bazel/toolchains:allow_nonstandard_protoc.flag_set": False,
29+
"//conditions:default": True,
30+
}),
31+
visibility = ["//visibility:public"],
32+
)
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"Validate that the protoc binary is authentic and not spoofed by a malicious actor."
2+
3+
load("//bazel/common:proto_common.bzl", "proto_common")
4+
load("//bazel/private:prebuilt_tool_integrity.bzl", "RELEASE_VERSION")
5+
load("//bazel/private:toolchain_helpers.bzl", "toolchains")
6+
7+
def _protoc_authenticity_impl(ctx):
8+
# When this flag is disabled, then users have no way to replace the protoc binary with their own toolchain registration.
9+
# Therefore there's no validation to perform.
10+
if not proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION:
11+
return [OutputGroupInfo(_validation = depset())]
12+
toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN]
13+
if not toolchain:
14+
fail("Protocol compiler toolchain could not be resolved.")
15+
proto_lang_toolchain_info = toolchain.proto
16+
validation_output = ctx.actions.declare_file("validation_output.txt")
17+
18+
ctx.actions.run_shell(
19+
mnemonic = "ProtocAuthenticityCheck",
20+
outputs = [validation_output],
21+
tools = [proto_lang_toolchain_info.proto_compiler],
22+
command = """\
23+
{protoc} --version > {validation_output}
24+
grep -q -e "-dev$" {validation_output} && {{
25+
echo 'WARNING: Detected a development version of protoc.
26+
Development versions are not validated for authenticity.
27+
To ensure a secure build, please use a released version of protoc.'
28+
exit 0
29+
}}
30+
grep -q "^libprotoc {RELEASE_VERSION}" {validation_output} || {{
31+
echo '{severity}: protoc version does not match protobuf Bazel module; we do not support this.
32+
It is considered undefined behavior that is expected to break in the future even if it appears to work today.'
33+
echo '{suppression_note}'
34+
echo 'Expected: libprotoc {RELEASE_VERSION}'
35+
echo -n 'Actual: '
36+
cat {validation_output}
37+
exit {mismatch_exit_code}
38+
}} >&2
39+
""".format(
40+
protoc = proto_lang_toolchain_info.proto_compiler.executable.path,
41+
validation_output = validation_output.path,
42+
RELEASE_VERSION = RELEASE_VERSION.removeprefix("v"),
43+
suppression_note = (
44+
"To suppress this error, run Bazel with --@com_google_protobuf//bazel/toolchains:allow_nonstandard_protoc" if ctx.attr.fail_on_mismatch else ""
45+
),
46+
mismatch_exit_code = 1 if ctx.attr.fail_on_mismatch else 0,
47+
severity = "ERROR" if ctx.attr.fail_on_mismatch else "INFO",
48+
),
49+
)
50+
return [OutputGroupInfo(_validation = depset([validation_output]))]
51+
52+
protoc_authenticity = rule(
53+
implementation = _protoc_authenticity_impl,
54+
fragments = ["proto"],
55+
attrs = {
56+
"fail_on_mismatch": attr.bool(
57+
default = True,
58+
doc = "If true, the build will fail when the protoc binary does not match the expected version.",
59+
),
60+
} | toolchains.if_legacy_toolchain({
61+
"_proto_compiler": attr.label(
62+
cfg = "exec",
63+
executable = True,
64+
allow_files = True,
65+
default = "//src/google/protobuf/compiler:protoc_minimal",
66+
),
67+
}),
68+
toolchains = toolchains.use_toolchain(toolchains.PROTO_TOOLCHAIN),
69+
)

bazel/toolchains/BUILD

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
2+
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
23

3-
package(default_applicable_licenses = ["//:license"])
4+
package(
5+
default_applicable_licenses = ["//:license"],
6+
default_visibility = ["//visibility:public"],
7+
)
48

59
bzl_library(
610
name = "proto_toolchain_bzl",
711
srcs = [
812
"proto_toolchain.bzl",
913
],
10-
visibility = ["//visibility:public"],
1114
deps = [
1215
"//bazel/private:proto_toolchain_rule_bzl",
1316
"//bazel/private:toolchain_helpers_bzl",
@@ -19,7 +22,6 @@ bzl_library(
1922
srcs = [
2023
"proto_lang_toolchain.bzl",
2124
],
22-
visibility = ["//visibility:public"],
2325
deps = [
2426
"//bazel/common:proto_common_bzl",
2527
"//bazel/private:proto_lang_toolchain_rule_bzl",
@@ -39,3 +41,26 @@ filegroup(
3941
"//bazel:__pkg__",
4042
],
4143
)
44+
45+
# The public API users set
46+
bool_flag(
47+
name = "prefer_prebuilt_protoc",
48+
# TODO: this should be True after the feature is vetted with some adoption
49+
build_setting_default = False,
50+
)
51+
52+
config_setting(
53+
name = "prefer_prebuilt_protoc.flag_set",
54+
flag_values = {":prefer_prebuilt_protoc": "true"},
55+
)
56+
57+
# The public API users set to disable the validation action failing.
58+
bool_flag(
59+
name = "allow_nonstandard_protoc",
60+
build_setting_default = False,
61+
)
62+
63+
config_setting(
64+
name = "allow_nonstandard_protoc.flag_set",
65+
flag_values = {":allow_nonstandard_protoc": "true"},
66+
)

0 commit comments

Comments
 (0)