Compare commits
1 commit
main
...
ci/bazel-h
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ab152e78c |
12 changed files with 841 additions and 0 deletions
12
.bazelrc
12
.bazelrc
|
|
@ -3,6 +3,18 @@
|
|||
# .bazelversion.
|
||||
common --enable_bzlmod
|
||||
common --color=no
|
||||
# C/C++ (and so cgo, -race and the Go stdlib) builds only with MODULE.bazel's
|
||||
# pinned LLVM toolchain: no host-detected local_config_cc to fall back to,
|
||||
# which would key actions to the client's gcc and lld and fail remote links
|
||||
# on workers without lld (scripts/bazel_hermetic_cc_test.go).
|
||||
common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
|
||||
|
||||
# The pinned remote-worker host in every action key: //platforms:rbe_worker's
|
||||
# worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and
|
||||
# rbe-west's schedulers run an action only on a worker that measured that
|
||||
# host (platforms/BUILD.bazel). Key-affecting, so unconditional: local,
|
||||
# fork-cache and remote runs hash alike.
|
||||
build --extra_execution_platforms=//platforms:rbe_worker
|
||||
|
||||
# ICU policy (engdocs/ICU-POLICY.md): build go-mysql-server with its pure-Go
|
||||
# regex backend so no target links libicu. MODULE.bazel carries the matching
|
||||
|
|
|
|||
|
|
@ -349,6 +349,7 @@ filegroup(
|
|||
"//issueops:repo_files",
|
||||
"//journalops:repo_files",
|
||||
"//memoryops:repo_files",
|
||||
"//platforms:repo_files",
|
||||
"//plugins/beads:repo_files",
|
||||
"//schema:repo_files",
|
||||
"//scripts:repo_files",
|
||||
|
|
@ -366,6 +367,7 @@ filegroup(
|
|||
"//test/conformance:repo_files",
|
||||
"//test/docsync:repo_files",
|
||||
"//test/testmainconvention:repo_files",
|
||||
"//third_party/cc_toolchain:repo_files",
|
||||
"//third_party/patches:repo_files",
|
||||
"//tools/bazel:repo_files",
|
||||
"//tools/docsmint:repo_files",
|
||||
|
|
|
|||
143
MODULE.bazel
143
MODULE.bazel
|
|
@ -7,6 +7,149 @@ bazel_dep(name = "platforms", version = "1.1.0")
|
|||
bazel_dep(name = "rules_go", version = "0.63.0")
|
||||
bazel_dep(name = "gazelle", version = "0.53.0")
|
||||
bazel_dep(name = "rules_shell", version = "0.6.1")
|
||||
bazel_dep(name = "toolchains_llvm", version = "1.11.0")
|
||||
|
||||
# Loaded by @llvm_dist_linux_x86_64's BUILD file (toolchains_llvm's template).
|
||||
bazel_dep(name = "rules_cc", version = "0.2.17")
|
||||
|
||||
# Hermetic C/C++ toolchain, ported from gastownhall/gascity. rules_go builds
|
||||
# the Go stdlib under -race and every cgo package (mattn/go-sqlite3, gozstd,
|
||||
# ...) with the registered C toolchain, and every Go compile depends on the
|
||||
# stdlib, so a host-detected toolchain (local_config_cc: CC=/usr/bin/gcc,
|
||||
# -fuse-ld=lld only where the CLIENT has lld, gcc-version-specific generated
|
||||
# inputs) keys those actions to the machine that ran the analysis and fails
|
||||
# remotely on workers without lld. Instead: LLVM pinned by sha256, plus a
|
||||
# sysroot of pinned Ubuntu 24.04 packages (the RBE workers' release, so
|
||||
# binaries load the same glibc and libstdc++ there), byte-identical on every
|
||||
# Linux x86_64 host. No ICU: beads builds go-mysql-server with its pure-Go
|
||||
# regex backend (engdocs/ICU-POLICY.md), so nothing may link libicu. .bazelrc
|
||||
# turns host detection off so nothing falls back to the host compiler;
|
||||
# scripts/bazel_hermetic_cc_test.go guards both.
|
||||
deb_sysroot = use_repo_rule("//third_party/cc_toolchain:deb_sysroot.bzl", "deb_sysroot")
|
||||
|
||||
deb_sysroot(
|
||||
name = "cc_sysroot_noble_amd64",
|
||||
mirrors = [
|
||||
"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",
|
||||
"https://archive.ubuntu.com/ubuntu/",
|
||||
],
|
||||
packages = {
|
||||
"pool/main/g/gcc-13/libgcc-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "cd689db2691edaa10f37329307292796bb599e722e0505c79e14caaa1fe9a93a",
|
||||
"pool/main/g/gcc-13/libstdc++-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "ee5633e863e19c3381ed97842ce35ed32ede96a3d1ae4e94c051d3036fe21347",
|
||||
"pool/main/g/gcc-14/libgcc-s1_14.2.0-4ubuntu2~24.04.1_amd64.deb": "aa7fadbe33b78bcf99885318040601c550c208929565b179891d9a3cc2aa68cd",
|
||||
"pool/main/g/gcc-14/libstdc++6_14.2.0-4ubuntu2~24.04.1_amd64.deb": "a51f8de7829211db961a31f02158058ad1a95f92ac6d0a5dff6350e2821c54c0",
|
||||
"pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "e13d5fcc1b2a86f75bca8e0026a8e39f24fe97ca86e92be79991f8697eb1306f",
|
||||
"pool/main/g/glibc/libc6_2.39-0ubuntu8.9_amd64.deb": "ff5557d99b51f761c4b7c92368b9cc45565eda17df9bf9eb4b134d09825008be",
|
||||
"pool/main/l/linux/linux-libc-dev_6.8.0-146.146_amd64.deb": "1a7bb04132d01cd14bea3d615e5861096ca1f70c4659902eb14b36c21993cadc",
|
||||
},
|
||||
remove = [
|
||||
"etc",
|
||||
"usr/bin",
|
||||
"usr/sbin",
|
||||
"usr/share",
|
||||
"usr/lib/x86_64-linux-gnu/gconv",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/crtoffload*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/lib*san*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libatomic*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libbacktrace*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libgcov*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libgomp*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libitm*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libquadmath*",
|
||||
"usr/lib/gcc/x86_64-linux-gnu/13/libstdc++*.a",
|
||||
],
|
||||
)
|
||||
|
||||
LLVM_VERSION = "22.1.8"
|
||||
|
||||
llvm_dist = use_repo_rule("//third_party/cc_toolchain:llvm_dist.bzl", "llvm_dist")
|
||||
|
||||
llvm_dist(
|
||||
name = "llvm_dist_linux_x86_64",
|
||||
llvm_version = LLVM_VERSION,
|
||||
members = [
|
||||
"bin/clang",
|
||||
"bin/clang++",
|
||||
"bin/clang-22",
|
||||
"bin/clang-cpp",
|
||||
"bin/ld.lld",
|
||||
"bin/ld64.lld",
|
||||
"bin/lld",
|
||||
"bin/llvm-ar",
|
||||
"bin/llvm-as",
|
||||
"bin/llvm-cov",
|
||||
"bin/llvm-install-name-tool",
|
||||
"bin/llvm-nm",
|
||||
"bin/llvm-objcopy",
|
||||
"bin/llvm-objdump",
|
||||
"bin/llvm-profdata",
|
||||
"bin/llvm-ranlib",
|
||||
"bin/llvm-readelf",
|
||||
"bin/llvm-readobj",
|
||||
"bin/llvm-strip",
|
||||
"bin/llvm-symbolizer",
|
||||
"bin/wasm-ld",
|
||||
"include/c++",
|
||||
"include/x86_64-unknown-linux-gnu",
|
||||
"lib/clang/22/include",
|
||||
"lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtbegin.o",
|
||||
"lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtend.o",
|
||||
"lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.builtins.a",
|
||||
# `bazel coverage` links cgo binaries with -fprofile-instr-generate,
|
||||
# which pulls in compiler-rt's profile runtime (~170KB).
|
||||
"lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a",
|
||||
],
|
||||
sha256 = "df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384",
|
||||
strip_prefix = "LLVM-22.1.8-Linux-X64",
|
||||
stub_tools = [
|
||||
"clang-format",
|
||||
"clang-tidy",
|
||||
"clangd",
|
||||
"llvm-dwp",
|
||||
],
|
||||
urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz"],
|
||||
)
|
||||
|
||||
# Every other host (macOS arm64, Linux arm64) runs toolchains_llvm's stock
|
||||
# LLVM_VERSION release for its platform, pinned by sha256 in toolchains_llvm's
|
||||
# distribution table and fetched only there. Once any toolchain_root is set,
|
||||
# toolchains_llvm creates no stock distribution of its own and fails every
|
||||
# build on a host no root covers. Not hermetic there: no sysroot, so cgo
|
||||
# uses the host's headers and libraries, as before.
|
||||
llvm_host_dist = use_repo_rule("@toolchains_llvm//toolchain:rules.bzl", "llvm")
|
||||
|
||||
llvm_host_dist(
|
||||
name = "llvm_dist_host",
|
||||
llvm_version = LLVM_VERSION,
|
||||
# The bare rule reads the per-host dict; llvm_toolchain's macro would
|
||||
# fill it in from llvm_version.
|
||||
llvm_versions = {"": LLVM_VERSION},
|
||||
)
|
||||
|
||||
llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")
|
||||
llvm.toolchain(
|
||||
name = "llvm_toolchain",
|
||||
llvm_version = LLVM_VERSION,
|
||||
# The sysroot's libstdc++, linked dynamically as host-gcc builds did.
|
||||
stdlib = {"linux-x86_64": "dynamic-stdc++"},
|
||||
)
|
||||
llvm.toolchain_root(
|
||||
name = "llvm_toolchain",
|
||||
label = "@llvm_dist_linux_x86_64//:BUILD.bazel",
|
||||
targets = ["linux-x86_64"],
|
||||
)
|
||||
llvm.toolchain_root(
|
||||
name = "llvm_toolchain",
|
||||
label = "@llvm_dist_host//:BUILD.bazel",
|
||||
)
|
||||
llvm.sysroot(
|
||||
name = "llvm_toolchain",
|
||||
label = "@cc_sysroot_noble_amd64//:sysroot",
|
||||
targets = ["linux-x86_64"],
|
||||
)
|
||||
use_repo(llvm, "llvm_toolchain")
|
||||
|
||||
register_toolchains("@llvm_toolchain//:all")
|
||||
|
||||
go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
|
||||
go_sdk.download(
|
||||
|
|
|
|||
44
MODULE.bazel.lock
generated
44
MODULE.bazel.lock
generated
|
|
@ -20,6 +20,10 @@
|
|||
"https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1",
|
||||
"https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4",
|
||||
"https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d",
|
||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f",
|
||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca",
|
||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456",
|
||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101",
|
||||
|
|
@ -35,10 +39,13 @@
|
|||
"https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.42.1/source.json": "fcd4396b2df85f64f2b3bb436ad870793ecf39180f1d796f913cc9276d355309",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b",
|
||||
"https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
|
||||
"https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8",
|
||||
"https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e",
|
||||
|
|
@ -56,6 +63,8 @@
|
|||
"https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb",
|
||||
"https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f",
|
||||
"https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd",
|
||||
"https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8",
|
||||
"https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2",
|
||||
"https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8",
|
||||
"https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350",
|
||||
"https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a",
|
||||
|
|
@ -70,12 +79,17 @@
|
|||
"https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108",
|
||||
"https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46",
|
||||
"https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713",
|
||||
"https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f",
|
||||
"https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5",
|
||||
"https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075",
|
||||
"https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0",
|
||||
"https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000",
|
||||
"https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902",
|
||||
"https://bcr.bazel.build/modules/mboworks_bzl/0.5.1/MODULE.bazel": "20c921aded4f4e00a6128dad47a843371c196ba8987474c11c8ba7b9da8c16f1",
|
||||
"https://bcr.bazel.build/modules/mboworks_bzl/0.5.1/source.json": "3d9086bb203475113a16afb0ca0c9b97fdb37fc6e5a62b604b60ffc2450dc8c5",
|
||||
"https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74",
|
||||
"https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9",
|
||||
"https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92",
|
||||
"https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83",
|
||||
"https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4",
|
||||
"https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85",
|
||||
|
|
@ -144,6 +158,7 @@
|
|||
"https://bcr.bazel.build/modules/rules_go/0.63.0/source.json": "5bf37b45c440ac03abad0cbfa8b8b8dd89cc4e82bc8305f468b8436fa58bae78",
|
||||
"https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
|
||||
"https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86",
|
||||
"https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963",
|
||||
"https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31",
|
||||
"https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a",
|
||||
"https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6",
|
||||
|
|
@ -190,6 +205,7 @@
|
|||
"https://bcr.bazel.build/modules/rules_python/1.7.0/source.json": "028a084b65dcf8f4dc4f82f8778dbe65df133f234b316828a82e060d81bdce32",
|
||||
"https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c",
|
||||
"https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b",
|
||||
"https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592",
|
||||
"https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b",
|
||||
"https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c",
|
||||
"https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca",
|
||||
|
|
@ -199,13 +215,21 @@
|
|||
"https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66",
|
||||
"https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
|
||||
"https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c",
|
||||
"https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd",
|
||||
"https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c",
|
||||
"https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5",
|
||||
"https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216",
|
||||
"https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91",
|
||||
"https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f",
|
||||
"https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b",
|
||||
"https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468",
|
||||
"https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c",
|
||||
"https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658",
|
||||
"https://bcr.bazel.build/modules/toolchains_llvm/1.11.0/MODULE.bazel": "c0b9c103deeaf0fc103934ff0a189273b4f9b82cc1622c59b67e1c3a83d6b5e6",
|
||||
"https://bcr.bazel.build/modules/toolchains_llvm/1.11.0/source.json": "d06e53520101c3890a094e5849b0ff16dde3a6f6531f25c997f340ebff34d569",
|
||||
"https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
|
||||
"https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072",
|
||||
"https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3",
|
||||
"https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
|
||||
"https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
|
||||
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca",
|
||||
|
|
@ -490,6 +514,26 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"@@toolchains_llvm+//toolchain/extensions:distributions.bzl%llvm_distributions": {
|
||||
"general": {
|
||||
"bzlTransitiveDigest": "RfYsFLwRh0rVY8ReiYgbOEyr84zuBs7mbcHq3dPY3jI=",
|
||||
"usagesDigest": "vl2VXD2E16aV3RG06i5tx7yyYASHiQFjvsiNZSDzQIg=",
|
||||
"recordedInputs": [],
|
||||
"generatedRepoSpecs": {
|
||||
"llvm_distributions_data": {
|
||||
"repoRuleId": "@@toolchains_llvm+//toolchain/internal:distributions_repo.bzl%llvm_distributions_repo",
|
||||
"attributes": {
|
||||
"srcs": [
|
||||
"@@toolchains_llvm+//toolchain/distributions:pre_github.jsonc",
|
||||
"@@toolchains_llvm+//toolchain/distributions:github_legacy.jsonc",
|
||||
"@@toolchains_llvm+//toolchain/distributions:github.jsonc",
|
||||
"@@toolchains_llvm+//toolchain/distributions:extra.jsonc"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"facts": {
|
||||
|
|
|
|||
49
platforms/BUILD.bazel
Normal file
49
platforms/BUILD.bazel
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
# rbe_worker: the execution platform of every build (.bazelrc selects it
|
||||
# unconditionally with --extra_execution_platforms, so every action and test
|
||||
# executes on it and local, fork-cache and remote runs hash alike).
|
||||
#
|
||||
# Test actions exec host tools (bash, git, coreutils, ...) and load the
|
||||
# shared libraries of every cgo binary they link (glibc, libstdc++), so the
|
||||
# worker host is an input to every cached result. Its exec_properties enter
|
||||
# each action's key: worker-env is the sha256 of tools/rbe/worker-env.txt,
|
||||
# the manifest of the pinned worker host.
|
||||
#
|
||||
# rbe-west's OSS pool is shared with gastownhall/gascity, which owns the
|
||||
# worker image: its tools/rbe/blacksmith-worker.sh advertises the sha256 of
|
||||
# the manifest it measures on its own host (gascity tools/rbe/worker-env),
|
||||
# and the schedulers match worker-env exactly, so these actions run only on
|
||||
# the pinned host. tools/rbe/worker-env.txt is a byte-for-byte copy of
|
||||
# gascity's: when gascity moves its pin (a new runner image, a package or Go
|
||||
# upgrade), copy its manifest here and update this sha256 in the same
|
||||
# commit, which is a new key for every action.
|
||||
# scripts/bazel_hermetic_cc_test.go keeps the pin and the manifest in step.
|
||||
platform(
|
||||
name = "rbe_worker",
|
||||
exec_properties = {
|
||||
"worker-env": "sha256:f4d6d0460877031a71cd0ddb4dc67e789c4e54fc18c0d888fa818c84f8cf61ed",
|
||||
},
|
||||
parents = ["@bazel_tools//tools:host_platform"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---
|
||||
|
||||
filegroup(
|
||||
name = "repo_files",
|
||||
srcs = glob(
|
||||
["**"],
|
||||
exclude = [
|
||||
"**/*.db",
|
||||
"**/*.exe",
|
||||
"**/*.out",
|
||||
"**/*.prof",
|
||||
"**/*.pyc",
|
||||
"**/*.test",
|
||||
"**/__pycache__/**",
|
||||
"**/node_modules/**",
|
||||
],
|
||||
),
|
||||
visibility = ["//:__pkg__"],
|
||||
)
|
||||
|
||||
# --- end repo_files ---
|
||||
|
|
@ -18,6 +18,7 @@ go_test(
|
|||
"bazel_cache_mode_test.go",
|
||||
"bazel_farm_workflow_test.go",
|
||||
"bazel_fork_mode_test.go",
|
||||
"bazel_hermetic_cc_test.go",
|
||||
"bazel_policy_test.go",
|
||||
"bazel_rbe_prewarm_test.go",
|
||||
"bazel_skip_allowlist_test.go",
|
||||
|
|
@ -43,6 +44,7 @@ go_test(
|
|||
"pr_preflight_test.go",
|
||||
"pr_risk_bazel_coverage_test.go",
|
||||
"pull_dolt_image_test.go",
|
||||
"rbe_worker_env_test.go",
|
||||
"release_script_test.go",
|
||||
"required_suite_test.go",
|
||||
"test_script_test.go",
|
||||
|
|
|
|||
273
scripts/bazel_hermetic_cc_test.go
Normal file
273
scripts/bazel_hermetic_cc_test.go
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
package scripts_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// rules_go builds the Go stdlib (under -race) and every cgo package with the
|
||||
// registered C toolchain, and every Go compile depends on the stdlib, so the
|
||||
// C toolchain is an input of nearly every action key. A host-detected toolchain
|
||||
// (local_config_cc) puts the host's gcc path, its lld availability and
|
||||
// gcc-version-specific generated files into those keys, so developer, fork
|
||||
// and CI runs never share results, and a client with lld (which the workers
|
||||
// lack) fails every remote cgo/-race link. MODULE.bazel therefore registers LLVM and
|
||||
// a sysroot fetched by sha256 (byte-identical on every Linux x86_64 host),
|
||||
// and .bazelrc turns host detection off so nothing falls back to the host
|
||||
// compiler silently.
|
||||
|
||||
const (
|
||||
hermeticCCModuleFile = "MODULE.bazel"
|
||||
hermeticCCDetectOff = "--repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1"
|
||||
hermeticCCToolchains = `register_toolchains("@llvm_toolchain//:all")`
|
||||
hermeticCCSysrootLabel = "@cc_sysroot_noble_amd64//:sysroot"
|
||||
hermeticCCLLVMLabel = "@llvm_dist_linux_x86_64//:BUILD.bazel"
|
||||
// Every other host's stock LLVM release (toolchains_llvm's sha256 table).
|
||||
hermeticCCHostLLVMLabel = "@llvm_dist_host//:BUILD.bazel"
|
||||
)
|
||||
|
||||
var (
|
||||
sha256HexRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
// snapshot.ubuntu.com serves each timestamp's archive state forever; a
|
||||
// moving mirror (archive.ubuntu.com) may only follow it as a fallback.
|
||||
ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`)
|
||||
)
|
||||
|
||||
// moduleCall returns the argument text of the single top-level call
|
||||
// `<fn>(\n...\n)` in MODULE.bazel whose arguments contain marker.
|
||||
func moduleCall(module, fn, marker string) (string, error) {
|
||||
var found []string
|
||||
for _, m := range regexp.MustCompile(`(?ms)^`+regexp.QuoteMeta(fn)+`\(\n(.*?)^\)`).FindAllStringSubmatch(module, -1) {
|
||||
if strings.Contains(m[1], marker) {
|
||||
found = append(found, m[1])
|
||||
}
|
||||
}
|
||||
if len(found) != 1 {
|
||||
return "", errors.New(hermeticCCModuleFile + ": want exactly one " + fn + "(...) naming " + marker)
|
||||
}
|
||||
return found[0], nil
|
||||
}
|
||||
|
||||
// quotedPairs returns the "key": "value" entries of a Starlark dict body.
|
||||
func quotedPairs(s string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, m := range regexp.MustCompile(`"([^"]+)":\s*"([^"]*)"`).FindAllStringSubmatch(s, -1) {
|
||||
out[m[1]] = m[2]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func checkHermeticCCModule(module string) []error {
|
||||
var errs []error
|
||||
for _, want := range []string{
|
||||
`bazel_dep(name = "toolchains_llvm", `,
|
||||
hermeticCCToolchains,
|
||||
} {
|
||||
if !strings.Contains(module, want) {
|
||||
errs = append(errs, errors.New(hermeticCCModuleFile+" must contain "+want))
|
||||
}
|
||||
}
|
||||
for _, m := range regexp.MustCompile(`(?s)register_toolchains\((.*?)\)`).FindAllStringSubmatch(module, -1) {
|
||||
if strings.Contains(m[1], "local_config_cc") || strings.Contains(m[1], "@bazel_tools//tools/cpp") {
|
||||
errs = append(errs, errors.New(hermeticCCModuleFile+" registers a host-detected C/C++ toolchain: "+m[0]))
|
||||
}
|
||||
}
|
||||
|
||||
if sysroot, err := moduleCall(module, "llvm.sysroot", hermeticCCSysrootLabel); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else if !strings.Contains(sysroot, `targets = ["linux-x86_64"]`) {
|
||||
errs = append(errs, errors.New("llvm.sysroot must apply to linux-x86_64"))
|
||||
}
|
||||
if _, err := moduleCall(module, "llvm.toolchain_root", hermeticCCLLVMLabel); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
// toolchains_llvm creates no stock distribution once any toolchain_root
|
||||
// is set, and fails every build on a host no root covers ("LLVM toolchain
|
||||
// root missing"), so the other hosts (macOS, Linux arm64) need a root
|
||||
// without targets.
|
||||
if root, err := moduleCall(module, "llvm.toolchain_root", hermeticCCHostLLVMLabel); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else if strings.Contains(root, "targets") {
|
||||
errs = append(errs, errors.New("llvm.toolchain_root naming "+hermeticCCHostLLVMLabel+" must be the fallback (no targets)"))
|
||||
}
|
||||
if dist, err := moduleCall(module, "llvm_host_dist", `name = "llvm_dist_host"`); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else if !strings.Contains(dist, "llvm_version = LLVM_VERSION,") || !strings.Contains(dist, `llvm_versions = {"": LLVM_VERSION},`) {
|
||||
// The bare `llvm` rule reads llvm_versions (llvm_toolchain's macro
|
||||
// fills it in from llvm_version; use_repo_rule does not).
|
||||
errs = append(errs, errors.New("llvm_host_dist must set llvm_version and llvm_versions to LLVM_VERSION, whose per-host sha256 toolchains_llvm pins"))
|
||||
}
|
||||
|
||||
if dist, err := moduleCall(module, "llvm_dist", `name = "llvm_dist_linux_x86_64"`); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else if m := regexp.MustCompile(`sha256 = "([^"]*)"`).FindStringSubmatch(dist); m == nil || !sha256HexRE.MatchString(m[1]) {
|
||||
errs = append(errs, errors.New("llvm_dist must pin its archive by sha256"))
|
||||
}
|
||||
|
||||
if sysroot, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else {
|
||||
mirrors := regexp.MustCompile(`(?s)mirrors = \[\s*"([^"]+)"`).FindStringSubmatch(sysroot)
|
||||
if mirrors == nil || !ubuntuSnapshotRE.MatchString(mirrors[1]) {
|
||||
errs = append(errs, errors.New("deb_sysroot's first mirror must be an immutable snapshot.ubuntu.com/ubuntu/<timestamp>/ root"))
|
||||
}
|
||||
body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(sysroot)
|
||||
if body == nil {
|
||||
errs = append(errs, errors.New("deb_sysroot has no packages"))
|
||||
} else {
|
||||
pkgs := quotedPairs(body[1])
|
||||
if len(pkgs) == 0 {
|
||||
errs = append(errs, errors.New("deb_sysroot has no packages"))
|
||||
}
|
||||
for path, sum := range pkgs {
|
||||
if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) {
|
||||
errs = append(errs, errors.New("deb_sysroot package "+path+" must be an amd64 pool .deb pinned by sha256"))
|
||||
}
|
||||
// ICU policy (engdocs/ICU-POLICY.md): nothing links libicu,
|
||||
// so the sysroot offers no ICU to link against.
|
||||
if strings.Contains(path, "/icu/") {
|
||||
errs = append(errs, errors.New("deb_sysroot package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// checkHermeticCCBazelRC: host detection is off unconditionally (every
|
||||
// command, every config) and nothing hands actions a host compiler.
|
||||
func checkHermeticCCBazelRC(rc string) []error {
|
||||
var errs []error
|
||||
off := false
|
||||
for _, line := range strings.Split(rc, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 || strings.HasPrefix(fields[0], "#") {
|
||||
continue
|
||||
}
|
||||
command, config, _ := strings.Cut(fields[0], ":")
|
||||
for _, flag := range fields[1:] {
|
||||
if strings.HasPrefix(flag, "#") {
|
||||
break
|
||||
}
|
||||
name, value, _ := strings.Cut(flag, "=")
|
||||
envName, _, _ := strings.Cut(value, "=")
|
||||
switch {
|
||||
case flag == hermeticCCDetectOff && command == "common" && config == "":
|
||||
off = true
|
||||
case envName == "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN" && flag != hermeticCCDetectOff:
|
||||
errs = append(errs, errors.New(".bazelrc: "+line+" re-enables host C/C++ toolchain detection"))
|
||||
case (name == "--repo_env" || name == "--action_env" || name == "--host_action_env") &&
|
||||
(envName == "CC" || envName == "CXX" || envName == "BAZEL_USE_CPP_ONLY_TOOLCHAIN"):
|
||||
errs = append(errs, errors.New(".bazelrc: "+line+" points C/C++ builds at a host compiler"))
|
||||
case (name == "--extra_toolchains" || name == "--crosstool_top" || name == "--host_crosstool_top") &&
|
||||
(strings.Contains(value, "local_config_cc") || strings.Contains(value, "@bazel_tools//tools/cpp")):
|
||||
errs = append(errs, errors.New(".bazelrc: "+line+" selects the host-detected C/C++ toolchain"))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !off {
|
||||
errs = append(errs, errors.New(".bazelrc must set `common "+hermeticCCDetectOff+"` unconditionally"))
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func TestBazelHermeticCCToolchain(t *testing.T) {
|
||||
root := bazelPolicyRoot(t)
|
||||
for _, err := range checkHermeticCCModule(readPolicyFile(t, root, hermeticCCModuleFile)) {
|
||||
t.Error(err)
|
||||
}
|
||||
for _, err := range checkHermeticCCBazelRC(readPolicyFile(t, root, ".bazelrc")) {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBazelHermeticCCToolchainGuards(t *testing.T) {
|
||||
sum := strings.Repeat("ab", 32)
|
||||
module := `bazel_dep(name = "rules_go", version = "0.63.0")
|
||||
bazel_dep(name = "toolchains_llvm", version = "1.11.0")
|
||||
|
||||
deb_sysroot(
|
||||
name = "cc_sysroot_noble_amd64",
|
||||
mirrors = [
|
||||
"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",
|
||||
"https://archive.ubuntu.com/ubuntu/",
|
||||
],
|
||||
packages = {
|
||||
"pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "` + sum + `",
|
||||
},
|
||||
)
|
||||
|
||||
llvm_dist(
|
||||
name = "llvm_dist_linux_x86_64",
|
||||
sha256 = "` + sum + `",
|
||||
)
|
||||
|
||||
llvm_host_dist(
|
||||
name = "llvm_dist_host",
|
||||
llvm_version = LLVM_VERSION,
|
||||
llvm_versions = {"": LLVM_VERSION},
|
||||
)
|
||||
|
||||
llvm.toolchain_root(
|
||||
name = "llvm_toolchain",
|
||||
label = "` + hermeticCCLLVMLabel + `",
|
||||
targets = ["linux-x86_64"],
|
||||
)
|
||||
llvm.toolchain_root(
|
||||
name = "llvm_toolchain",
|
||||
label = "` + hermeticCCHostLLVMLabel + `",
|
||||
)
|
||||
llvm.sysroot(
|
||||
name = "llvm_toolchain",
|
||||
label = "` + hermeticCCSysrootLabel + `",
|
||||
targets = ["linux-x86_64"],
|
||||
)
|
||||
|
||||
` + hermeticCCToolchains + "\n"
|
||||
if errs := checkHermeticCCModule(module); len(errs) != 0 {
|
||||
t.Fatalf("good MODULE.bazel fixture: %v", errs)
|
||||
}
|
||||
for name, bad := range map[string]string{
|
||||
"no toolchains_llvm": strings.Replace(module, `bazel_dep(name = "toolchains_llvm", version = "1.11.0")`, "", 1),
|
||||
"no registration": strings.Replace(module, hermeticCCToolchains, "", 1),
|
||||
"second registration": module + `register_toolchains("@local_config_cc//:all")` + "\n",
|
||||
"no sysroot": strings.Replace(module, "llvm.sysroot(", "llvm.other(", 1),
|
||||
"host LLVM": strings.Replace(module, "llvm.toolchain_root(", "llvm.other(", 1),
|
||||
"no other-host root": strings.Replace(module, "label = \""+hermeticCCHostLLVMLabel+"\",", "", 1),
|
||||
"other-host root scoped": strings.Replace(module, "label = \""+hermeticCCHostLLVMLabel+"\",", "label = \""+hermeticCCHostLLVMLabel+"\",\n targets = [\"darwin-aarch64\"],", 1),
|
||||
"no other-host dist": strings.Replace(module, "llvm_host_dist(", "other(", 1),
|
||||
"other-host dist version": strings.Replace(module, "llvm_version = LLVM_VERSION", `llvm_version = "17.0.6"`, 1),
|
||||
"other-host dist versions": strings.Replace(module, `llvm_versions = {"": LLVM_VERSION},`, "", 1),
|
||||
"unpinned LLVM": strings.Replace(module, ` sha256 = "`+sum+`",`+"\n)", "\n)", 1),
|
||||
"moving mirror first": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",`, "", 1),
|
||||
"unpinned package": strings.Replace(module, `_amd64.deb": "`+sum+`"`, `_amd64.deb": ""`, 1),
|
||||
"ICU in sysroot": strings.Replace(module, " packages = {\n", " packages = {\n \"pool/main/i/icu/libicu74_74.2-1ubuntu3.1_amd64.deb\": \""+sum+"\",\n", 1),
|
||||
"sysroot other targets": strings.Replace(module, "label = \""+hermeticCCSysrootLabel+"\",\n targets = [\"linux-x86_64\"]", "label = \""+hermeticCCSysrootLabel+"\",\n targets = []", 1),
|
||||
} {
|
||||
if len(checkHermeticCCModule(bad)) == 0 {
|
||||
t.Errorf("%s: expected an error for MODULE.bazel fixture:\n%s", name, bad)
|
||||
}
|
||||
}
|
||||
|
||||
rc := "common --enable_bzlmod\ncommon " + hermeticCCDetectOff + "\ntest --test_output=errors\n"
|
||||
if errs := checkHermeticCCBazelRC(rc); len(errs) != 0 {
|
||||
t.Fatalf("good .bazelrc fixture: %v", errs)
|
||||
}
|
||||
for name, bad := range map[string]string{
|
||||
"detection on": strings.Replace(rc, "common "+hermeticCCDetectOff+"\n", "", 1),
|
||||
"only for build": strings.Replace(rc, "common "+hermeticCCDetectOff, "build "+hermeticCCDetectOff, 1),
|
||||
"only in a config": strings.Replace(rc, "common "+hermeticCCDetectOff, "common:ci "+hermeticCCDetectOff, 1),
|
||||
"commented out": strings.Replace(rc, "common "+hermeticCCDetectOff, "# common "+hermeticCCDetectOff, 1),
|
||||
"re-enabled": rc + "build:remote-exec --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0\n",
|
||||
"host CC": rc + "build --repo_env=CC=/usr/bin/gcc\n",
|
||||
"host CC action env": rc + "build --action_env=CC\n",
|
||||
"local_config_cc": rc + "build --extra_toolchains=@local_config_cc//:all\n",
|
||||
"bazel_tools crosstool": rc + "build --crosstool_top=@bazel_tools//tools/cpp:toolchain\n",
|
||||
} {
|
||||
if len(checkHermeticCCBazelRC(bad)) == 0 {
|
||||
t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
119
scripts/rbe_worker_env_test.go
Normal file
119
scripts/rbe_worker_env_test.go
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
package scripts_test
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Test actions exec host tools and load the shared libraries of every cgo
|
||||
// binary they link, so the remote worker host is an input to every result
|
||||
// rbe-west caches. //platforms:rbe_worker puts it into the action key: its
|
||||
// worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and
|
||||
// rbe-west's schedulers run an action only on a worker that advertises that
|
||||
// exact value. The OSS pool and its manifest are gastownhall/gascity's
|
||||
// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a copy of the
|
||||
// manifest so the pin is reviewable and moves only with it.
|
||||
|
||||
const (
|
||||
rbeWorkerPlatformBuild = "platforms/BUILD.bazel"
|
||||
rbeWorkerEnvManifest = "tools/rbe/worker-env.txt"
|
||||
rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker"
|
||||
)
|
||||
|
||||
var (
|
||||
rbeWorkerPlatformRE = regexp.MustCompile(`(?s)\nplatform\(\n name = "rbe_worker",\n(.*?)\n\)\n`)
|
||||
rbeExecPropsRE = regexp.MustCompile(`(?s)exec_properties = \{\n(.*?)\n \},`)
|
||||
rbeExecPropRE = regexp.MustCompile(`^\s*"([^"]+)": "([^"]*)",$`)
|
||||
workerEnvPinRE = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
)
|
||||
|
||||
// checkRBEWorkerPlatform: the rbe_worker platform's only exec property is
|
||||
// worker-env, pinned to the sha256 of manifest. Any other property would be
|
||||
// one no OSS worker advertises, and no action would ever schedule.
|
||||
func checkRBEWorkerPlatform(build, manifest string) []error {
|
||||
m := rbeWorkerPlatformRE.FindStringSubmatch(build)
|
||||
if m == nil {
|
||||
return []error{errors.New(rbeWorkerPlatformBuild + ": no platform rbe_worker")}
|
||||
}
|
||||
props := rbeExecPropsRE.FindStringSubmatch(m[1])
|
||||
if props == nil {
|
||||
return []error{errors.New(rbeWorkerPlatformBuild + ": platform rbe_worker has no exec_properties")}
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, line := range strings.Split(props[1], "\n") {
|
||||
e := rbeExecPropRE.FindStringSubmatch(line)
|
||||
if e == nil {
|
||||
return []error{errors.New(rbeWorkerPlatformBuild + ": unexpected exec_properties line " + line)}
|
||||
}
|
||||
got[e[1]] = e[2]
|
||||
}
|
||||
pin := got["worker-env"]
|
||||
if len(got) != 1 || !workerEnvPinRE.MatchString(pin) {
|
||||
return []error{errors.New(rbeWorkerPlatformBuild + ": rbe_worker exec_properties must be worker-env=sha256:<hex> alone")}
|
||||
}
|
||||
sum := sha256.Sum256([]byte(manifest))
|
||||
if want := "sha256:" + hex.EncodeToString(sum[:]); pin != want {
|
||||
return []error{errors.New(rbeWorkerPlatformBuild + " pins worker-env=" + pin + ", but " +
|
||||
rbeWorkerEnvManifest + " hashes to " + want + ": commit the manifest and its sha256 together")}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkRBEWorkerSelected: every command executes on rbe_worker (the flag is
|
||||
// key-affecting, so it may not depend on a config or command).
|
||||
func checkRBEWorkerSelected(rc string) error {
|
||||
for _, line := range strings.Split(rc, "\n") {
|
||||
if strings.TrimSpace(line) == "build "+rbeWorkerPlatformFlag {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New(".bazelrc must set `build " + rbeWorkerPlatformFlag + "` unconditionally")
|
||||
}
|
||||
|
||||
func TestRBEWorkerPlatformPinsWorkerEnv(t *testing.T) {
|
||||
root := bazelPolicyRoot(t)
|
||||
for _, err := range checkRBEWorkerPlatform(readPolicyFile(t, root, rbeWorkerPlatformBuild), readPolicyFile(t, root, rbeWorkerEnvManifest)) {
|
||||
t.Error(err)
|
||||
}
|
||||
if err := checkRBEWorkerSelected(readPolicyFile(t, root, ".bazelrc")); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRBEWorkerPlatformGuards(t *testing.T) {
|
||||
manifest := "arch x86_64\nos ubuntu 24.04\n"
|
||||
sum := sha256.Sum256([]byte(manifest))
|
||||
pin := "sha256:" + hex.EncodeToString(sum[:])
|
||||
build := "# header\nplatform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n parents = [\"@bazel_tools//tools:host_platform\"],\n)\n"
|
||||
if errs := checkRBEWorkerPlatform(build, manifest); len(errs) != 0 {
|
||||
t.Fatalf("good platform fixture: %v", errs)
|
||||
}
|
||||
for name, bad := range map[string][2]string{
|
||||
"manifest moved": {build, manifest + "pkg git 1\n"},
|
||||
"no platform": {strings.Replace(build, `name = "rbe_worker"`, `name = "other"`, 1), manifest},
|
||||
"extra property": {strings.Replace(build, " },", " \"pool\": \"x\",\n },", 1), manifest},
|
||||
"not a sha": {strings.Replace(build, pin, "latest", 1), manifest},
|
||||
} {
|
||||
if len(checkRBEWorkerPlatform(bad[0], bad[1])) == 0 {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
|
||||
rc := "common --enable_bzlmod\nbuild " + rbeWorkerPlatformFlag + "\n"
|
||||
if err := checkRBEWorkerSelected(rc); err != nil {
|
||||
t.Fatalf("good .bazelrc fixture: %v", err)
|
||||
}
|
||||
for name, bad := range map[string]string{
|
||||
"missing": "common --enable_bzlmod\n",
|
||||
"only in a config": strings.Replace(rc, "build "+rbeWorkerPlatformFlag, "build:remote-exec "+rbeWorkerPlatformFlag, 1),
|
||||
"commented out": strings.Replace(rc, "build "+rbeWorkerPlatformFlag, "# build "+rbeWorkerPlatformFlag, 1),
|
||||
} {
|
||||
if checkRBEWorkerSelected(bad) == nil {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
25
third_party/cc_toolchain/BUILD.bazel
vendored
Normal file
25
third_party/cc_toolchain/BUILD.bazel
vendored
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# Repository rules behind the hermetic C/C++ toolchain registered in
|
||||
# MODULE.bazel: llvm_dist.bzl (pinned LLVM) and deb_sysroot.bzl (pinned
|
||||
# Ubuntu sysroot). Ported from gastownhall/gascity; keep them in sync.
|
||||
|
||||
# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---
|
||||
|
||||
filegroup(
|
||||
name = "repo_files",
|
||||
srcs = glob(
|
||||
["**"],
|
||||
exclude = [
|
||||
"**/*.db",
|
||||
"**/*.exe",
|
||||
"**/*.out",
|
||||
"**/*.prof",
|
||||
"**/*.pyc",
|
||||
"**/*.test",
|
||||
"**/__pycache__/**",
|
||||
"**/node_modules/**",
|
||||
],
|
||||
),
|
||||
visibility = ["//:__pkg__"],
|
||||
)
|
||||
|
||||
# --- end repo_files ---
|
||||
80
third_party/cc_toolchain/deb_sysroot.bzl
vendored
Normal file
80
third_party/cc_toolchain/deb_sysroot.bzl
vendored
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""deb_sysroot: a C/C++ sysroot assembled from pinned Debian/Ubuntu packages.
|
||||
|
||||
Every package is fetched by sha256, so the sysroot is byte-identical on every
|
||||
host. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc,
|
||||
libstdc++ and ICU headers and link libraries that cgo targets need, instead
|
||||
of whatever the host has under /usr.
|
||||
"""
|
||||
|
||||
# Trims what no compile or link needs, then makes the sysroot self-contained:
|
||||
# package symlinks may be absolute (outside a chroot they would resolve
|
||||
# against the HOST), and glibc's linker scripts name the pre-usr-merge /lib
|
||||
# and /lib64, which the sysroot only has under /usr.
|
||||
_POSTPROCESS = """
|
||||
set -euo pipefail
|
||||
shopt -s globstar nullglob
|
||||
rm -rf -- $REMOVE
|
||||
find . -type l -lname '/*' -print0 | while IFS= read -r -d '' l; do
|
||||
ln -sfn "$(realpath -m --relative-to="$(dirname "$l")" "$PWD$(readlink "$l")")" "$l"
|
||||
done
|
||||
find . -xtype l -delete
|
||||
grep -rlI --include='*.so' 'GROUP' usr/lib | while IFS= read -r s; do
|
||||
sed -i -e 's@ /lib/@ /usr/lib/@g' -e 's@ /lib64/@ /usr/lib64/@g' "$s"
|
||||
done
|
||||
"""
|
||||
|
||||
def _deb_sysroot_impl(rctx):
|
||||
for path, sha256 in rctx.attr.packages.items():
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
deb = "_debs/" + name
|
||||
rctx.download(
|
||||
url = [mirror + path for mirror in rctx.attr.mirrors],
|
||||
output = deb,
|
||||
sha256 = sha256,
|
||||
)
|
||||
unpacked = "_debs/" + name + ".d"
|
||||
rctx.extract(deb, output = unpacked)
|
||||
data = [f for f in rctx.path(unpacked).readdir() if f.basename.startswith("data.tar")]
|
||||
if len(data) != 1:
|
||||
fail("%s: expected one data.tar.* member, found %s" % (name, data))
|
||||
rctx.extract(data[0], output = ".")
|
||||
rctx.delete("_debs")
|
||||
|
||||
res = rctx.execute(
|
||||
["bash", "-c", _POSTPROCESS],
|
||||
environment = {"REMOVE": " ".join(rctx.attr.remove)},
|
||||
)
|
||||
if res.return_code != 0:
|
||||
fail("deb_sysroot: post-processing failed:\n" + res.stderr)
|
||||
|
||||
rctx.file("BUILD.bazel", """\
|
||||
filegroup(
|
||||
name = "sysroot",
|
||||
srcs = glob(
|
||||
["**"],
|
||||
exclude = [
|
||||
"BUILD.bazel",
|
||||
"REPO.bazel",
|
||||
],
|
||||
),
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
""")
|
||||
return rctx.repo_metadata(reproducible = True)
|
||||
|
||||
deb_sysroot = repository_rule(
|
||||
implementation = _deb_sysroot_impl,
|
||||
attrs = {
|
||||
"packages": attr.string_dict(
|
||||
mandatory = True,
|
||||
doc = "Pool path (relative to each mirror) to the .deb's sha256.",
|
||||
),
|
||||
"mirrors": attr.string_list(
|
||||
mandatory = True,
|
||||
doc = "Archive roots tried in order; the first is an immutable snapshot.",
|
||||
),
|
||||
"remove": attr.string_list(
|
||||
doc = "Bash globs (globstar) dropped after unpacking: docs, unused runtimes and archives.",
|
||||
),
|
||||
},
|
||||
)
|
||||
54
third_party/cc_toolchain/llvm_dist.bzl
vendored
Normal file
54
third_party/cc_toolchain/llvm_dist.bzl
vendored
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""llvm_dist: the slice of an official LLVM release that C/C++ builds use.
|
||||
|
||||
LLVM's Linux release archive unpacks to ~12GB (MLIR, Flang, LLDB, clangd,
|
||||
...), and toolchains_llvm puts every bin/ file into each cgo action's inputs.
|
||||
This repository keeps clang, lld, the binutils replacements and the clang
|
||||
resource headers (~700MB), fetched from the same pinned archive by sha256,
|
||||
and lays them out as toolchains_llvm's own distribution repository does, so
|
||||
it plugs in through `llvm.toolchain_root`.
|
||||
"""
|
||||
|
||||
def _llvm_dist_impl(rctx):
|
||||
archive = "_llvm.tar.xz"
|
||||
rctx.download(url = rctx.attr.urls, output = archive, sha256 = rctx.attr.sha256)
|
||||
res = rctx.execute(
|
||||
["tar", "-xJf", archive, "--strip-components=1"] +
|
||||
[rctx.attr.strip_prefix + "/" + m for m in rctx.attr.members],
|
||||
timeout = 3600,
|
||||
)
|
||||
if res.return_code != 0:
|
||||
fail("llvm_dist: extracting %s failed:\n%s" % (rctx.attr.urls[0], res.stderr))
|
||||
rctx.delete(archive)
|
||||
|
||||
# toolchains_llvm symlinks a fixed tool list into its toolchain package;
|
||||
# tools this repository leaves out get a stub that fails loudly if run.
|
||||
for tool in rctx.attr.stub_tools:
|
||||
rctx.file(
|
||||
"bin/" + tool,
|
||||
"#!/bin/sh\necho '%s is not part of this LLVM slice (third_party/cc_toolchain/llvm_dist.bzl)' >&2\nexit 1\n" % tool,
|
||||
executable = True,
|
||||
)
|
||||
|
||||
major = rctx.attr.llvm_version.split(".")[0]
|
||||
rctx.file(
|
||||
"BUILD.bazel",
|
||||
rctx.read(Label("@toolchains_llvm//toolchain:BUILD.llvm_repo.tpl")).format(LLVM_VERSION = major),
|
||||
)
|
||||
return rctx.repo_metadata(reproducible = True)
|
||||
|
||||
llvm_dist = repository_rule(
|
||||
implementation = _llvm_dist_impl,
|
||||
attrs = {
|
||||
"llvm_version": attr.string(mandatory = True),
|
||||
"urls": attr.string_list(mandatory = True),
|
||||
"sha256": attr.string(mandatory = True),
|
||||
"strip_prefix": attr.string(mandatory = True),
|
||||
"members": attr.string_list(
|
||||
mandatory = True,
|
||||
doc = "Archive paths (below strip_prefix) to extract.",
|
||||
),
|
||||
"stub_tools": attr.string_list(
|
||||
doc = "bin/ tools toolchains_llvm links to but no action here runs.",
|
||||
),
|
||||
},
|
||||
)
|
||||
38
tools/rbe/worker-env.txt
Normal file
38
tools/rbe/worker-env.txt
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
arch x86_64
|
||||
dolt dolt version 2.1.8
|
||||
go go version go1.26.6 linux/amd64
|
||||
os ubuntu 24.04
|
||||
pkg bash 5.2.21-2ubuntu4
|
||||
pkg cmake 3.28.3-1build7
|
||||
pkg coreutils 9.4-3ubuntu6.1
|
||||
pkg diffutils 1:3.10-1build1
|
||||
pkg findutils 4.9.0-5build1
|
||||
pkg git 1:2.52.0-0ppa1~ubuntu24.04.3
|
||||
pkg grep 3.11-4build1
|
||||
pkg jq 1.7.1-3ubuntu0.24.04.2
|
||||
pkg libbz2-dev 1.0.8-5.1ubuntu0.1
|
||||
pkg libc6 2.39-0ubuntu8.6
|
||||
pkg libexpat1-dev 2.6.1-2ubuntu0.6
|
||||
pkg libffi-dev 3.4.6-1build1
|
||||
pkg libgcc-s1 14.2.0-4ubuntu2~24.04.1
|
||||
pkg libicu74 74.2-1ubuntu3.1
|
||||
pkg liblzma-dev 5.6.1+really5.4.5-1ubuntu0.3
|
||||
pkg liblzma5 5.6.1+really5.4.5-1ubuntu0.3
|
||||
pkg libncurses-dev 6.4+20240113-1ubuntu2.2
|
||||
pkg libreadline-dev 8.2-4build1
|
||||
pkg libsqlite3-dev 3.45.1-1ubuntu2.8
|
||||
pkg libstdc++6 14.2.0-4ubuntu2~24.04.1
|
||||
pkg libxml2 2.9.14+dfsg-1.3ubuntu3.9
|
||||
pkg libxml2-dev 2.9.14+dfsg-1.3ubuntu3.9
|
||||
pkg lsof 4.95.0-1build3
|
||||
pkg make 4.3-4.1build2
|
||||
pkg procps 2:4.0.4-4ubuntu3.2
|
||||
pkg python3-dev 3.12.3-0ubuntu2.1
|
||||
pkg sed 4.9-2build1
|
||||
pkg sqlite3 3.45.1-1ubuntu2.8
|
||||
pkg tar 1.35+dfsg-3build1
|
||||
pkg tmux 3.4-1ubuntu0.1
|
||||
pkg util-linux 2.39.3-9ubuntu6.4
|
||||
pkg xz-utils 5.6.1+really5.4.5-1ubuntu0.3
|
||||
pkg zlib1g 1:1.3.dfsg-3.1ubuntu2.2
|
||||
pkg zlib1g-dev 1:1.3.dfsg-3.1ubuntu2.2
|
||||
Loading…
Add table
Add a link
Reference in a new issue