diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 92dc96635..8bb03ddd1 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -39,15 +39,18 @@ runs: # unverified repo contents cache is off (write-bazelrc.sh) and the Bazel # binary is not cached, so a poisoned entry (a cache written by an # untrusted run) cannot change what executes. v3: v2 also held Bazelisk's - # download and the contents cache. + # download and the contents cache. v4: MODULE.bazel joins the key because + # the lock does not record non-registry repository rules (llvm_dist), so + # v3 kept restoring the 1.94 GB LLVM .tar.xz after the switch to the + # 149 MB slice. - name: Restore Bazel runner cache id: restore uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/bazel-ci-cache - key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + key: bazel-repo-v4-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }} restore-keys: | - bazel-repo-v3-${{ runner.os }}- + bazel-repo-v4-${{ runner.os }}- - name: Install Bazelisk shell: bash diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh index 07bed3644..0101669b4 100755 --- a/.github/scripts/bazel-gate.sh +++ b/.github/scripts/bazel-gate.sh @@ -41,7 +41,7 @@ case "${1:-}" in skips=() if [[ "$valid" == true ]]; then case "$mode" in - skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE BAZEL_CMD_DOLT) ;; + skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_RELEASE_CROSS BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE BAZEL_CMD_DOLT) ;; local) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE BAZEL_CMD_DOLT) ;; cache) skips+=(BAZEL_EMBEDDED BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; esac diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 625466eb2..6564862f0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,8 +1,9 @@ # The Bazel lanes: the only CI run of the Linux Go test tiers. Runs # `bazel test //... --config=ci`, the PR-core selection (race, -short, dolt -# skip), plus one job per tier (pure-Go and js/wasm, the dolt-server lane, -# the embedded-Dolt, proxied-server and server-Dolt storage tiers that -# pr-risk.yml used to run, integration, the cmd/bd Dolt-server tier). +# skip), plus one job per tier (pure-Go and js/wasm, the release +# cross-compile, the dolt-server lane, the embedded-Dolt, proxied-server and +# server-Dolt storage tiers that pr-risk.yml used to run, integration, the +# cmd/bd Dolt-server tier). # # PRs and merge groups run it through pr.yml's `bazel` job (workflow_call). # The legacy go test jobs these lanes mirrored are retired (ga-96smfk.22), @@ -24,9 +25,10 @@ # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate # requires the call's aggregate result (BAZEL) and bazel-test, bazel-pure, -# bazel-embedded, bazel-integration, bazel-doltserver, bazel-proxied and -# bazel-server-storage (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, -# BAZEL_INTEGRATION, BAZEL_DOLTSERVER, BAZEL_PROXIED, BAZEL_SERVER_STORAGE); +# bazel-release-cross, bazel-embedded, bazel-integration, bazel-doltserver, +# bazel-proxied and bazel-server-storage (BAZEL_TEST, BAZEL_PURE, +# BAZEL_RELEASE_CROSS, BAZEL_EMBEDDED, BAZEL_INTEGRATION, BAZEL_DOLTSERVER, +# BAZEL_PROXIED, BAZEL_SERVER_STORAGE); # bazel-cmd-dolt (the cmd/bd Dolt-server tier, BAZEL_CMD_DOLT) is advisory: # job-level continue-on-error, and pr.yml requires it only while its # BAZEL_CMD_DOLT_REQUIRED flag is "true" (pr.yml then also passes @@ -167,8 +169,8 @@ # gates no longer consume this artifact (they build their own bd); it is # kept for the F3.5.3 SHA256SUMS comparison and for debugging. # -# Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about -# 2 GB (Bazel's content-addressable --repository_cache, re-hashed on every +# Runner cache: one entry per (.bazelversion, MODULE.bazel, MODULE.bazel.lock) +# triple, about 1 GB (Bazel's content-addressable --repository_cache, re-hashed on every # hit; never the Bazel binary or the repo contents cache), written # only by a push to main that missed it. No --disk_cache: remote runs use the # remote cache, and local runs would need about 7 GB more. @@ -330,6 +332,9 @@ on: bazel-pure: description: "bazel-pure's job.status" value: ${{ jobs.bazel-pure.outputs.result }} + bazel-release-cross: + description: "bazel-release-cross's job.status" + value: ${{ jobs.bazel-release-cross.outputs.result }} bazel-embedded: description: "bazel-embedded's job.status" value: ${{ jobs.bazel-embedded.outputs.result }} @@ -931,15 +936,15 @@ jobs: retention-days: 7 # Only a push to main that missed the exact key writes the runner cache. - # Its content (--repository_cache only, never the rc - # or credentials) is fixed by .bazelversion and MODULE.bazel.lock, so one - # entry per lock is enough and later pushes do not churn the quota. + # Its content (--repository_cache only, never the rc or credentials) is + # fixed by .bazelversion, MODULE.bazel and MODULE.bazel.lock, so one + # entry per triple is enough and later pushes do not churn the quota. - name: Save Bazel runner cache if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/bazel-ci-cache - key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + key: bazel-repo-v4-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }} # A redacted, schema'd summary of this invocation's BEP, exec log and # profile: timing, per-mnemonic cache classes, cold actions, test @@ -1107,19 +1112,6 @@ jobs: --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-build-pure.log.zst" --experimental_build_event_upload_strategy=local - # Every release target (scripts/ci/release-targets.txt) on every PR, so - # a platform-gating mistake is caught here instead of at tag time - # (#5662): the whole tree, cgo off, one remote `bazel build - # --platforms=...` per target (the script's header has the coverage - # boundary: the cgo release builds are not compiled here). nogo - # (//tools/nogo) validates every one of those compiles, so this is also - # the lint and vet gate for files only other platforms build - # (//go:build windows, darwin, !linux ...; engdocs/LINTING.md). - - name: Cross-compile every release target (--config=release-cross) - run: | - set -euo pipefail - ./scripts/ci/bazel-release-cross-compile.sh - # `go build` with CGO_ENABLED=0 rejects a pure binary whose imports reach # dolt's cgo-only store/nbs (gozstd). Bazel compiles gozstd's pure stubs # instead, whose init panics (third_party/patches/gozstd_nocgo.patch), so @@ -1212,6 +1204,65 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # Every release target (scripts/ci/release-targets.txt) on every PR, so a + # platform-gating mistake is caught here instead of at tag time (#5662): + # the whole tree, cgo off, one remote `bazel build --platforms=...` per + # target (the script's header has the coverage boundary: the cgo release + # builds are not compiled here). nogo (//tools/nogo) validates every one of + # those compiles, so this is also the lint and vet gate for files only + # other platforms build (//go:build windows, darwin, !linux ...; + # engdocs/LINTING.md). A job of its own rather than a step of bazel-pure: + # release_cross is a split transition that shares no analysis with the pure + # invocations, and as a step it was about a minute on the longest cached + # lane. Same runner and skip rule as bazel-pure (it runs in every mode but + # skip). + bazel-release-cross: + name: release cross-compile + needs: rbe + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode != 'skip' }} + timeout-minutes: 90 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} + + - name: Cross-compile every release target (--config=release-cross) + run: | + set -euo pipefail + ./scripts/ci/bazel-release-cross-compile.sh + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's embedded-Dolt tier ("Test (Embedded Dolt Cmd N/20)", # "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt Conformance - # core/audit)") as --config=embedded: the embedded-tagged variants of diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6cefbbbe6..628ed5dcc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1327,9 +1327,10 @@ jobs: } # The Bazel lanes (bazel.yml): the only pre-merge run of every Linux Go - # test tier (PR core, pure-Go and js/wasm, the dolt-server lane, the - # embedded-Dolt, proxied-server and server-Dolt storage tiers that - # pr-risk.yml used to run, integration and the cmd/bd Dolt-server tier). + # test tier (PR core, pure-Go and js/wasm, the release cross-compile, the + # dolt-server lane, the embedded-Dolt, proxied-server and server-Dolt + # storage tiers that pr-risk.yml used to run, integration and the cmd/bd + # Dolt-server tier). # ci-gate requires each of them to have run remotely on rbe-west and # passed, on every pull_request and merge group: same-repo PRs and merge # groups in mode remote, fork and Dependabot PRs through rbe-west's mint @@ -1406,6 +1407,7 @@ jobs: BAZEL BAZEL_TEST BAZEL_PURE + BAZEL_RELEASE_CROSS BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER @@ -1443,6 +1445,7 @@ jobs: # skipped, which passes only in the modes bazel-gate.sh allows. BAZEL_TEST: ${{ needs.bazel.outputs.bazel-test || 'skipped' }} BAZEL_PURE: ${{ needs.bazel.outputs.bazel-pure || 'skipped' }} + BAZEL_RELEASE_CROSS: ${{ needs.bazel.outputs.bazel-release-cross || 'skipped' }} BAZEL_EMBEDDED: ${{ needs.bazel.outputs.bazel-embedded || 'skipped' }} BAZEL_INTEGRATION: ${{ needs.bazel.outputs.bazel-integration || 'skipped' }} BAZEL_DOLTSERVER: ${{ needs.bazel.outputs.bazel-doltserver || 'skipped' }} diff --git a/MODULE.bazel b/MODULE.bazel index b429ea297..4daf1ae58 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -111,15 +111,24 @@ llvm_dist( # which pulls in compiler-rt's profile runtime (~170KB). "lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a", ], - sha256 = "df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384", - strip_prefix = "LLVM-22.1.8-Linux-X64", + # LLVM-22.1.8-Linux-X64.tar.xz (sha256 df0e1ecf...0006384), sliced to the + # members above by tools/cc_toolchain/repack_llvm.sh: 149 MB instead of + # 1.94 GB, and no single-threaded xz pass on every cold runner. The files + # are byte-identical to the release's, so no action key changes. + # tools/cc_toolchain/llvm_members.txt must list the same members. + sha256 = "de6147773999137b70ef3d7bd8ecb1e5b91f2a161d02fd09aec591b84516bec5", + sliced = True, stub_tools = [ "clang-format", "clang-tidy", "clangd", "llvm-dwp", ], - urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz"], + # The sha256 pin, not the host, guarantees the bytes. gascity's release + # asset for now; a beads mirror of the same bytes can be listed first. + urls = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM-22.1.8-Linux-X64-slice.tar.zst", + ], ) # Every other host (macOS arm64, Linux arm64) runs toolchains_llvm's stock @@ -195,6 +204,23 @@ go_deps.gazelle_override( ], path = "github.com/dolthub/go-mysql-server", ) + +# protoc-gen-validate ships validate/validate.pb.go, so its Go library can +# compile from the checked-in code. Its own validate/BUILD instead makes the +# Go library a go_proto_library, so every lane built protoc from source just +# to regenerate that file: about 4.5k C++ compile, link and archive actions, +# each carrying the LLVM toolchain as inputs. The module reaches the graph +# only through grpc xds and the cloud SDKs. Gazelle regenerates its BUILD +# files with proto rules off; build_file_name makes it write BUILD.bazel (which +# Bazel prefers) rather than merge into the upstream BUILD files. +go_deps.gazelle_override( + build_file_generation = "on", + directives = [ + "gazelle:build_file_name BUILD.bazel", + "gazelle:proto disable", + ], + path = "github.com/envoyproxy/protoc-gen-validate", +) go_deps.module_override( patch_strip = 1, patches = ["//third_party/patches:eventsapi_schema_build.patch"], diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 8bc7ebb4e..fb22a7410 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -55,9 +55,9 @@ Current PR-related workflow names: outputs (`rbe-enabled` is `true` in `remote`, `fork-ro` and `fork-rw`); every lane exports its `job.status` as an output named after the job. `pr.yml`'s gate requires the - call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, - `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and - `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go + call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, + `BAZEL_RELEASE_CROSS`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, + `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go test tiers: the legacy jobs they mirrored in `pr.yml` and `pr-risk.yml` are retired (ga-96smfk.22; see [Legacy Tier Retirement](#legacy-tier-retirement-d2)), so the gate also @@ -899,8 +899,9 @@ Required` requires them to have run remotely and passed. `TestPinnedDoltCLIMatchesContainerImage`. The release-target cross-compilation (formerly pr.yml's `check-release-target-cross-compilation`, `go build ./...` with - `CGO_ENABLED=0` per target) is a step of bazel.yml's `bazel-pure` lane - (`BAZEL_PURE`): `scripts/ci/bazel-release-cross-compile.sh` runs one + `CGO_ENABLED=0` per target) is bazel.yml's `bazel-release-cross` lane + (`BAZEL_RELEASE_CROSS`), split out of `bazel-pure` so it runs in + parallel with it: `scripts/ci/bazel-release-cross-compile.sh` runs one remote `bazel build //tools/bazel:release_cross`, every `go_library` and `go_binary` for each row of `scripts/ci/release-targets.txt`, plus `//tools/bazel:pure_bd_has_no_cgo_only_deps` (a pure bd must not link @@ -910,7 +911,7 @@ Required` requires them to have run remotely and passed. equal to the toolchain's) and the golangci-lint linters `.golangci.yml` enables run as nogo (`//tools/nogo`) beside every compile of every Bazel lane: natively in `bazel test //... --config=ci`, and for every release - platform in the `bazel-pure` lane's release cross-compile + platform in the `bazel-release-cross` lane (engdocs/LINTING.md). The former `scripts-go-checks` (`Go checks (vet)`) and `pr-lint-wrapper` (`PR Lint (native|windows|darwin)`) jobs are retired. @@ -1116,7 +1117,7 @@ scope, not this slice's. `scripts/ci/check-release-cross-compile.sh `, which builds every target in its group sequentially and reports every failure before exiting non-zero, so a PR touching two platforms at once sees both failures in one - log instead of needing a per-target re-run. (Since retired: bazel.yml's `bazel-pure` + log instead of needing a per-target re-run. (Since retired: bazel.yml's `bazel-release-cross` lane builds the same manifest with Bazel, `--platforms` per target.) - **`advisory-reports` fold.** `build-examples` and `complexity-report` (both already advisory: neither was in ci-gate's `needs`/`CI_GATE_REQUIRED`) diff --git a/engdocs/LINTING.md b/engdocs/LINTING.md index 15a8c4bdf..991a6d0a3 100644 --- a/engdocs/LINTING.md +++ b/engdocs/LINTING.md @@ -42,8 +42,8 @@ Where it gates: - **bazel.yml's `test` lane** (required through `CI Gate / Required`): `bazel test //... --config=ci` validates every package natively. -- **bazel.yml's `pure` lane** (required): its release cross-compile - (`scripts/ci/bazel-release-cross-compile.sh`) builds +- **bazel.yml's `release cross-compile` lane** (`bazel-release-cross`, + required): `scripts/ci/bazel-release-cross-compile.sh` builds `//tools/bazel:release_cross`, every `go_library` and `go_binary` split-transitioned to each release platform without cgo, and nogo validates each of those compiles. So `//go:build windows`, `darwin`, diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 7a2a62915..9adaed89f 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -36,7 +36,8 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or |---|---|---| | Test (`bazel-test`) | `make test`, i.e. `bazel test //... --config=ci` | PR Core's selection: race, `-short`, skips. Includes nogo, gofmt and the repository guards. The default gate for every Go change. | | Lint, all platforms | `make ci-pr-lint` | nogo natively plus the windows/amd64 and darwin/arm64 passes. `make lint-changed` covers only your changed packages. | -| Pure-Go (`bazel-pure`) | `bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test` | cgo off. The job's cmd/bd test subset (`PURE_CMD_BD_TESTS`), release cross-compile and js/wasm step are in bazel.yml. | +| Pure-Go (`bazel-pure`) | `bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test` | cgo off. The job's cmd/bd test subset (`PURE_CMD_BD_TESTS`) and js/wasm step are in bazel.yml. | +| Release cross-compile (`bazel-release-cross`) | `./scripts/ci/bazel-release-cross-compile.sh` | Every `go_library`/`go_binary` for each row of `scripts/ci/release-targets.txt`, cgo off, with nogo. | | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 16 shards of the integration-tagged cmd/bd suite. | diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go index bd0a74721..335d0e648 100644 --- a/scripts/bazel_hermetic_cc_test.go +++ b/scripts/bazel_hermetic_cc_test.go @@ -26,6 +26,9 @@ const ( hermeticCCLLVMLabel = "@llvm_dist_linux_x86_64//:BUILD.bazel" // Every other host's stock LLVM release (toolchains_llvm's sha256 table). hermeticCCHostLLVMLabel = "@llvm_dist_host//:BUILD.bazel" + // The members list tools/cc_toolchain/repack_llvm.sh slices the release + // to; it must equal llvm_dist's members (MODULE.bazel). + hermeticCCLLVMMembersFile = "tools/cc_toolchain/llvm_members.txt" ) var ( @@ -33,6 +36,11 @@ var ( // snapshot.ubuntu.com serves each timestamp's archive state forever; a // moving mirror (archive.ubuntu.com) may only follow it as a fallback. ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`) + // A sliced llvm_dist archive is a repack (repack_llvm.sh), not an + // upstream release, so it may only come from a gastownhall release: + // beads' own mirror or gascity's original asset (same bytes, same sha256). + llvmSliceURLRE = regexp.MustCompile(`^https://github\.com/gastownhall/(beads|gascity)/releases/download/toolchain-llvm-[0-9.]+-slice-[0-9]+/[A-Za-z0-9._-]+\.tar\.zst$`) + quotedStringRE = regexp.MustCompile(`"([^"]*)"`) ) // moduleCall returns the argument text of the single top-level call @@ -136,6 +144,46 @@ func checkHermeticCCModule(module string) []error { return errs } +// checkHermeticLLVMSlice: a sliced llvm_dist downloads only from a +// gastownhall release, and its members equal the repack script's list, so +// the pinned slice can be rebuilt from upstream and checked file by file. +func checkHermeticLLVMSlice(module, membersFile string) []error { + dist, err := moduleCall(module, "llvm_dist", `name = "llvm_dist_linux_x86_64"`) + if err != nil { + return []error{err} + } + if !strings.Contains(dist, "sliced = True,") { + return nil + } + var errs []error + urls := regexp.MustCompile(`(?s)urls = \[(.*?)\]`).FindStringSubmatch(dist) + if urls == nil || len(quotedStringRE.FindAllStringSubmatch(urls[1], -1)) == 0 { + errs = append(errs, errors.New("sliced llvm_dist has no urls")) + } else { + for _, u := range quotedStringRE.FindAllStringSubmatch(urls[1], -1) { + if !llvmSliceURLRE.MatchString(u[1]) { + errs = append(errs, errors.New("sliced llvm_dist url "+u[1]+" is not a gastownhall/beads or gastownhall/gascity release .tar.zst")) + } + } + } + var members []string + if body := regexp.MustCompile(`(?s)members = \[(.*?)\]`).FindStringSubmatch(dist); body != nil { + for _, m := range quotedStringRE.FindAllStringSubmatch(body[1], -1) { + members = append(members, m[1]) + } + } + var listed []string + for _, line := range strings.Split(membersFile, "\n") { + if line = strings.TrimSpace(line); line != "" { + listed = append(listed, line) + } + } + if len(members) == 0 || strings.Join(members, "\n") != strings.Join(listed, "\n") { + errs = append(errs, errors.New("llvm_dist members must equal "+hermeticCCLLVMMembersFile+" (same order), or the pinned slice cannot be rebuilt by repack_llvm.sh")) + } + return errs +} + // checkHermeticCCBazelRC: host detection is off unconditionally (every // command, every config) and nothing hands actions a host compiler. func checkHermeticCCBazelRC(rc string) []error { @@ -175,7 +223,11 @@ func checkHermeticCCBazelRC(rc string) []error { func TestBazelHermeticCCToolchain(t *testing.T) { root := bazelPolicyRoot(t) - for _, err := range checkHermeticCCModule(readPolicyFile(t, root, hermeticCCModuleFile)) { + module := readPolicyFile(t, root, hermeticCCModuleFile) + for _, err := range checkHermeticCCModule(module) { + t.Error(err) + } + for _, err := range checkHermeticLLVMSlice(module, readPolicyFile(t, root, hermeticCCLLVMMembersFile)) { t.Error(err) } for _, err := range checkHermeticCCBazelRC(readPolicyFile(t, root, ".bazelrc")) { @@ -271,3 +323,46 @@ llvm.sysroot( } } } + +func TestBazelHermeticLLVMSliceGuards(t *testing.T) { + sum := strings.Repeat("ab", 32) + slice := "https://github.com/gastownhall/gascity/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM-22.1.8-Linux-X64-slice.tar.zst" + mirror := "https://github.com/gastownhall/beads/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM-22.1.8-Linux-X64-slice.tar.zst" + module := `llvm_dist( + name = "llvm_dist_linux_x86_64", + members = [ + "bin/clang", + # a comment + "lib/clang/22/include", + ], + sha256 = "` + sum + `", + sliced = True, + urls = [ + "` + mirror + `", + "` + slice + `", + ], +) +` + members := "bin/clang\nlib/clang/22/include\n" + if errs := checkHermeticLLVMSlice(module, members); len(errs) != 0 { + t.Fatalf("good sliced llvm_dist fixture: %v", errs) + } + unsliced := strings.Replace(strings.Replace(module, " sliced = True,\n", "", 1), slice, "https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz", 1) + if errs := checkHermeticLLVMSlice(unsliced, members); len(errs) != 0 { + t.Fatalf("unsliced llvm_dist needs no slice host or members file: %v", errs) + } + for name, bad := range map[string][2]string{ + "other host": {strings.Replace(module, slice, "https://example.com/LLVM-22.1.8-Linux-X64-slice.tar.zst", 1), members}, + "other repo": {strings.Replace(module, "gastownhall/gascity", "someone/gascity", 1), members}, + "xz release": {strings.Replace(module, slice, "https://github.com/gastownhall/gascity/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM.tar.xz", 1), members}, + "no urls": {strings.Replace(strings.Replace(module, `"`+slice+`",`, "", 1), `"`+mirror+`",`, "", 1), members}, + "member missing": {module, "bin/clang\n"}, + "member extra": {module, members + "bin/lld\n"}, + "member reordered": {module, "lib/clang/22/include\nbin/clang\n"}, + "empty member file": {module, ""}, + } { + if len(checkHermeticLLVMSlice(bad[0], bad[1])) == 0 { + t.Errorf("%s: expected an error for sliced llvm_dist fixture:\n%s\nmembers:\n%s", name, bad[0], bad[1]) + } + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go index d94a94c16..e76b9ad02 100644 --- a/scripts/ci_analytics_workflow_test.go +++ b/scripts/ci_analytics_workflow_test.go @@ -29,10 +29,11 @@ var ciAnalyticsLaneJobs = map[string]string{ // TestBazelLaneIsGatedAlongsideLegacy's gated/advisory/flag-gated split, so // a new job in bazel.yml can't silently go unaccounted for here either. var ciAnalyticsExcludedJobs = map[string]string{ - "rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize", - "rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane", - "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", - "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", + "rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize", + "rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane", + "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", + "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", + "bazel-release-cross": "one bazel-release-cross-compile.sh run with no BEP/exec-log/profile capture, as when it was a bazel-pure step", } // ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index d3cdeab5b..f5479a463 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -125,37 +125,55 @@ func TestPRCIGateDropsRetiredPolicyAndLintJobs(t *testing.T) { } } -// TestReleaseCrossCompileRunsInBazelPureLane pins the release-target -// cross-compilation gate into bazel.yml's pure-Go lane, whose job.status -// ci-gate requires (BAZEL_PURE), and keeps the retired pr.yml job from -// coming back beside it: a second, `go build` copy of the gate would run -// every target twice. -func TestReleaseCrossCompileRunsInBazelPureLane(t *testing.T) { +// TestReleaseCrossCompileRunsInItsOwnBazelLane pins the release-target +// cross-compilation gate into bazel.yml's bazel-release-cross lane, whose +// job.status ci-gate requires (BAZEL_RELEASE_CROSS), and keeps it from +// running twice: not back in bazel-pure (where it was a step on the longest +// cached lane), and not as the retired pr.yml `go build` job beside it. +func TestReleaseCrossCompileRunsInItsOwnBazelLane(t *testing.T) { const ( retiredJob = "check-release-target-cross-compilation" retiredToken = "CHECK_RELEASE_TARGET_CROSS_COMPILATION" stepName = "Cross-compile every release target (--config=release-cross)" + script = "./scripts/ci/bazel-release-cross-compile.sh" ) pr := readCIWorkflow(t, "pr.yml") if _, ok := pr.Jobs[retiredJob]; ok { - t.Errorf("pr.yml still defines %s; bazel.yml's %s lane runs the release cross-compilation", retiredJob, bazelPureJobName) + t.Errorf("pr.yml still defines %s; bazel.yml's %s lane runs the release cross-compilation", retiredJob, bazelReleaseCrossJobName) } gate := pr.job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env if contains(gate.Needs, retiredJob) || gateEnv[retiredToken] != "" || contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), retiredToken) { t.Errorf("ci-gate still wires the retired %s job", retiredJob) } - if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_PURE") { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_PURE, the lane that cross-compiles the release targets") + if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_RELEASE_CROSS") { + t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_RELEASE_CROSS, the lane that cross-compiles the release targets") } - run := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName).step(t, stepName).Run + bazel := readCIWorkflow(t, bazelWorkflowName) + for name, job := range bazel.Jobs { + if name == bazelReleaseCrossJobName { + continue + } + for _, step := range job.Steps { + if strings.Contains(step.Run, script) { + t.Errorf("%s step %q runs %s too; only %s cross-compiles the release targets", name, step.Name, script, bazelReleaseCrossJobName) + } + } + } + cross := bazel.job(t, bazelReleaseCrossJobName) + pure := bazel.job(t, bazelPureJobName) + if cross.RunsOn != pure.RunsOn || cross.If != pure.If || !slices.Equal(cross.Needs, pure.Needs) { + t.Errorf("%s runs-on/if/needs = %q / %q / %v, want %s's %q / %q / %v", + bazelReleaseCrossJobName, cross.RunsOn, cross.If, cross.Needs, bazelPureJobName, pure.RunsOn, pure.If, pure.Needs) + } + run := cross.step(t, stepName).Run for _, required := range []string{ "set -euo pipefail", - "./scripts/ci/bazel-release-cross-compile.sh", + script, } { if !strings.Contains(run, required) { - t.Errorf("%s step %q does not contain %q:\n%s", bazelPureJobName, stepName, required, run) + t.Errorf("%s step %q does not contain %q:\n%s", bazelReleaseCrossJobName, stepName, required, run) } } @@ -1712,16 +1730,19 @@ func captureOne(t *testing.T, pattern, body, source string) string { // --- Bazel lane (.github/workflows/bazel.yml, gated through pr.yml) ---------- const ( - bazelWorkflowName = "bazel.yml" - bazelJobName = "bazel-test" - bazelPureJobName = "bazel-pure" - bazelDoltJobName = "bazel-doltserver" - bazelEmbedJobName = "bazel-embedded" - bazelRBEJobName = "rbe" - bazelIntegJobName = "bazel-integration" - bazelProxiedJobName = "bazel-proxied" - bazelServerJobName = "bazel-server-storage" - bazelCmdDoltJobName = "bazel-cmd-dolt" + bazelWorkflowName = "bazel.yml" + bazelJobName = "bazel-test" + bazelPureJobName = "bazel-pure" + // The release-target cross-compilation, split out of bazel-pure so it + // runs in parallel with it (same runner and skip rule). + bazelReleaseCrossJobName = "bazel-release-cross" + bazelDoltJobName = "bazel-doltserver" + bazelEmbedJobName = "bazel-embedded" + bazelRBEJobName = "rbe" + bazelIntegJobName = "bazel-integration" + bazelProxiedJobName = "bazel-proxied" + bazelServerJobName = "bazel-server-storage" + bazelCmdDoltJobName = "bazel-cmd-dolt" // F3: the MCP and npm package gates, moved here from pr.yml so they need // only the rbe job. Unlike every other lane they do not mirror a Bazel // config; pr.yml opts them in with package-gates: "on". @@ -1746,8 +1767,8 @@ const ( // Same SHA/comment already used in this repo for this action (rbe- // prewarm's mint step, update-flake-lock.yml). appTokenActionSHA = "bcd2ba49218906704ab6c1aa796996da409d3eb1" - bazelCacheKeyPrefix = "bazel-repo-v3-${{ runner.os }}-" - bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }}" + bazelCacheKeyPrefix = "bazel-repo-v4-${{ runner.os }}-" + bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }}" bazelCachePath = "${{ runner.temp }}/bazel-ci-cache" // Save only from a push to main that missed the exact key: the content is // fixed by the key, so re-saving every push only churns the quota. @@ -1758,7 +1779,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} +var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -1848,10 +1869,12 @@ var bazelWorkflowTriggers = []string{"push", "workflow_call", "workflow_dispatch // TestBazelGateSimulation), for the recorded reason. A new job in bazel.yml // must be added to one of the two (TestBazelLaneIsGatedAlongsideLegacy). var bazelLaneGateIDs = map[string]string{ - bazelJobName: "BAZEL_TEST", - bazelPureJobName: "BAZEL_PURE", - bazelEmbedJobName: "BAZEL_EMBEDDED", - bazelDoltJobName: "BAZEL_DOLTSERVER", + bazelJobName: "BAZEL_TEST", + bazelPureJobName: "BAZEL_PURE", + // Every release target cross-compiled, cgo off (and nogo over each). + bazelReleaseCrossJobName: "BAZEL_RELEASE_CROSS", + bazelEmbedJobName: "BAZEL_EMBEDDED", + bazelDoltJobName: "BAZEL_DOLTSERVER", // Remote-only PR Risk tiers (fork and Dependabot PRs rely on // pr-risk.yml's legacy jobs, like the embedded tier's). bazelProxiedJobName: "BAZEL_PROXIED", diff --git a/third_party/cc_toolchain/llvm_dist.bzl b/third_party/cc_toolchain/llvm_dist.bzl index 994aeef39..93fab6b23 100644 --- a/third_party/cc_toolchain/llvm_dist.bzl +++ b/third_party/cc_toolchain/llvm_dist.bzl @@ -9,16 +9,26 @@ it plugs in through `llvm.toolchain_root`. """ def _llvm_dist_impl(rctx): - archive = "_llvm.tar.xz" - rctx.download(url = rctx.attr.urls, output = archive, sha256 = rctx.attr.sha256) - res = rctx.execute( - ["tar", "-xJf", archive, "--strip-components=1"] + - [rctx.attr.strip_prefix + "/" + m for m in rctx.attr.members], - timeout = 3600, - ) - if res.return_code != 0: - fail("llvm_dist: extracting %s failed:\n%s" % (rctx.attr.urls[0], res.stderr)) - rctx.delete(archive) + if rctx.attr.sliced: + # A members-only, zstd archive of the same release (tools/cc_toolchain/ + # repack_llvm.sh): the same bytes as the tar -xJf below, extracted by + # Bazel itself in seconds instead of a minute-plus of single-threaded xz + # on every cold CI runner. Action keys are unchanged (identical files). + rctx.download_and_extract(url = rctx.attr.urls, sha256 = rctx.attr.sha256, type = "tar.zst") + missing = [m for m in rctx.attr.members if not rctx.path(m).exists] + if missing: + fail("llvm_dist: the sliced archive lacks %s" % ", ".join(missing)) + else: + archive = "_llvm.tar.xz" + rctx.download(url = rctx.attr.urls, output = archive, sha256 = rctx.attr.sha256) + res = rctx.execute( + ["tar", "-xJf", archive, "--strip-components=1"] + + [rctx.attr.strip_prefix + "/" + m for m in rctx.attr.members], + timeout = 3600, + ) + if res.return_code != 0: + fail("llvm_dist: extracting %s failed:\n%s" % (rctx.attr.urls[0], res.stderr)) + rctx.delete(archive) # toolchains_llvm symlinks a fixed tool list into its toolchain package; # tools this repository leaves out get a stub that fails loudly if run. @@ -42,7 +52,10 @@ llvm_dist = repository_rule( "llvm_version": attr.string(mandatory = True), "urls": attr.string_list(mandatory = True), "sha256": attr.string(mandatory = True), - "strip_prefix": attr.string(mandatory = True), + "strip_prefix": attr.string(doc = "The release archive's top directory (unsliced archives only)."), + "sliced": attr.bool( + doc = "urls is a members-only .tar.zst made by repack_llvm.sh, not the release .tar.xz.", + ), "members": attr.string_list( mandatory = True, doc = "Archive paths (below strip_prefix) to extract.", diff --git a/tools/cc_toolchain/llvm_members.txt b/tools/cc_toolchain/llvm_members.txt new file mode 100644 index 000000000..8733e050e --- /dev/null +++ b/tools/cc_toolchain/llvm_members.txt @@ -0,0 +1,28 @@ +bin/clang +bin/clang++ +bin/clang-22 +bin/clang-cpp +bin/ld.lld +bin/ld64.lld +bin/lld +bin/llvm-ar +bin/llvm-as +bin/llvm-cov +bin/llvm-install-name-tool +bin/llvm-nm +bin/llvm-objcopy +bin/llvm-objdump +bin/llvm-profdata +bin/llvm-ranlib +bin/llvm-readelf +bin/llvm-readobj +bin/llvm-strip +bin/llvm-symbolizer +bin/wasm-ld +include/c++ +include/x86_64-unknown-linux-gnu +lib/clang/22/include +lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtbegin.o +lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtend.o +lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.builtins.a +lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a diff --git a/tools/cc_toolchain/repack_llvm.sh b/tools/cc_toolchain/repack_llvm.sh new file mode 100755 index 000000000..73978d8aa --- /dev/null +++ b/tools/cc_toolchain/repack_llvm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Deterministic members-only repack of the pinned LLVM release for beads' +# hermetic C toolchain (MODULE.bazel llvm_dist "llvm_dist_linux_x86_64"). +# +# repack_llvm.sh MEMBERS_FILE OUT.tar.zst +# +# MEMBERS_FILE is one archive path per line (below the strip prefix); this +# directory's llvm_members.txt matches MODULE.bazel's llvm_dist_linux_x86_64 +# "members" exactly, so: +# tools/cc_toolchain/repack_llvm.sh tools/cc_toolchain/llvm_members.txt \ +# LLVM-22.1.8-Linux-X64-slice.tar.zst +# rebuilds and verifies the pinned slice. Changing MODULE.bazel's members +# list requires updating llvm_members.txt the same way, and vice versa. +# +# Downloads the upstream release (sha256-pinned, as MODULE.bazel pins it), +# extracts exactly the listed members (paths below the strip prefix), and +# writes a reproducible .tar.zst: sorted names, mtime 0, uid/gid 0, no xattrs. +# The extracted files are byte-identical to what llvm_dist.bzl extracts today, +# so every action key that reads them is unchanged; only the archive (URL, +# sha256, format) changes. Re-running the script yields the same sha256. +set -euo pipefail +members=${1:?members file} +out=${2:?output .tar.zst} +ver=22.1.8 +url="https://github.com/llvm/llvm-project/releases/download/llvmorg-${ver}/LLVM-${ver}-Linux-X64.tar.xz" +sha=df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384 +prefix="LLVM-${ver}-Linux-X64" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +curl -fsSL -o "$work/llvm.tar.xz" "$url" +echo "$sha $work/llvm.tar.xz" | sha256sum -c - >/dev/null +mkdir "$work/x" +# xz is one stream: this is the slow, single-threaded step CI pays today. +# shellcheck disable=SC2046 # intentional: one archive path per tar argument +tar -xJf "$work/llvm.tar.xz" -C "$work/x" --strip-components=1 \ + $(sed "s#^#${prefix}/#" "$members") +tar -C "$work/x" --sort=name --mtime='@0' --owner=0 --group=0 --numeric-owner \ + --no-xattrs --no-acls --no-selinux -cf - . | zstd -19 -T0 -q -o "$out" -f +sha256sum "$out"