From fa3d50eb19dd81f1d98a298eb4721c77e8eacce5 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 4 Oct 2026 12:26:54 -0700 Subject: [PATCH] ci(bazel): zstd for the anonymous fork cache when it advertises it (rbe-west E4) (#7202) * ci(bazel): zstd cache transfers for the anonymous fork cache only, behind RBE_CACHE_ZSTD (E4) E4 of the rbe-west egress reduction (design A.5), reworked for the read-only zstd server shape. rbe-west's trusted MAIN and OSS schedulers (:443 to :50052) and rbe-fork (:8444) advertise no compressor, and Bazel refuses a remote that does not advertise zstd. So only the anonymous fork cache (rbe-cache :8443, nativelink-anon, which keeps zstd) may ask for it. - **`write-bazelrc.sh`:** no `--remote_cache_compression` on the trusted remote-exec or rbe-fork paths. In mode cache, with `RBE_CACHE_ZSTD=1`, it adds `build:fork-cache --remote_cache_compression`. Empty or 0 adds nothing; another value warns and stays off, rather than failing every cache lane. - **`bazel.yml`:** every setup-bazel caller passes `RBE_CACHE_ZSTD: ${{ needs.rbe.outputs.mode == 'cache' && vars.RBE_CACHE_ZSTD || '' }}`, so only cache runs see the variable. Merging changes nothing, and rollback is the variable, with no revert. - **`.bazelrc`:** fork-cache no longer sets the flag; a comment points at the gate. Tests: - `TestBazelCacheModeReachesTheRC` evaluates `vars.` right-hand sides and simulates `RBE_CACHE_ZSTD=1`. Mode cache's rc carries exactly the fork-cache zstd line; the remote, fork-ro, fork-rw and local modes carry no compression at all. - `TestSetupBazelRCWriter` runs `RBE_CACHE_ZSTD` = 1, 0, empty and a bad value in mode cache, and remote-exec with 1 (never compressed). `TestSetupBazelRCWriterForkMode` pins rbe-fork's exact lines with `RBE_CACHE_ZSTD=1`. - The bazel.yml setup-env pin includes `RBE_CACHE_ZSTD`. - `checkBazelrcForkCache` rejects the flag in any `.bazelrc` config, with fork-cache and remote-exec fixtures. - New `TestBazelRemoteCacheCompressionOnlyForkCache` scans `.bazelrc`, the workflows and the local action files. The flag must appear once: the writer's fork-cache echo. **Merge order:** after rbe-west's anonymous process advertises zstd. Even then the flag stays off until `RBE_CACHE_ZSTD=1` is set. If rbe-cache stops advertising zstd, fork-cache's fallback flags make lanes build locally. **Action keys:** unchanged; the flag only changes ByteStream resource names (`compressed-blobs/zstd/`). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): zstd for the fork cache by a GetCapabilities probe, not a repository variable (E4) Review finding 3: GitHub gives fork pull_request runs no repository variables (bazel.yml: forks "see no secrets and no vars"), so vars.RBE_CACHE_ZSTD was always empty exactly where the switch matters. Fork run 37220594372 logs RBE_VAR_ON: false. The variable could never turn zstd on for fork-cache clients, and once on it could not follow rbe-cache rolling zstd back. - New setup-bazel/cache-zstd-probe.sh, a byte copy of gascity's tools/rbe/cache-zstd-probe.sh: one unary REAPI GetCapabilities call (instance oss) to rbe-cache :8443 over curl's HTTP/2, decoded with python3's standard library. It exits 0 only when cache_capabilities.supported_compressors lists ZSTD, the field Bazel checks. Any other result exits nonzero: not listed, an HTTP or gRPC error, a refused connection, a timeout (connect 3s, total 5s), an unparsable answer, or no curl or python3. It writes nothing to stdout and one verdict line to stderr. No new action and no new dependency. - write-bazelrc.sh: in mode cache, `if bash .../cache-zstd-probe.sh` adds build:fork-cache --remote_cache_compression. The probe never fails the script. RBE_CACHE_ZSTD and its bad-value warning are gone. - bazel.yml: the ten RBE_CACHE_ZSTD env lines are gone (back to the pre-E4 setup env). Tests: - New TestCacheZstdProbe runs the real script against a TLS HTTP/2 stand-in for rbe-cache, which checks the exact request. Passes: zstd advertised (packed, and unpacked among others). Fails: rbe-cache's real 2026-10-04 answer (no compressors), deflate only, zstd for batch updates only, zstd outside cache_capabilities, a gRPC error trailer, no grpc-status, trailers-only UNIMPLEMENTED, HTTP 502, a compressed or truncated message, a truncated field, not gRPC, a timeout, a refused connection, an untrusted certificate. It also pins the probe's endpoint and instance to .bazelrc's fork-cache. - TestSetupBazelRCWriter: the fork cache gets exactly the zstd line after --config=fork-cache against a zstd stand-in, and none against the live answer or a refused port. Remote-exec and rbe-fork, with a zstd stand-in, never probe and never carry compression. - TestBazelCacheModeReachesTheRC again rejects any vars. in setup-bazel's env, and no mode's rc carries compression (the probe is refused there). - The writer harnesses default RBE_CACHE_PROBE_URL to a refused loopback port, so no test reaches rbe-cache. Live check, 2026-10-04: rbe-cache answers without supported_compressors, so the writer's mode cache rc has no flag until the E1 cutover. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 3 + .../actions/setup-bazel/cache-zstd-probe.sh | 119 +++++++++ .github/actions/setup-bazel/write-bazelrc.sh | 17 +- scripts/bazel_cache_mode_test.go | 251 +++++++++++++++++- scripts/bazel_fork_mode_test.go | 3 +- scripts/bazel_policy_test.go | 47 ++++ scripts/ci_workflow_test.go | 55 ++++ 7 files changed, 491 insertions(+), 4 deletions(-) create mode 100644 .github/actions/setup-bazel/cache-zstd-probe.sh diff --git a/.bazelrc b/.bazelrc index 9d514ce3e..58a3caeaa 100644 --- a/.bazelrc +++ b/.bazelrc @@ -344,6 +344,9 @@ build:fork-cache --remote_retries=2 build:fork-cache --experimental_circuit_breaker_strategy=failure # The farm admits 16 connections per source IP. build:fork-cache --remote_max_connections=8 +# No --remote_cache_compression here: setup-bazel's write-bazelrc.sh adds it +# for fork-cache while rbe-cache advertises zstd (cache-zstd-probe.sh), so the +# server decides and rollback needs no change here. # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. diff --git a/.github/actions/setup-bazel/cache-zstd-probe.sh b/.github/actions/setup-bazel/cache-zstd-probe.sh new file mode 100644 index 000000000..4e74368eb --- /dev/null +++ b/.github/actions/setup-bazel/cache-zstd-probe.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Does rbe-west's anonymous read-only cache (rbe-cache :8443, .bazelrc's +# fork-cache) advertise zstd right now? Exits 0 only when its REAPI +# GetCapabilities answer lists ZSTD in cache_capabilities.supported_compressors, +# the field Bazel checks before it accepts --remote_cache_compression (it +# refuses a remote that does not list it). Anything else exits nonzero: not +# listed, an HTTP or gRPC error, a refused connection, a timeout, an answer +# that does not parse, no curl or python3. The caller then leaves the flag +# off, and never fails on it. +# +# So the server decides, at run time: when rbe-west turns zstd on, fork-cache +# runs use it; when it turns it off (rollback), they stop. No repository +# variable (fork pull_request runs see none) and no revert. +# +# One unary gRPC call over curl's HTTP/2, decoded by python3's standard +# library: no grpcurl, no new action. Nothing goes to stdout (callers write +# their rc from it); one verdict line goes to stderr. A byte copy lives in +# gascity (tools/rbe/) and beads (.github/actions/setup-bazel/); keep in sync. +# +# Tests only: RBE_CACHE_PROBE_URL (a stand-in server) and +# RBE_CACHE_PROBE_MAX_TIME (seconds). +set -uo pipefail +url=${RBE_CACHE_PROBE_URL:-https://rbe-cache.ops.gascity.com:8443} +max_time=${RBE_CACHE_PROBE_MAX_TIME:-5} + +no() { + echo "rbe-cache zstd probe: $*; the fork cache stays identity" >&2 + exit 1 +} +command -v curl >/dev/null 2>&1 || no "no curl" +command -v python3 >/dev/null 2>&1 || no "no python3" +tmp=$(mktemp -d) || no "no temp dir" +trap 'rm -rf "$tmp"' EXIT + +# GetCapabilitiesRequest{instance_name: "oss"} (fork-cache's +# --remote_instance_name) as one uncompressed gRPC message. +printf '\000\000\000\000\005\012\003oss' >"$tmp/req" || no "cannot write the request" +code=$(curl -sS --http2 --connect-timeout 3 --max-time "$max_time" \ + -H 'content-type: application/grpc' -H 'te: trailers' \ + --data-binary @"$tmp/req" -D "$tmp/head" -o "$tmp/body" -w '%{http_code}' \ + "$url/build.bazel.remote.execution.v2.Capabilities/GetCapabilities" 2>"$tmp/err") || + no "$(tr '\n' ' ' <"$tmp/err")" +[ "$code" = 200 ] || no "HTTP $code" +# grpc-status is a trailer; curl writes trailers after the headers. +st=$(tr -d '\r' <"$tmp/head" | sed -n 's/^grpc-status: *//Ip' | tail -1) +[ "$st" = 0 ] || no "gRPC status ${st:-missing}" + +why=$(python3 -I - "$tmp/body" <<'PY' +import sys + +ZSTD = 1 # build.bazel.remote.execution.v2.Compressor.Value.ZSTD + + +def varint(b, i): + v = s = 0 + while True: + if i >= len(b) or s > 63: + raise ValueError("truncated varint") + c = b[i] + i += 1 + v |= (c & 0x7F) << s + s += 7 + if c < 0x80: + return v, i + + +def fields(b): + i = 0 + while i < len(b): + key, i = varint(b, i) + num, wire = key >> 3, key & 7 + if wire == 0: + v, i = varint(b, i) + elif wire == 1: + v, i = b[i:i + 8], i + 8 + elif wire == 2: + n, i = varint(b, i) + v, i = b[i:i + n], i + n + elif wire == 5: + v, i = b[i:i + 4], i + 4 + else: + raise ValueError("wire type %d" % wire) + if i > len(b): + raise ValueError("truncated field %d" % num) + yield num, wire, v + + +try: + body = open(sys.argv[1], "rb").read() + if len(body) < 5 or body[0] != 0: + raise ValueError("not an uncompressed gRPC message") + n = int.from_bytes(body[1:5], "big") + if len(body) != 5 + n: + raise ValueError("gRPC length %d, %d bytes" % (n, len(body) - 5)) + compressors = [] + for num, wire, v in fields(body[5:]): + if num != 1 or wire != 2: # ServerCapabilities.cache_capabilities + continue + for cnum, cwire, cv in fields(v): + if cnum != 6: # CacheCapabilities.supported_compressors + continue + if cwire == 0: + compressors.append(cv) + elif cwire == 2: # packed + j = 0 + while j < len(cv): + x, j = varint(cv, j) + compressors.append(x) +except (OSError, ValueError) as e: + print("unreadable answer (%s)" % e) + sys.exit(2) +if ZSTD in compressors: + print("advertises zstd") + sys.exit(0) +print("zstd not advertised (supported_compressors %s)" % compressors) +sys.exit(1) +PY +) || no "${why:-python3 failed}" +echo "rbe-cache zstd probe: $why; the fork cache uses zstd" >&2 diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh index 2436d976d..5dca6562f 100644 --- a/.github/actions/setup-bazel/write-bazelrc.sh +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -27,7 +27,12 @@ # # BAZEL_FORK_CACHE=true (bazel.yml mode "cache": rbe=cache dispatches, and # fork/Dependabot runs while rbe-fork is closed) appends --config=fork-cache: -# .bazelrc's credential-free, read-only rbe-west cache. +# .bazelrc's credential-free, read-only rbe-west cache. While that cache +# advertises zstd (cache-zstd-probe.sh asks its GetCapabilities; any failure +# means no) it also asks for zstd transfers (--remote_cache_compression). A +# probe, not a repository variable: fork pull_request runs see no vars. No +# other mode ever asks, because only the anonymous cache can advertise zstd, +# and Bazel refuses a remote that does not. # # RBE_FORK_CERT_FILE + RBE_FORK_KEY_FILE + RBE_FORK_ENDPOINT + RBE_FORK_INSTANCE # (bazel.yml modes fork-ro/fork-rw: fork-credential.sh's outputs, a @@ -222,7 +227,15 @@ elif [[ "$set_fields" -ne 0 ]]; then echo "setup-bazel: remote execution is partially configured; BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY must be set together (or none, for local execution)" >&2 exit 1 elif [[ "$fork_cache" == true ]]; then - echo "build --config=fork-cache" >>"$rc" + # zstd for the anonymous cache's transfers while it advertises zstd: + # transport only, so action keys are unchanged. The probe never fails + # this script; it only decides the line. + { + echo "build --config=fork-cache" + if bash "$(dirname "${BASH_SOURCE[0]}")/cache-zstd-probe.sh"; then + echo "build:fork-cache --remote_cache_compression" + fi + } >>"$rc" cache=true echo "setup-bazel: read-only remote cache (rbe-cache); executing locally" else diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index e942f7378..7ee9e653e 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -1,14 +1,23 @@ package scripts_test import ( + "bytes" "encoding/base64" + "encoding/binary" + "encoding/pem" + "errors" + "io" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "regexp" "sort" "strings" + "sync/atomic" "testing" + "time" ) // Mode cache, end to end, for every way a bazel.yml run can start: the rbe @@ -47,7 +56,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // setup-bazel's env values: `needs.rbe.outputs.X == 'v' && RHS || ''`, // RHS a secret or a literal. Anything else fails the test, so the - // simulation cannot drift from the workflow. + // simulation cannot drift from the workflow; in particular no repository + // variable (fork pull_request runs see none). setupExpr := regexp.MustCompile(`^\$\{\{ needs\.rbe\.outputs\.(enabled|mode) == '([a-z]+)' && (?:secrets\.([A-Z_]+)|'([^']*)') \|\| '' \}\}$`) evalSetup := func(t *testing.T, expr, mode, enabled, tier string, haveSecrets bool) string { t.Helper() @@ -100,6 +110,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "bazel-ci-cache"), "BAZEL_CI_SECRET_DIR=" + secretDir, + // Mode cache's zstd probe: a refused port, never rbe-cache. + "RBE_CACHE_PROBE_URL=" + refusedProbeURL, } for k, v := range env { cmd.Env = append(cmd.Env, k+"="+v) @@ -323,6 +335,13 @@ func checkModeRC(t *testing.T, lane, mode, outputs, rc string, files []string, l t.Errorf("%s (mode %s): rc lacks %q:\n%s", lane, mode, strings.TrimSpace(want), rc) } } + // zstd: no mode here asks for compression. Mode cache's probe is + // refused here (TestSetupBazelRCWriter runs it passing); the trusted + // schedulers and rbe-fork advertise none, and Bazel refuses such a + // remote. + if has("remote_cache_compression") { + t.Errorf("%s (mode %s): rc asks for compression:\n%s", lane, mode, rc) + } credentialFree := func() { for _, bad := range []string{"remote_executor", "remote-exec", "tls_", "remote_instance_name", "--remote_upload_local_results", "--remote_cache", "--remote_header", "--bes_", "farm.invalid"} { if has(bad) { @@ -382,3 +401,233 @@ func checkModeRC(t *testing.T, lane, mode, outputs, rc string, files []string, l t.Errorf("%s runs in mode %q", lane, mode) } } + +// cacheZstdProbe: setup-bazel's zstd probe for the fork cache (a byte copy +// of gascity's tools/rbe/cache-zstd-probe.sh). +const cacheZstdProbe = setupBazelActionDir + "/cache-zstd-probe.sh" + +// refusedProbeURL: a loopback port nothing listens on, the probe's URL in +// tests that do not serve one (a refused connection: no flag). +const refusedProbeURL = "https://127.0.0.1:1" + +// GetCapabilities bodies for a stand-in rbe-cache. capsLive is what rbe-cache +// answered on 2026-10-04, before it advertised zstd (cache_capabilities: +// SHA256 and BLAKE3, action cache read-only, 64 MiB batches, symlinks +// allowed; API 2.0 to 2.3); capsZstd is the same with supported_compressors +// [ZSTD]. +var ( + capsLiveCache = []byte{0x0a, 0x02, 0x01, 0x09, 0x12, 0x02, 0x08, 0x01, 0x20, 0x80, 0x80, 0x04, 0x28, 0x01} + capsLiveAPIVersions = []byte{0x22, 0x02, 0x08, 0x02, 0x2a, 0x04, 0x08, 0x02, 0x10, 0x03} + capsLive = capsWithCache(capsLiveCache) + capsZstd = capsWithCache(pbBytes(6, []byte{1}), capsLiveCache) + // GetCapabilitiesRequest{instance_name: "oss"}, fork-cache's instance. + capsRequest = grpcMessage(pbBytes(1, []byte("oss"))) +) + +// capsWithCache: a ServerCapabilities with the live API versions and a +// cache_capabilities of the given fields. +func capsWithCache(cache ...[]byte) []byte { + return append(pbBytes(1, bytes.Join(cache, nil)), capsLiveAPIVersions...) +} + +func pbBytes(field int, b []byte) []byte { + out := binary.AppendUvarint(nil, uint64(field)<<3|2) + return append(binary.AppendUvarint(out, uint64(len(b))), b...) +} + +func pbVarint(field int, v uint64) []byte { + return binary.AppendUvarint(binary.AppendUvarint(nil, uint64(field)<<3), v) +} + +// grpcMessage frames m as one uncompressed gRPC message. +func grpcMessage(m []byte) []byte { + return append(binary.BigEndian.AppendUint32([]byte{0}, uint32(len(m))), m...) +} + +// capsAnswer is how the stand-in rbe-cache answers GetCapabilities: HTTP +// status (0: 200), body, and grpc-status ("": none; with no body, in the +// headers, as gRPC's trailers-only errors are), after delay. +type capsAnswer struct { + status int + grpc string + body []byte + delay time.Duration +} + +// serveCapabilities answers a like rbe-cache's Caddy, over TLS and HTTP/2, +// after checking the request is the probe's GetCapabilities for instance +// oss. It returns the probe's RBE_CACHE_PROBE_URL, a CA file for curl's +// CURL_CA_BUNDLE, and the request count. +func serveCapabilities(t *testing.T, a capsAnswer) (url, caFile string, hits *atomic.Int32) { + t.Helper() + hits = new(atomic.Int32) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + body, _ := io.ReadAll(r.Body) + const path = "/build.bazel.remote.execution.v2.Capabilities/GetCapabilities" + if r.ProtoMajor != 2 || r.Method != http.MethodPost || r.URL.Path != path || + r.Header.Get("Content-Type") != "application/grpc" || !bytes.Equal(body, capsRequest) { + t.Errorf("probe sent %s %s %s, content-type %q, body %x; want HTTP/2 POST %s, application/grpc, %x", + r.Proto, r.Method, r.URL.Path, r.Header.Get("Content-Type"), body, path, capsRequest) + w.WriteHeader(http.StatusBadRequest) + return + } + select { + case <-time.After(a.delay): + case <-r.Context().Done(): + return + } + w.Header().Set("Content-Type", "application/grpc") + trailer := a.grpc != "" && a.body != nil + if trailer { + w.Header().Set("Trailer", "Grpc-Status") + } else if a.grpc != "" { + w.Header().Set("Grpc-Status", a.grpc) + } + status := a.status + if status == 0 { + status = http.StatusOK + } + w.WriteHeader(status) + _, _ = w.Write(a.body) + // Streamed, as gRPC servers answer: Go drops the trailer of a + // response it can give a content-length. + w.(http.Flusher).Flush() + if trailer { + w.Header().Set("Grpc-Status", a.grpc) + } + })) + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + caFile = filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0o644); err != nil { + t.Fatal(err) + } + return srv.URL, caFile, hits +} + +// requireCacheZstdProbeTools: the probe needs curl and python3 (without +// them it leaves the flag off, which the refused-probe tests cover). +func requireCacheZstdProbeTools(t *testing.T) { + t.Helper() + requireHostTool(t, "curl") + requireHostTool(t, "python3") +} + +// TestCacheZstdProbe runs cache-zstd-probe.sh against a stand-in rbe-cache. +// Only an answer listing ZSTD in cache_capabilities.supported_compressors +// (the field Bazel checks) passes; every other answer, error, refusal or +// timeout fails, and write-bazelrc.sh then writes no flag. The probe never +// writes stdout (write-bazelrc.sh appends its stdout to the rc) and always +// says why on stderr. +func TestCacheZstdProbe(t *testing.T) { + bash := requireHostTool(t, "bash") + requireCacheZstdProbeTools(t) + root := sourceRepoRoot(t) + script := filepath.Join(root, cacheZstdProbe) + + // The probe asks what fork-cache uses: rbe-cache, instance oss. + probeText := readPolicyFile(t, root, cacheZstdProbe) + for _, want := range []string{ + "url=${RBE_CACHE_PROBE_URL:-https://rbe-cache.ops.gascity.com:8443}\n", + `printf '\000\000\000\000\005\012\003oss'`, + "--connect-timeout 3 --max-time \"$max_time\"", + "max_time=${RBE_CACHE_PROBE_MAX_TIME:-5}\n", + } { + if !strings.Contains(probeText, want) { + t.Errorf("%s lacks %q", cacheZstdProbe, want) + } + } + if !bytes.Equal(capsRequest, []byte("\x00\x00\x00\x00\x05\x0a\x03oss")) { + t.Fatalf("capsRequest = %x", capsRequest) + } + bazelrc := readPolicyFile(t, root, ".bazelrc") + for _, want := range []string{"\nbuild:fork-cache --remote_cache=" + forkCacheEndpoint + "\n", "\nbuild:fork-cache --remote_instance_name=" + forkCacheInstance + "\n"} { + if !strings.Contains(bazelrc, want) { + t.Errorf(".bazelrc lacks %q, which the probe asks", strings.TrimSpace(want)) + } + } + if forkCacheEndpoint != "grpcs://rbe-cache.ops.gascity.com:8443" || forkCacheInstance != "oss" { + t.Errorf("fork-cache is %s instance %s; the probe asks rbe-cache.ops.gascity.com:8443 instance oss", forkCacheEndpoint, forkCacheInstance) + } + + zstd := grpcMessage(capsZstd) + cases := []struct { + name string + answer capsAnswer + want bool + }{ + {"rbe-cache before zstd", capsAnswer{grpc: "0", body: grpcMessage(capsLive)}, false}, + {"zstd advertised", capsAnswer{grpc: "0", body: zstd}, true}, + {"zstd among others, unpacked", capsAnswer{grpc: "0", body: grpcMessage(capsWithCache(capsLiveCache, pbVarint(6, 2), pbVarint(6, 1)))}, true}, + {"deflate only", capsAnswer{grpc: "0", body: grpcMessage(capsWithCache(capsLiveCache, pbBytes(6, []byte{2})))}, false}, + {"zstd for batch updates only", capsAnswer{grpc: "0", body: grpcMessage(capsWithCache(capsLiveCache, pbBytes(7, []byte{1})))}, false}, + {"zstd outside cache_capabilities", capsAnswer{grpc: "0", body: grpcMessage(append(capsLive, pbBytes(2, pbBytes(6, []byte{1}))...))}, false}, + {"gRPC error after the answer", capsAnswer{grpc: "13", body: zstd}, false}, + {"no grpc-status", capsAnswer{body: zstd}, false}, + {"trailers-only UNIMPLEMENTED", capsAnswer{grpc: "12"}, false}, + {"HTTP 502", capsAnswer{status: http.StatusBadGateway, grpc: "0", body: zstd}, false}, + {"compressed message", capsAnswer{grpc: "0", body: append([]byte{1}, zstd[1:]...)}, false}, + {"truncated message", capsAnswer{grpc: "0", body: zstd[:len(zstd)-1]}, false}, + {"truncated field", capsAnswer{grpc: "0", body: grpcMessage(capsZstd[:4])}, false}, + {"not gRPC", capsAnswer{grpc: "0", body: []byte("bad gateway")}, false}, + {"timeout", capsAnswer{grpc: "0", body: zstd, delay: 10 * time.Second}, false}, + } + run := func(t *testing.T, env ...string) (bool, string, string) { + t.Helper() + cmd := exec.Command(bash, script) + cmd.Dir = t.TempDir() + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH"), "RBE_CACHE_PROBE_MAX_TIME=1"}, env...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + var exit *exec.ExitError + if err != nil && !errors.As(err, &exit) { + t.Fatalf("run %s: %v", cacheZstdProbe, err) + } + return err == nil, stdout.String(), stderr.String() + } + check := func(t *testing.T, ok bool, stdout, stderr string, want bool) { + t.Helper() + if ok != want { + t.Errorf("probe passed: %v, want %v; stderr:\n%s", ok, want, stderr) + } + if stdout != "" { + t.Errorf("probe wrote stdout %q; write-bazelrc.sh would append it to the rc", stdout) + } + verdict := "; the fork cache stays identity\n" + if want { + verdict = "; the fork cache uses zstd\n" + } + if !strings.HasPrefix(stderr, "rbe-cache zstd probe: ") || !strings.HasSuffix(stderr, verdict) { + t.Errorf("probe stderr %q; want one verdict line ending %q", stderr, verdict) + } + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + url, ca, hits := serveCapabilities(t, c.answer) + start := time.Now() + ok, stdout, stderr := run(t, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca) + check(t, ok, stdout, stderr, c.want) + if hits.Load() != 1 { + t.Errorf("probe asked %d times, want 1", hits.Load()) + } + if d := time.Since(start); d > 5*time.Second { + t.Errorf("probe took %v; RBE_CACHE_PROBE_MAX_TIME=1 bounds it", d) + } + }) + } + t.Run("refused", func(t *testing.T) { + ok, stdout, stderr := run(t, "RBE_CACHE_PROBE_URL="+refusedProbeURL) + check(t, ok, stdout, stderr, false) + }) + t.Run("untrusted certificate", func(t *testing.T) { + url, _, hits := serveCapabilities(t, capsAnswer{grpc: "0", body: zstd}) + ok, stdout, stderr := run(t, "RBE_CACHE_PROBE_URL="+url) + check(t, ok, stdout, stderr, false) + if hits.Load() != 0 { + t.Errorf("probe completed a request through an untrusted certificate") + } + }) +} diff --git a/scripts/bazel_fork_mode_test.go b/scripts/bazel_fork_mode_test.go index 0ee2bf7fa..9cc3f9250 100644 --- a/scripts/bazel_fork_mode_test.go +++ b/scripts/bazel_fork_mode_test.go @@ -367,7 +367,8 @@ func TestSetupBazelRCWriterForkMode(t *testing.T) { cmd := exec.Command(bash, script) cmd.Dir = ws cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_WORKSPACE=" + ws, "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), - "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret} + "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret, + "RBE_CACHE_PROBE_URL=" + refusedProbeURL} for k, v := range env { cmd.Env = append(cmd.Env, k+"="+v) } diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 7e6f3c4e0..acc5ae0b7 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -342,6 +342,13 @@ var publicForkCacheLines = map[string]bool{ const ( forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" forkCacheInstance = "oss" + // zstd cache transfers: only the anonymous fork cache can advertise a + // compressor, so only fork-cache may ask for one, and only while + // write-bazelrc.sh's cache-zstd-probe.sh finds it advertised (a probe, + // not a repository variable: fork pull_request runs see no vars). + // Trusted remote-exec and rbe-fork never: their schedulers advertise + // none, and Bazel then refuses the remote. + forkCacheZstdLine = "build:fork-cache --remote_cache_compression" ) // publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint @@ -1258,6 +1265,9 @@ func checkBazelrcForkCache(bazelrc string) []error { var errs []error var opts []bazelrcOption for _, o := range parseBazelrcOptions(bazelrc) { + if strings.Contains(o.flag, "remote_cache_compression") { + errs = append(errs, errors.New(o.source()+" sets "+o.flag+"; only setup-bazel's generated rc may, for fork-cache while rbe-cache advertises zstd")) + } switch { case o.config == "fork-cache": opts = append(opts, o) @@ -1415,6 +1425,8 @@ func TestBazelForkCacheConfig(t *testing.T) { "slow timeout": good + "build:fork-cache --remote_timeout=60\n", "zero timeout": good + "build:fork-cache --remote_timeout=0\n", "duration timeout": good + "build:fork-cache --remote_timeout=15s\n", + "zstd in .bazelrc": good + "build:fork-cache --remote_cache_compression\n", + "trusted zstd in .bazelrc": good + "build:remote-exec --remote_cache_compression\n", "plain build expands": good + "build --config=fork-cache\n", "plain common expands": good + "common --config=fork-cache\n", "plain upload": good + "build --remote_upload_local_results\n", @@ -1539,3 +1551,38 @@ func TestBazelrcIntegrationLaneFixtures(t *testing.T) { } } } + +// TestBazelRemoteCacheCompressionOnlyForkCache: --remote_cache_compression +// appears in one place, write-bazelrc.sh's fork-cache line behind +// the zstd probe. No .bazelrc config, workflow or other action file may set +// it: every other remote (rbe-west's trusted schedulers on :443, rbe-fork on +// :8444) advertises no compressor, and Bazel then refuses the remote. +func TestBazelRemoteCacheCompressionOnlyForkCache(t *testing.T) { + root := bazelPolicyRoot(t) + files := []string{".bazelrc"} + for _, pattern := range []string{".github/workflows/*.yml", ".github/workflows/*.yaml", ".github/actions/*/*"} { + m, err := filepath.Glob(filepath.Join(root, pattern)) + if err != nil { + t.Fatal(err) + } + for _, f := range m { + rel, err := filepath.Rel(root, f) + if err != nil { + t.Fatal(err) + } + files = append(files, rel) + } + } + var found []string + for _, f := range files { + for i, line := range strings.Split(readPolicyFile(t, root, f), "\n") { + if strings.Contains(line, "remote_cache_compression") && !strings.HasPrefix(strings.TrimSpace(line), "#") { + found = append(found, f+":"+strconv.Itoa(i+1)+": "+strings.TrimSpace(line)) + } + } + } + want := setupBazelActionDir + "/write-bazelrc.sh:" + if len(found) != 1 || !strings.HasPrefix(found[0], want) || !strings.HasSuffix(found[0], `echo "`+forkCacheZstdLine+`"`) { + t.Errorf("--remote_cache_compression set at %q; want only %s echo %q", found, want, forkCacheZstdLine) + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 3d93f8cdf..086d8f0a1 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5902,6 +5902,9 @@ func TestSetupBazelRCWriter(t *testing.T) { "GITHUB_OUTPUT=" + out, "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret, + // The fork cache's zstd probe: a refused port unless extra + // names a stand-in rbe-cache; never rbe-cache itself. + "RBE_CACHE_PROBE_URL=" + refusedProbeURL, }, extra...) logs, err := cmd.CombinedOutput() outputs, _ := os.ReadFile(out) @@ -5948,6 +5951,58 @@ func TestSetupBazelRCWriter(t *testing.T) { t.Errorf("log lacks the read-only cache notice:\n%s", logs) } }) + // The fork cache asks for zstd exactly while rbe-cache advertises it + // (cache-zstd-probe.sh against a stand-in; TestCacheZstdProbe runs every + // other answer): the line follows --config=fork-cache, and the lane + // never fails on the probe. Remote-exec and rbe-fork never probe. + t.Run("fork cache zstd follows rbe-cache", func(t *testing.T) { + requireCacheZstdProbeTools(t) + for answer, wantLine := range map[string]bool{"zstd": true, "identity": false, "refused": false} { + env := []string{"BAZEL_FORK_CACHE=true"} + if answer != "refused" { + body := capsZstd + if answer == "identity" { + body = capsLive + } + url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(body)}) + env = append(env, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca) + } + outputs, rc, logs, err := run(t, env...) + if err != nil { + t.Fatalf("%s: err=%v\n%s", answer, err, logs) + } + if got := strings.Contains(rc, "\nbuild --config=fork-cache\n"+forkCacheZstdLine+"\n"); got != wantLine || strings.Count(rc, "remote_cache_compression") != map[bool]int{true: 1}[wantLine] { + t.Errorf("rbe-cache %s: rc = %q; want %q right after --config=fork-cache: %v", answer, rc, forkCacheZstdLine, wantLine) + } + if strings.Contains(rc, "remote-exec") || !strings.Contains(outputs, "cache=true") { + t.Errorf("rbe-cache %s: outputs = %q rc = %q; want cache=true and no remote-exec", answer, outputs, rc) + } + if !strings.Contains(logs, "rbe-cache zstd probe: ") { + t.Errorf("rbe-cache %s: log lacks the probe's verdict:\n%s", answer, logs) + } + } + }) + t.Run("remote-exec and rbe-fork never ask for zstd", func(t *testing.T) { + requireCacheZstdProbeTools(t) + url, ca, hits := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(capsZstd)}) + probe := []string{"RBE_CACHE_PROBE_URL=" + url, "CURL_CA_BUNDLE=" + ca} + for name, env := range map[string][]string{ + "remote-exec": {executor, "RBE_TLS_CERT=" + pem("CERTIFICATE"), "RBE_TLS_KEY=" + pem("PRIVATE KEY")}, + "rbe-fork": {"RBE_FORK_CERT_FILE=/tmp/s/fork.crt", "RBE_FORK_KEY_FILE=/tmp/s/fork.key", + "RBE_FORK_ENDPOINT=grpcs://rbe-fork.ops.gascity.com:8444", "RBE_FORK_INSTANCE=oss-fork"}, + } { + _, rc, logs, err := run(t, append(env, probe...)...) + if err != nil { + t.Fatalf("%s: err=%v\n%s", name, err, logs) + } + if strings.Contains(rc, "remote_cache_compression") { + t.Errorf("%s rc asks for compression; rbe-west's schedulers and rbe-fork advertise none:\n%s", name, rc) + } + } + if n := hits.Load(); n != 0 { + t.Errorf("remote-exec or rbe-fork probed rbe-cache %d times; only the fork cache may", n) + } + }) t.Run("rejects fork cache with the secrets", func(t *testing.T) { _, rc, logs, err := run(t, "BAZEL_FORK_CACHE=true", executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY")) if err == nil || !strings.Contains(logs, "mutually exclusive") {