diff --git a/.bazelrc b/.bazelrc index 9d514ce3e..58a3caeaa 100644 --- a/.bazelrc +++ b/.bazelrc @@ -344,6 +344,9 @@ build:fork-cache --remote_retries=2 build:fork-cache --experimental_circuit_breaker_strategy=failure # The farm admits 16 connections per source IP. build:fork-cache --remote_max_connections=8 +# No --remote_cache_compression here: setup-bazel's write-bazelrc.sh adds it +# for fork-cache while rbe-cache advertises zstd (cache-zstd-probe.sh), so the +# server decides and rollback needs no change here. # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. diff --git a/.github/actions/setup-bazel/cache-zstd-probe.sh b/.github/actions/setup-bazel/cache-zstd-probe.sh new file mode 100644 index 000000000..4e74368eb --- /dev/null +++ b/.github/actions/setup-bazel/cache-zstd-probe.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Does rbe-west's anonymous read-only cache (rbe-cache :8443, .bazelrc's +# fork-cache) advertise zstd right now? Exits 0 only when its REAPI +# GetCapabilities answer lists ZSTD in cache_capabilities.supported_compressors, +# the field Bazel checks before it accepts --remote_cache_compression (it +# refuses a remote that does not list it). Anything else exits nonzero: not +# listed, an HTTP or gRPC error, a refused connection, a timeout, an answer +# that does not parse, no curl or python3. The caller then leaves the flag +# off, and never fails on it. +# +# So the server decides, at run time: when rbe-west turns zstd on, fork-cache +# runs use it; when it turns it off (rollback), they stop. No repository +# variable (fork pull_request runs see none) and no revert. +# +# One unary gRPC call over curl's HTTP/2, decoded by python3's standard +# library: no grpcurl, no new action. Nothing goes to stdout (callers write +# their rc from it); one verdict line goes to stderr. A byte copy lives in +# gascity (tools/rbe/) and beads (.github/actions/setup-bazel/); keep in sync. +# +# Tests only: RBE_CACHE_PROBE_URL (a stand-in server) and +# RBE_CACHE_PROBE_MAX_TIME (seconds). +set -uo pipefail +url=${RBE_CACHE_PROBE_URL:-https://rbe-cache.ops.gascity.com:8443} +max_time=${RBE_CACHE_PROBE_MAX_TIME:-5} + +no() { + echo "rbe-cache zstd probe: $*; the fork cache stays identity" >&2 + exit 1 +} +command -v curl >/dev/null 2>&1 || no "no curl" +command -v python3 >/dev/null 2>&1 || no "no python3" +tmp=$(mktemp -d) || no "no temp dir" +trap 'rm -rf "$tmp"' EXIT + +# GetCapabilitiesRequest{instance_name: "oss"} (fork-cache's +# --remote_instance_name) as one uncompressed gRPC message. +printf '\000\000\000\000\005\012\003oss' >"$tmp/req" || no "cannot write the request" +code=$(curl -sS --http2 --connect-timeout 3 --max-time "$max_time" \ + -H 'content-type: application/grpc' -H 'te: trailers' \ + --data-binary @"$tmp/req" -D "$tmp/head" -o "$tmp/body" -w '%{http_code}' \ + "$url/build.bazel.remote.execution.v2.Capabilities/GetCapabilities" 2>"$tmp/err") || + no "$(tr '\n' ' ' <"$tmp/err")" +[ "$code" = 200 ] || no "HTTP $code" +# grpc-status is a trailer; curl writes trailers after the headers. +st=$(tr -d '\r' <"$tmp/head" | sed -n 's/^grpc-status: *//Ip' | tail -1) +[ "$st" = 0 ] || no "gRPC status ${st:-missing}" + +why=$(python3 -I - "$tmp/body" <<'PY' +import sys + +ZSTD = 1 # build.bazel.remote.execution.v2.Compressor.Value.ZSTD + + +def varint(b, i): + v = s = 0 + while True: + if i >= len(b) or s > 63: + raise ValueError("truncated varint") + c = b[i] + i += 1 + v |= (c & 0x7F) << s + s += 7 + if c < 0x80: + return v, i + + +def fields(b): + i = 0 + while i < len(b): + key, i = varint(b, i) + num, wire = key >> 3, key & 7 + if wire == 0: + v, i = varint(b, i) + elif wire == 1: + v, i = b[i:i + 8], i + 8 + elif wire == 2: + n, i = varint(b, i) + v, i = b[i:i + n], i + n + elif wire == 5: + v, i = b[i:i + 4], i + 4 + else: + raise ValueError("wire type %d" % wire) + if i > len(b): + raise ValueError("truncated field %d" % num) + yield num, wire, v + + +try: + body = open(sys.argv[1], "rb").read() + if len(body) < 5 or body[0] != 0: + raise ValueError("not an uncompressed gRPC message") + n = int.from_bytes(body[1:5], "big") + if len(body) != 5 + n: + raise ValueError("gRPC length %d, %d bytes" % (n, len(body) - 5)) + compressors = [] + for num, wire, v in fields(body[5:]): + if num != 1 or wire != 2: # ServerCapabilities.cache_capabilities + continue + for cnum, cwire, cv in fields(v): + if cnum != 6: # CacheCapabilities.supported_compressors + continue + if cwire == 0: + compressors.append(cv) + elif cwire == 2: # packed + j = 0 + while j < len(cv): + x, j = varint(cv, j) + compressors.append(x) +except (OSError, ValueError) as e: + print("unreadable answer (%s)" % e) + sys.exit(2) +if ZSTD in compressors: + print("advertises zstd") + sys.exit(0) +print("zstd not advertised (supported_compressors %s)" % compressors) +sys.exit(1) +PY +) || no "${why:-python3 failed}" +echo "rbe-cache zstd probe: $why; the fork cache uses zstd" >&2 diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh index 2436d976d..5dca6562f 100644 --- a/.github/actions/setup-bazel/write-bazelrc.sh +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -27,7 +27,12 @@ # # BAZEL_FORK_CACHE=true (bazel.yml mode "cache": rbe=cache dispatches, and # fork/Dependabot runs while rbe-fork is closed) appends --config=fork-cache: -# .bazelrc's credential-free, read-only rbe-west cache. +# .bazelrc's credential-free, read-only rbe-west cache. While that cache +# advertises zstd (cache-zstd-probe.sh asks its GetCapabilities; any failure +# means no) it also asks for zstd transfers (--remote_cache_compression). A +# probe, not a repository variable: fork pull_request runs see no vars. No +# other mode ever asks, because only the anonymous cache can advertise zstd, +# and Bazel refuses a remote that does not. # # RBE_FORK_CERT_FILE + RBE_FORK_KEY_FILE + RBE_FORK_ENDPOINT + RBE_FORK_INSTANCE # (bazel.yml modes fork-ro/fork-rw: fork-credential.sh's outputs, a @@ -222,7 +227,15 @@ elif [[ "$set_fields" -ne 0 ]]; then echo "setup-bazel: remote execution is partially configured; BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY must be set together (or none, for local execution)" >&2 exit 1 elif [[ "$fork_cache" == true ]]; then - echo "build --config=fork-cache" >>"$rc" + # zstd for the anonymous cache's transfers while it advertises zstd: + # transport only, so action keys are unchanged. The probe never fails + # this script; it only decides the line. + { + echo "build --config=fork-cache" + if bash "$(dirname "${BASH_SOURCE[0]}")/cache-zstd-probe.sh"; then + echo "build:fork-cache --remote_cache_compression" + fi + } >>"$rc" cache=true echo "setup-bazel: read-only remote cache (rbe-cache); executing locally" else diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index e942f7378..7ee9e653e 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -1,14 +1,23 @@ package scripts_test import ( + "bytes" "encoding/base64" + "encoding/binary" + "encoding/pem" + "errors" + "io" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "regexp" "sort" "strings" + "sync/atomic" "testing" + "time" ) // Mode cache, end to end, for every way a bazel.yml run can start: the rbe @@ -47,7 +56,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // setup-bazel's env values: `needs.rbe.outputs.X == 'v' && RHS || ''`, // RHS a secret or a literal. Anything else fails the test, so the - // simulation cannot drift from the workflow. + // simulation cannot drift from the workflow; in particular no repository + // variable (fork pull_request runs see none). setupExpr := regexp.MustCompile(`^\$\{\{ needs\.rbe\.outputs\.(enabled|mode) == '([a-z]+)' && (?:secrets\.([A-Z_]+)|'([^']*)') \|\| '' \}\}$`) evalSetup := func(t *testing.T, expr, mode, enabled, tier string, haveSecrets bool) string { t.Helper() @@ -100,6 +110,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "bazel-ci-cache"), "BAZEL_CI_SECRET_DIR=" + secretDir, + // Mode cache's zstd probe: a refused port, never rbe-cache. + "RBE_CACHE_PROBE_URL=" + refusedProbeURL, } for k, v := range env { cmd.Env = append(cmd.Env, k+"="+v) @@ -323,6 +335,13 @@ func checkModeRC(t *testing.T, lane, mode, outputs, rc string, files []string, l t.Errorf("%s (mode %s): rc lacks %q:\n%s", lane, mode, strings.TrimSpace(want), rc) } } + // zstd: no mode here asks for compression. Mode cache's probe is + // refused here (TestSetupBazelRCWriter runs it passing); the trusted + // schedulers and rbe-fork advertise none, and Bazel refuses such a + // remote. + if has("remote_cache_compression") { + t.Errorf("%s (mode %s): rc asks for compression:\n%s", lane, mode, rc) + } credentialFree := func() { for _, bad := range []string{"remote_executor", "remote-exec", "tls_", "remote_instance_name", "--remote_upload_local_results", "--remote_cache", "--remote_header", "--bes_", "farm.invalid"} { if has(bad) { @@ -382,3 +401,233 @@ func checkModeRC(t *testing.T, lane, mode, outputs, rc string, files []string, l t.Errorf("%s runs in mode %q", lane, mode) } } + +// cacheZstdProbe: setup-bazel's zstd probe for the fork cache (a byte copy +// of gascity's tools/rbe/cache-zstd-probe.sh). +const cacheZstdProbe = setupBazelActionDir + "/cache-zstd-probe.sh" + +// refusedProbeURL: a loopback port nothing listens on, the probe's URL in +// tests that do not serve one (a refused connection: no flag). +const refusedProbeURL = "https://127.0.0.1:1" + +// GetCapabilities bodies for a stand-in rbe-cache. capsLive is what rbe-cache +// answered on 2026-10-04, before it advertised zstd (cache_capabilities: +// SHA256 and BLAKE3, action cache read-only, 64 MiB batches, symlinks +// allowed; API 2.0 to 2.3); capsZstd is the same with supported_compressors +// [ZSTD]. +var ( + capsLiveCache = []byte{0x0a, 0x02, 0x01, 0x09, 0x12, 0x02, 0x08, 0x01, 0x20, 0x80, 0x80, 0x04, 0x28, 0x01} + capsLiveAPIVersions = []byte{0x22, 0x02, 0x08, 0x02, 0x2a, 0x04, 0x08, 0x02, 0x10, 0x03} + capsLive = capsWithCache(capsLiveCache) + capsZstd = capsWithCache(pbBytes(6, []byte{1}), capsLiveCache) + // GetCapabilitiesRequest{instance_name: "oss"}, fork-cache's instance. + capsRequest = grpcMessage(pbBytes(1, []byte("oss"))) +) + +// capsWithCache: a ServerCapabilities with the live API versions and a +// cache_capabilities of the given fields. +func capsWithCache(cache ...[]byte) []byte { + return append(pbBytes(1, bytes.Join(cache, nil)), capsLiveAPIVersions...) +} + +func pbBytes(field int, b []byte) []byte { + out := binary.AppendUvarint(nil, uint64(field)<<3|2) + return append(binary.AppendUvarint(out, uint64(len(b))), b...) +} + +func pbVarint(field int, v uint64) []byte { + return binary.AppendUvarint(binary.AppendUvarint(nil, uint64(field)<<3), v) +} + +// grpcMessage frames m as one uncompressed gRPC message. +func grpcMessage(m []byte) []byte { + return append(binary.BigEndian.AppendUint32([]byte{0}, uint32(len(m))), m...) +} + +// capsAnswer is how the stand-in rbe-cache answers GetCapabilities: HTTP +// status (0: 200), body, and grpc-status ("": none; with no body, in the +// headers, as gRPC's trailers-only errors are), after delay. +type capsAnswer struct { + status int + grpc string + body []byte + delay time.Duration +} + +// serveCapabilities answers a like rbe-cache's Caddy, over TLS and HTTP/2, +// after checking the request is the probe's GetCapabilities for instance +// oss. It returns the probe's RBE_CACHE_PROBE_URL, a CA file for curl's +// CURL_CA_BUNDLE, and the request count. +func serveCapabilities(t *testing.T, a capsAnswer) (url, caFile string, hits *atomic.Int32) { + t.Helper() + hits = new(atomic.Int32) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + body, _ := io.ReadAll(r.Body) + const path = "/build.bazel.remote.execution.v2.Capabilities/GetCapabilities" + if r.ProtoMajor != 2 || r.Method != http.MethodPost || r.URL.Path != path || + r.Header.Get("Content-Type") != "application/grpc" || !bytes.Equal(body, capsRequest) { + t.Errorf("probe sent %s %s %s, content-type %q, body %x; want HTTP/2 POST %s, application/grpc, %x", + r.Proto, r.Method, r.URL.Path, r.Header.Get("Content-Type"), body, path, capsRequest) + w.WriteHeader(http.StatusBadRequest) + return + } + select { + case <-time.After(a.delay): + case <-r.Context().Done(): + return + } + w.Header().Set("Content-Type", "application/grpc") + trailer := a.grpc != "" && a.body != nil + if trailer { + w.Header().Set("Trailer", "Grpc-Status") + } else if a.grpc != "" { + w.Header().Set("Grpc-Status", a.grpc) + } + status := a.status + if status == 0 { + status = http.StatusOK + } + w.WriteHeader(status) + _, _ = w.Write(a.body) + // Streamed, as gRPC servers answer: Go drops the trailer of a + // response it can give a content-length. + w.(http.Flusher).Flush() + if trailer { + w.Header().Set("Grpc-Status", a.grpc) + } + })) + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + caFile = filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0o644); err != nil { + t.Fatal(err) + } + return srv.URL, caFile, hits +} + +// requireCacheZstdProbeTools: the probe needs curl and python3 (without +// them it leaves the flag off, which the refused-probe tests cover). +func requireCacheZstdProbeTools(t *testing.T) { + t.Helper() + requireHostTool(t, "curl") + requireHostTool(t, "python3") +} + +// TestCacheZstdProbe runs cache-zstd-probe.sh against a stand-in rbe-cache. +// Only an answer listing ZSTD in cache_capabilities.supported_compressors +// (the field Bazel checks) passes; every other answer, error, refusal or +// timeout fails, and write-bazelrc.sh then writes no flag. The probe never +// writes stdout (write-bazelrc.sh appends its stdout to the rc) and always +// says why on stderr. +func TestCacheZstdProbe(t *testing.T) { + bash := requireHostTool(t, "bash") + requireCacheZstdProbeTools(t) + root := sourceRepoRoot(t) + script := filepath.Join(root, cacheZstdProbe) + + // The probe asks what fork-cache uses: rbe-cache, instance oss. + probeText := readPolicyFile(t, root, cacheZstdProbe) + for _, want := range []string{ + "url=${RBE_CACHE_PROBE_URL:-https://rbe-cache.ops.gascity.com:8443}\n", + `printf '\000\000\000\000\005\012\003oss'`, + "--connect-timeout 3 --max-time \"$max_time\"", + "max_time=${RBE_CACHE_PROBE_MAX_TIME:-5}\n", + } { + if !strings.Contains(probeText, want) { + t.Errorf("%s lacks %q", cacheZstdProbe, want) + } + } + if !bytes.Equal(capsRequest, []byte("\x00\x00\x00\x00\x05\x0a\x03oss")) { + t.Fatalf("capsRequest = %x", capsRequest) + } + bazelrc := readPolicyFile(t, root, ".bazelrc") + for _, want := range []string{"\nbuild:fork-cache --remote_cache=" + forkCacheEndpoint + "\n", "\nbuild:fork-cache --remote_instance_name=" + forkCacheInstance + "\n"} { + if !strings.Contains(bazelrc, want) { + t.Errorf(".bazelrc lacks %q, which the probe asks", strings.TrimSpace(want)) + } + } + if forkCacheEndpoint != "grpcs://rbe-cache.ops.gascity.com:8443" || forkCacheInstance != "oss" { + t.Errorf("fork-cache is %s instance %s; the probe asks rbe-cache.ops.gascity.com:8443 instance oss", forkCacheEndpoint, forkCacheInstance) + } + + zstd := grpcMessage(capsZstd) + cases := []struct { + name string + answer capsAnswer + want bool + }{ + {"rbe-cache before zstd", capsAnswer{grpc: "0", body: grpcMessage(capsLive)}, false}, + {"zstd advertised", capsAnswer{grpc: "0", body: zstd}, true}, + {"zstd among others, unpacked", capsAnswer{grpc: "0", body: grpcMessage(capsWithCache(capsLiveCache, pbVarint(6, 2), pbVarint(6, 1)))}, true}, + {"deflate only", capsAnswer{grpc: "0", body: grpcMessage(capsWithCache(capsLiveCache, pbBytes(6, []byte{2})))}, false}, + {"zstd for batch updates only", capsAnswer{grpc: "0", body: grpcMessage(capsWithCache(capsLiveCache, pbBytes(7, []byte{1})))}, false}, + {"zstd outside cache_capabilities", capsAnswer{grpc: "0", body: grpcMessage(append(capsLive, pbBytes(2, pbBytes(6, []byte{1}))...))}, false}, + {"gRPC error after the answer", capsAnswer{grpc: "13", body: zstd}, false}, + {"no grpc-status", capsAnswer{body: zstd}, false}, + {"trailers-only UNIMPLEMENTED", capsAnswer{grpc: "12"}, false}, + {"HTTP 502", capsAnswer{status: http.StatusBadGateway, grpc: "0", body: zstd}, false}, + {"compressed message", capsAnswer{grpc: "0", body: append([]byte{1}, zstd[1:]...)}, false}, + {"truncated message", capsAnswer{grpc: "0", body: zstd[:len(zstd)-1]}, false}, + {"truncated field", capsAnswer{grpc: "0", body: grpcMessage(capsZstd[:4])}, false}, + {"not gRPC", capsAnswer{grpc: "0", body: []byte("bad gateway")}, false}, + {"timeout", capsAnswer{grpc: "0", body: zstd, delay: 10 * time.Second}, false}, + } + run := func(t *testing.T, env ...string) (bool, string, string) { + t.Helper() + cmd := exec.Command(bash, script) + cmd.Dir = t.TempDir() + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH"), "RBE_CACHE_PROBE_MAX_TIME=1"}, env...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + var exit *exec.ExitError + if err != nil && !errors.As(err, &exit) { + t.Fatalf("run %s: %v", cacheZstdProbe, err) + } + return err == nil, stdout.String(), stderr.String() + } + check := func(t *testing.T, ok bool, stdout, stderr string, want bool) { + t.Helper() + if ok != want { + t.Errorf("probe passed: %v, want %v; stderr:\n%s", ok, want, stderr) + } + if stdout != "" { + t.Errorf("probe wrote stdout %q; write-bazelrc.sh would append it to the rc", stdout) + } + verdict := "; the fork cache stays identity\n" + if want { + verdict = "; the fork cache uses zstd\n" + } + if !strings.HasPrefix(stderr, "rbe-cache zstd probe: ") || !strings.HasSuffix(stderr, verdict) { + t.Errorf("probe stderr %q; want one verdict line ending %q", stderr, verdict) + } + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + url, ca, hits := serveCapabilities(t, c.answer) + start := time.Now() + ok, stdout, stderr := run(t, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca) + check(t, ok, stdout, stderr, c.want) + if hits.Load() != 1 { + t.Errorf("probe asked %d times, want 1", hits.Load()) + } + if d := time.Since(start); d > 5*time.Second { + t.Errorf("probe took %v; RBE_CACHE_PROBE_MAX_TIME=1 bounds it", d) + } + }) + } + t.Run("refused", func(t *testing.T) { + ok, stdout, stderr := run(t, "RBE_CACHE_PROBE_URL="+refusedProbeURL) + check(t, ok, stdout, stderr, false) + }) + t.Run("untrusted certificate", func(t *testing.T) { + url, _, hits := serveCapabilities(t, capsAnswer{grpc: "0", body: zstd}) + ok, stdout, stderr := run(t, "RBE_CACHE_PROBE_URL="+url) + check(t, ok, stdout, stderr, false) + if hits.Load() != 0 { + t.Errorf("probe completed a request through an untrusted certificate") + } + }) +} diff --git a/scripts/bazel_fork_mode_test.go b/scripts/bazel_fork_mode_test.go index 0ee2bf7fa..9cc3f9250 100644 --- a/scripts/bazel_fork_mode_test.go +++ b/scripts/bazel_fork_mode_test.go @@ -367,7 +367,8 @@ func TestSetupBazelRCWriterForkMode(t *testing.T) { cmd := exec.Command(bash, script) cmd.Dir = ws cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_WORKSPACE=" + ws, "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), - "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret} + "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret, + "RBE_CACHE_PROBE_URL=" + refusedProbeURL} for k, v := range env { cmd.Env = append(cmd.Env, k+"="+v) } diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 7e6f3c4e0..acc5ae0b7 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -342,6 +342,13 @@ var publicForkCacheLines = map[string]bool{ const ( forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" forkCacheInstance = "oss" + // zstd cache transfers: only the anonymous fork cache can advertise a + // compressor, so only fork-cache may ask for one, and only while + // write-bazelrc.sh's cache-zstd-probe.sh finds it advertised (a probe, + // not a repository variable: fork pull_request runs see no vars). + // Trusted remote-exec and rbe-fork never: their schedulers advertise + // none, and Bazel then refuses the remote. + forkCacheZstdLine = "build:fork-cache --remote_cache_compression" ) // publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint @@ -1258,6 +1265,9 @@ func checkBazelrcForkCache(bazelrc string) []error { var errs []error var opts []bazelrcOption for _, o := range parseBazelrcOptions(bazelrc) { + if strings.Contains(o.flag, "remote_cache_compression") { + errs = append(errs, errors.New(o.source()+" sets "+o.flag+"; only setup-bazel's generated rc may, for fork-cache while rbe-cache advertises zstd")) + } switch { case o.config == "fork-cache": opts = append(opts, o) @@ -1415,6 +1425,8 @@ func TestBazelForkCacheConfig(t *testing.T) { "slow timeout": good + "build:fork-cache --remote_timeout=60\n", "zero timeout": good + "build:fork-cache --remote_timeout=0\n", "duration timeout": good + "build:fork-cache --remote_timeout=15s\n", + "zstd in .bazelrc": good + "build:fork-cache --remote_cache_compression\n", + "trusted zstd in .bazelrc": good + "build:remote-exec --remote_cache_compression\n", "plain build expands": good + "build --config=fork-cache\n", "plain common expands": good + "common --config=fork-cache\n", "plain upload": good + "build --remote_upload_local_results\n", @@ -1539,3 +1551,38 @@ func TestBazelrcIntegrationLaneFixtures(t *testing.T) { } } } + +// TestBazelRemoteCacheCompressionOnlyForkCache: --remote_cache_compression +// appears in one place, write-bazelrc.sh's fork-cache line behind +// the zstd probe. No .bazelrc config, workflow or other action file may set +// it: every other remote (rbe-west's trusted schedulers on :443, rbe-fork on +// :8444) advertises no compressor, and Bazel then refuses the remote. +func TestBazelRemoteCacheCompressionOnlyForkCache(t *testing.T) { + root := bazelPolicyRoot(t) + files := []string{".bazelrc"} + for _, pattern := range []string{".github/workflows/*.yml", ".github/workflows/*.yaml", ".github/actions/*/*"} { + m, err := filepath.Glob(filepath.Join(root, pattern)) + if err != nil { + t.Fatal(err) + } + for _, f := range m { + rel, err := filepath.Rel(root, f) + if err != nil { + t.Fatal(err) + } + files = append(files, rel) + } + } + var found []string + for _, f := range files { + for i, line := range strings.Split(readPolicyFile(t, root, f), "\n") { + if strings.Contains(line, "remote_cache_compression") && !strings.HasPrefix(strings.TrimSpace(line), "#") { + found = append(found, f+":"+strconv.Itoa(i+1)+": "+strings.TrimSpace(line)) + } + } + } + want := setupBazelActionDir + "/write-bazelrc.sh:" + if len(found) != 1 || !strings.HasPrefix(found[0], want) || !strings.HasSuffix(found[0], `echo "`+forkCacheZstdLine+`"`) { + t.Errorf("--remote_cache_compression set at %q; want only %s echo %q", found, want, forkCacheZstdLine) + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 3d93f8cdf..086d8f0a1 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5902,6 +5902,9 @@ func TestSetupBazelRCWriter(t *testing.T) { "GITHUB_OUTPUT=" + out, "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret, + // The fork cache's zstd probe: a refused port unless extra + // names a stand-in rbe-cache; never rbe-cache itself. + "RBE_CACHE_PROBE_URL=" + refusedProbeURL, }, extra...) logs, err := cmd.CombinedOutput() outputs, _ := os.ReadFile(out) @@ -5948,6 +5951,58 @@ func TestSetupBazelRCWriter(t *testing.T) { t.Errorf("log lacks the read-only cache notice:\n%s", logs) } }) + // The fork cache asks for zstd exactly while rbe-cache advertises it + // (cache-zstd-probe.sh against a stand-in; TestCacheZstdProbe runs every + // other answer): the line follows --config=fork-cache, and the lane + // never fails on the probe. Remote-exec and rbe-fork never probe. + t.Run("fork cache zstd follows rbe-cache", func(t *testing.T) { + requireCacheZstdProbeTools(t) + for answer, wantLine := range map[string]bool{"zstd": true, "identity": false, "refused": false} { + env := []string{"BAZEL_FORK_CACHE=true"} + if answer != "refused" { + body := capsZstd + if answer == "identity" { + body = capsLive + } + url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(body)}) + env = append(env, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca) + } + outputs, rc, logs, err := run(t, env...) + if err != nil { + t.Fatalf("%s: err=%v\n%s", answer, err, logs) + } + if got := strings.Contains(rc, "\nbuild --config=fork-cache\n"+forkCacheZstdLine+"\n"); got != wantLine || strings.Count(rc, "remote_cache_compression") != map[bool]int{true: 1}[wantLine] { + t.Errorf("rbe-cache %s: rc = %q; want %q right after --config=fork-cache: %v", answer, rc, forkCacheZstdLine, wantLine) + } + if strings.Contains(rc, "remote-exec") || !strings.Contains(outputs, "cache=true") { + t.Errorf("rbe-cache %s: outputs = %q rc = %q; want cache=true and no remote-exec", answer, outputs, rc) + } + if !strings.Contains(logs, "rbe-cache zstd probe: ") { + t.Errorf("rbe-cache %s: log lacks the probe's verdict:\n%s", answer, logs) + } + } + }) + t.Run("remote-exec and rbe-fork never ask for zstd", func(t *testing.T) { + requireCacheZstdProbeTools(t) + url, ca, hits := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(capsZstd)}) + probe := []string{"RBE_CACHE_PROBE_URL=" + url, "CURL_CA_BUNDLE=" + ca} + for name, env := range map[string][]string{ + "remote-exec": {executor, "RBE_TLS_CERT=" + pem("CERTIFICATE"), "RBE_TLS_KEY=" + pem("PRIVATE KEY")}, + "rbe-fork": {"RBE_FORK_CERT_FILE=/tmp/s/fork.crt", "RBE_FORK_KEY_FILE=/tmp/s/fork.key", + "RBE_FORK_ENDPOINT=grpcs://rbe-fork.ops.gascity.com:8444", "RBE_FORK_INSTANCE=oss-fork"}, + } { + _, rc, logs, err := run(t, append(env, probe...)...) + if err != nil { + t.Fatalf("%s: err=%v\n%s", name, err, logs) + } + if strings.Contains(rc, "remote_cache_compression") { + t.Errorf("%s rc asks for compression; rbe-west's schedulers and rbe-fork advertise none:\n%s", name, rc) + } + } + if n := hits.Load(); n != 0 { + t.Errorf("remote-exec or rbe-fork probed rbe-cache %d times; only the fork cache may", n) + } + }) t.Run("rejects fork cache with the secrets", func(t *testing.T) { _, rc, logs, err := run(t, "BAZEL_FORK_CACHE=true", executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY")) if err == nil || !strings.Contains(logs, "mutually exclusive") {