beads/scripts/tracked_files_test.go
Julian Knutsen f53010a23b
ci(bazel): remote repo contents cache: main-push seed, lane readers, nightly verify (#7453)
Port gascity's remote repo contents cache wiring (gascity #7467, #7478) to
bazel.yml, gated on the repository variable RBE_REPO_CONTENTS_CACHE
(unset/off: nothing; seed: rrc-seed on push to main; canary: + bazel-test
reads; on: + every test lane reads). Mode remote only.

- rbe job: an rrc step turns the variable into the rrc output.
- Readers: the eight test lanes append startup
  --experimental_remote_repo_contents_cache and common
  --loading_phase_threads=64 to .bazelrc.local before Set up Bazel.
- rrc-seed (push to main, id-token: write): a 30-minute
  rbe-rrc-writer-beads certificate from rbe-west's mint
  (.github/scripts/rrc-writer-credential.sh), then every lane's command
  (.github/scripts/rrc-lane-commands.txt) with --nobuild, --remote_cache
  only and --remote_upload_local_results (.github/scripts/rrc-run-lanes.sh).
- rrc-verify (schedule/dispatch, issues: write): cold fetch of every lane
  and tools/bazel/rrc_verify.py against the cache; a mismatch opens an
  rrc-verify issue.
- pr.yml, nightly.yml and bazel-farm.yml grant id-token: write and
  issues: write on their bazel.yml call (startup_failure otherwise);
  TestBazelWorkflowCallersGrantEveryRequestedPermission guards it.

rrc-writer-credential.sh, rrc_verify.py and rrc_verify_test.py are byte
copies of gascity's, pinned by sha256.

Refs ga-vnycm2.29.

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-10 02:50:26 +00:00

306 lines
13 KiB
Go

package scripts_test
import (
"bufio"
"bytes"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
)
// Scans of every tracked file: //scripts:tracked_files_test, the one target
// here that declares //:repo_files (the whole checkout), so that every other
// target can declare only the partition it reads and stay cached when an
// unrelated file changes. Keep this file to tests that really read every
// tracked file, and keep them cheap: any edit anywhere re-runs them.
// trackedFilesMin is the fewest files a listing of //:repo_files may hold
// (the checkout has several thousand).
const trackedFilesMin = 1000
// --- no remote-execution endpoints in tracked files -------------------------
// Remote-execution endpoints, credentials and TLS material belong in a
// gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the
// workspace, never in the repository. The scan covers committed bazelrc-like
// files, Markdown docs and .github/** (plan §3.8) and flags:
// - any remote/BES/TLS flag below whose value is a literal (not a
// <placeholder>, a $VARIABLE/${{ expression }}, empty, or loopback);
// - in rc and .github files, any gRPC URL that is not loopback. Markdown is
// exempt from this rule because beads documents OTel gRPC exporters.
//
// The one exception is publicForkCacheLines.
var (
remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` +
`bes_backend|bes_results_url|bes_header|tls_client_certificate|tls_client_key|tls_certificate`
// `--flag=value` is recognized everywhere; `--flag value` only in rc and
// workflow files, where prose ("--remote_executor and ...") does not occur.
remoteFlagEqRe = regexp.MustCompile(`--(` + remoteFlagNames + `)=("[^"]*"|'[^']*'|[^\s"'` + "`" + `]*)`)
remoteFlagSpaceRe = regexp.MustCompile(`--(` + remoteFlagNames + `)\s+("[^"]*"|'[^']*'|[^\s"'` + "`" + `]+)`)
remoteEndpointRe = regexp.MustCompile(`\bgrpcs?://[^\s"'<>)\]]+`)
loopbackValueRe = regexp.MustCompile(`^(?:[a-z][a-z0-9+.-]*://)?(?:127\.0\.0\.1|localhost|\[::1\])(?:[:/]|$)`)
// remoteScanPrefilter: a file lacking all of these cannot produce a hit.
remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")}
)
// publicForkCacheLines: the endpoint lines of .bazelrc's fork-cache config,
// public by design. rbe-west's rbe-cache endpoint is anonymous and read-only
// (action-cache and CAS reads; writes and Execute are refused by the farm),
// so there is no credential to leak. Exactly these lines, trimmed, and only
// in .bazelrc; the same line anywhere else, or any other endpoint, is still a
// hit. Assembled so this file never contains a literal endpoint.
var publicForkCacheLines = map[string]bool{
"build:fork-cache --remote_cache=" + forkCacheEndpoint: true,
"build:fork-cache --remote_instance_name=" + forkCacheInstance: true,
}
// publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint
// rbe-fork-mint may hand a fork run (rbe-fork, :8444), so a compromised
// mint cannot point fork builds elsewhere. It is public and carries no
// credential: allowed as exactly this line, in exactly that file (a byte
// copy of gascity's tools/rbe/fork-credential.sh).
const (
publicRBEForkPinFile = ".github/actions/setup-bazel/fork-credential.sh"
publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}`
)
// publicRRCWriterPin: rrc-writer-credential.sh (a byte copy of gascity's)
// pins the one endpoint rbe-west's mint may hand the remote repo contents
// cache writer (rbe-west, :443), so a compromised mint cannot point the
// writer's uploads elsewhere. Public, no credential: allowed as exactly
// this line, in exactly that file.
const (
publicRRCWriterPinFile = ".github/scripts/rrc-writer-credential.sh"
publicRRCWriterPinLine = `ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-west\.ops\.gascity\.com(:443)?$'}`
)
type endpointHit struct {
path string
line int
what string
}
// remoteScanKind classifies a tracked path for the endpoint scan.
func remoteScanKind(rel string) (scan, strict bool) {
base := filepath.Base(rel)
switch {
case strings.Contains(base, "bazelrc"), strings.HasPrefix(rel, ".github/"):
return true, true
case strings.HasSuffix(base, ".md"):
return true, false
}
return false, false
}
func allowedRemoteValue(v string) bool {
v = strings.Trim(v, `"'`)
if i := strings.Index(v, "://"); i >= 0 {
v = v[i+len("://"):]
}
return v == "" || strings.HasPrefix(v, "<") || strings.HasPrefix(v, "$") || loopbackValueRe.MatchString(v)
}
// findRemoteEndpoints reports literal remote endpoints/credentials in content.
// strict enables the whitespace-separated flag form and the gRPC URL rule.
func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit {
if bytes.IndexByte(content, 0) >= 0 { // binary
return nil
}
relevant := false
for _, needle := range remoteScanPrefilter {
if bytes.Contains(content, needle) {
relevant = true
break
}
}
if !relevant {
return nil
}
flagRes := []*regexp.Regexp{remoteFlagEqRe}
if strict {
flagRes = append(flagRes, remoteFlagSpaceRe)
}
var hits []endpointHit
scanner := bufio.NewScanner(bytes.NewReader(content))
scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
for n := 1; scanner.Scan(); n++ {
line := scanner.Text()
if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] {
continue
}
if path == publicRBEForkPinFile && line == publicRBEForkPinLine ||
path == publicRRCWriterPinFile && line == publicRRCWriterPinLine {
continue
}
for _, re := range flagRes {
for _, m := range re.FindAllStringSubmatch(line, -1) {
if !allowedRemoteValue(m[2]) {
hits = append(hits, endpointHit{path: path, line: n, what: m[0]})
}
}
}
if strict {
for _, url := range remoteEndpointRe.FindAllString(line, -1) {
if !allowedRemoteValue(url) {
hits = append(hits, endpointHit{path: path, line: n, what: url})
}
}
}
}
return hits
}
func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) {
// Fixtures are assembled at runtime so this file never contains a literal
// endpoint itself.
scheme := "grpc" + "s://"
flag := func(name string) string { return "--" + name }
for _, bad := range []string{
"build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe.example.com:443\n",
"build:remote-exec " + flag("remote_executor") + "=rbe.example.com:443\n",
"build:remote-exec " + flag("remote_executor") + " rbe.example.com:443\n",
"build " + flag("remote_cache") + "=https://cache.example.com\n",
"build " + flag("bes_backend") + "=bes.example.com\n",
"build " + flag("bes_results_url") + "=https://results.example.com/inv/\n",
"build " + flag("remote_header") + "=x-api-key=abc123\n",
"build " + flag("tls_client_certificate") + "=/etc/rbe/client.crt\n",
"build " + flag("tls_client_key") + "=\"/etc/rbe/client.key\"\n",
"# farm: " + scheme + "10.0.0.5:8980\n",
"REMOTE=" + "grpc" + "://cache.internal:9092\n",
} {
if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 {
t.Errorf("endpoint fixture not detected: %q", bad)
}
}
for _, ok := range []string{
"build:remote-exec " + flag("remote_executor") + "=" + scheme + "<endpoint>\n",
"build:remote-exec " + flag("remote_executor") + "=<endpoint>\n",
"build:remote-exec " + flag("tls_client_certificate") + "=<path>\n",
" run: echo \"build " + flag("remote_executor") + "=${{ secrets.BAZEL_REMOTE_EXECUTOR }}\" >> \"$RUNNER_TEMP/ci.bazelrc\"\n",
" " + flag("tls_client_key") + " \"$RUNNER_TEMP/rbe.key\"\n",
" " + flag("remote_executor") + "=" + scheme + "${{ secrets.RBE_HOST }}\n",
flag("remote_cache") + "=" + "grpc" + "://127.0.0.1:50052\n",
flag("remote_cache") + "=localhost:9092\n",
flag("remote_cache") + "=\n",
"build " + flag("remote_timeout") + "=600\n",
"use grpc for transport\n",
} {
if hits := findRemoteEndpoints(".bazelrc", []byte(ok), true); len(hits) != 0 {
t.Errorf("allowed fixture flagged: %q -> %v", ok, hits)
}
}
// The fork cache's public endpoint: exactly its two .bazelrc lines.
forkCache := "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n" +
" build:fork-cache " + flag("remote_instance_name") + "=oss \n"
if hits := findRemoteEndpoints(".bazelrc", []byte(forkCache), true); len(hits) != 0 {
t.Errorf("fork-cache endpoint lines flagged in .bazelrc: %v", hits)
}
for _, path := range []string{"tools/ci.bazelrc", ".bazelrc.local", "user.bazelrc", ".github/workflows/x.yml", ".github/actions/setup-bazel/x.sh"} {
if hits := findRemoteEndpoints(path, []byte(forkCache), true); len(hits) < 2 {
t.Errorf("fork-cache endpoint lines in %s: hits %v, want both lines flagged (allowlisted in .bazelrc only)", path, hits)
}
}
for _, bad := range []string{
"build:fork-cache " + flag("remote_cache") + "=" + scheme + "other.example:8443\n",
"build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "/x\n",
"build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + " " + flag("remote_header") + "=x-api-key=abc\n",
"build " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n",
"build:remote-exec " + flag("remote_executor") + "=" + forkCacheEndpoint + "\n",
"build:fork-cache " + flag("remote_instance_name") + "=beads\n",
"# see " + forkCacheEndpoint + "\n",
} {
if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 {
t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad)
}
}
// rbe-fork's endpoint pin: that line in that file only; anything else
// naming the endpoint, there or elsewhere, is flagged.
if hits := findRemoteEndpoints(publicRBEForkPinFile, []byte("set -eu\n"+publicRBEForkPinLine+"\n"), true); len(hits) != 0 {
t.Errorf("rbe-fork endpoint pin flagged in %s: %v", publicRBEForkPinFile, hits)
}
for _, path := range []string{".github/actions/setup-bazel/write-bazelrc.sh", ".github/workflows/bazel.yml", ".bazelrc", "tools/rbe/fork-credential.sh"} {
if len(findRemoteEndpoints(path, []byte(publicRBEForkPinLine+"\n"), true)) == 0 {
t.Errorf("rbe-fork endpoint pin not flagged in %s (allowlisted in %s only)", path, publicRBEForkPinFile)
}
}
for _, bad := range []string{
" " + publicRBEForkPinLine,
strings.Replace(publicRBEForkPinLine, "8444", "443", 1),
`ENDPOINT=` + "grpc" + `s://rbe-fork.ops.gascity.com:8444`,
"build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe-fork.ops.gascity.com:8444",
} {
if len(findRemoteEndpoints(publicRBEForkPinFile, []byte(bad+"\n"), true)) == 0 {
t.Errorf("non-allowlisted rbe-fork line not detected in %s: %q", publicRBEForkPinFile, bad)
}
}
// Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine;
// a literal flag value is not.
for _, ok := range []string{
"Set OTEL_EXPORTER_OTLP_ENDPOINT=" + "grpc" + "://collector:4317\n",
"Pass " + flag("remote_executor") + " and TLS flags from user.bazelrc.\n",
} {
if hits := findRemoteEndpoints("docs/x.md", []byte(ok), false); len(hits) != 0 {
t.Errorf("allowed Markdown fixture flagged: %q -> %v", ok, hits)
}
}
if len(findRemoteEndpoints("docs/x.md", []byte("bazel build "+flag("remote_executor")+"=rbe.example.com:443 //...\n"), false)) == 0 {
t.Error("literal remote_executor in Markdown not detected")
}
for rel, want := range map[string][2]bool{
".bazelrc": {true, true},
"tools/ci.bazelrc": {true, true},
".bazelrc.local": {true, true},
".github/workflows/x.yml": {true, true},
"docs/BAZEL.md": {true, false},
"issues.jsonl": {false, false},
"internal/telemetry/x.go": {false, false},
} {
if scan, strict := remoteScanKind(rel); scan != want[0] || strict != want[1] {
t.Errorf("remoteScanKind(%q) = %v,%v, want %v,%v", rel, scan, strict, want[0], want[1])
}
}
root := bazelPolicyRoot(t)
var hits []endpointHit
for _, rel := range repoFiles(t, root, trackedFilesMin) {
scan, strict := remoteScanKind(rel)
if !scan {
continue
}
content, err := os.ReadFile(filepath.Join(root, rel))
if err != nil {
t.Fatalf("read %s: %v", rel, err)
}
hits = append(hits, findRemoteEndpoints(rel, content, strict)...)
}
for _, h := range hits {
t.Errorf("%s:%d: remote endpoint or credential %q in a tracked file; it belongs in a gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the workspace", h.path, h.line, h.what)
}
}
// TestNoLocalPlanPathsInTrackedFiles: tracked files must not point readers
// at a maintainer's private, out-of-repo planning notes (a home-directory
// planning-notes tree), which no other contributor can open. Cite an
// in-repo doc, a bead, or a PR instead. The needle is assembled at runtime so
// this file does not match itself.
func TestNoLocalPlanPathsInTrackedFiles(t *testing.T) {
root := bazelPolicyRoot(t)
needle := []byte("beads-" + "bazel-plan")
for _, rel := range repoFiles(t, root, trackedFilesMin) {
content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel)))
if err != nil {
t.Fatalf("read %s: %v", rel, err)
}
if bytes.IndexByte(content, 0) >= 0 {
continue // binary, as git grep -I
}
for n, line := range bytes.Split(content, []byte("\n")) {
if bytes.Contains(line, needle) {
t.Errorf("%s:%d: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", rel, n+1, needle)
}
}
}
}