ci(bazel): remote repo contents cache: main-push seed, lane readers, nightly verify (#7453)

Port gascity's remote repo contents cache wiring (gascity #7467, #7478) to
bazel.yml, gated on the repository variable RBE_REPO_CONTENTS_CACHE
(unset/off: nothing; seed: rrc-seed on push to main; canary: + bazel-test
reads; on: + every test lane reads). Mode remote only.

- rbe job: an rrc step turns the variable into the rrc output.
- Readers: the eight test lanes append startup
  --experimental_remote_repo_contents_cache and common
  --loading_phase_threads=64 to .bazelrc.local before Set up Bazel.
- rrc-seed (push to main, id-token: write): a 30-minute
  rbe-rrc-writer-beads certificate from rbe-west's mint
  (.github/scripts/rrc-writer-credential.sh), then every lane's command
  (.github/scripts/rrc-lane-commands.txt) with --nobuild, --remote_cache
  only and --remote_upload_local_results (.github/scripts/rrc-run-lanes.sh).
- rrc-verify (schedule/dispatch, issues: write): cold fetch of every lane
  and tools/bazel/rrc_verify.py against the cache; a mismatch opens an
  rrc-verify issue.
- pr.yml, nightly.yml and bazel-farm.yml grant id-token: write and
  issues: write on their bazel.yml call (startup_failure otherwise);
  TestBazelWorkflowCallersGrantEveryRequestedPermission guards it.

rrc-writer-credential.sh, rrc_verify.py and rrc_verify_test.py are byte
copies of gascity's, pinned by sha256.

Refs ga-vnycm2.29.

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Julian Knutsen 2026-10-10 02:50:26 +00:00 • committed by GitHub
commit f53010a23b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 2310 additions and 30 deletions

15
.github/scripts/rrc-lane-commands.txt vendored Normal file
View file

@ -0,0 +1,15 @@
# Every bazel.yml lane's loading and analysis, for the remote repo contents
# cache: rrc-seed runs each line with --nobuild on a push to main (uploading
# the repositories it fetches) and rrc-verify runs each cold every night.
# One `bazel` command per line (no shell quoting); # starts a comment.
# scripts/bazel_rrc_test.go checks that every --config a reading lane's
# bazel command uses appears here.
test //... --config=ci
test //... --config=embedded
test //... --config=integration
test //... --config=doltserver
test //... --config=doltserver-proxied
test //... --config=doltserver-integration
test //... --config=doltserver-cmd
build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test
build --config=js-wasm //internal/hooks:hooks_test

51
.github/scripts/rrc-run-lanes.sh vendored Executable file
View file

@ -0,0 +1,51 @@
#!/usr/bin/env bash
# Loading and analysis of every bazel.yml lane, for the remote repo contents
# cache (bazel.yml's rrc-seed and rrc-verify jobs).
#
# Usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...]
#
# Runs `bazel STARTUP_FLAG... <line> --nobuild COMMAND_FLAG...` for each
# command line in .github/scripts/rrc-lane-commands.txt (or RRC_LANE_COMMANDS),
# in order, stopping at the first failure. --nobuild: repository rules,
# loading and analysis only, never a build or test action.
#
# Bazel exits 1 after `test --nobuild` even when loading and analysis succeed
# ("Couldn't start the build. Unable to run tests"). A command passes when it
# exits 0, or exits non-zero having printed "Build completed successfully"
# and no ERROR line but that one.
set -euo pipefail
startup=()
while [ $# -gt 0 ] && [ "$1" != -- ]; do
startup+=("$1")
shift
done
[ $# -gt 0 ] || {
echo "usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...]" >&2
exit 2
}
shift
flags=("$@")
file=${RRC_LANE_COMMANDS:-$(dirname "${BASH_SOURCE[0]}")/rrc-lane-commands.txt}
mapfile -t cmds < <(sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$file" | grep -v '^[[:space:]]*$' || true)
if [ "${#cmds[@]}" -eq 0 ]; then
echo "::error::no lane commands in $file"
exit 1
fi
log=$(mktemp)
trap 'rm -f "$log"' EXIT
summary=${GITHUB_STEP_SUMMARY:-/dev/null}
for cmd in "${cmds[@]}"; do
read -r -a args <<<"$cmd"
start=$(date +%s)
rc=0
bazel ${startup[@]+"${startup[@]}"} "${args[@]}" --nobuild ${flags[@]+"${flags[@]}"} </dev/null 2>&1 | tee "$log" || rc=$?
if [ "$rc" -ne 0 ] && grep -q '^INFO: Build completed successfully' "$log" &&
! grep '^ERROR:' "$log" | grep -qv "^ERROR: Couldn't start the build. Unable to run tests$"; then
rc=0
fi
echo "rrc: \`bazel $cmd\`: exit $rc, $(($(date +%s) - start))s" | tee -a "$summary"
[ "$rc" -eq 0 ] || exit "$rc"
done

113
.github/scripts/rrc-writer-credential.sh vendored Executable file
View file

@ -0,0 +1,113 @@
#!/usr/bin/env bash
# A 30-minute rbe-west client certificate for the remote repo contents cache
# writer (bazel.yml's rrc-seed job; engdocs/design/bazel-remote-repo-contents-cache.md).
#
# The job trades its GitHub Actions OIDC token (audience rbe-rrc-writer) for a
# certificate CN=rbe-rrc-writer from rbe-west's mint. The mint signs only for
# this repository's bazel.yml on a push to main (repository, ref, event_name
# and job_workflow_ref claims); pull_request, fork and Dependabot runs cannot
# get one (no OIDC token, or a refs/pull/N/merge one). On rbe-west that
# certificate reaches only GetCapabilities, the CAS and an action cache front
# (rrc-gate) that admits repo-contents entries and refuses everything else, so
# it can never write an execution result or execute anything.
#
# The key is an EC P-256 key (PKCS#8: Bazel's Netty TLS refuses a SEC1 "EC
# PRIVATE KEY") generated here in BAZEL_CI_SECRET_DIR, outside the workspace;
# it never leaves the runner. The OIDC token is masked. Outputs (to
# $GITHUB_OUTPUT when set, else stdout): cert, key, endpoint, instance; none
# when the mint answers 503 or cannot be reached (seeding switched off on
# rbe-west: infra nativelink-cas/west/README.md "Client contract"), which
# exits 0 with a warning.
#
# Tests only: RBE_RRC_MINT (a stand-in mint) and RBE_RRC_ENDPOINT_RE.
set -euo pipefail
MINT=${RBE_RRC_MINT:-https://rbe-mint.ops.gascity.com:8444}
ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^grpcs://rbe-west\.ops\.gascity\.com(:443)?$'}
AUDIENCE=rbe-rrc-writer
dir=${BAZEL_CI_SECRET_DIR:?BAZEL_CI_SECRET_DIR is required}
out=${GITHUB_OUTPUT:-/dev/stdout}
if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]; then
echo "::error title=rrc writer::no OIDC token request URL; the job needs permissions: id-token: write" >&2
exit 1
fi
install -d -m 0700 "$dir"
umask 077
oidc=$(curl -sS --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors \
-H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$AUDIENCE" | jq -r '.value // empty')
if [ -z "$oidc" ]; then
echo "::error title=rrc writer::GitHub returned no OIDC token" >&2
exit 1
fi
printf '::add-mask::%s\n' "$oidc"
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/rrc-writer.key" 2>/dev/null
csr=$(openssl req -new -key "$dir/rrc-writer.key" -subj "/CN=rbe-rrc-writer-request")
body=$(jq -cn --arg csr "$csr" '{csr_pem: $csr}')
reply="$dir/rrc-mint.json"
code=000
for attempt in 1 2 3 4; do
# --connect-timeout: a closed gate drops the SYN; fail fast, not in 60 s.
code=$(curl -sS -o "$reply" -w '%{http_code}' --connect-timeout 5 --max-time 60 \
-H "Authorization: Bearer $oidc" -H 'content-type: application/json' \
--data "$body" "$MINT/v1/rrc-writer/cert") || code=000
# Retry what may pass on its own: a rate limit (429), the mint could not
# reach GitHub's JWKS (502), or the network (000). Everything else is an
# answer.
case "$code" in 429 | 502 | 000) ;; *) break ;; esac
[ "$attempt" -eq 4 ] || sleep $((attempt * 10))
done
case "$code" in
200) ;;
503 | 000)
# The mint's switch is off, its CA is not installed, or rbe-west's fork
# edge is closed: seeding is off, not broken. No outputs, so the seed
# step skips; the push to main never fails on it.
echo "::warning title=rrc writer: seeding off (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)"
exit 0
;;
*)
echo "::error title=rrc writer mint refused (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)"
exit 1
;;
esac
endpoint=$(jq -r '.endpoint // empty' "$reply")
instance=$(jq -r '.instance // empty' "$reply")
[[ $endpoint =~ $ENDPOINT_RE ]] || {
echo "::error::rrc writer mint returned endpoint '$endpoint'" >&2
exit 1
}
[ "$instance" = oss ] || {
echo "::error::rrc writer mint returned instance '$instance', want oss" >&2
exit 1
}
jq -r '.cert_pem // empty' "$reply" >"$dir/rrc-writer.crt"
subject=$(openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -nameopt RFC2253) || {
echo "::error::rrc writer mint returned no certificate" >&2
exit 1
}
rdns=$(tr ',' '\n' <<<"${subject#subject=}")
cn=$(sed -n 's/^CN=//p' <<<"$rdns")
org=$(sed -n 's/^O=//p' <<<"$rdns")
if ! [[ $cn =~ ^rbe-rrc-writer(-[a-z0-9]+)?$ ]] || [ "$org" != gascity ]; then
echo "::error::rrc writer mint returned a certificate for '$subject'" >&2
exit 1
fi
# The certificate must belong to the key generated above.
if [ "$(openssl x509 -in "$dir/rrc-writer.crt" -noout -pubkey | openssl sha256)" != "$(openssl pkey -in "$dir/rrc-writer.key" -pubout | openssl sha256)" ]; then
echo "::error::rrc writer mint returned a certificate for another key" >&2
exit 1
fi
openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -enddate
rm -f "$reply"
{
echo "cert=$dir/rrc-writer.crt"
echo "key=$dir/rrc-writer.key"
echo "endpoint=$endpoint"
echo "instance=$instance"
} >>"$out"

View file

@ -120,8 +120,18 @@ jobs:
name: Bazel Farm
needs: authorize
if: ${{ needs.authorize.outputs.allowed == 'true' }}
# A called workflow's job gets at most this call's token permissions, and
# GitHub checks them when the run starts, even for a job whose if: skips
# it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks
# id-token: write (push to main only) and rrc-verify issues: write
# (schedule and dispatch only). Neither runs for this event, every other
# bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC
# token but a main-push bazel.yml one
# (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission).
permissions:
contents: read
id-token: write
issues: write
uses: ./.github/workflows/bazel.yml
# Exactly the four RBE secrets: never the rbe-prewarm app private key
# (B1, security review of bdef342d5). This pull_request_target run is

View file

@ -424,6 +424,7 @@ jobs:
enabled: ${{ steps.decide.outputs.enabled }}
mode: ${{ steps.decide.outputs.mode }}
tier: ${{ steps.decide.outputs.tier }}
rrc: ${{ steps.rrc.outputs.rrc }}
steps:
# The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts).
# A push, merge_group or dispatch has no pull_request, so FORK is false.
@ -496,6 +497,37 @@ jobs:
} >> "$GITHUB_OUTPUT"
echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY"
# Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported
# from gascity's bazel.yml; gascity
# engdocs/design/bazel-remote-repo-contents-cache.md): extracted
# external repositories (go_deps, the Go SDK, the LLVM slice) come from
# rbe-west's action cache instead of being re-fetched by every cold lane
# client. vars.RBE_REPO_CONTENTS_CACHE rolls it out:
# unset/off nothing (the default);
# seed main pushes seed the cache (rrc-seed); no lane reads it;
# canary seed, and bazel-test reads;
# on seed, and every lane in .github/scripts/rrc-lane-commands.txt
# reads.
# Mode remote only: fork pull_request runs see no vars, and only
# rbe-west's trusted edge serves the entries. Any other value is off,
# with a warning, so a typo never fails the required gate. Like decide,
# this step runs no repository code (it comes before the checkout).
- name: Remote repo contents cache
id: rrc
env:
RRC: ${{ vars.RBE_REPO_CONTENTS_CACHE }}
MODE: ${{ steps.decide.outputs.mode }}
run: |
rrc=off
case "$RRC" in
"" | off) ;;
seed | canary | on)
if [ "$MODE" = remote ]; then rrc=$RRC; fi ;;
*) echo "::warning title=RBE_REPO_CONTENTS_CACHE::unknown value '$RRC' (want off, seed, canary or on); treated as off" ;;
esac
echo "rrc=$rrc" >> "$GITHUB_OUTPUT"
echo "Remote repo contents cache: **$rrc**" >> "$GITHUB_STEP_SUMMARY"
# The worker-env preflight (see the job's comment): the PR's own copy,
# so a PR that syncs it passes. Only the two paths it compares.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@ -785,6 +817,22 @@ jobs:
- name: Verify restored Go modules
run: go mod verify
# The remote repo contents cache, read-only (the rbe job's rrc output:
# canary, bazel-test; on, every lane). Each cached repository costs
# several sequential round trips, resolved on Skyframe's loading
# threads (default: the CPU count), so without 64 of them a cached read
# is slower than fetching. Both lines are key neutral (startup and
# loading only). Reading only: setup-bazel's rc keeps
# --noremote_upload_local_results, so nothing here writes; rrc-seed
# writes, on push to main. Before Set up Bazel: no server starts first.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && (needs.rbe.outputs.rrc == 'on' || needs.rbe.outputs.rrc == 'canary') }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -1045,6 +1093,15 @@ jobs:
with:
node-version: '24'
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -1333,6 +1390,15 @@ jobs:
persist-credentials: false
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -1505,6 +1571,15 @@ jobs:
persist-credentials: false
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -1650,6 +1725,15 @@ jobs:
persist-credentials: false
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -1770,6 +1854,15 @@ jobs:
persist-credentials: false
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -1922,6 +2015,15 @@ jobs:
persist-credentials: false
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -2085,6 +2187,15 @@ jobs:
persist-credentials: false
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
# The remote repo contents cache, read-only, as in bazel-test.
- name: Remote repo contents cache (read)
if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }}
run: |
{
echo 'startup --experimental_remote_repo_contents_cache'
echo 'common --loading_phase_threads=64'
} >> .bazelrc.local
- name: Set up Bazel
id: bazel
uses: ./.github/actions/setup-bazel
@ -2432,3 +2543,138 @@ jobs:
env:
JOB_STATUS: ${{ job.status }}
run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"
# The remote repo contents cache's only writer (ga-vnycm2.29, ported from
# gascity's bazel.yml). On a push to main, with the rbe job's rrc not off,
# run every lane's loading and analysis (.github/scripts/rrc-lane-commands.txt,
# --nobuild): only main's repository rules and Starlark run (gazelle's
# fetch_repo, the Go SDK and LLVM downloads, extraction: what every lane
# already runs on its runner), never a build or test action. Bazel uploads
# each reproducible repository's tree (--remote_upload_local_results) with
# a 30-minute rbe-rrc-writer-beads certificate that rbe-west's mint signs
# only for this workflow on a push to main
# (.github/scripts/rrc-writer-credential.sh); rbe-west's rrc-gate admits
# repo-contents entries from it and refuses every other action cache
# write, and the certificate cannot Execute. Cache only (--remote_cache,
# no executor). Not a lane: no result output, never in pr.yml's gate
# (which never sees it anyway: push only), so a failed seed only leaves the
# lanes fetching. setup-bazel runs in local mode (no secrets): its rc, the
# runner cache and the pinned Bazel, nothing else. The only bazel.yml job
# with id-token: write; every caller must grant it (GitHub checks a called
# workflow's job permissions at startup, even for a job that skips), and
# the mint refuses any token but a main-push bazel.yml one.
rrc-seed:
name: repo contents cache seed
needs: rbe
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 30
permissions:
contents: read
id-token: write # the rbe-rrc-writer-beads certificate (GitHub OIDC)
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Bazel
uses: ./.github/actions/setup-bazel
- name: rbe-rrc-writer certificate
id: writer
env:
BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret
run: bash .github/scripts/rrc-writer-credential.sh
- name: Seed the remote repo contents cache
if: ${{ steps.writer.outputs.cert != '' }}
env:
CERT: ${{ steps.writer.outputs.cert }}
KEY: ${{ steps.writer.outputs.key }}
ENDPOINT: ${{ steps.writer.outputs.endpoint }}
INSTANCE: ${{ steps.writer.outputs.instance }}
run: >-
bash .github/scripts/rrc-run-lanes.sh --experimental_remote_repo_contents_cache --
--loading_phase_threads=64
"--remote_cache=$ENDPOINT" "--remote_instance_name=$INSTANCE"
"--tls_client_certificate=$CERT" "--tls_client_key=$KEY"
--remote_upload_local_results
- name: Remove the writer key
if: ${{ always() }}
run: rm -f -- "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.key" "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.crt"
# Nightly check of the remote repo contents cache (gascity design R1/R2). A
# cached repository tree is trusted by every reader, and nothing on the
# reader checks it against what its rule produces. So, from nightly.yml (or
# a dispatch), fetch every lane's repositories cold (rrc-lane-commands.txt
# with --nobuild and no repo contents cache, Bazel set up exactly as a
# lane) and compare each repository's tree and marker with the cached
# entry a lane would read for the same key (tools/bazel/rrc_verify.py,
# reads only, with the lanes' certificate and the endpoint from
# setup-bazel's rc: no secret is referenced outside setup-bazel). A
# mismatch is a poisoned entry or a rule falsely marked reproducible: the
# job fails and opens (or comments on) an rrc-verify issue. Not a lane:
# never gated. The only bazel.yml job with issues: write; every caller
# must grant it, as for rrc-seed's id-token.
rrc-verify:
name: repo contents cache verify
needs: rbe
if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 45
permissions:
contents: read
issues: write # the rrc-verify alert issue
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Bazel
uses: ./.github/actions/setup-bazel
env:
BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }}
RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }}
RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }}
RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }}
RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }}
- name: Cold fetch (every lane, no repo contents cache)
run: bash .github/scripts/rrc-run-lanes.sh -- --loading_phase_threads=64
- name: Compare with the cache
id: verify
run: |
set -euo pipefail
secret_dir="$RUNNER_TEMP/bazel-ci-secret"
prefix='build:remote-exec --remote_executor='
endpoint=$(grep -F -m1 "$prefix" "$secret_dir/ci.bazelrc" || true)
endpoint=${endpoint#"$prefix"}
[ -n "$endpoint" ] || { echo "::error::setup-bazel's rc names no executor"; exit 1; }
python3 tools/bazel/rrc_verify.py \
--output-base "$(bazel info output_base)" \
--endpoint "$endpoint" --instance oss \
--cert "$secret_dir/client.crt" --key "$secret_dir/client.key" \
--bazel-version "$(tr -d '[:space:]' < .bazelversion)" \
--json "$RUNNER_TEMP/rrc-verify.json" --summary "$GITHUB_STEP_SUMMARY"
- name: Upload the report
if: ${{ always() && hashFiles(format('{0}/rrc-verify.json', runner.temp)) != '' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
overwrite: true # a job re-run is the same run: replace, not 409
name: rrc-verify-${{ github.run_attempt }}
path: ${{ runner.temp }}/rrc-verify.json
retention-days: 30
- name: Alert (rrc-verify issue)
if: ${{ failure() && steps.verify.outcome == 'failure' }}
env:
GH_TOKEN: ${{ github.token }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
title="rrc-verify: remote repo contents cache differs from a cold fetch"
body="The nightly rrc-verify run found cached repository trees that differ from a cold fetch, or could not check them: $RUN_URL (summary and the rrc-verify report artifact).
A mismatch is a poisoned entry or a repository rule falsely marked reproducible (gascity engdocs/design/bazel-remote-repo-contents-cache.md, R1/R2). Stop readers first: set the repository variable RBE_REPO_CONTENTS_CACHE to off. Then follow gascity/infra nativelink-cas/west/README.md (remote repo contents cache writer, poisoned entries)."
gh label create rrc-verify -R "$GITHUB_REPOSITORY" --color B60205 \
--description "Remote repo contents cache verification failures" 2>/dev/null || true
n=$(gh issue list -R "$GITHUB_REPOSITORY" --label rrc-verify --state open --json number --jq '.[0].number // empty')
if [ -n "$n" ]; then
gh issue comment "$n" -R "$GITHUB_REPOSITORY" --body "Again: $RUN_URL"
else
gh issue create -R "$GITHUB_REPOSITORY" --label rrc-verify --title "$title" --body "$body"
fi

View file

@ -25,6 +25,18 @@ jobs:
# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md).
bazel:
name: Bazel (fresh, full detail)
# A called workflow's job gets at most this call's token permissions, and
# GitHub checks them when the run starts, even for a job whose if: skips
# it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks
# id-token: write (push to main only) and rrc-verify issues: write
# (schedule and dispatch only). rrc-verify runs here (its alert issue);
# rrc-seed never does, every other bazel.yml job keeps contents: read,
# and rbe-west's mint refuses any OIDC token but a main-push bazel.yml one
# (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission).
permissions:
contents: read
id-token: write
issues: write
uses: ./.github/workflows/bazel.yml
secrets:
RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }}

View file

@ -1347,8 +1347,18 @@ jobs:
# build their own bd via this call's rbe job.
bazel:
name: Bazel
# A called workflow's job gets at most this call's token permissions, and
# GitHub checks them when the run starts, even for a job whose if: skips
# it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks
# id-token: write (push to main only) and rrc-verify issues: write
# (schedule and dispatch only). Neither runs for this event, every other
# bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC
# token but a main-push bazel.yml one
# (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission).
permissions:
contents: read
id-token: write
issues: write
uses: ./.github/workflows/bazel.yml
secrets:
RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }}

View file

@ -620,6 +620,38 @@ job's `if:` or rotating the key, in either of two ways:
step's own kill switch, checked before any network call. Deleting the
variable does not disable pre-warming; it restores the default of 1.
## Remote Repo Contents Cache
A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go
SDK, the LLVM slice) before its first action. Bazel 9's remote repo contents
cache serves those extracted trees from rbe-west's `oss` action cache instead
(ported from gascity's `bazel.yml`; design: gascity
`engdocs/design/bazel-remote-repo-contents-cache.md`). The repository variable
`RBE_REPO_CONTENTS_CACHE` rolls it out, in mode `remote` only:
| value | effect |
|---|---|
| unset / `off` | nothing (default) |
| `seed` | each push to main runs `bazel.yml`'s `rrc-seed` job: every lane's command in `.github/scripts/rrc-lane-commands.txt` with `--nobuild`, uploading repository trees with a 30-minute `rbe-rrc-writer-beads` certificate that rbe-west's mint signs only for that job (GitHub OIDC; `.github/scripts/rrc-writer-credential.sh`). rbe-west's `rrc-gate` admits only repo-contents entries from it |
| `canary` | `seed`, and the `test` lane reads the cache |
| `on` | `seed`, and every lane but the release cross-compile and the package gates reads it |
Readers append `startup --experimental_remote_repo_contents_cache` and
`common --loading_phase_threads=64` to `.bazelrc.local` before setting up
Bazel; both are key neutral, and the lanes keep
`--noremote_upload_local_results`. While the variable is not `off`,
nightly.yml's call also runs `rrc-verify`: a cold fetch of every lane,
compared with the cached entries by `tools/bazel/rrc_verify.py`; a mismatch
fails the job and opens an `rrc-verify` issue.
`rrc-seed` asks for `id-token: write` and `rrc-verify` for `issues: write`, so
every caller of `bazel.yml` (pr.yml, nightly.yml, bazel-farm.yml) grants both:
GitHub checks a called workflow's job permissions when the run starts, even
for jobs that skip. Neither job runs on a pull_request, merge_group or
pull_request_target event, and every other `bazel.yml` job keeps
`contents: read`. Rollback: set the variable to `off` (or `seed`, to keep the
cache warm without readers).
## Required Check Contract
After the aggregate checks are verified on the branch, branch protection or the

View file

@ -41,6 +41,7 @@ go_test(
"bazel_hermetic_cc_test.go",
"bazel_policy_test.go",
"bazel_rbe_prewarm_test.go",
"bazel_rrc_test.go",
"ci_actions_policy_test.go",
"ci_analytics_workflow_test.go",
"ci_blacksmith_runner_test.go",

View file

@ -212,8 +212,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) {
var lanes []string
for name, job := range workflow.Jobs {
if name == bazelRBEJobName {
continue
if name == bazelRBEJobName || isBazelRRCJob(name) {
continue // not lanes: never in a PR call (bazel_rrc_test.go)
}
runs := bazelLaneRunModes(t, name, job.If, s.with)[mode]
switch {

View file

@ -94,7 +94,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) {
}
// Permissions: contents: read at the workflow and on every job, and
// nothing else anywhere in the file.
// nothing else anywhere in the file, but the farm call's grant of what
// bazel.yml's rrc jobs ask for (bazelCallPermissions).
readOnly := map[string]any{"contents": "read"}
if !reflect.DeepEqual(doc.Permissions, readOnly) {
t.Errorf("workflow permissions = %v, want %v", doc.Permissions, readOnly)
@ -103,8 +104,12 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) {
var jobs []string
for name, job := range workflow.Jobs {
jobs = append(jobs, name)
if !reflect.DeepEqual(job.Permissions, readOnly) {
t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, readOnly)
want := readOnly
if name == "farm" {
want = bazelCallPermissions
}
if !reflect.DeepEqual(job.Permissions, want) {
t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, want)
}
if len(job.Env) != 0 {
t.Errorf("job %s env = %v; want none", name, job.Env)
@ -116,7 +121,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) {
}
secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`)
walkYAML(root, "", func(path string, key bool, value string) {
if strings.Contains(path, "permissions") && !key && value != "read" {
if strings.Contains(path, "permissions") && !key && value != "read" &&
!(strings.HasPrefix(path, ".jobs.farm.permissions.") && bazelCallPermissions[strings.TrimPrefix(path, ".jobs.farm.permissions.")] == value) {
t.Errorf("%s = %q; only read permissions are allowed", path, value)
}
if key && value == "continue-on-error" {
@ -227,6 +233,19 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) {
}
}
// bazelCallPermissions: every bazel.yml call's token permissions (farm,
// pr.yml, nightly.yml). A called workflow's job gets at most its caller's,
// and GitHub checks them when the run starts, even for a job whose if:
// skips it, so each call grants what bazel.yml's rrc jobs ask for:
// rrc-seed's id-token: write (push to main only) and rrc-verify's issues:
// write (schedule and dispatch only). Neither runs on pull_request,
// merge_group or pull_request_target, and every other bazel.yml job, every
// lane that runs PR or fork code included, keeps contents: read
// (TestBazelWorkflowPermissionsReadOnly), so no PR-authored step ever holds
// either; rbe-west's mint refuses any OIDC token but a main-push bazel.yml
// one anyway.
var bazelCallPermissions = map[string]any{"contents": "read", "id-token": "write", "issues": "write"}
// bazel.yml's side of the farm: the pinned checkout, the one decision input,
// and nothing a pull_request_target run could hand a privileged consumer.
func TestBazelWorkflowForkFarmInputs(t *testing.T) {
@ -246,6 +265,15 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) {
if actionFamily(step.Uses) != "actions/checkout" {
continue
}
// The rrc jobs run on push to main, schedule and dispatch only,
// never on pull_request_target: they check out the event's ref,
// with no fork opt-in.
if isBazelRRCJob(name) {
if want := map[string]string{"persist-credentials": "false"}; !reflect.DeepEqual(step.With, want) {
t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want)
}
continue
}
checkouts++
want := map[string]string{"ref": bazelCheckoutRef, "persist-credentials": "false", "allow-unsafe-pr-checkout": bazelAllowUnsafeCheckout}
// F3: package-mcp/package-npm's detect step diffs PR_BASE_SHA
@ -271,8 +299,8 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) {
// shared gascity credential). rbe's checkout (sparse) comes after its
// decide step, for the worker-env preflight
// (TestBazelRBEWorkerEnvPreflight).
if checkouts != len(workflow.Jobs)-1 {
t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1)
if want := len(workflow.Jobs) - 1 - len(bazelRRCJobs); checkouts != want {
t.Errorf("%d checkouts in %s, want one per job excluding %s and the rrc jobs (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, want)
}
if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue {
t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue)
@ -617,7 +645,11 @@ func TestBazelFarmCacheModeStaysReadOnly(t *testing.T) {
}
// bazel.yml's token: exactly contents: read, at the top and on any job that
// declares permissions (a call cannot exceed its caller's, but pin it).
// declares permissions (a call cannot exceed its caller's, but pin it),
// except the rrc jobs: rrc-seed's id-token: write and rrc-verify's issues:
// write (pinned by TestBazelRRCSeedJob and TestBazelRRCVerifyJob; neither
// runs on a pull_request, merge_group or pull_request_target event, so the
// callers' wider grants reach no lane and no fork-authored code).
func TestBazelWorkflowPermissionsReadOnly(t *testing.T) {
var doc struct {
Permissions any `yaml:"permissions"`
@ -630,6 +662,9 @@ func TestBazelWorkflowPermissionsReadOnly(t *testing.T) {
t.Errorf("%s permissions = %v, want exactly %v", bazelWorkflowName, doc.Permissions, readOnly)
}
for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs {
if isBazelRRCJob(name) {
continue
}
if job.Permissions != nil && !reflect.DeepEqual(job.Permissions, readOnly) {
t.Errorf("%s job %s permissions = %v, want none or exactly %v", bazelWorkflowName, name, job.Permissions, readOnly)
}

783
scripts/bazel_rrc_test.go Normal file
View file

@ -0,0 +1,783 @@
package scripts_test
import (
"crypto/sha256"
"encoding/hex"
"os"
"os/exec"
"path/filepath"
"reflect"
"regexp"
"slices"
"sort"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
// Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported from
// gascity's bazel.yml, gascity #7467 and #7478). bazel.yml's rbe job turns
// vars.RBE_REPO_CONTENTS_CACHE into its rrc output (off, seed, canary, on);
// each reading lane's reader step appends bazelRRCReadLines to
// .bazelrc.local; the rrc-seed job, on push to main only, runs every lane's
// loading and analysis (.github/scripts/rrc-lane-commands.txt, through
// rrc-run-lanes.sh) and uploads repository trees with a 30-minute
// rbe-rrc-writer-beads certificate (rrc-writer-credential.sh); the
// rrc-verify job, nightly, compares the cache with a cold fetch
// (tools/bazel/rrc_verify.py).
const (
bazelRRCReadStep = "Remote repo contents cache (read)"
bazelRRCModeStepID = "rrc"
bazelRRCVar = "vars.RBE_REPO_CONTENTS_CACHE"
bazelRRCSeedJob = "rrc-seed"
bazelRRCSeedJobIf = "${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}"
bazelRRCVerifyJob = "rrc-verify"
bazelRRCVerifyJobIf = "${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}"
bazelRRCCanaryLane = "bazel-test"
bazelRRCCredential = ".github/scripts/rrc-writer-credential.sh"
bazelRRCRunLanes = ".github/scripts/rrc-run-lanes.sh"
bazelRRCLaneCmds = ".github/scripts/rrc-lane-commands.txt"
bazelRRCVerifyTool = "tools/bazel/rrc_verify.py"
bazelRRCVerifyTest = "tools/bazel/rrc_verify_test.py"
bazelSetupBazelUses = "./.github/actions/setup-bazel"
)
// bazelRRCReadLanes: the lanes that read the cache under `on` (bazel-test
// alone under `canary`). Not bazel-release-cross (other target platforms,
// not seeded) nor the package gates.
var bazelRRCReadLanes = []string{
"bazel-cmd-dolt", "bazel-doltserver", "bazel-embedded", "bazel-integration",
"bazel-proxied", "bazel-pure", "bazel-server-storage", "bazel-test",
}
// bazelRRCReadLines: the reader's .bazelrc.local lines; both are key neutral
// (a startup option, and loading parallelism).
var bazelRRCReadLines = []string{
"startup --experimental_remote_repo_contents_cache",
"common --loading_phase_threads=64",
}
// bazelRRCJobs: bazel.yml's jobs that are not lanes (no result output, never
// gated): the remote repo contents cache's writer and its nightly check.
var bazelRRCJobs = []string{bazelRRCSeedJob, bazelRRCVerifyJob}
// isBazelRRCJob reports whether a bazel.yml job is one of bazelRRCJobs.
func isBazelRRCJob(name string) bool { return slices.Contains(bazelRRCJobs, name) }
// evalRRCIf evaluates a job or step if: with ctx, failing the test on an
// expression the evaluator cannot read.
func evalRRCIf(t *testing.T, expr string, ctx map[string]string) bool {
t.Helper()
v, err := evalGHExpr(expr, ctx)
if err != nil {
t.Fatalf("evalGHExpr(%q): %v", expr, err)
}
return ghTruthy(v)
}
// runRRCScript runs a step's script (or a script file) under GitHub's bash
// flags in dir with env, returning its combined output.
func runRRCScript(t *testing.T, dir, script string, env map[string]string) (string, error) {
t.Helper()
cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script)
cmd.Dir = dir
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir}
for k, v := range env {
cmd.Env = append(cmd.Env, k+"="+v)
}
out, err := cmd.CombinedOutput()
return string(out), err
}
// rrcOutputs reads a $GITHUB_OUTPUT file (key=value lines).
func rrcOutputs(t *testing.T, path string) map[string]string {
t.Helper()
out := map[string]string{}
data, err := os.ReadFile(path)
if os.IsNotExist(err) {
return out
}
if err != nil {
t.Fatal(err)
}
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
if k, v, ok := strings.Cut(line, "="); ok {
out[k] = v
}
}
return out
}
func rrcStep(t *testing.T, job ciWorkflowJob, what string, pick func(ciWorkflowStep) bool) ciWorkflowStep {
t.Helper()
var found []ciWorkflowStep
for _, s := range job.Steps {
if pick(s) {
found = append(found, s)
}
}
if len(found) != 1 {
t.Fatalf("want one %s step, got %d", what, len(found))
}
return found[0]
}
// TestBazelRRCModeStep runs the rbe job's rrc step for every variable value
// and execution mode: only mode remote enables anything, unset and off are
// off, and an unknown value is off with a warning (a typo never fails the
// required gate). It reads no secret and runs before the checkout.
func TestBazelRRCModeStep(t *testing.T) {
rbe := readCIWorkflow(t, bazelWorkflowName).Jobs[bazelRBEJobName]
if got := rbe.Outputs["rrc"]; got != "${{ steps.rrc.outputs.rrc }}" {
t.Errorf("rbe job output rrc = %q", got)
}
step := rrcStep(t, rbe, "rrc", func(s ciWorkflowStep) bool { return s.ID == bazelRRCModeStepID })
if want := map[string]string{"RRC": "${{ " + bazelRRCVar + " }}", "MODE": "${{ steps.decide.outputs.mode }}"}; !reflect.DeepEqual(step.Env, want) {
t.Errorf("rrc step env %v, want %v", step.Env, want)
}
if step.If != "" {
t.Errorf("rrc step if %q; it must always set the output", step.If)
}
for i, s := range rbe.Steps {
if s.Uses != "" {
t.Errorf("rbe step %d (%s) precedes nothing it should: the rrc step must come before any checkout", i, s.Uses)
}
if s.ID == bazelRRCModeStepID {
break
}
}
for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local", "skip"} {
for value, want := range map[string]string{"": "off", "off": "off", "seed": "seed", "canary": "canary", "on": "on", "On": "off", "true": "off", "on;x": "off"} {
if mode != "remote" {
want = "off"
}
dir := t.TempDir()
output := filepath.Join(dir, "output")
out, err := runRRCScript(t, dir, step.Run, map[string]string{
"RRC": value, "MODE": mode, "GITHUB_OUTPUT": output, "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"),
})
if err != nil {
t.Errorf("mode %s value %q: %v\n%s", mode, value, err, out)
continue
}
if got := rrcOutputs(t, output)["rrc"]; got != want {
t.Errorf("mode %s value %q: rrc=%q, want %q", mode, value, got, want)
}
unknown := !slices.Contains([]string{"", "off", "seed", "canary", "on"}, value)
if warned := strings.Contains(out, "::warning"); warned != unknown {
t.Errorf("mode %s value %q: warning %v, want %v:\n%s", mode, value, warned, unknown, out)
}
}
}
}
// TestBazelRRCReadSteps: exactly bazelRRCReadLanes read, in mode remote only
// (fork and local runs never: only rbe-west's trusted edge serves the
// entries), bazel-test under canary and every one of them under on; each
// writes exactly bazelRRCReadLines, never mentions uploads, and comes before
// the job's Set up Bazel (so no Bazel server starts without it).
func TestBazelRRCReadSteps(t *testing.T) {
wf := readCIWorkflow(t, bazelWorkflowName)
var readers []string
for name, job := range wf.Jobs {
read, setup := -1, -1
for i, s := range job.Steps {
switch {
case s.Name == bazelRRCReadStep:
if read >= 0 {
t.Errorf("%s has two %q steps", name, bazelRRCReadStep)
}
read = i
case s.Uses == bazelSetupBazelUses && setup < 0:
setup = i
}
}
if read < 0 {
continue
}
readers = append(readers, name)
if setup < 0 || read > setup {
t.Errorf("%s: %q at step %d, Set up Bazel at %d; the reader must come first", name, bazelRRCReadStep, read, setup)
}
step := job.Steps[read]
for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local"} {
for _, rrc := range []string{"off", "seed", "canary", "on"} {
want := mode == "remote" && (rrc == "on" || (rrc == "canary" && name == bazelRRCCanaryLane))
if got := evalRRCIf(t, step.If, map[string]string{"needs.rbe.outputs.mode": mode, "needs.rbe.outputs.rrc": rrc}); got != want {
t.Errorf("%s mode %s rrc %s: reader runs %v, want %v", name, mode, rrc, got, want)
}
}
}
dir := t.TempDir()
if out, err := runRRCScript(t, dir, step.Run, nil); err != nil {
t.Fatalf("%s reader: %v\n%s", name, err, out)
}
got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, ".bazelrc.local")), "\n")
if !slices.Equal(got, bazelRRCReadLines) {
t.Errorf("%s reader writes %q, want %q", name, got, bazelRRCReadLines)
}
if strings.Contains(step.Run, "upload") || len(step.Env) != 0 {
t.Errorf("%s reader mentions uploads or takes env; lanes only read", name)
}
}
sort.Strings(readers)
if !slices.Equal(readers, bazelRRCReadLanes) {
t.Errorf("lanes with a reader step %v, want %v", readers, bazelRRCReadLanes)
}
}
// bazelRRCLaneCommands parses rrc-lane-commands.txt as rrc-run-lanes.sh does.
func bazelRRCLaneCommands(t *testing.T, root string) []string {
t.Helper()
var cmds []string
for _, line := range strings.Split(readPolicyFile(t, root, bazelRRCLaneCmds), "\n") {
line, _, _ = strings.Cut(line, "#")
if line = strings.TrimSpace(line); line != "" {
cmds = append(cmds, line)
}
}
return cmds
}
// TestBazelRRCLaneCommandsCoverEveryReader: rrc-seed seeds (and rrc-verify
// checks) what the readers fetch, so every --config a reading lane passes to
// `bazel test` or `bazel build` has a command in rrc-lane-commands.txt, and
// every command there is a reading lane's.
func TestBazelRRCLaneCommandsCoverEveryReader(t *testing.T) {
root := sourceRepoRoot(t)
seeded := map[string]bool{}
for _, cmd := range bazelRRCLaneCommands(t, root) {
f := strings.Fields(cmd)
if f[0] != "test" && f[0] != "build" {
t.Errorf("lane command %q: want a test or build command", cmd)
}
for _, a := range f[1:] {
if c, ok := strings.CutPrefix(a, "--config="); ok {
seeded[c] = true
}
}
}
used := map[string]bool{}
invocation := regexp.MustCompile(`\bbazel (?:test|build)\b[^\n]*(?:\\\n[^\n]*)*`)
config := regexp.MustCompile(`--config=([a-z0-9-]+)`)
wf := readCIWorkflow(t, bazelWorkflowName)
for _, lane := range bazelRRCReadLanes {
for _, s := range wf.Jobs[lane].Steps {
for _, inv := range invocation.FindAllString(s.Run, -1) {
for _, m := range config.FindAllStringSubmatch(inv, -1) {
if m[1] != "sole-run" && m[1] != "fresh" { // test-result options, nothing fetched
used[m[1]] = true
}
}
}
}
}
if len(used) < len(bazelRRCReadLanes) {
t.Fatalf("found the configs %v in %d reading lanes; the scan is broken", used, len(bazelRRCReadLanes))
}
for c := range used {
if !seeded[c] {
t.Errorf("a reading lane runs bazel with --config=%s, which %s does not seed", c, bazelRRCLaneCmds)
}
}
for c := range seeded {
if !used[c] {
t.Errorf("%s seeds --config=%s, which no reading lane uses", bazelRRCLaneCmds, c)
}
}
}
// TestRRCRunLanes runs rrc-run-lanes.sh with a stub bazel: one --nobuild
// invocation per lane command, in order, startup flags first and command
// flags after; `test --nobuild`'s exit 1 after a successful analysis
// passes, a failed analysis fails, and an empty command list fails.
func TestRRCRunLanes(t *testing.T) {
root := sourceRepoRoot(t)
script := filepath.Join(root, bazelRRCRunLanes)
dir := t.TempDir()
bin := filepath.Join(dir, "bin")
if err := os.MkdirAll(bin, 0o755); err != nil {
t.Fatal(err)
}
calls := filepath.Join(dir, "calls")
stub := "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >>" + calls + "\n" +
"if [ -n \"${BAZEL_TEST_FAIL:-}\" ]; then echo 'ERROR: no such package'; echo 'ERROR: Build did NOT complete successfully'; exit 1; fi\n" +
"case \"$*\" in *' build '*) echo 'INFO: Build completed successfully, 0 total actions'; exit 0 ;; esac\n" +
"echo 'INFO: Build completed successfully, 0 total actions'\necho \"ERROR: Couldn't start the build. Unable to run tests\"\nexit 1\n"
if err := os.WriteFile(filepath.Join(bin, "bazel"), []byte(stub), 0o755); err != nil {
t.Fatal(err)
}
env := map[string]string{
"PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"),
"GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"),
}
run := "bash " + script + " --experimental_remote_repo_contents_cache -- --loading_phase_threads=64 --remote_upload_local_results"
if out, err := runRRCScript(t, dir, run, env); err != nil {
t.Fatalf("rrc-run-lanes.sh: %v\n%s", err, out)
}
var want []string
for _, cmd := range bazelRRCLaneCommands(t, root) {
want = append(want, "--experimental_remote_repo_contents_cache "+cmd+" --nobuild --loading_phase_threads=64 --remote_upload_local_results")
}
if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); !slices.Equal(got, want) {
t.Errorf("bazel calls:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if err := os.Remove(calls); err != nil {
t.Fatal(err)
}
if out, err := runRRCScript(t, dir, "bash "+script+" --", env); err != nil {
t.Fatalf("without flags: %v\n%s", err, out)
}
if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); len(got) == 0 || !strings.HasPrefix(got[0], "test ") || !strings.HasSuffix(got[0], " --nobuild") {
t.Errorf("without flags, first call %q", got)
}
failEnv := map[string]string{"BAZEL_TEST_FAIL": "1"}
for k, v := range env {
failEnv[k] = v
}
if out, err := runRRCScript(t, dir, run, failEnv); err == nil {
t.Errorf("a failed analysis passed:\n%s", out)
}
empty := filepath.Join(dir, "empty.txt")
if err := os.WriteFile(empty, []byte("# nothing\n\n"), 0o644); err != nil {
t.Fatal(err)
}
emptyEnv := map[string]string{"RRC_LANE_COMMANDS": empty}
for k, v := range env {
emptyEnv[k] = v
}
if out, err := runRRCScript(t, dir, run, emptyEnv); err == nil {
t.Errorf("no lane commands passed; it must fail rather than seed nothing:\n%s", out)
}
if out, err := runRRCScript(t, dir, "bash "+script+" --flag", env); err == nil {
t.Errorf("no -- separator passed:\n%s", out)
}
}
// TestBazelRRCSeedJob: the only writer runs on push to main in mode remote
// with rrc not off, holds the only id-token permission in bazel.yml, reads
// no secret (its credential is the OIDC-minted certificate), sets Bazel up
// in local mode, seeds only with a certificate, uploads to the cache alone
// (no executor), and always removes the key.
func TestBazelRRCSeedJob(t *testing.T) {
wf := readCIWorkflow(t, bazelWorkflowName)
job, ok := wf.Jobs[bazelRRCSeedJob]
if !ok {
t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCSeedJob)
}
if job.If != bazelRRCSeedJobIf {
t.Errorf("%s if %q, want %q", bazelRRCSeedJob, job.If, bazelRRCSeedJobIf)
}
if !slices.Equal(job.Needs, []string{bazelRBEJobName}) || job.Outputs != nil {
t.Errorf("%s needs %v outputs %v; it needs the rbe job alone and reports nothing to the gate", bazelRRCSeedJob, job.Needs, job.Outputs)
}
for _, c := range []struct {
event, ref, mode, rrc string
want bool
}{
{"push", "refs/heads/main", "remote", "seed", true},
{"push", "refs/heads/main", "remote", "canary", true},
{"push", "refs/heads/main", "remote", "on", true},
{"push", "refs/heads/main", "remote", "off", false},
{"push", "refs/heads/main", "cache", "on", false},
{"push", "refs/heads/other", "remote", "on", false},
{"pull_request", "refs/pull/1/merge", "remote", "on", false},
{"pull_request_target", "refs/heads/main", "remote", "on", false},
{"merge_group", "refs/heads/gh-readonly-queue/main/pr-1-abc", "remote", "on", false},
{"workflow_dispatch", "refs/heads/main", "remote", "on", false},
{"schedule", "refs/heads/main", "remote", "on", false},
} {
got := evalRRCIf(t, job.If, map[string]string{
"github.event_name": c.event, "github.ref": c.ref, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc,
})
if got != c.want {
t.Errorf("%+v: seed runs %v, want %v", c, got, c.want)
}
}
if want := map[string]any{"contents": "read", "id-token": "write"}; !reflect.DeepEqual(job.Permissions, want) {
t.Errorf("%s permissions %v, want %v", bazelRRCSeedJob, job.Permissions, want)
}
raw := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)
if text := bazelRRCJobText(t, raw, bazelRRCSeedJob); strings.Contains(text, "secrets.") {
t.Errorf("%s reads a secret; its only credential is the OIDC-minted writer certificate", bazelRRCSeedJob)
}
setup := rrcStep(t, job, "setup-bazel", func(s ciWorkflowStep) bool { return s.Uses == bazelSetupBazelUses })
if len(setup.Env) != 0 {
t.Errorf("%s setup-bazel env %v; local mode (no executor, no certificate) only", bazelRRCSeedJob, setup.Env)
}
cred := rrcStep(t, job, "writer certificate", func(s ciWorkflowStep) bool { return s.ID == "writer" })
if cred.Run != "bash "+bazelRRCCredential {
t.Errorf("writer step runs %q, want bash %s", cred.Run, bazelRRCCredential)
}
seed := rrcStep(t, job, "seed", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, bazelRRCRunLanes) })
if seed.If != "${{ steps.writer.outputs.cert != '' }}" {
t.Errorf("seed step if %q; it must skip when the mint switched seeding off (no certificate)", seed.If)
}
for _, want := range []string{
"bash " + bazelRRCRunLanes + " --experimental_remote_repo_contents_cache --", "--loading_phase_threads=64",
`"--remote_cache=$ENDPOINT"`, `"--remote_instance_name=$INSTANCE"`,
`"--tls_client_certificate=$CERT"`, `"--tls_client_key=$KEY"`, "--remote_upload_local_results",
} {
if !strings.Contains(seed.Run, want) {
t.Errorf("seed step lacks %q:\n%s", want, seed.Run)
}
}
if strings.Contains(seed.Run, "remote_executor") {
t.Errorf("seed step names an executor; the writer certificate is cache only")
}
cleanup := rrcStep(t, job, "key removal", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "rrc-writer.key") })
if cleanup.If != "${{ always() }}" {
t.Errorf("the writer key removal runs if %q; it must always run", cleanup.If)
}
}
// bazelRRCJobText returns a job's raw text in a workflow.
func bazelRRCJobText(t *testing.T, raw, job string) string {
t.Helper()
start := strings.Index(raw, "\n "+job+":\n")
if start < 0 {
t.Fatalf("cannot find the %s job's text", job)
}
text := raw[start+1:]
if next := regexp.MustCompile(`\n [a-z0-9-]+:\n`).FindStringIndex(text); next != nil {
text = text[:next[0]]
}
return text
}
// TestBazelRRCVerifyJob: the nightly check runs from nightly.yml (schedule)
// or a dispatch while the cache is in use, fetches every lane cold without
// the repo contents cache, compares with rrc_verify.py using the lanes'
// read-only certificate and setup-bazel's endpoint (no secret outside
// setup-bazel), and alerts through an issue only when the comparison fails.
func TestBazelRRCVerifyJob(t *testing.T) {
wf := readCIWorkflow(t, bazelWorkflowName)
job, ok := wf.Jobs[bazelRRCVerifyJob]
if !ok {
t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCVerifyJob)
}
if job.If != bazelRRCVerifyJobIf {
t.Errorf("%s if %q, want %q", bazelRRCVerifyJob, job.If, bazelRRCVerifyJobIf)
}
if job.Outputs != nil {
t.Errorf("%s outputs %v; it reports nothing to the gate", bazelRRCVerifyJob, job.Outputs)
}
for _, c := range []struct {
event, mode, rrc string
want bool
}{
{"schedule", "remote", "on", true},
{"schedule", "remote", "seed", true},
{"workflow_dispatch", "remote", "canary", true},
{"schedule", "remote", "off", false},
{"schedule", "cache", "on", false},
{"push", "remote", "on", false},
{"pull_request", "remote", "on", false},
{"pull_request_target", "remote", "on", false},
{"merge_group", "remote", "on", false},
} {
got := evalRRCIf(t, job.If, map[string]string{
"github.event_name": c.event, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc,
})
if got != c.want {
t.Errorf("%+v: verify runs %v, want %v", c, got, c.want)
}
}
if want := map[string]any{"contents": "read", "issues": "write"}; !reflect.DeepEqual(job.Permissions, want) {
t.Errorf("%s permissions %v, want %v", bazelRRCVerifyJob, job.Permissions, want)
}
for _, s := range job.Steps {
if strings.Contains(s.Run, "experimental_remote_repo_contents_cache") || strings.Contains(s.Run, ".bazelrc.local") ||
strings.Contains(s.Run, "remote_upload_local_results") {
t.Errorf("%s step %q reads or writes the repo contents cache; the cold fetch must not", bazelRRCVerifyJob, s.Name)
}
for k, v := range s.Env {
if strings.Contains(v, "secrets.") && s.Uses != bazelSetupBazelUses {
t.Errorf("%s step %q env %s reads a secret; only setup-bazel may", bazelRRCVerifyJob, s.Name, k)
}
}
}
fetch := rrcStep(t, job, "cold fetch", func(s ciWorkflowStep) bool { return strings.HasPrefix(s.Name, "Cold fetch") })
if fetch.Run != "bash "+bazelRRCRunLanes+" -- --loading_phase_threads=64" {
t.Errorf("cold fetch runs %q; it must run every lane's command with no startup flag", fetch.Run)
}
verify := rrcStep(t, job, "verify", func(s ciWorkflowStep) bool { return s.ID == "verify" })
for _, want := range []string{"python3 " + bazelRRCVerifyTool, `--cert "$secret_dir/client.crt"`, `--key "$secret_dir/client.key"`, "--instance oss", ".bazelversion", "remote_executor="} {
if !strings.Contains(verify.Run, want) {
t.Errorf("verify step lacks %q", want)
}
}
alert := rrcStep(t, job, "alert", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "gh issue") })
if alert.If != "${{ failure() && steps.verify.outcome == 'failure' }}" {
t.Errorf("alert step if %q; it must alert only on a failed comparison", alert.If)
}
if strings.Contains(alert.Run, "${{") {
t.Errorf("alert step interpolates an expression into its script; pass it through env")
}
}
// permissionRank orders GitHub token permission levels so a grant can be
// compared with a request: write covers read, read covers none.
var permissionRank = map[string]int{"": 0, "none": 0, "read": 1, "write": 2}
type permWorkflow struct {
Jobs map[string]struct {
Uses string `yaml:"uses"`
Permissions map[string]string `yaml:"permissions"`
} `yaml:"jobs"`
}
// TestBazelWorkflowCallersGrantEveryRequestedPermission guards against
// startup_failure (gascity #7478): GitHub validates a reusable workflow's
// job permissions when the caller starts, including jobs whose `if:` would
// skip, so every workflow that calls bazel.yml must grant at least the
// highest level any bazel.yml job requests for each scope.
func TestBazelWorkflowCallersGrantEveryRequestedPermission(t *testing.T) {
root := sourceRepoRoot(t)
load := func(name string) permWorkflow {
var wf permWorkflow
if err := yaml.Unmarshal([]byte(readPolicyFile(t, root, ".github/workflows/"+name)), &wf); err != nil {
t.Fatalf("parse %s: %v", name, err)
}
return wf
}
need := map[string]string{}
for _, job := range load(bazelWorkflowName).Jobs {
for scope, level := range job.Permissions {
if permissionRank[level] > permissionRank[need[scope]] {
need[scope] = level
}
}
}
if need["id-token"] != "write" || need["issues"] != "write" {
t.Fatalf("bazel.yml requests %v; the parser is broken or the rrc jobs are gone", need)
}
entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows"))
if err != nil {
t.Fatal(err)
}
var callers []string
for _, e := range entries {
name := e.Name()
if name == bazelWorkflowName || (!strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml")) {
continue
}
for jobName, job := range load(name).Jobs {
if job.Uses != "./.github/workflows/"+bazelWorkflowName {
continue
}
callers = append(callers, name)
var missing []string
for scope, level := range need {
if permissionRank[job.Permissions[scope]] < permissionRank[level] {
missing = append(missing, scope+": "+level)
}
}
sort.Strings(missing)
if len(missing) > 0 {
t.Errorf("%s job %q calls %s without granting %s (startup_failure)", name, jobName, bazelWorkflowName, strings.Join(missing, ", "))
}
}
}
sort.Strings(callers)
if want := []string{"bazel-farm.yml", "nightly.yml", "pr.yml"}; !slices.Equal(callers, want) {
t.Errorf("bazel.yml callers %v, want %v", callers, want)
}
}
// rrcMintCurlStub stands in for curl in rrc-writer-credential.sh. The OIDC
// request (its URL carries audience=) logs the audience and authorization
// and answers a token. A mint request logs its Authorization header and
// answers as the space-separated RBE_TEST_MINT says, one word per request
// (the last repeats): ok (a certificate for the runner's key, CN
// rbe-rrc-writer-beads O=gascity), cn, endpoint, instance, otherkey, nocert
// (each wrong in that one way), 000 (connection refused) or an HTTP status
// with an error body.
const rrcMintCurlStub = `#!/usr/bin/env bash
set -euo pipefail
out= url= headers=()
while [ $# -gt 0 ]; do
case "$1" in
-o) out=$2; shift 2 ;;
-H) headers+=("$2"); shift 2 ;;
-w | --data | --connect-timeout | --max-time | --retry) shift 2 ;;
-*) shift ;;
*) url=$1; shift ;;
esac
done
case "$url" in
*audience=*)
echo "oidc audience=${url##*audience=} auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG"
echo '{"value": "oidc-jwt-value"}'
exit 0
;;
esac
echo "mint $url auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG"
n=$(grep -c '^mint ' "$RBE_TEST_MINT_LOG")
read -r -a answers <<<"$RBE_TEST_MINT"
i=$((n - 1)); [ "$i" -lt "${#answers[@]}" ] || i=$((${#answers[@]} - 1))
answer=${answers[$i]}
key="$BAZEL_CI_SECRET_DIR/rrc-writer.key" cn=rbe-rrc-writer-beads endpoint=grpcs://rbe-west.ops.gascity.com:443 instance=oss
case "$answer" in
ok | cn | endpoint | instance | otherkey | nocert) ;;
000) echo "curl: (7) Failed to connect" >&2; exit 7 ;;
*) printf '{"error": "stub %s"}' "$answer" >"$out"; printf '%s' "$answer"; exit 0 ;;
esac
case "$answer" in
cn) cn=rbe-ci ;;
endpoint) endpoint=grpcs://attacker.example:443 ;;
instance) instance=main ;;
otherkey) openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out other.key 2>/dev/null; key=other.key ;;
esac
pem=
if [ "$answer" != nocert ]; then
pem=$(openssl req -new -x509 -key "$key" -subj "/O=gascity/CN=$cn" -days 1 2>/dev/null)
fi
jq -cn --arg pem "$pem" --arg e "$endpoint" --arg i "$instance" --arg cn "$cn" \
'{cert_pem: $pem, endpoint: $e, instance: $i, cn: $cn}' >"$out"
printf 200
`
// TestRRCWriterCredential runs rrc-writer-credential.sh against a stubbed
// curl (rrcMintCurlStub): it asks GitHub for an OIDC token with audience
// rbe-rrc-writer, masks it, sends it to the mint, retries only 429, 502 and
// the network, treats 503 or an unreachable mint as seeding off (no
// outputs, exit 0), and accepts only a certificate for the key it generated,
// CN rbe-rrc-writer[-x] O=gascity, for rbe-west's endpoint and instance oss.
func TestRRCWriterCredential(t *testing.T) {
for _, tool := range []string{"bash", "jq", "openssl"} {
requireHostTool(t, tool)
}
script := filepath.Join(sourceRepoRoot(t), bazelRRCCredential)
for _, c := range []struct {
answers string
ok, off bool
mints int
}{
{"ok", true, false, 1},
{"429 ok", true, false, 2},
{"502 ok", true, false, 2},
{"000 ok", true, false, 2},
{"502", false, false, 4},
{"403", false, false, 1},
{"401", false, false, 1},
{"409", false, false, 1},
{"cn", false, false, 1},
{"endpoint", false, false, 1},
{"instance", false, false, 1},
{"otherkey", false, false, 1},
{"nocert", false, false, 1},
{"503", true, true, 1},
{"000", true, true, 4},
} {
dir := t.TempDir()
bin := filepath.Join(dir, "bin")
if err := os.MkdirAll(bin, 0o755); err != nil {
t.Fatal(err)
}
for name, body := range map[string]string{"curl": rrcMintCurlStub, "sleep": "#!/bin/sh\nexit 0\n"} {
if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0o755); err != nil {
t.Fatal(err)
}
}
secret := filepath.Join(t.TempDir(), "secret")
output := filepath.Join(dir, "output")
out, err := runRRCScript(t, dir, "bash "+script, map[string]string{
"PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"),
"BAZEL_CI_SECRET_DIR": secret,
"GITHUB_OUTPUT": output,
"ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.invalid/oidc?api-version=2.0",
"ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token",
"RBE_TEST_MINT": c.answers,
"RBE_TEST_MINT_LOG": filepath.Join(dir, "mint.log"),
})
if (err == nil) != c.ok {
t.Errorf("%q: ok=%v, want %v\n%s", c.answers, err == nil, c.ok, out)
continue
}
log := readPolicyFile(t, dir, "mint.log")
if !strings.Contains(log, "oidc audience=rbe-rrc-writer auth=Authorization: bearer request-token\n") {
t.Errorf("%q: OIDC request %q", c.answers, log)
}
if got := strings.Count(log, "mint https://rbe-mint.ops.gascity.com:8444/v1/rrc-writer/cert auth=Authorization: Bearer oidc-jwt-value\n"); got != c.mints {
t.Errorf("%q: %d mint requests with the OIDC token, want %d:\n%s", c.answers, got, c.mints, log)
}
if !strings.Contains(out, "::add-mask::oidc-jwt-value") {
t.Errorf("%q: the OIDC token is not masked:\n%s", c.answers, out)
}
if !c.ok {
continue
}
if c.off {
if _, err := os.Stat(output); !os.IsNotExist(err) {
t.Errorf("%q: wrote outputs (%v); the seed must skip", c.answers, err)
}
if !strings.Contains(out, "seeding off (HTTP "+c.answers[len(c.answers)-3:]+")") {
t.Errorf("%q: no seeding-off warning:\n%s", c.answers, out)
}
continue
}
got := rrcOutputs(t, output)
for name, want := range map[string]string{
"cert": filepath.Join(secret, "rrc-writer.crt"), "key": filepath.Join(secret, "rrc-writer.key"),
"endpoint": "grpcs://rbe-west.ops.gascity.com:443", "instance": "oss",
} {
if got[name] != want {
t.Errorf("%q: output %s=%q, want %q", c.answers, name, got[name], want)
}
}
if key := readPolicyFile(t, secret, "rrc-writer.key"); !strings.Contains(key, "-----BEGIN PRIVATE KEY-----") {
t.Errorf("%q: key is not PKCS#8 (Bazel's TLS refuses SEC1)", c.answers)
}
if fi, err := os.Stat(filepath.Join(secret, "rrc-writer.key")); err != nil || fi.Mode().Perm() != 0o600 {
t.Errorf("%q: key mode %v (%v), want 0600", c.answers, fi, err)
}
if _, err := os.Stat(filepath.Join(secret, "rrc-mint.json")); !os.IsNotExist(err) {
t.Errorf("%q: the mint reply stays on disk (%v)", c.answers, err)
}
}
dir := t.TempDir()
if out, err := runRRCScript(t, dir, "bash "+script, map[string]string{
"BAZEL_CI_SECRET_DIR": filepath.Join(t.TempDir(), "secret"), "GITHUB_OUTPUT": filepath.Join(dir, "output"),
}); err == nil || !strings.Contains(out, "id-token: write") {
t.Errorf("without an OIDC request URL: err %v, want a failure naming id-token: write\n%s", err, out)
}
}
// The writer credential script and rrc_verify.py with its unit test are byte
// copies of gascity's (gascity 78c0e5c6f198306b96fe854c18b4d72f16395ce7,
// #7467), canonical there like tools/bazel/ci_analytics_extract.py: the
// pins catch a local edit that would drift out of lockstep.
var bazelRRCVendored = map[string]string{
bazelRRCCredential: "021788a724876bd1df7e5c117a4b3e1e60c5b25bb18bca472d61c35a08cb1fac",
bazelRRCVerifyTool: "00d50e569e2a50d8ef3d2740460de7337b71afd14dc76d2d2442dbd162236f94",
bazelRRCVerifyTest: "d934f13b466660f3148cf3c8dc7273803b4417f2a52f4edc2dd2f43afb93bfaa",
}
func TestBazelRRCVendoredFromGascity(t *testing.T) {
root := sourceRepoRoot(t)
for path, want := range bazelRRCVendored {
sum := sha256.Sum256([]byte(readPolicyFile(t, root, path)))
if got := hex.EncodeToString(sum[:]); got != want {
t.Errorf("%s sha256 = %s, want %s: it drifted from gascity's copy, or gascity changed it and this pin (and the copy) needs updating", path, got, want)
}
}
}
// TestRRCVerifyUnitTests runs gascity's rrc_verify_test.py in place: the
// AC key construction pinned against Bazel 9.3.0's own writes, the
// roll-forward of intermediate entries, a poisoned entry, and a GUID for
// .bazelversion's release.
func TestRRCVerifyUnitTests(t *testing.T) {
root := sourceRepoRoot(t)
python := requireHostTool(t, "python3")
cmd := exec.Command(python, filepath.Join(root, bazelRRCVerifyTest), "-v")
cmd.Dir = filepath.Join(root, "tools", "bazel")
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("%s failed: %v\n%s", bazelRRCVerifyTest, err, out)
}
}

View file

@ -34,6 +34,8 @@ var ciAnalyticsExcludedJobs = map[string]string{
"package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract",
"package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract",
"bazel-release-cross": "one bazel-release-cross-compile.sh run with no BEP/exec-log/profile capture, as when it was a bazel-pure step",
"rrc-seed": "the remote repo contents cache writer: --nobuild analysis on push to main, no test lane to summarize",
"rrc-verify": "the remote repo contents cache's nightly check: --nobuild cold fetches, no test lane to summarize",
}
// ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }})

View file

@ -580,8 +580,8 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) {
for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} {
ctx := map[string]string{"needs.rbe.outputs.mode": mode}
for name, job := range workflow.Jobs {
if name == bazelRBEJobName {
continue
if name == bazelRBEJobName || isBazelRRCJob(name) {
continue // rrc jobs: push/schedule only, never a fork (literal label)
}
want := "ubuntu-latest"
if mode == "remote" {

View file

@ -1779,7 +1779,7 @@ const (
// bazel.yml's jobs: the rbe job that decides the execution mode, the
// --config=ci lane, one job per CI job a Bazel config mirrors, and the two
// package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings.
var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName}
var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob}
// The lanes that only run remotely (skipped unless the rbe job chose remote);
// bazel-integration runs remotely or with the read-only cache (bazelIntegIf);
@ -2148,6 +2148,9 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) {
t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName)
}
if isBazelRRCJob(name) {
continue // runner, if and setup-bazel: bazel_rrc_test.go
}
// F3: the package gates use larger runners (bazelPackageRunsOn) and
// their own if (the caller's package-gates input, not the rbe job's
// mode - they never skip for execution-mode reasons).
@ -2711,8 +2714,8 @@ func TestBazelLaneIsGated(t *testing.T) {
"rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}",
}
for name, job := range workflow.Jobs {
if name == bazelRBEJobName {
continue
if name == bazelRBEJobName || isBazelRRCJob(name) {
continue // the rrc jobs never run in a PR call (bazel_rrc_test.go)
}
_, gated := bazelLaneGateIDs[name]
_, advisory := bazelAdvisoryLanes[name]
@ -2756,14 +2759,15 @@ func TestBazelLaneIsGated(t *testing.T) {
}
// One caller on PR events: pr.yml's bazel job, with the four RBE secrets
// only, read-only contents, and no if (the rbe job decides).
// only, bazelCallPermissions (contents: read, plus what the rrc jobs,
// which never run on a PR, ask for), and no if (the rbe job decides).
pr := readCIWorkflow(t, "pr.yml")
bazel := pr.job(t, "bazel")
if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 {
t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName)
}
if !reflect.DeepEqual(bazel.Permissions, map[string]any{"contents": "read"}) {
t.Errorf("pr.yml bazel job permissions = %v, want contents: read", bazel.Permissions)
if !reflect.DeepEqual(bazel.Permissions, bazelCallPermissions) {
t.Errorf("pr.yml bazel job permissions = %v, want %v", bazel.Permissions, bazelCallPermissions)
}
if !reflect.DeepEqual(bazel.With, bazelPRCallWith) {
t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith)
@ -3039,6 +3043,11 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string
// bazelAdvisoryLanes' name-specific exception in
// TestBazelGateSimulation below).
return map[string]bool{"remote": true}
case bazelRRCSeedJobIf, bazelRRCVerifyJobIf:
// The remote repo contents cache's writer (push to main) and
// nightly check (schedule, dispatch): never in a pull_request,
// merge_group or pull_request_target call, in any mode.
return map[string]bool{}
}
t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr)
return nil
@ -4182,12 +4191,13 @@ func TestBazelDoltServerTiers(t *testing.T) {
}{
// +2 over the earlier counts: the CI analytics summary/upload steps
// (S3; see bazel-test's "CI analytics summary" comment), added to
// every lane job ahead of the result recorder.
{bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 10},
// every lane job ahead of the result recorder. +1: the remote repo
// contents cache reader (TestBazelRRCReadSteps).
{bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 11},
// One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost-
// latency-study.md recommendation 4), since this is the tier whose
// last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs.
{bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 11},
{bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 12},
} {
job := workflow.job(t, c.job)
if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 {
@ -4195,7 +4205,7 @@ func TestBazelDoltServerTiers(t *testing.T) {
}
assertBazelTierStep(t, job, c.job, c.config)
if n := len(job.Steps); n != c.wantSteps {
t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps)
t.Errorf("%s has %d steps, want %d (checkout, the rrc reader, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps)
}
logs := job.step(t, "Upload test logs")
if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") {
@ -4787,12 +4797,15 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) {
"enabled": "${{ steps.decide.outputs.enabled }}",
"mode": "${{ steps.decide.outputs.mode }}",
"tier": "${{ steps.decide.outputs.tier }}",
// The remote repo contents cache's switch, from decide's mode
// (TestBazelRRCModeStep).
"rrc": "${{ steps.rrc.outputs.rrc }}",
}
if !reflect.DeepEqual(job.Outputs, wantOutputs) {
t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs)
}
if len(job.Steps) != 3 {
t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps))
if len(job.Steps) != 4 {
t.Fatalf("%s has %d steps, want the decision step, the rrc step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps))
}
step := job.Steps[0]
wantEnv := map[string]string{

View file

@ -367,21 +367,21 @@ func TestRBEWorkerEnvSync(t *testing.T) {
// mode the lanes read is decide's alone.
func TestBazelRBEWorkerEnvPreflight(t *testing.T) {
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName)
if len(job.Steps) != 3 || job.Steps[0].ID != "decide" {
t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID)
if len(job.Steps) != 4 || job.Steps[0].ID != "decide" || job.Steps[1].ID != bazelRRCModeStepID {
t.Fatalf("%s steps = %d (first %q), want decide, rrc (TestBazelRRCModeStep), the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID)
}
if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) {
t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName)
}
checkout, check := job.Steps[1], job.Steps[2]
checkout, check := job.Steps[2], job.Steps[3]
if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout {
t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s",
t.Errorf("%s step 3: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s",
bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf)
}
wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"}
if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) ||
!strings.Contains(check.Run, "tools/rbe/worker-env-sync main") {
t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main",
t.Errorf("%s step 4: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main",
bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv)
}
@ -427,8 +427,11 @@ func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) {
}
for name, job := range nightly.Jobs {
want := any(nil)
if name == nightlyWorkerEnvJob {
switch name {
case nightlyWorkerEnvJob:
want = map[string]any{"contents": "read", "issues": "write"}
case "bazel": // the bazel.yml call: what its rrc jobs ask for
want = bazelCallPermissions
}
if !reflect.DeepEqual(job.Permissions, want) {
t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want)

View file

@ -67,6 +67,16 @@ const (
publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}`
)
// publicRRCWriterPin: rrc-writer-credential.sh (a byte copy of gascity's)
// pins the one endpoint rbe-west's mint may hand the remote repo contents
// cache writer (rbe-west, :443), so a compromised mint cannot point the
// writer's uploads elsewhere. Public, no credential: allowed as exactly
// this line, in exactly that file.
const (
publicRRCWriterPinFile = ".github/scripts/rrc-writer-credential.sh"
publicRRCWriterPinLine = `ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-west\.ops\.gascity\.com(:443)?$'}`
)
type endpointHit struct {
path string
line int
@ -121,7 +131,8 @@ func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit
if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] {
continue
}
if path == publicRBEForkPinFile && line == publicRBEForkPinLine {
if path == publicRBEForkPinFile && line == publicRBEForkPinLine ||
path == publicRRCWriterPinFile && line == publicRRCWriterPinLine {
continue
}
for _, re := range flagRes {

620
tools/bazel/rrc_verify.py Normal file
View file

@ -0,0 +1,620 @@
#!/usr/bin/env python3
"""Verify rbe-west's remote repo contents cache against a cold fetch (ga-vnycm2.26).
Bazel's --experimental_remote_repo_contents_cache stores each reproducible
external repository as the ActionResult of a synthetic, never executed action
keyed by the repository rule's predeclared-inputs hash
(RemoteRepoContentsCacheImpl, engdocs/design/bazel-remote-repo-contents-cache.md).
Nothing on the reader checks that the cached tree is what the rule produces:
that is the work the cache skips. So, nightly, this job fetches every lane's
repositories cold, without the cache, and compares each repository's tree and
marker file with the entry a lane would read for the same key. A mismatch is
a poisoned entry or a repository rule falsely marked reproducible (design
R1/R2); both must be looked at by a human.
Inputs: the cold output base (setup-bazel's rc: --repo_contents_cache= keeps
each repository in external/<name> with its marker external/@<name>.marker)
and read access to the cache (rbe-west's trusted edge with the CI client
certificate, or any REAPI endpoint). For each marker, it walks the action
cache as the reader does: the final entry under the predeclared hash, or
intermediate entries whose stdout lists batches of recorded inputs, whose
values it takes from the cold marker and rolls into the next key
(Fingerprint: sha256 over length-prefixed UTF-8 strings). The final entry's
marker must equal the cold marker byte for byte, and its tree (the Tree
proto, read from the CAS) must hold the same names, file digests, executable
bits and symlink targets as the cold directory.
Transport: unary and server-streaming gRPC over curl's HTTP/2 with the
standard library's protobuf-free wire decoding, as tools/rbe/cache-zstd-probe.sh
does: no grpc package on a job that holds a farm credential.
Exit status: 0 every cached repository matches; 1 a mismatch or an invalid
entry; 2 a usage error or too many transport failures to tell.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import dataclasses
import hashlib
import json
import os
import stat
import subprocess
import sys
import tempfile
from typing import Callable, Iterable, Optional
# RemoteRepoContentsCacheImpl.GUID per Bazel release: the synthetic Command's
# only argument, so every key depends on it. A .bazelversion bump adds its
# release here (scripts/bazel_rrc_test.go checks .bazelversion has one).
BAZEL_RRC_GUIDS = {
"9.3.0": "06a53d89-9f52-46ed-8064-f7af6ba27769",
}
MARKER_FILE_PATH = ".recorded_inputs"
REPO_DIRECTORY_PATH = "repo_contents"
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
GRPC_NOT_FOUND = 5
MAX_CHAIN = 64 # intermediate hops per repository
MAX_DIFFS = 20 # per repository, in the report
# --- protobuf wire format ---------------------------------------------------
def _varint(n: int) -> bytes:
out = bytearray()
while True:
b = n & 0x7F
n >>= 7
if n:
out.append(b | 0x80)
else:
out.append(b)
return bytes(out)
def _field(num: int, payload: bytes) -> bytes:
return _varint(num << 3 | 2) + _varint(len(payload)) + payload
def _field_varint(num: int, v: int) -> bytes:
return _varint(num << 3) + _varint(v)
def _read_varint(b: bytes, i: int) -> tuple[int, int]:
v = s = 0
while True:
if i >= len(b) or s > 63:
raise ValueError("truncated varint")
c = b[i]
i += 1
v |= (c & 0x7F) << s
s += 7
if c < 0x80:
return v, i
def fields(b: bytes) -> Iterable[tuple[int, int, object]]:
"""Yields (field number, wire type, value) for one message."""
i = 0
while i < len(b):
key, i = _read_varint(b, i)
num, wire = key >> 3, key & 7
if wire == 0:
v, i = _read_varint(b, i)
elif wire == 1:
v, i = b[i : i + 8], i + 8
elif wire == 2:
n, i = _read_varint(b, i)
v, i = b[i : i + n], i + n
elif wire == 5:
v, i = b[i : i + 4], i + 4
else:
raise ValueError("wire type %d" % wire)
if i > len(b):
raise ValueError("truncated field %d" % num)
yield num, wire, v
@dataclasses.dataclass(frozen=True)
class Digest:
hash: str
size: int
def encode(self) -> bytes:
out = _field(1, self.hash.encode())
if self.size:
out += _field_varint(2, self.size)
return out
@staticmethod
def decode(b: bytes) -> "Digest":
h, size = "", 0
for num, _, v in fields(b):
if num == 1:
h = bytes(v).decode()
elif num == 2:
size = int(v)
return Digest(h, size)
@staticmethod
def of(data: bytes) -> "Digest":
return Digest(hashlib.sha256(data).hexdigest(), len(data))
def command_bytes(guid: str) -> bytes:
"""RemoteRepoContentsCacheImpl.COMMAND, serialized in field order."""
return (
_field(1, guid.encode()) # arguments
+ _field(3, MARKER_FILE_PATH.encode()) # output_files
+ _field(4, REPO_DIRECTORY_PATH.encode()) # output_directories
+ _field(5, b"") # platform: the default instance, set
+ _field(7, MARKER_FILE_PATH.encode()) # output_paths
+ _field(7, REPO_DIRECTORY_PATH.encode())
)
def action_digest(guid: str, input_hash: str) -> Digest:
"""The AC key the cache uses for input_hash (buildAction)."""
action = (
_field(1, Digest.of(command_bytes(guid)).encode()) # command_digest
+ _field(2, Digest(EMPTY_SHA256, 0).encode()) # input_root_digest
+ _field(9, input_hash.encode("latin-1")) # salt
+ _field(10, b"") # platform
)
return Digest.of(action)
def _bazel_string(s: str) -> bytes:
# Bazel holds file contents read as ISO-8859-1 in Java Strings and
# Fingerprint.addString writes those Strings as UTF-8.
return s.encode("utf-8")
def roll_forward(hash_: str, recorded_input_with_value: str) -> str:
"""RemoteRepoContentsCacheImpl.rollForwardHash."""
out = hashlib.sha256()
for s in (hash_, recorded_input_with_value):
b = _bazel_string(s)
out.update(_varint(len(b)) + b)
return out.hexdigest()
# --- REAPI messages ---------------------------------------------------------
@dataclasses.dataclass
class ActionResult:
exit_code: int = 0
output_files: list = dataclasses.field(default_factory=list) # (path, Digest, is_executable, contents)
output_directories: list = dataclasses.field(default_factory=list) # (path, tree Digest)
symlinks: int = 0
stdout_raw: bytes = b""
stdout_digest: Optional[Digest] = None
stderr_size: int = 0
@staticmethod
def decode(b: bytes) -> "ActionResult":
r = ActionResult()
for num, _, v in fields(b):
if num == 2:
path, dg, exe, contents = "", None, False, b""
for n2, _, v2 in fields(v):
if n2 == 1:
path = bytes(v2).decode()
elif n2 == 2:
dg = Digest.decode(v2)
elif n2 == 4:
exe = bool(v2)
elif n2 == 5:
contents = bytes(v2)
r.output_files.append((path, dg, exe, contents))
elif num == 3:
path, tree = "", None
for n2, _, v2 in fields(v):
if n2 == 1:
path = bytes(v2).decode()
elif n2 == 3:
tree = Digest.decode(v2)
r.output_directories.append((path, tree))
elif num == 4:
r.exit_code = int(v)
elif num == 5:
r.stdout_raw = bytes(v)
elif num == 6:
r.stdout_digest = Digest.decode(v)
elif num in (7, 8):
r.stderr_size += len(v)
elif num in (10, 11, 12):
r.symlinks += 1
return r
@dataclasses.dataclass
class Directory:
files: dict # name -> (Digest, is_executable)
dirs: dict # name -> Digest
symlinks: dict # name -> target
@staticmethod
def decode(b: bytes) -> "Directory":
d = Directory({}, {}, {})
for num, _, v in fields(b):
name, dg, exe, target = "", None, False, ""
for n2, _, v2 in fields(v):
if n2 == 1:
name = bytes(v2).decode("utf-8", "surrogateescape")
elif n2 == 2 and num in (1, 2):
dg = Digest.decode(v2)
elif n2 == 2 and num == 3:
target = bytes(v2).decode("utf-8", "surrogateescape")
elif n2 == 4 and num == 1:
exe = bool(v2)
if num == 1:
d.files[name] = (dg, exe)
elif num == 2:
d.dirs[name] = dg
elif num == 3:
d.symlinks[name] = target
return d
def decode_tree(b: bytes) -> tuple[Directory, dict]:
"""Tree -> (root, {Directory digest hash: Directory})."""
root, children = None, {}
for num, _, v in fields(b):
if num == 1:
root = Directory.decode(v)
elif num == 2:
children[hashlib.sha256(v).hexdigest()] = Directory.decode(v)
if root is None:
raise ValueError("Tree without a root")
return root, children
# --- transport ----------------------------------------------------------------
class RPCError(Exception):
def __init__(self, code: int, message: str):
super().__init__("gRPC status %d: %s" % (code, message))
self.code = code
def _frame(msg: bytes) -> bytes:
return b"\0" + len(msg).to_bytes(4, "big") + msg
def _unframe(body: bytes) -> list[bytes]:
out, i = [], 0
while i < len(body):
if i + 5 > len(body) or body[i] != 0:
raise ValueError("not an uncompressed gRPC frame")
n = int.from_bytes(body[i + 1 : i + 5], "big")
out.append(body[i + 5 : i + 5 + n])
i += 5 + n
if i != len(body):
raise ValueError("truncated gRPC frame")
return out
class CurlTransport:
"""gRPC over curl's HTTP/2: grpcs:// with optional mTLS, or grpc://."""
def __init__(self, endpoint: str, cert: str = "", key: str = "", ca: str = "", timeout: int = 60):
if endpoint.startswith("grpcs://"):
self.base = "https://" + endpoint[len("grpcs://") :]
self.tls = True
elif endpoint.startswith("grpc://"):
self.base = "http://" + endpoint[len("grpc://") :]
self.tls = False
else:
raise ValueError("endpoint must be grpcs:// or grpc://, got %r" % endpoint)
self.base = self.base.rstrip("/")
self.cert, self.key, self.ca, self.timeout = cert, key, ca, timeout
def call(self, method: str, request: bytes) -> list[bytes]:
with tempfile.TemporaryDirectory() as tmp:
req, head, body = (os.path.join(tmp, n) for n in ("req", "head", "body"))
with open(req, "wb") as f:
f.write(_frame(request))
cmd = ["curl", "-sS", "--connect-timeout", "10", "--max-time", str(self.timeout)]
cmd += ["--http2"] if self.tls else ["--http2-prior-knowledge"]
if self.cert:
cmd += ["--cert", self.cert, "--key", self.key]
if self.ca:
cmd += ["--cacert", self.ca]
cmd += [
"-H", "content-type: application/grpc", "-H", "te: trailers",
"--data-binary", "@" + req, "-D", head, "-o", body, "-w", "%{http_code}",
self.base + "/" + method,
] # fmt: skip
p = subprocess.run(cmd, capture_output=True, text=True, check=False)
if p.returncode != 0:
raise RPCError(-1, "curl exit %d: %s" % (p.returncode, p.stderr.strip()))
if p.stdout != "200":
raise RPCError(-1, "HTTP %s" % p.stdout)
headers = read_bytes(head).decode("latin-1").replace("\r", "")
status, message = None, ""
for line in headers.split("\n"):
k, _, v = line.partition(":")
if k.strip().lower() == "grpc-status":
status = int(v.strip())
elif k.strip().lower() == "grpc-message":
message = v.strip()
if status is None:
raise RPCError(-1, "no grpc-status")
if status != 0:
raise RPCError(status, message)
return _unframe(read_bytes(body))
class Cache:
"""The few REAPI reads the reader makes."""
def __init__(self, rpc: Callable[[str, bytes], list[bytes]], instance: str):
self.rpc, self.instance = rpc, instance
def action_result(self, key: Digest) -> Optional[ActionResult]:
req = (
_field(1, self.instance.encode())
+ _field(2, key.encode())
+ _field_varint(3, 1) # inline_stdout
+ _field(5, MARKER_FILE_PATH.encode()) # inline_output_files
)
try:
msgs = self.rpc("build.bazel.remote.execution.v2.ActionCache/GetActionResult", req)
except RPCError as e:
if e.code == GRPC_NOT_FOUND:
return None
raise
if len(msgs) != 1:
raise RPCError(-1, "GetActionResult: %d messages" % len(msgs))
return ActionResult.decode(msgs[0])
def blob(self, d: Digest) -> bytes:
if d.size == 0:
return b""
name = "%s/blobs/%s/%d" % (self.instance, d.hash, d.size) if self.instance else "blobs/%s/%d" % (d.hash, d.size)
msgs = self.rpc("google.bytestream.ByteStream/Read", _field(1, name.encode()))
data = b"".join(bytes(v) for m in msgs for num, _, v in fields(m) if num == 10)
if Digest.of(data) != d:
raise RPCError(-1, "blob %s/%d: got %d bytes that do not match" % (d.hash, d.size, len(data)))
return data
# --- verification ---------------------------------------------------------------
@dataclasses.dataclass
class Result:
repo: str
status: str # match, mismatch, invalid, not_cached, error
detail: list = dataclasses.field(default_factory=list)
def read_marker(path: str) -> tuple[str, dict]:
"""(predeclared hash, {recorded input: its 'input value' line})."""
text = read_bytes(path).decode("latin-1")
lines = text.split("\n")
recorded = {}
for line in lines[1:]:
if not line:
continue
inp, sep, _ = line.partition(" ")
if sep:
recorded[inp] = line
return lines[0].strip(), recorded
def find_final(cache: Cache, guid: str, predeclared: str, recorded: dict) -> tuple[Optional[ActionResult], list]:
"""Walks the AC from predeclared like fetchFinalCacheEntry; returns (final entry or None, invalid reasons)."""
invalid, current, seen = [], [predeclared], set()
for _ in range(MAX_CHAIN):
nxt = []
for h in current:
if h in seen:
continue
seen.add(h)
ar = cache.action_result(action_digest(guid, h))
if ar is None:
continue
if ar.exit_code != 0:
invalid.append("entry %s: exit code %d" % (h, ar.exit_code))
continue
if len(ar.output_files) == 1 and len(ar.output_directories) == 1 and ar.symlinks == 0:
return ar, invalid
if ar.output_files or ar.output_directories or ar.symlinks or not (ar.stdout_raw or (ar.stdout_digest and ar.stdout_digest.size)):
invalid.append("entry %s: neither a final nor an intermediate result" % h)
continue
stdout = ar.stdout_raw if ar.stdout_raw else cache.blob(ar.stdout_digest)
for batch in stdout.decode("latin-1").splitlines():
rolling = h
for inp in batch.split(" "):
line = recorded.get(inp)
if line is None:
break # the cold fetch recorded other inputs: not this branch
rolling = roll_forward(rolling, line)
else:
nxt.append(rolling)
if not nxt:
return None, invalid
current = nxt
invalid.append("more than %d intermediate hops" % MAX_CHAIN)
return None, invalid
def read_bytes(path: str) -> bytes:
with open(path, "rb") as f:
return f.read()
def _sha256_file(path: str) -> tuple[str, int]:
h, n = hashlib.sha256(), 0
with open(path, "rb") as f:
while chunk := f.read(1 << 20):
h.update(chunk)
n += len(chunk)
return h.hexdigest(), n
def compare_tree(root: Directory, children: dict, local: str) -> list:
"""Differences between the cached tree and the cold directory local."""
diffs = []
def walk(d: Directory, path: str, rel: str) -> None:
if len(diffs) >= MAX_DIFFS:
return
try:
names = set(os.listdir(path))
except OSError as e:
diffs.append("%s: cannot list the cold directory (%s)" % (rel or ".", e))
return
remote = set(d.files) | set(d.dirs) | set(d.symlinks)
for n in sorted(remote - names):
diffs.append("%s: only in the cache" % os.path.join(rel, n))
for n in sorted(names - remote):
diffs.append("%s: only in the cold fetch" % os.path.join(rel, n))
for n in sorted(remote & names):
if len(diffs) >= MAX_DIFFS:
return
p, r = os.path.join(path, n), os.path.join(rel, n)
st = os.lstat(p)
if n in d.symlinks:
if not stat.S_ISLNK(st.st_mode):
diffs.append("%s: a symlink in the cache, not in the cold fetch" % r)
elif os.readlink(p) != d.symlinks[n]:
diffs.append("%s: symlink to %r in the cache, %r cold" % (r, d.symlinks[n], os.readlink(p)))
continue
try:
st = os.stat(p) # the upload follows symlinks it does not keep
except OSError as e:
diffs.append("%s: dangling in the cold fetch (%s)" % (r, e))
continue
if n in d.files:
dg, exe = d.files[n]
if not stat.S_ISREG(st.st_mode):
diffs.append("%s: a file in the cache, not in the cold fetch" % r)
continue
h, size = _sha256_file(p)
if dg is None or (h, size) != (dg.hash, dg.size):
diffs.append("%s: content differs (cache %s/%s, cold %s/%d)" % (r, dg and dg.hash, dg and dg.size, h, size))
if exe != bool(st.st_mode & stat.S_IXUSR):
diffs.append("%s: executable bit differs (cache %s)" % (r, exe))
else:
if not stat.S_ISDIR(st.st_mode):
diffs.append("%s: a directory in the cache, not in the cold fetch" % r)
continue
child = children.get(d.dirs[n].hash)
if child is None:
diffs.append("%s: the Tree lacks directory %s" % (r, d.dirs[n].hash))
continue
walk(child, p, r)
walk(root, local, "")
return diffs
def verify_repo(cache: Cache, guid: str, external: str, name: str) -> Result:
marker = os.path.join(external, "@%s.marker" % name)
try:
predeclared, recorded = read_marker(marker)
final, invalid = find_final(cache, guid, predeclared, recorded)
if invalid:
return Result(name, "invalid", invalid)
if final is None:
return Result(name, "not_cached")
diffs = []
path, mdg, _, contents = final.output_files[0]
dpath, tree = final.output_directories[0]
if path != MARKER_FILE_PATH or dpath != REPO_DIRECTORY_PATH or tree is None or mdg is None:
return Result(name, "invalid", ["final entry outputs %r and %r" % (path, dpath)])
cached_marker = contents if contents else cache.blob(mdg)
if cached_marker != read_bytes(marker):
diffs.append("%s: marker file differs" % MARKER_FILE_PATH)
root, children = decode_tree(cache.blob(tree))
diffs += compare_tree(root, children, os.path.join(external, name))
return Result(name, "mismatch" if diffs else "match", diffs[:MAX_DIFFS])
except (RPCError, ValueError, OSError) as e:
return Result(name, "error", [str(e)])
def repos(external: str) -> list:
out = []
for f in sorted(os.listdir(external)):
if f.startswith("@") and f.endswith(".marker") and os.path.isdir(os.path.join(external, f[1:-7])):
out.append(f[1:-7])
return out
def verify(cache: Cache, guid: str, output_base: str, jobs: int) -> list:
external = os.path.join(output_base, "external")
names = repos(external)
with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
return list(pool.map(lambda n: verify_repo(cache, guid, external, n), names))
def summarize(results: list) -> dict:
counts = {}
for r in results:
counts[r.status] = counts.get(r.status, 0) + 1
return counts
def exit_status(results: list, max_error_fraction: float) -> int:
counts = summarize(results)
if counts.get("mismatch") or counts.get("invalid"):
return 1
if not results or counts.get("error", 0) > max_error_fraction * len(results):
return 2
return 0
def main(argv: Optional[list] = None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("--output-base", required=True, help="the cold fetch's output base")
ap.add_argument("--endpoint", required=True, help="grpcs://host:port or grpc://host:port")
ap.add_argument("--instance", default="oss")
ap.add_argument("--cert", default="")
ap.add_argument("--key", default="")
ap.add_argument("--ca", default="")
ap.add_argument("--bazel-version", required=True, help="the release that wrote the entries (.bazelversion)")
ap.add_argument("--jobs", type=int, default=16)
ap.add_argument("--max-error-fraction", type=float, default=0.01, help="transport errors tolerated before exit 2")
ap.add_argument("--json", default="", help="write the full report here")
ap.add_argument("--summary", default="", help="append a Markdown summary here (GITHUB_STEP_SUMMARY)")
a = ap.parse_args(argv)
guid = BAZEL_RRC_GUIDS.get(a.bazel_version)
if guid is None:
print("rrc-verify: no GUID for Bazel %s; add RemoteRepoContentsCacheImpl.GUID to BAZEL_RRC_GUIDS" % a.bazel_version, file=sys.stderr)
return 2
if bool(a.cert) != bool(a.key):
print("rrc-verify: --cert and --key go together", file=sys.stderr)
return 2
cache = Cache(CurlTransport(a.endpoint, a.cert, a.key, a.ca).call, a.instance)
results = verify(cache, guid, a.output_base, a.jobs)
counts = summarize(results)
bad = [r for r in results if r.status in ("mismatch", "invalid", "error")]
lines = ["rrc-verify: %d repositories: %s" % (len(results), ", ".join("%s %d" % kv for kv in sorted(counts.items())))]
for r in bad:
lines.append(" %s %s: %s" % (r.status, r.repo, "; ".join(r.detail)))
print("\n".join(lines))
if a.json:
with open(a.json, "w") as f:
json.dump({"counts": counts, "results": [dataclasses.asdict(r) for r in results if r.status != "match"]}, f, indent=1)
if a.summary:
with open(a.summary, "a") as f:
f.write("### Remote repo contents cache verification\n\n")
f.write("| status | repositories |\n|---|---|\n")
for k, v in sorted(counts.items()):
f.write("| %s | %d |\n" % (k, v))
for r in bad[:50]:
f.write("\n- **%s** `%s`: %s" % (r.status, r.repo, "; ".join(r.detail)[:500]))
f.write("\n")
return exit_status(results, a.max_error_fraction)
if __name__ == "__main__":
sys.exit(main())

View file

@ -0,0 +1,323 @@
"""Tests for tools/bazel/rrc_verify.py (ga-vnycm2.26).
The golden keys below come from Bazel 9.3.0 itself: it wrote the entries
(`bazel test --nobuild --experimental_remote_repo_contents_cache
--remote_upload_local_results` into a throwaway NativeLink 1.7.1), and
rrc_verify found 1124 of 1124 cached repositories under the keys it computes.
Everything else runs against an in-memory cache.
"""
import hashlib
import os
import stat
import tempfile
import unittest
import rrc_verify as v
GUID = v.BAZEL_RRC_GUIDS["9.3.0"]
# gazelle++go_deps+com_github_aleksi_pointer, a go_repository: its marker
# (predeclared hash, then recorded inputs) and the chain of intermediate
# entries Bazel 9.3.0 wrote for it, one batch per hop.
POINTER_MARKER = (
"d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b\n"
"FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n"
"REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache gazelle++non_module_deps+bazel_gazelle_go_repository_cache\n"
"FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n"
"REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools +go_repository_tools+gazelle_go_repository_tools\n"
"FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo 8b1288089311dc63c3d4b04c433d9ba11df5b09b5e8fdb1f8db0804f360ca6c0\n"
"ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE \\0\n"
"FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle 1b764f6d9ad08e324bdcab429c3105815b8914f289d594acfc791bdac2bc2b2c\n"
"FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE cc988347dd32755bac20223ea21cd982da6da90d9113dee8ccd4c4b88b3983db\n"
)
# (input hash, its AC key, the intermediate entry's stdout)
POINTER_CHAIN = [
("d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b",
"add188cbb3ec90ac802738e80e42516a17ba75e7563efffa80f64fee38edf866",
"FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache\n"),
("8bc57512298f35dd223a75991454f01b20e0581fc5b3cb9ad7e26521a3b89db2",
"5a07101d8884c3f86496450121a048660f511b33076205395149ef6c369ef719",
"FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools\n"),
("07b34597f53e981474b3e4952e51c5caa2ba3877f61ad0cc2fe39c84f51acc8f",
"f7eac27fd1a860cc4563bf18f42f0672d533e171341e7293aae8b5f5d8cdebc3",
"FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE\n"),
("1ef1aa4b4cb73c42f39465f8feba3ae846c412e5a0a7f2790264264c1941df6e",
"531c6579bec5634c2744fdd5b90ea123739d431991472877bffe3548c86697b1",
"FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle\n"),
("edc467b59551f6058310c401062c51315e9a1c3b2bc8ec0a57d7411d7ca4b6dc",
"94a6fc01bd63b6c1506919e676603784eb73e9049b121c4580aaf6e75893d276",
"FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE\n"),
] # fmt: skip
POINTER_FINAL_HASH = "3ceee7642907f9b23db2d1944d05314384151ca509d432eca651bdee2d61997c"
POINTER_FINAL_KEY = v.Digest("b0ca33d505c031b1ee5aa17138c14135803b0989ef76f3203999a316a933e2f2", 206)
# --- an in-memory cache ---------------------------------------------------------
def enc_directory(files=(), dirs=(), symlinks=()):
out = b""
for name, dg, exe in sorted(files):
node = v._field(1, name.encode()) + v._field(2, dg.encode())
if exe:
node += v._field_varint(4, 1)
out += v._field(1, node)
for name, dg in sorted(dirs):
out += v._field(2, v._field(1, name.encode()) + v._field(2, dg.encode()))
for name, target in sorted(symlinks):
out += v._field(3, v._field(1, name.encode()) + v._field(2, target.encode()))
return out
class FakeCache:
"""An AC (input hash -> ActionResult bytes) and a CAS, behind Cache's rpc."""
def __init__(self):
self.ac, self.cas = {}, {}
def put_blob(self, data):
d = v.Digest.of(data)
self.cas[d.hash] = data
return d
def tree_of(self, path):
"""Uploads path's files and returns the Tree digest (as Bazel would)."""
children = []
def directory(p):
files, dirs, links = [], [], []
for n in sorted(os.listdir(p)):
q = os.path.join(p, n)
st = os.lstat(q)
if stat.S_ISLNK(st.st_mode):
links.append((n, os.readlink(q)))
elif stat.S_ISDIR(st.st_mode):
raw = directory(q)
children.append(raw)
dirs.append((n, v.Digest.of(raw)))
else:
data = v.read_bytes(q)
files.append((n, self.put_blob(data), bool(st.st_mode & stat.S_IXUSR)))
return enc_directory(files, dirs, links)
root = directory(path)
tree = v._field(1, root) + b"".join(v._field(2, c) for c in children)
return self.put_blob(tree)
def put_final(self, input_hash, marker, tree, exit_code=0):
md = self.put_blob(marker)
ar = (
v._field(2, v._field(1, v.MARKER_FILE_PATH.encode()) + v._field(2, md.encode()))
+ v._field(3, v._field(1, v.REPO_DIRECTORY_PATH.encode()) + v._field(3, tree.encode()))
)
if exit_code:
ar += v._field_varint(4, exit_code)
self.ac[input_hash] = ar
def put_intermediate(self, input_hash, stdout, raw=True):
if raw:
self.ac[input_hash] = v._field(5, stdout.encode()) + v._field(6, v.Digest.of(stdout.encode()).encode())
else:
self.ac[input_hash] = v._field(6, self.put_blob(stdout.encode()).encode())
def rpc(self, method, request):
if method.endswith("ActionCache/GetActionResult"):
key = None
for num, _, val in v.fields(request):
if num == 2:
key = v.Digest.decode(val)
for h, ar in self.ac.items():
if v.action_digest(GUID, h) == key:
return [ar]
raise v.RPCError(v.GRPC_NOT_FOUND, "not found")
if method.endswith("ByteStream/Read"):
name = bytes(next(val for num, _, val in v.fields(request) if num == 1)).decode()
_, _, h, size = name.split("/")
data = self.cas.get(h)
if data is None:
raise v.RPCError(v.GRPC_NOT_FOUND, "no blob")
return [v._field(10, data[: len(data) // 2]), v._field(10, data[len(data) // 2 :])]
raise AssertionError(method)
def write(path, data, mode=0o644):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as f:
f.write(data)
os.chmod(path, mode)
class Fixture:
"""A cold output base with one repository, cached as fetched."""
def __init__(self, test, name="repo", marker=b"h0\n"):
self.tmp = tempfile.TemporaryDirectory()
test.addCleanup(self.tmp.cleanup)
self.ext = os.path.join(self.tmp.name, "external")
self.name = name
self.dir = os.path.join(self.ext, name)
write(os.path.join(self.dir, "BUILD.bazel"), b"filegroup(name = 'x')\n")
write(os.path.join(self.dir, "bin", "tool"), b"#!/bin/sh\n", 0o755)
write(os.path.join(self.dir, "sub", "deep", "data.txt"), b"data\n")
os.makedirs(os.path.join(self.dir, "empty"))
os.symlink("bin/tool", os.path.join(self.dir, "link"))
self.marker = marker
write(os.path.join(self.ext, "@%s.marker" % name), marker)
self.cache = FakeCache()
def seed(self, input_hash="h0"):
self.cache.put_final(input_hash, self.marker, self.cache.tree_of(self.dir))
def verify(self):
return v.verify_repo(v.Cache(self.cache.rpc, "oss"), GUID, self.ext, self.name)
class KeysTest(unittest.TestCase):
def test_action_digest_matches_bazel_9_3_0(self):
for h, key, _ in POINTER_CHAIN:
self.assertEqual(v.action_digest(GUID, h).hash, key)
self.assertEqual(v.action_digest(GUID, POINTER_FINAL_HASH), POINTER_FINAL_KEY)
def test_roll_forward_matches_bazel_9_3_0(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
path = os.path.join(tmp.name, "m")
write(path, POINTER_MARKER.encode())
predeclared, recorded = v.read_marker(path)
self.assertEqual(predeclared, POINTER_CHAIN[0][0])
h = predeclared
for i, (want, _, stdout) in enumerate(POINTER_CHAIN):
self.assertEqual(h, want, "hop %d" % i)
for inp in stdout.strip().split(" "):
h = v.roll_forward(h, recorded[inp])
self.assertEqual(h, POINTER_FINAL_HASH)
def test_guid_for_every_release_is_a_uuid(self):
for release, guid in v.BAZEL_RRC_GUIDS.items():
self.assertRegex(guid, r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", release)
def test_bazelversion_has_a_guid(self):
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(v.__file__))))
path = os.path.join(root, ".bazelversion")
version = v.read_bytes(path).decode().strip()
self.assertIn(version, v.BAZEL_RRC_GUIDS, "a .bazelversion bump must add RemoteRepoContentsCacheImpl.GUID")
class VerifyRepoTest(unittest.TestCase):
def test_match(self):
f = Fixture(self)
f.seed()
r = f.verify()
self.assertEqual((r.status, r.detail), ("match", []))
def test_not_cached(self):
f = Fixture(self)
self.assertEqual(f.verify().status, "not_cached")
def test_content_poisoned(self):
f = Fixture(self)
f.seed()
write(os.path.join(f.dir, "sub", "deep", "data.txt"), b"other\n")
r = f.verify()
self.assertEqual(r.status, "mismatch")
self.assertIn("sub/deep/data.txt: content differs", " ".join(r.detail))
def test_extra_and_missing_files(self):
f = Fixture(self)
f.seed()
write(os.path.join(f.dir, "init.go"), b"package x\n")
os.remove(os.path.join(f.dir, "BUILD.bazel"))
detail = " ".join(f.verify().detail)
self.assertIn("init.go: only in the cold fetch", detail)
self.assertIn("BUILD.bazel: only in the cache", detail)
def test_executable_bit(self):
f = Fixture(self)
f.seed()
os.chmod(os.path.join(f.dir, "bin", "tool"), 0o644)
self.assertIn("bin/tool: executable bit differs", " ".join(f.verify().detail))
def test_symlink_target(self):
f = Fixture(self)
f.seed()
os.remove(os.path.join(f.dir, "link"))
os.symlink("sub", os.path.join(f.dir, "link"))
self.assertIn("link: symlink to", " ".join(f.verify().detail))
def test_empty_directory(self):
f = Fixture(self)
f.seed()
os.rmdir(os.path.join(f.dir, "empty"))
self.assertIn("empty: only in the cache", " ".join(f.verify().detail))
def test_marker_differs(self):
f = Fixture(self)
f.seed()
write(os.path.join(f.ext, "@repo.marker"), b"h0\nENV:X 1\n")
r = f.verify()
self.assertEqual(r.status, "mismatch")
self.assertIn("marker file differs", " ".join(r.detail))
def test_intermediate_chain(self):
marker = b"p0\nENV:A 1\nFILE:@@x//y abc\nENV:B \\0\n"
f = Fixture(self, marker=marker)
h1 = v.roll_forward(v.roll_forward("p0", "ENV:A 1"), "FILE:@@x//y abc")
h2 = v.roll_forward(h1, "ENV:B \\0")
f.cache.put_intermediate("p0", "ENV:Z\nENV:A FILE:@@x//y\n") # the first batch is another fetch's
f.cache.put_intermediate(h1, "ENV:B\n", raw=False)
f.seed(h2)
r = f.verify()
self.assertEqual((r.status, r.detail), ("match", []))
def test_intermediate_without_a_matching_batch(self):
f = Fixture(self, marker=b"p0\nENV:A 1\n")
f.cache.put_intermediate("p0", "ENV:Q\n")
f.seed(v.roll_forward("p0", "ENV:A 2")) # another value of A: not this fetch's
self.assertEqual(f.verify().status, "not_cached")
def test_invalid_exit_code(self):
f = Fixture(self)
f.cache.put_final("h0", f.marker, f.cache.tree_of(f.dir), exit_code=1)
self.assertEqual(f.verify().status, "invalid")
def test_invalid_shape(self):
f = Fixture(self)
f.cache.ac["h0"] = v._field_varint(4, 0) + v._field(7, b"stderr")
self.assertEqual(f.verify().status, "invalid")
def test_tampered_blob_is_an_error(self):
f = Fixture(self)
f.seed()
tree = next(d for d in f.cache.cas if f.cache.cas[d].startswith(b"\x0a"))
f.cache.cas[tree] = f.cache.cas[tree] + b"\x00"
self.assertEqual(f.verify().status, "error")
class ReportTest(unittest.TestCase):
def test_exit_status(self):
R = v.Result
self.assertEqual(v.exit_status([R("a", "match"), R("b", "not_cached")], 0.01), 0)
self.assertEqual(v.exit_status([R("a", "match"), R("b", "mismatch")], 0.01), 1)
self.assertEqual(v.exit_status([R("a", "invalid")], 0.01), 1)
self.assertEqual(v.exit_status([R("a", "error")] + [R(str(i), "match") for i in range(200)], 0.01), 0)
self.assertEqual(v.exit_status([R("a", "error"), R("b", "match")], 0.01), 2)
self.assertEqual(v.exit_status([], 0.01), 2)
def test_repos_lists_marked_repositories(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
ext = os.path.join(tmp.name, "external")
for n in ("a", "b+c"):
os.makedirs(os.path.join(ext, n))
write(os.path.join(ext, "@%s.marker" % n), b"h\n")
os.makedirs(os.path.join(ext, "unmarked"))
write(os.path.join(ext, "@gone.marker"), b"h\n")
self.assertEqual(v.repos(ext), ["a", "b+c"])
def test_unknown_release_is_a_usage_error(self):
self.assertEqual(v.main(["--output-base", "/x", "--endpoint", "grpc://x:1", "--bazel-version", "0.0.0"]), 2)
if __name__ == "__main__":
unittest.main()