diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt new file mode 100644 index 000000000..08bd7eb5e --- /dev/null +++ b/.github/scripts/rrc-lane-commands.txt @@ -0,0 +1,15 @@ +# Every bazel.yml lane's loading and analysis, for the remote repo contents +# cache: rrc-seed runs each line with --nobuild on a push to main (uploading +# the repositories it fetches) and rrc-verify runs each cold every night. +# One `bazel` command per line (no shell quoting); # starts a comment. +# scripts/bazel_rrc_test.go checks that every --config a reading lane's +# bazel command uses appears here. +test //... --config=ci +test //... --config=embedded +test //... --config=integration +test //... --config=doltserver +test //... --config=doltserver-proxied +test //... --config=doltserver-integration +test //... --config=doltserver-cmd +build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test +build --config=js-wasm //internal/hooks:hooks_test diff --git a/.github/scripts/rrc-run-lanes.sh b/.github/scripts/rrc-run-lanes.sh new file mode 100755 index 000000000..d5f329a84 --- /dev/null +++ b/.github/scripts/rrc-run-lanes.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Loading and analysis of every bazel.yml lane, for the remote repo contents +# cache (bazel.yml's rrc-seed and rrc-verify jobs). +# +# Usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...] +# +# Runs `bazel STARTUP_FLAG... --nobuild COMMAND_FLAG...` for each +# command line in .github/scripts/rrc-lane-commands.txt (or RRC_LANE_COMMANDS), +# in order, stopping at the first failure. --nobuild: repository rules, +# loading and analysis only, never a build or test action. +# +# Bazel exits 1 after `test --nobuild` even when loading and analysis succeed +# ("Couldn't start the build. Unable to run tests"). A command passes when it +# exits 0, or exits non-zero having printed "Build completed successfully" +# and no ERROR line but that one. +set -euo pipefail + +startup=() +while [ $# -gt 0 ] && [ "$1" != -- ]; do + startup+=("$1") + shift +done +[ $# -gt 0 ] || { + echo "usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...]" >&2 + exit 2 +} +shift +flags=("$@") + +file=${RRC_LANE_COMMANDS:-$(dirname "${BASH_SOURCE[0]}")/rrc-lane-commands.txt} +mapfile -t cmds < <(sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$file" | grep -v '^[[:space:]]*$' || true) +if [ "${#cmds[@]}" -eq 0 ]; then + echo "::error::no lane commands in $file" + exit 1 +fi + +log=$(mktemp) +trap 'rm -f "$log"' EXIT +summary=${GITHUB_STEP_SUMMARY:-/dev/null} +for cmd in "${cmds[@]}"; do + read -r -a args <<<"$cmd" + start=$(date +%s) + rc=0 + bazel ${startup[@]+"${startup[@]}"} "${args[@]}" --nobuild ${flags[@]+"${flags[@]}"} &1 | tee "$log" || rc=$? + if [ "$rc" -ne 0 ] && grep -q '^INFO: Build completed successfully' "$log" && + ! grep '^ERROR:' "$log" | grep -qv "^ERROR: Couldn't start the build. Unable to run tests$"; then + rc=0 + fi + echo "rrc: \`bazel $cmd\`: exit $rc, $(($(date +%s) - start))s" | tee -a "$summary" + [ "$rc" -eq 0 ] || exit "$rc" +done diff --git a/.github/scripts/rrc-writer-credential.sh b/.github/scripts/rrc-writer-credential.sh new file mode 100755 index 000000000..a650c7ddc --- /dev/null +++ b/.github/scripts/rrc-writer-credential.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# A 30-minute rbe-west client certificate for the remote repo contents cache +# writer (bazel.yml's rrc-seed job; engdocs/design/bazel-remote-repo-contents-cache.md). +# +# The job trades its GitHub Actions OIDC token (audience rbe-rrc-writer) for a +# certificate CN=rbe-rrc-writer from rbe-west's mint. The mint signs only for +# this repository's bazel.yml on a push to main (repository, ref, event_name +# and job_workflow_ref claims); pull_request, fork and Dependabot runs cannot +# get one (no OIDC token, or a refs/pull/N/merge one). On rbe-west that +# certificate reaches only GetCapabilities, the CAS and an action cache front +# (rrc-gate) that admits repo-contents entries and refuses everything else, so +# it can never write an execution result or execute anything. +# +# The key is an EC P-256 key (PKCS#8: Bazel's Netty TLS refuses a SEC1 "EC +# PRIVATE KEY") generated here in BAZEL_CI_SECRET_DIR, outside the workspace; +# it never leaves the runner. The OIDC token is masked. Outputs (to +# $GITHUB_OUTPUT when set, else stdout): cert, key, endpoint, instance; none +# when the mint answers 503 or cannot be reached (seeding switched off on +# rbe-west: infra nativelink-cas/west/README.md "Client contract"), which +# exits 0 with a warning. +# +# Tests only: RBE_RRC_MINT (a stand-in mint) and RBE_RRC_ENDPOINT_RE. +set -euo pipefail +MINT=${RBE_RRC_MINT:-https://rbe-mint.ops.gascity.com:8444} +ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^grpcs://rbe-west\.ops\.gascity\.com(:443)?$'} +AUDIENCE=rbe-rrc-writer +dir=${BAZEL_CI_SECRET_DIR:?BAZEL_CI_SECRET_DIR is required} +out=${GITHUB_OUTPUT:-/dev/stdout} + +if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]; then + echo "::error title=rrc writer::no OIDC token request URL; the job needs permissions: id-token: write" >&2 + exit 1 +fi + +install -d -m 0700 "$dir" +umask 077 + +oidc=$(curl -sS --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$AUDIENCE" | jq -r '.value // empty') +if [ -z "$oidc" ]; then + echo "::error title=rrc writer::GitHub returned no OIDC token" >&2 + exit 1 +fi +printf '::add-mask::%s\n' "$oidc" + +openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/rrc-writer.key" 2>/dev/null +csr=$(openssl req -new -key "$dir/rrc-writer.key" -subj "/CN=rbe-rrc-writer-request") +body=$(jq -cn --arg csr "$csr" '{csr_pem: $csr}') + +reply="$dir/rrc-mint.json" +code=000 +for attempt in 1 2 3 4; do + # --connect-timeout: a closed gate drops the SYN; fail fast, not in 60 s. + code=$(curl -sS -o "$reply" -w '%{http_code}' --connect-timeout 5 --max-time 60 \ + -H "Authorization: Bearer $oidc" -H 'content-type: application/json' \ + --data "$body" "$MINT/v1/rrc-writer/cert") || code=000 + # Retry what may pass on its own: a rate limit (429), the mint could not + # reach GitHub's JWKS (502), or the network (000). Everything else is an + # answer. + case "$code" in 429 | 502 | 000) ;; *) break ;; esac + [ "$attempt" -eq 4 ] || sleep $((attempt * 10)) +done +case "$code" in +200) ;; +503 | 000) + # The mint's switch is off, its CA is not installed, or rbe-west's fork + # edge is closed: seeding is off, not broken. No outputs, so the seed + # step skips; the push to main never fails on it. + echo "::warning title=rrc writer: seeding off (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 0 + ;; +*) + echo "::error title=rrc writer mint refused (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 1 + ;; +esac + +endpoint=$(jq -r '.endpoint // empty' "$reply") +instance=$(jq -r '.instance // empty' "$reply") +[[ $endpoint =~ $ENDPOINT_RE ]] || { + echo "::error::rrc writer mint returned endpoint '$endpoint'" >&2 + exit 1 +} +[ "$instance" = oss ] || { + echo "::error::rrc writer mint returned instance '$instance', want oss" >&2 + exit 1 +} +jq -r '.cert_pem // empty' "$reply" >"$dir/rrc-writer.crt" +subject=$(openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -nameopt RFC2253) || { + echo "::error::rrc writer mint returned no certificate" >&2 + exit 1 +} +rdns=$(tr ',' '\n' <<<"${subject#subject=}") +cn=$(sed -n 's/^CN=//p' <<<"$rdns") +org=$(sed -n 's/^O=//p' <<<"$rdns") +if ! [[ $cn =~ ^rbe-rrc-writer(-[a-z0-9]+)?$ ]] || [ "$org" != gascity ]; then + echo "::error::rrc writer mint returned a certificate for '$subject'" >&2 + exit 1 +fi +# The certificate must belong to the key generated above. +if [ "$(openssl x509 -in "$dir/rrc-writer.crt" -noout -pubkey | openssl sha256)" != "$(openssl pkey -in "$dir/rrc-writer.key" -pubout | openssl sha256)" ]; then + echo "::error::rrc writer mint returned a certificate for another key" >&2 + exit 1 +fi +openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -enddate +rm -f "$reply" +{ + echo "cert=$dir/rrc-writer.crt" + echo "key=$dir/rrc-writer.key" + echo "endpoint=$endpoint" + echo "instance=$instance" +} >>"$out" diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index 7ab2c0a7b..caeb0ad2f 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -120,8 +120,18 @@ jobs: name: Bazel Farm needs: authorize if: ${{ needs.authorize.outputs.allowed == 'true' }} + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). Neither runs for this event, every other + # bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC + # token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). permissions: contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml # Exactly the four RBE secrets: never the rbe-prewarm app private key # (B1, security review of bdef342d5). This pull_request_target run is diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index eeb98a410..ce9bf223b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -424,6 +424,7 @@ jobs: enabled: ${{ steps.decide.outputs.enabled }} mode: ${{ steps.decide.outputs.mode }} tier: ${{ steps.decide.outputs.tier }} + rrc: ${{ steps.rrc.outputs.rrc }} steps: # The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts). # A push, merge_group or dispatch has no pull_request, so FORK is false. @@ -496,6 +497,37 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + # Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported + # from gascity's bazel.yml; gascity + # engdocs/design/bazel-remote-repo-contents-cache.md): extracted + # external repositories (go_deps, the Go SDK, the LLVM slice) come from + # rbe-west's action cache instead of being re-fetched by every cold lane + # client. vars.RBE_REPO_CONTENTS_CACHE rolls it out: + # unset/off nothing (the default); + # seed main pushes seed the cache (rrc-seed); no lane reads it; + # canary seed, and bazel-test reads; + # on seed, and every lane in .github/scripts/rrc-lane-commands.txt + # reads. + # Mode remote only: fork pull_request runs see no vars, and only + # rbe-west's trusted edge serves the entries. Any other value is off, + # with a warning, so a typo never fails the required gate. Like decide, + # this step runs no repository code (it comes before the checkout). + - name: Remote repo contents cache + id: rrc + env: + RRC: ${{ vars.RBE_REPO_CONTENTS_CACHE }} + MODE: ${{ steps.decide.outputs.mode }} + run: | + rrc=off + case "$RRC" in + "" | off) ;; + seed | canary | on) + if [ "$MODE" = remote ]; then rrc=$RRC; fi ;; + *) echo "::warning title=RBE_REPO_CONTENTS_CACHE::unknown value '$RRC' (want off, seed, canary or on); treated as off" ;; + esac + echo "rrc=$rrc" >> "$GITHUB_OUTPUT" + echo "Remote repo contents cache: **$rrc**" >> "$GITHUB_STEP_SUMMARY" + # The worker-env preflight (see the job's comment): the PR's own copy, # so a PR that syncs it passes. Only the two paths it compares. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -785,6 +817,22 @@ jobs: - name: Verify restored Go modules run: go mod verify + # The remote repo contents cache, read-only (the rbe job's rrc output: + # canary, bazel-test; on, every lane). Each cached repository costs + # several sequential round trips, resolved on Skyframe's loading + # threads (default: the CPU count), so without 64 of them a cached read + # is slower than fetching. Both lines are key neutral (startup and + # loading only). Reading only: setup-bazel's rc keeps + # --noremote_upload_local_results, so nothing here writes; rrc-seed + # writes, on push to main. Before Set up Bazel: no server starts first. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && (needs.rbe.outputs.rrc == 'on' || needs.rbe.outputs.rrc == 'canary') }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1045,6 +1093,15 @@ jobs: with: node-version: '24' + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1333,6 +1390,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1505,6 +1571,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1650,6 +1725,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1770,6 +1854,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1922,6 +2015,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -2085,6 +2187,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -2432,3 +2543,138 @@ jobs: env: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # The remote repo contents cache's only writer (ga-vnycm2.29, ported from + # gascity's bazel.yml). On a push to main, with the rbe job's rrc not off, + # run every lane's loading and analysis (.github/scripts/rrc-lane-commands.txt, + # --nobuild): only main's repository rules and Starlark run (gazelle's + # fetch_repo, the Go SDK and LLVM downloads, extraction: what every lane + # already runs on its runner), never a build or test action. Bazel uploads + # each reproducible repository's tree (--remote_upload_local_results) with + # a 30-minute rbe-rrc-writer-beads certificate that rbe-west's mint signs + # only for this workflow on a push to main + # (.github/scripts/rrc-writer-credential.sh); rbe-west's rrc-gate admits + # repo-contents entries from it and refuses every other action cache + # write, and the certificate cannot Execute. Cache only (--remote_cache, + # no executor). Not a lane: no result output, never in pr.yml's gate + # (which never sees it anyway: push only), so a failed seed only leaves the + # lanes fetching. setup-bazel runs in local mode (no secrets): its rc, the + # runner cache and the pinned Bazel, nothing else. The only bazel.yml job + # with id-token: write; every caller must grant it (GitHub checks a called + # workflow's job permissions at startup, even for a job that skips), and + # the mint refuses any token but a main-push bazel.yml one. + rrc-seed: + name: repo contents cache seed + needs: rbe + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + id-token: write # the rbe-rrc-writer-beads certificate (GitHub OIDC) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Bazel + uses: ./.github/actions/setup-bazel + - name: rbe-rrc-writer certificate + id: writer + env: + BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + run: bash .github/scripts/rrc-writer-credential.sh + - name: Seed the remote repo contents cache + if: ${{ steps.writer.outputs.cert != '' }} + env: + CERT: ${{ steps.writer.outputs.cert }} + KEY: ${{ steps.writer.outputs.key }} + ENDPOINT: ${{ steps.writer.outputs.endpoint }} + INSTANCE: ${{ steps.writer.outputs.instance }} + run: >- + bash .github/scripts/rrc-run-lanes.sh --experimental_remote_repo_contents_cache -- + --loading_phase_threads=64 + "--remote_cache=$ENDPOINT" "--remote_instance_name=$INSTANCE" + "--tls_client_certificate=$CERT" "--tls_client_key=$KEY" + --remote_upload_local_results + - name: Remove the writer key + if: ${{ always() }} + run: rm -f -- "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.key" "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.crt" + + # Nightly check of the remote repo contents cache (gascity design R1/R2). A + # cached repository tree is trusted by every reader, and nothing on the + # reader checks it against what its rule produces. So, from nightly.yml (or + # a dispatch), fetch every lane's repositories cold (rrc-lane-commands.txt + # with --nobuild and no repo contents cache, Bazel set up exactly as a + # lane) and compare each repository's tree and marker with the cached + # entry a lane would read for the same key (tools/bazel/rrc_verify.py, + # reads only, with the lanes' certificate and the endpoint from + # setup-bazel's rc: no secret is referenced outside setup-bazel). A + # mismatch is a poisoned entry or a rule falsely marked reproducible: the + # job fails and opens (or comments on) an rrc-verify issue. Not a lane: + # never gated. The only bazel.yml job with issues: write; every caller + # must grant it, as for rrc-seed's id-token. + rrc-verify: + name: repo contents cache verify + needs: rbe + if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + permissions: + contents: read + issues: write # the rrc-verify alert issue + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + - name: Cold fetch (every lane, no repo contents cache) + run: bash .github/scripts/rrc-run-lanes.sh -- --loading_phase_threads=64 + - name: Compare with the cache + id: verify + run: | + set -euo pipefail + secret_dir="$RUNNER_TEMP/bazel-ci-secret" + prefix='build:remote-exec --remote_executor=' + endpoint=$(grep -F -m1 "$prefix" "$secret_dir/ci.bazelrc" || true) + endpoint=${endpoint#"$prefix"} + [ -n "$endpoint" ] || { echo "::error::setup-bazel's rc names no executor"; exit 1; } + python3 tools/bazel/rrc_verify.py \ + --output-base "$(bazel info output_base)" \ + --endpoint "$endpoint" --instance oss \ + --cert "$secret_dir/client.crt" --key "$secret_dir/client.key" \ + --bazel-version "$(tr -d '[:space:]' < .bazelversion)" \ + --json "$RUNNER_TEMP/rrc-verify.json" --summary "$GITHUB_STEP_SUMMARY" + - name: Upload the report + if: ${{ always() && hashFiles(format('{0}/rrc-verify.json', runner.temp)) != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: rrc-verify-${{ github.run_attempt }} + path: ${{ runner.temp }}/rrc-verify.json + retention-days: 30 + - name: Alert (rrc-verify issue) + if: ${{ failure() && steps.verify.outcome == 'failure' }} + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="rrc-verify: remote repo contents cache differs from a cold fetch" + body="The nightly rrc-verify run found cached repository trees that differ from a cold fetch, or could not check them: $RUN_URL (summary and the rrc-verify report artifact). + + A mismatch is a poisoned entry or a repository rule falsely marked reproducible (gascity engdocs/design/bazel-remote-repo-contents-cache.md, R1/R2). Stop readers first: set the repository variable RBE_REPO_CONTENTS_CACHE to off. Then follow gascity/infra nativelink-cas/west/README.md (remote repo contents cache writer, poisoned entries)." + gh label create rrc-verify -R "$GITHUB_REPOSITORY" --color B60205 \ + --description "Remote repo contents cache verification failures" 2>/dev/null || true + n=$(gh issue list -R "$GITHUB_REPOSITORY" --label rrc-verify --state open --json number --jq '.[0].number // empty') + if [ -n "$n" ]; then + gh issue comment "$n" -R "$GITHUB_REPOSITORY" --body "Again: $RUN_URL" + else + gh issue create -R "$GITHUB_REPOSITORY" --label rrc-verify --title "$title" --body "$body" + fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 56f6a23e7..7bd190a2d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,6 +25,18 @@ jobs: # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md). bazel: name: Bazel (fresh, full detail) + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). rrc-verify runs here (its alert issue); + # rrc-seed never does, every other bazel.yml job keeps contents: read, + # and rbe-west's mint refuses any OIDC token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). + permissions: + contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9483e2b36..22dc8c23e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1347,8 +1347,18 @@ jobs: # build their own bd via this call's rbe job. bazel: name: Bazel + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). Neither runs for this event, every other + # bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC + # token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). permissions: contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index c0c340b6d..ce91016b0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -620,6 +620,38 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. +## Remote Repo Contents Cache + +A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go +SDK, the LLVM slice) before its first action. Bazel 9's remote repo contents +cache serves those extracted trees from rbe-west's `oss` action cache instead +(ported from gascity's `bazel.yml`; design: gascity +`engdocs/design/bazel-remote-repo-contents-cache.md`). The repository variable +`RBE_REPO_CONTENTS_CACHE` rolls it out, in mode `remote` only: + +| value | effect | +|---|---| +| unset / `off` | nothing (default) | +| `seed` | each push to main runs `bazel.yml`'s `rrc-seed` job: every lane's command in `.github/scripts/rrc-lane-commands.txt` with `--nobuild`, uploading repository trees with a 30-minute `rbe-rrc-writer-beads` certificate that rbe-west's mint signs only for that job (GitHub OIDC; `.github/scripts/rrc-writer-credential.sh`). rbe-west's `rrc-gate` admits only repo-contents entries from it | +| `canary` | `seed`, and the `test` lane reads the cache | +| `on` | `seed`, and every lane but the release cross-compile and the package gates reads it | + +Readers append `startup --experimental_remote_repo_contents_cache` and +`common --loading_phase_threads=64` to `.bazelrc.local` before setting up +Bazel; both are key neutral, and the lanes keep +`--noremote_upload_local_results`. While the variable is not `off`, +nightly.yml's call also runs `rrc-verify`: a cold fetch of every lane, +compared with the cached entries by `tools/bazel/rrc_verify.py`; a mismatch +fails the job and opens an `rrc-verify` issue. + +`rrc-seed` asks for `id-token: write` and `rrc-verify` for `issues: write`, so +every caller of `bazel.yml` (pr.yml, nightly.yml, bazel-farm.yml) grants both: +GitHub checks a called workflow's job permissions when the run starts, even +for jobs that skip. Neither job runs on a pull_request, merge_group or +pull_request_target event, and every other `bazel.yml` job keeps +`contents: read`. Rollback: set the variable to `off` (or `seed`, to keep the +cache warm without readers). + ## Required Check Contract After the aggregate checks are verified on the branch, branch protection or the diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a91b79346..08cf03e48 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -41,6 +41,7 @@ go_test( "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", + "bazel_rrc_test.go", "ci_actions_policy_test.go", "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 7ee9e653e..8212d4efd 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -212,8 +212,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { var lanes []string for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // not lanes: never in a PR call (bazel_rrc_test.go) } runs := bazelLaneRunModes(t, name, job.If, s.with)[mode] switch { diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index 36a40b5b0..887130463 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -94,7 +94,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } // Permissions: contents: read at the workflow and on every job, and - // nothing else anywhere in the file. + // nothing else anywhere in the file, but the farm call's grant of what + // bazel.yml's rrc jobs ask for (bazelCallPermissions). readOnly := map[string]any{"contents": "read"} if !reflect.DeepEqual(doc.Permissions, readOnly) { t.Errorf("workflow permissions = %v, want %v", doc.Permissions, readOnly) @@ -103,8 +104,12 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { var jobs []string for name, job := range workflow.Jobs { jobs = append(jobs, name) - if !reflect.DeepEqual(job.Permissions, readOnly) { - t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, readOnly) + want := readOnly + if name == "farm" { + want = bazelCallPermissions + } + if !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, want) } if len(job.Env) != 0 { t.Errorf("job %s env = %v; want none", name, job.Env) @@ -116,7 +121,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { - if strings.Contains(path, "permissions") && !key && value != "read" { + if strings.Contains(path, "permissions") && !key && value != "read" && + !(strings.HasPrefix(path, ".jobs.farm.permissions.") && bazelCallPermissions[strings.TrimPrefix(path, ".jobs.farm.permissions.")] == value) { t.Errorf("%s = %q; only read permissions are allowed", path, value) } if key && value == "continue-on-error" { @@ -227,6 +233,19 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } } +// bazelCallPermissions: every bazel.yml call's token permissions (farm, +// pr.yml, nightly.yml). A called workflow's job gets at most its caller's, +// and GitHub checks them when the run starts, even for a job whose if: +// skips it, so each call grants what bazel.yml's rrc jobs ask for: +// rrc-seed's id-token: write (push to main only) and rrc-verify's issues: +// write (schedule and dispatch only). Neither runs on pull_request, +// merge_group or pull_request_target, and every other bazel.yml job, every +// lane that runs PR or fork code included, keeps contents: read +// (TestBazelWorkflowPermissionsReadOnly), so no PR-authored step ever holds +// either; rbe-west's mint refuses any OIDC token but a main-push bazel.yml +// one anyway. +var bazelCallPermissions = map[string]any{"contents": "read", "id-token": "write", "issues": "write"} + // bazel.yml's side of the farm: the pinned checkout, the one decision input, // and nothing a pull_request_target run could hand a privileged consumer. func TestBazelWorkflowForkFarmInputs(t *testing.T) { @@ -246,6 +265,15 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { if actionFamily(step.Uses) != "actions/checkout" { continue } + // The rrc jobs run on push to main, schedule and dispatch only, + // never on pull_request_target: they check out the event's ref, + // with no fork opt-in. + if isBazelRRCJob(name) { + if want := map[string]string{"persist-credentials": "false"}; !reflect.DeepEqual(step.With, want) { + t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) + } + continue + } checkouts++ want := map[string]string{"ref": bazelCheckoutRef, "persist-credentials": "false", "allow-unsafe-pr-checkout": bazelAllowUnsafeCheckout} // F3: package-mcp/package-npm's detect step diffs PR_BASE_SHA @@ -271,8 +299,8 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { // shared gascity credential). rbe's checkout (sparse) comes after its // decide step, for the worker-env preflight // (TestBazelRBEWorkerEnvPreflight). - if checkouts != len(workflow.Jobs)-1 { - t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1) + if want := len(workflow.Jobs) - 1 - len(bazelRRCJobs); checkouts != want { + t.Errorf("%d checkouts in %s, want one per job excluding %s and the rrc jobs (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, want) } if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue { t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue) @@ -617,7 +645,11 @@ func TestBazelFarmCacheModeStaysReadOnly(t *testing.T) { } // bazel.yml's token: exactly contents: read, at the top and on any job that -// declares permissions (a call cannot exceed its caller's, but pin it). +// declares permissions (a call cannot exceed its caller's, but pin it), +// except the rrc jobs: rrc-seed's id-token: write and rrc-verify's issues: +// write (pinned by TestBazelRRCSeedJob and TestBazelRRCVerifyJob; neither +// runs on a pull_request, merge_group or pull_request_target event, so the +// callers' wider grants reach no lane and no fork-authored code). func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { var doc struct { Permissions any `yaml:"permissions"` @@ -630,6 +662,9 @@ func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { t.Errorf("%s permissions = %v, want exactly %v", bazelWorkflowName, doc.Permissions, readOnly) } for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if isBazelRRCJob(name) { + continue + } if job.Permissions != nil && !reflect.DeepEqual(job.Permissions, readOnly) { t.Errorf("%s job %s permissions = %v, want none or exactly %v", bazelWorkflowName, name, job.Permissions, readOnly) } diff --git a/scripts/bazel_rrc_test.go b/scripts/bazel_rrc_test.go new file mode 100644 index 000000000..f423b7337 --- /dev/null +++ b/scripts/bazel_rrc_test.go @@ -0,0 +1,783 @@ +package scripts_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported from +// gascity's bazel.yml, gascity #7467 and #7478). bazel.yml's rbe job turns +// vars.RBE_REPO_CONTENTS_CACHE into its rrc output (off, seed, canary, on); +// each reading lane's reader step appends bazelRRCReadLines to +// .bazelrc.local; the rrc-seed job, on push to main only, runs every lane's +// loading and analysis (.github/scripts/rrc-lane-commands.txt, through +// rrc-run-lanes.sh) and uploads repository trees with a 30-minute +// rbe-rrc-writer-beads certificate (rrc-writer-credential.sh); the +// rrc-verify job, nightly, compares the cache with a cold fetch +// (tools/bazel/rrc_verify.py). + +const ( + bazelRRCReadStep = "Remote repo contents cache (read)" + bazelRRCModeStepID = "rrc" + bazelRRCVar = "vars.RBE_REPO_CONTENTS_CACHE" + bazelRRCSeedJob = "rrc-seed" + bazelRRCSeedJobIf = "${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}" + bazelRRCVerifyJob = "rrc-verify" + bazelRRCVerifyJobIf = "${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}" + bazelRRCCanaryLane = "bazel-test" + bazelRRCCredential = ".github/scripts/rrc-writer-credential.sh" + bazelRRCRunLanes = ".github/scripts/rrc-run-lanes.sh" + bazelRRCLaneCmds = ".github/scripts/rrc-lane-commands.txt" + bazelRRCVerifyTool = "tools/bazel/rrc_verify.py" + bazelRRCVerifyTest = "tools/bazel/rrc_verify_test.py" + bazelSetupBazelUses = "./.github/actions/setup-bazel" +) + +// bazelRRCReadLanes: the lanes that read the cache under `on` (bazel-test +// alone under `canary`). Not bazel-release-cross (other target platforms, +// not seeded) nor the package gates. +var bazelRRCReadLanes = []string{ + "bazel-cmd-dolt", "bazel-doltserver", "bazel-embedded", "bazel-integration", + "bazel-proxied", "bazel-pure", "bazel-server-storage", "bazel-test", +} + +// bazelRRCReadLines: the reader's .bazelrc.local lines; both are key neutral +// (a startup option, and loading parallelism). +var bazelRRCReadLines = []string{ + "startup --experimental_remote_repo_contents_cache", + "common --loading_phase_threads=64", +} + +// bazelRRCJobs: bazel.yml's jobs that are not lanes (no result output, never +// gated): the remote repo contents cache's writer and its nightly check. +var bazelRRCJobs = []string{bazelRRCSeedJob, bazelRRCVerifyJob} + +// isBazelRRCJob reports whether a bazel.yml job is one of bazelRRCJobs. +func isBazelRRCJob(name string) bool { return slices.Contains(bazelRRCJobs, name) } + +// evalRRCIf evaluates a job or step if: with ctx, failing the test on an +// expression the evaluator cannot read. +func evalRRCIf(t *testing.T, expr string, ctx map[string]string) bool { + t.Helper() + v, err := evalGHExpr(expr, ctx) + if err != nil { + t.Fatalf("evalGHExpr(%q): %v", expr, err) + } + return ghTruthy(v) +} + +// runRRCScript runs a step's script (or a script file) under GitHub's bash +// flags in dir with env, returning its combined output. +func runRRCScript(t *testing.T, dir, script string, env map[string]string) (string, error) { + t.Helper() + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Dir = dir + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + out, err := cmd.CombinedOutput() + return string(out), err +} + +// rrcOutputs reads a $GITHUB_OUTPUT file (key=value lines). +func rrcOutputs(t *testing.T, path string) map[string]string { + t.Helper() + out := map[string]string{} + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return out + } + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + out[k] = v + } + } + return out +} + +func rrcStep(t *testing.T, job ciWorkflowJob, what string, pick func(ciWorkflowStep) bool) ciWorkflowStep { + t.Helper() + var found []ciWorkflowStep + for _, s := range job.Steps { + if pick(s) { + found = append(found, s) + } + } + if len(found) != 1 { + t.Fatalf("want one %s step, got %d", what, len(found)) + } + return found[0] +} + +// TestBazelRRCModeStep runs the rbe job's rrc step for every variable value +// and execution mode: only mode remote enables anything, unset and off are +// off, and an unknown value is off with a warning (a typo never fails the +// required gate). It reads no secret and runs before the checkout. +func TestBazelRRCModeStep(t *testing.T) { + rbe := readCIWorkflow(t, bazelWorkflowName).Jobs[bazelRBEJobName] + if got := rbe.Outputs["rrc"]; got != "${{ steps.rrc.outputs.rrc }}" { + t.Errorf("rbe job output rrc = %q", got) + } + step := rrcStep(t, rbe, "rrc", func(s ciWorkflowStep) bool { return s.ID == bazelRRCModeStepID }) + if want := map[string]string{"RRC": "${{ " + bazelRRCVar + " }}", "MODE": "${{ steps.decide.outputs.mode }}"}; !reflect.DeepEqual(step.Env, want) { + t.Errorf("rrc step env %v, want %v", step.Env, want) + } + if step.If != "" { + t.Errorf("rrc step if %q; it must always set the output", step.If) + } + for i, s := range rbe.Steps { + if s.Uses != "" { + t.Errorf("rbe step %d (%s) precedes nothing it should: the rrc step must come before any checkout", i, s.Uses) + } + if s.ID == bazelRRCModeStepID { + break + } + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local", "skip"} { + for value, want := range map[string]string{"": "off", "off": "off", "seed": "seed", "canary": "canary", "on": "on", "On": "off", "true": "off", "on;x": "off"} { + if mode != "remote" { + want = "off" + } + dir := t.TempDir() + output := filepath.Join(dir, "output") + out, err := runRRCScript(t, dir, step.Run, map[string]string{ + "RRC": value, "MODE": mode, "GITHUB_OUTPUT": output, "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"), + }) + if err != nil { + t.Errorf("mode %s value %q: %v\n%s", mode, value, err, out) + continue + } + if got := rrcOutputs(t, output)["rrc"]; got != want { + t.Errorf("mode %s value %q: rrc=%q, want %q", mode, value, got, want) + } + unknown := !slices.Contains([]string{"", "off", "seed", "canary", "on"}, value) + if warned := strings.Contains(out, "::warning"); warned != unknown { + t.Errorf("mode %s value %q: warning %v, want %v:\n%s", mode, value, warned, unknown, out) + } + } + } +} + +// TestBazelRRCReadSteps: exactly bazelRRCReadLanes read, in mode remote only +// (fork and local runs never: only rbe-west's trusted edge serves the +// entries), bazel-test under canary and every one of them under on; each +// writes exactly bazelRRCReadLines, never mentions uploads, and comes before +// the job's Set up Bazel (so no Bazel server starts without it). +func TestBazelRRCReadSteps(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + var readers []string + for name, job := range wf.Jobs { + read, setup := -1, -1 + for i, s := range job.Steps { + switch { + case s.Name == bazelRRCReadStep: + if read >= 0 { + t.Errorf("%s has two %q steps", name, bazelRRCReadStep) + } + read = i + case s.Uses == bazelSetupBazelUses && setup < 0: + setup = i + } + } + if read < 0 { + continue + } + readers = append(readers, name) + if setup < 0 || read > setup { + t.Errorf("%s: %q at step %d, Set up Bazel at %d; the reader must come first", name, bazelRRCReadStep, read, setup) + } + step := job.Steps[read] + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local"} { + for _, rrc := range []string{"off", "seed", "canary", "on"} { + want := mode == "remote" && (rrc == "on" || (rrc == "canary" && name == bazelRRCCanaryLane)) + if got := evalRRCIf(t, step.If, map[string]string{"needs.rbe.outputs.mode": mode, "needs.rbe.outputs.rrc": rrc}); got != want { + t.Errorf("%s mode %s rrc %s: reader runs %v, want %v", name, mode, rrc, got, want) + } + } + } + dir := t.TempDir() + if out, err := runRRCScript(t, dir, step.Run, nil); err != nil { + t.Fatalf("%s reader: %v\n%s", name, err, out) + } + got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, ".bazelrc.local")), "\n") + if !slices.Equal(got, bazelRRCReadLines) { + t.Errorf("%s reader writes %q, want %q", name, got, bazelRRCReadLines) + } + if strings.Contains(step.Run, "upload") || len(step.Env) != 0 { + t.Errorf("%s reader mentions uploads or takes env; lanes only read", name) + } + } + sort.Strings(readers) + if !slices.Equal(readers, bazelRRCReadLanes) { + t.Errorf("lanes with a reader step %v, want %v", readers, bazelRRCReadLanes) + } +} + +// bazelRRCLaneCommands parses rrc-lane-commands.txt as rrc-run-lanes.sh does. +func bazelRRCLaneCommands(t *testing.T, root string) []string { + t.Helper() + var cmds []string + for _, line := range strings.Split(readPolicyFile(t, root, bazelRRCLaneCmds), "\n") { + line, _, _ = strings.Cut(line, "#") + if line = strings.TrimSpace(line); line != "" { + cmds = append(cmds, line) + } + } + return cmds +} + +// TestBazelRRCLaneCommandsCoverEveryReader: rrc-seed seeds (and rrc-verify +// checks) what the readers fetch, so every --config a reading lane passes to +// `bazel test` or `bazel build` has a command in rrc-lane-commands.txt, and +// every command there is a reading lane's. +func TestBazelRRCLaneCommandsCoverEveryReader(t *testing.T) { + root := sourceRepoRoot(t) + seeded := map[string]bool{} + for _, cmd := range bazelRRCLaneCommands(t, root) { + f := strings.Fields(cmd) + if f[0] != "test" && f[0] != "build" { + t.Errorf("lane command %q: want a test or build command", cmd) + } + for _, a := range f[1:] { + if c, ok := strings.CutPrefix(a, "--config="); ok { + seeded[c] = true + } + } + } + used := map[string]bool{} + invocation := regexp.MustCompile(`\bbazel (?:test|build)\b[^\n]*(?:\\\n[^\n]*)*`) + config := regexp.MustCompile(`--config=([a-z0-9-]+)`) + wf := readCIWorkflow(t, bazelWorkflowName) + for _, lane := range bazelRRCReadLanes { + for _, s := range wf.Jobs[lane].Steps { + for _, inv := range invocation.FindAllString(s.Run, -1) { + for _, m := range config.FindAllStringSubmatch(inv, -1) { + if m[1] != "sole-run" && m[1] != "fresh" { // test-result options, nothing fetched + used[m[1]] = true + } + } + } + } + } + if len(used) < len(bazelRRCReadLanes) { + t.Fatalf("found the configs %v in %d reading lanes; the scan is broken", used, len(bazelRRCReadLanes)) + } + for c := range used { + if !seeded[c] { + t.Errorf("a reading lane runs bazel with --config=%s, which %s does not seed", c, bazelRRCLaneCmds) + } + } + for c := range seeded { + if !used[c] { + t.Errorf("%s seeds --config=%s, which no reading lane uses", bazelRRCLaneCmds, c) + } + } +} + +// TestRRCRunLanes runs rrc-run-lanes.sh with a stub bazel: one --nobuild +// invocation per lane command, in order, startup flags first and command +// flags after; `test --nobuild`'s exit 1 after a successful analysis +// passes, a failed analysis fails, and an empty command list fails. +func TestRRCRunLanes(t *testing.T) { + root := sourceRepoRoot(t) + script := filepath.Join(root, bazelRRCRunLanes) + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + calls := filepath.Join(dir, "calls") + stub := "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >>" + calls + "\n" + + "if [ -n \"${BAZEL_TEST_FAIL:-}\" ]; then echo 'ERROR: no such package'; echo 'ERROR: Build did NOT complete successfully'; exit 1; fi\n" + + "case \"$*\" in *' build '*) echo 'INFO: Build completed successfully, 0 total actions'; exit 0 ;; esac\n" + + "echo 'INFO: Build completed successfully, 0 total actions'\necho \"ERROR: Couldn't start the build. Unable to run tests\"\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "bazel"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + env := map[string]string{ + "PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"), + } + run := "bash " + script + " --experimental_remote_repo_contents_cache -- --loading_phase_threads=64 --remote_upload_local_results" + if out, err := runRRCScript(t, dir, run, env); err != nil { + t.Fatalf("rrc-run-lanes.sh: %v\n%s", err, out) + } + var want []string + for _, cmd := range bazelRRCLaneCommands(t, root) { + want = append(want, "--experimental_remote_repo_contents_cache "+cmd+" --nobuild --loading_phase_threads=64 --remote_upload_local_results") + } + if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); !slices.Equal(got, want) { + t.Errorf("bazel calls:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + if err := os.Remove(calls); err != nil { + t.Fatal(err) + } + if out, err := runRRCScript(t, dir, "bash "+script+" --", env); err != nil { + t.Fatalf("without flags: %v\n%s", err, out) + } + if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); len(got) == 0 || !strings.HasPrefix(got[0], "test ") || !strings.HasSuffix(got[0], " --nobuild") { + t.Errorf("without flags, first call %q", got) + } + + failEnv := map[string]string{"BAZEL_TEST_FAIL": "1"} + for k, v := range env { + failEnv[k] = v + } + if out, err := runRRCScript(t, dir, run, failEnv); err == nil { + t.Errorf("a failed analysis passed:\n%s", out) + } + empty := filepath.Join(dir, "empty.txt") + if err := os.WriteFile(empty, []byte("# nothing\n\n"), 0o644); err != nil { + t.Fatal(err) + } + emptyEnv := map[string]string{"RRC_LANE_COMMANDS": empty} + for k, v := range env { + emptyEnv[k] = v + } + if out, err := runRRCScript(t, dir, run, emptyEnv); err == nil { + t.Errorf("no lane commands passed; it must fail rather than seed nothing:\n%s", out) + } + if out, err := runRRCScript(t, dir, "bash "+script+" --flag", env); err == nil { + t.Errorf("no -- separator passed:\n%s", out) + } +} + +// TestBazelRRCSeedJob: the only writer runs on push to main in mode remote +// with rrc not off, holds the only id-token permission in bazel.yml, reads +// no secret (its credential is the OIDC-minted certificate), sets Bazel up +// in local mode, seeds only with a certificate, uploads to the cache alone +// (no executor), and always removes the key. +func TestBazelRRCSeedJob(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + job, ok := wf.Jobs[bazelRRCSeedJob] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCSeedJob) + } + if job.If != bazelRRCSeedJobIf { + t.Errorf("%s if %q, want %q", bazelRRCSeedJob, job.If, bazelRRCSeedJobIf) + } + if !slices.Equal(job.Needs, []string{bazelRBEJobName}) || job.Outputs != nil { + t.Errorf("%s needs %v outputs %v; it needs the rbe job alone and reports nothing to the gate", bazelRRCSeedJob, job.Needs, job.Outputs) + } + for _, c := range []struct { + event, ref, mode, rrc string + want bool + }{ + {"push", "refs/heads/main", "remote", "seed", true}, + {"push", "refs/heads/main", "remote", "canary", true}, + {"push", "refs/heads/main", "remote", "on", true}, + {"push", "refs/heads/main", "remote", "off", false}, + {"push", "refs/heads/main", "cache", "on", false}, + {"push", "refs/heads/other", "remote", "on", false}, + {"pull_request", "refs/pull/1/merge", "remote", "on", false}, + {"pull_request_target", "refs/heads/main", "remote", "on", false}, + {"merge_group", "refs/heads/gh-readonly-queue/main/pr-1-abc", "remote", "on", false}, + {"workflow_dispatch", "refs/heads/main", "remote", "on", false}, + {"schedule", "refs/heads/main", "remote", "on", false}, + } { + got := evalRRCIf(t, job.If, map[string]string{ + "github.event_name": c.event, "github.ref": c.ref, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc, + }) + if got != c.want { + t.Errorf("%+v: seed runs %v, want %v", c, got, c.want) + } + } + if want := map[string]any{"contents": "read", "id-token": "write"}; !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("%s permissions %v, want %v", bazelRRCSeedJob, job.Permissions, want) + } + raw := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName) + if text := bazelRRCJobText(t, raw, bazelRRCSeedJob); strings.Contains(text, "secrets.") { + t.Errorf("%s reads a secret; its only credential is the OIDC-minted writer certificate", bazelRRCSeedJob) + } + setup := rrcStep(t, job, "setup-bazel", func(s ciWorkflowStep) bool { return s.Uses == bazelSetupBazelUses }) + if len(setup.Env) != 0 { + t.Errorf("%s setup-bazel env %v; local mode (no executor, no certificate) only", bazelRRCSeedJob, setup.Env) + } + cred := rrcStep(t, job, "writer certificate", func(s ciWorkflowStep) bool { return s.ID == "writer" }) + if cred.Run != "bash "+bazelRRCCredential { + t.Errorf("writer step runs %q, want bash %s", cred.Run, bazelRRCCredential) + } + seed := rrcStep(t, job, "seed", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, bazelRRCRunLanes) }) + if seed.If != "${{ steps.writer.outputs.cert != '' }}" { + t.Errorf("seed step if %q; it must skip when the mint switched seeding off (no certificate)", seed.If) + } + for _, want := range []string{ + "bash " + bazelRRCRunLanes + " --experimental_remote_repo_contents_cache --", "--loading_phase_threads=64", + `"--remote_cache=$ENDPOINT"`, `"--remote_instance_name=$INSTANCE"`, + `"--tls_client_certificate=$CERT"`, `"--tls_client_key=$KEY"`, "--remote_upload_local_results", + } { + if !strings.Contains(seed.Run, want) { + t.Errorf("seed step lacks %q:\n%s", want, seed.Run) + } + } + if strings.Contains(seed.Run, "remote_executor") { + t.Errorf("seed step names an executor; the writer certificate is cache only") + } + cleanup := rrcStep(t, job, "key removal", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "rrc-writer.key") }) + if cleanup.If != "${{ always() }}" { + t.Errorf("the writer key removal runs if %q; it must always run", cleanup.If) + } +} + +// bazelRRCJobText returns a job's raw text in a workflow. +func bazelRRCJobText(t *testing.T, raw, job string) string { + t.Helper() + start := strings.Index(raw, "\n "+job+":\n") + if start < 0 { + t.Fatalf("cannot find the %s job's text", job) + } + text := raw[start+1:] + if next := regexp.MustCompile(`\n [a-z0-9-]+:\n`).FindStringIndex(text); next != nil { + text = text[:next[0]] + } + return text +} + +// TestBazelRRCVerifyJob: the nightly check runs from nightly.yml (schedule) +// or a dispatch while the cache is in use, fetches every lane cold without +// the repo contents cache, compares with rrc_verify.py using the lanes' +// read-only certificate and setup-bazel's endpoint (no secret outside +// setup-bazel), and alerts through an issue only when the comparison fails. +func TestBazelRRCVerifyJob(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + job, ok := wf.Jobs[bazelRRCVerifyJob] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCVerifyJob) + } + if job.If != bazelRRCVerifyJobIf { + t.Errorf("%s if %q, want %q", bazelRRCVerifyJob, job.If, bazelRRCVerifyJobIf) + } + if job.Outputs != nil { + t.Errorf("%s outputs %v; it reports nothing to the gate", bazelRRCVerifyJob, job.Outputs) + } + for _, c := range []struct { + event, mode, rrc string + want bool + }{ + {"schedule", "remote", "on", true}, + {"schedule", "remote", "seed", true}, + {"workflow_dispatch", "remote", "canary", true}, + {"schedule", "remote", "off", false}, + {"schedule", "cache", "on", false}, + {"push", "remote", "on", false}, + {"pull_request", "remote", "on", false}, + {"pull_request_target", "remote", "on", false}, + {"merge_group", "remote", "on", false}, + } { + got := evalRRCIf(t, job.If, map[string]string{ + "github.event_name": c.event, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc, + }) + if got != c.want { + t.Errorf("%+v: verify runs %v, want %v", c, got, c.want) + } + } + if want := map[string]any{"contents": "read", "issues": "write"}; !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("%s permissions %v, want %v", bazelRRCVerifyJob, job.Permissions, want) + } + for _, s := range job.Steps { + if strings.Contains(s.Run, "experimental_remote_repo_contents_cache") || strings.Contains(s.Run, ".bazelrc.local") || + strings.Contains(s.Run, "remote_upload_local_results") { + t.Errorf("%s step %q reads or writes the repo contents cache; the cold fetch must not", bazelRRCVerifyJob, s.Name) + } + for k, v := range s.Env { + if strings.Contains(v, "secrets.") && s.Uses != bazelSetupBazelUses { + t.Errorf("%s step %q env %s reads a secret; only setup-bazel may", bazelRRCVerifyJob, s.Name, k) + } + } + } + fetch := rrcStep(t, job, "cold fetch", func(s ciWorkflowStep) bool { return strings.HasPrefix(s.Name, "Cold fetch") }) + if fetch.Run != "bash "+bazelRRCRunLanes+" -- --loading_phase_threads=64" { + t.Errorf("cold fetch runs %q; it must run every lane's command with no startup flag", fetch.Run) + } + verify := rrcStep(t, job, "verify", func(s ciWorkflowStep) bool { return s.ID == "verify" }) + for _, want := range []string{"python3 " + bazelRRCVerifyTool, `--cert "$secret_dir/client.crt"`, `--key "$secret_dir/client.key"`, "--instance oss", ".bazelversion", "remote_executor="} { + if !strings.Contains(verify.Run, want) { + t.Errorf("verify step lacks %q", want) + } + } + alert := rrcStep(t, job, "alert", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "gh issue") }) + if alert.If != "${{ failure() && steps.verify.outcome == 'failure' }}" { + t.Errorf("alert step if %q; it must alert only on a failed comparison", alert.If) + } + if strings.Contains(alert.Run, "${{") { + t.Errorf("alert step interpolates an expression into its script; pass it through env") + } +} + +// permissionRank orders GitHub token permission levels so a grant can be +// compared with a request: write covers read, read covers none. +var permissionRank = map[string]int{"": 0, "none": 0, "read": 1, "write": 2} + +type permWorkflow struct { + Jobs map[string]struct { + Uses string `yaml:"uses"` + Permissions map[string]string `yaml:"permissions"` + } `yaml:"jobs"` +} + +// TestBazelWorkflowCallersGrantEveryRequestedPermission guards against +// startup_failure (gascity #7478): GitHub validates a reusable workflow's +// job permissions when the caller starts, including jobs whose `if:` would +// skip, so every workflow that calls bazel.yml must grant at least the +// highest level any bazel.yml job requests for each scope. +func TestBazelWorkflowCallersGrantEveryRequestedPermission(t *testing.T) { + root := sourceRepoRoot(t) + load := func(name string) permWorkflow { + var wf permWorkflow + if err := yaml.Unmarshal([]byte(readPolicyFile(t, root, ".github/workflows/"+name)), &wf); err != nil { + t.Fatalf("parse %s: %v", name, err) + } + return wf + } + need := map[string]string{} + for _, job := range load(bazelWorkflowName).Jobs { + for scope, level := range job.Permissions { + if permissionRank[level] > permissionRank[need[scope]] { + need[scope] = level + } + } + } + if need["id-token"] != "write" || need["issues"] != "write" { + t.Fatalf("bazel.yml requests %v; the parser is broken or the rrc jobs are gone", need) + } + entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + var callers []string + for _, e := range entries { + name := e.Name() + if name == bazelWorkflowName || (!strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml")) { + continue + } + for jobName, job := range load(name).Jobs { + if job.Uses != "./.github/workflows/"+bazelWorkflowName { + continue + } + callers = append(callers, name) + var missing []string + for scope, level := range need { + if permissionRank[job.Permissions[scope]] < permissionRank[level] { + missing = append(missing, scope+": "+level) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Errorf("%s job %q calls %s without granting %s (startup_failure)", name, jobName, bazelWorkflowName, strings.Join(missing, ", ")) + } + } + } + sort.Strings(callers) + if want := []string{"bazel-farm.yml", "nightly.yml", "pr.yml"}; !slices.Equal(callers, want) { + t.Errorf("bazel.yml callers %v, want %v", callers, want) + } +} + +// rrcMintCurlStub stands in for curl in rrc-writer-credential.sh. The OIDC +// request (its URL carries audience=) logs the audience and authorization +// and answers a token. A mint request logs its Authorization header and +// answers as the space-separated RBE_TEST_MINT says, one word per request +// (the last repeats): ok (a certificate for the runner's key, CN +// rbe-rrc-writer-beads O=gascity), cn, endpoint, instance, otherkey, nocert +// (each wrong in that one way), 000 (connection refused) or an HTTP status +// with an error body. +const rrcMintCurlStub = `#!/usr/bin/env bash +set -euo pipefail +out= url= headers=() +while [ $# -gt 0 ]; do + case "$1" in + -o) out=$2; shift 2 ;; + -H) headers+=("$2"); shift 2 ;; + -w | --data | --connect-timeout | --max-time | --retry) shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done +case "$url" in +*audience=*) + echo "oidc audience=${url##*audience=} auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG" + echo '{"value": "oidc-jwt-value"}' + exit 0 + ;; +esac +echo "mint $url auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG" +n=$(grep -c '^mint ' "$RBE_TEST_MINT_LOG") +read -r -a answers <<<"$RBE_TEST_MINT" +i=$((n - 1)); [ "$i" -lt "${#answers[@]}" ] || i=$((${#answers[@]} - 1)) +answer=${answers[$i]} +key="$BAZEL_CI_SECRET_DIR/rrc-writer.key" cn=rbe-rrc-writer-beads endpoint=grpcs://rbe-west.ops.gascity.com:443 instance=oss +case "$answer" in +ok | cn | endpoint | instance | otherkey | nocert) ;; +000) echo "curl: (7) Failed to connect" >&2; exit 7 ;; +*) printf '{"error": "stub %s"}' "$answer" >"$out"; printf '%s' "$answer"; exit 0 ;; +esac +case "$answer" in +cn) cn=rbe-ci ;; +endpoint) endpoint=grpcs://attacker.example:443 ;; +instance) instance=main ;; +otherkey) openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out other.key 2>/dev/null; key=other.key ;; +esac +pem= +if [ "$answer" != nocert ]; then + pem=$(openssl req -new -x509 -key "$key" -subj "/O=gascity/CN=$cn" -days 1 2>/dev/null) +fi +jq -cn --arg pem "$pem" --arg e "$endpoint" --arg i "$instance" --arg cn "$cn" \ + '{cert_pem: $pem, endpoint: $e, instance: $i, cn: $cn}' >"$out" +printf 200 +` + +// TestRRCWriterCredential runs rrc-writer-credential.sh against a stubbed +// curl (rrcMintCurlStub): it asks GitHub for an OIDC token with audience +// rbe-rrc-writer, masks it, sends it to the mint, retries only 429, 502 and +// the network, treats 503 or an unreachable mint as seeding off (no +// outputs, exit 0), and accepts only a certificate for the key it generated, +// CN rbe-rrc-writer[-x] O=gascity, for rbe-west's endpoint and instance oss. +func TestRRCWriterCredential(t *testing.T) { + for _, tool := range []string{"bash", "jq", "openssl"} { + requireHostTool(t, tool) + } + script := filepath.Join(sourceRepoRoot(t), bazelRRCCredential) + for _, c := range []struct { + answers string + ok, off bool + mints int + }{ + {"ok", true, false, 1}, + {"429 ok", true, false, 2}, + {"502 ok", true, false, 2}, + {"000 ok", true, false, 2}, + {"502", false, false, 4}, + {"403", false, false, 1}, + {"401", false, false, 1}, + {"409", false, false, 1}, + {"cn", false, false, 1}, + {"endpoint", false, false, 1}, + {"instance", false, false, 1}, + {"otherkey", false, false, 1}, + {"nocert", false, false, 1}, + {"503", true, true, 1}, + {"000", true, true, 4}, + } { + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range map[string]string{"curl": rrcMintCurlStub, "sleep": "#!/bin/sh\nexit 0\n"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + secret := filepath.Join(t.TempDir(), "secret") + output := filepath.Join(dir, "output") + out, err := runRRCScript(t, dir, "bash "+script, map[string]string{ + "PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "BAZEL_CI_SECRET_DIR": secret, + "GITHUB_OUTPUT": output, + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.invalid/oidc?api-version=2.0", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + "RBE_TEST_MINT": c.answers, + "RBE_TEST_MINT_LOG": filepath.Join(dir, "mint.log"), + }) + if (err == nil) != c.ok { + t.Errorf("%q: ok=%v, want %v\n%s", c.answers, err == nil, c.ok, out) + continue + } + log := readPolicyFile(t, dir, "mint.log") + if !strings.Contains(log, "oidc audience=rbe-rrc-writer auth=Authorization: bearer request-token\n") { + t.Errorf("%q: OIDC request %q", c.answers, log) + } + if got := strings.Count(log, "mint https://rbe-mint.ops.gascity.com:8444/v1/rrc-writer/cert auth=Authorization: Bearer oidc-jwt-value\n"); got != c.mints { + t.Errorf("%q: %d mint requests with the OIDC token, want %d:\n%s", c.answers, got, c.mints, log) + } + if !strings.Contains(out, "::add-mask::oidc-jwt-value") { + t.Errorf("%q: the OIDC token is not masked:\n%s", c.answers, out) + } + if !c.ok { + continue + } + if c.off { + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Errorf("%q: wrote outputs (%v); the seed must skip", c.answers, err) + } + if !strings.Contains(out, "seeding off (HTTP "+c.answers[len(c.answers)-3:]+")") { + t.Errorf("%q: no seeding-off warning:\n%s", c.answers, out) + } + continue + } + got := rrcOutputs(t, output) + for name, want := range map[string]string{ + "cert": filepath.Join(secret, "rrc-writer.crt"), "key": filepath.Join(secret, "rrc-writer.key"), + "endpoint": "grpcs://rbe-west.ops.gascity.com:443", "instance": "oss", + } { + if got[name] != want { + t.Errorf("%q: output %s=%q, want %q", c.answers, name, got[name], want) + } + } + if key := readPolicyFile(t, secret, "rrc-writer.key"); !strings.Contains(key, "-----BEGIN PRIVATE KEY-----") { + t.Errorf("%q: key is not PKCS#8 (Bazel's TLS refuses SEC1)", c.answers) + } + if fi, err := os.Stat(filepath.Join(secret, "rrc-writer.key")); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("%q: key mode %v (%v), want 0600", c.answers, fi, err) + } + if _, err := os.Stat(filepath.Join(secret, "rrc-mint.json")); !os.IsNotExist(err) { + t.Errorf("%q: the mint reply stays on disk (%v)", c.answers, err) + } + } + + dir := t.TempDir() + if out, err := runRRCScript(t, dir, "bash "+script, map[string]string{ + "BAZEL_CI_SECRET_DIR": filepath.Join(t.TempDir(), "secret"), "GITHUB_OUTPUT": filepath.Join(dir, "output"), + }); err == nil || !strings.Contains(out, "id-token: write") { + t.Errorf("without an OIDC request URL: err %v, want a failure naming id-token: write\n%s", err, out) + } +} + +// The writer credential script and rrc_verify.py with its unit test are byte +// copies of gascity's (gascity 78c0e5c6f198306b96fe854c18b4d72f16395ce7, +// #7467), canonical there like tools/bazel/ci_analytics_extract.py: the +// pins catch a local edit that would drift out of lockstep. +var bazelRRCVendored = map[string]string{ + bazelRRCCredential: "021788a724876bd1df7e5c117a4b3e1e60c5b25bb18bca472d61c35a08cb1fac", + bazelRRCVerifyTool: "00d50e569e2a50d8ef3d2740460de7337b71afd14dc76d2d2442dbd162236f94", + bazelRRCVerifyTest: "d934f13b466660f3148cf3c8dc7273803b4417f2a52f4edc2dd2f43afb93bfaa", +} + +func TestBazelRRCVendoredFromGascity(t *testing.T) { + root := sourceRepoRoot(t) + for path, want := range bazelRRCVendored { + sum := sha256.Sum256([]byte(readPolicyFile(t, root, path))) + if got := hex.EncodeToString(sum[:]); got != want { + t.Errorf("%s sha256 = %s, want %s: it drifted from gascity's copy, or gascity changed it and this pin (and the copy) needs updating", path, got, want) + } + } +} + +// TestRRCVerifyUnitTests runs gascity's rrc_verify_test.py in place: the +// AC key construction pinned against Bazel 9.3.0's own writes, the +// roll-forward of intermediate entries, a poisoned entry, and a GUID for +// .bazelversion's release. +func TestRRCVerifyUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, filepath.Join(root, bazelRRCVerifyTest), "-v") + cmd.Dir = filepath.Join(root, "tools", "bazel") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s failed: %v\n%s", bazelRRCVerifyTest, err, out) + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go index e76b9ad02..1dfbb353a 100644 --- a/scripts/ci_analytics_workflow_test.go +++ b/scripts/ci_analytics_workflow_test.go @@ -34,6 +34,8 @@ var ciAnalyticsExcludedJobs = map[string]string{ "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", "bazel-release-cross": "one bazel-release-cross-compile.sh run with no BEP/exec-log/profile capture, as when it was a bazel-pure step", + "rrc-seed": "the remote repo contents cache writer: --nobuild analysis on push to main, no test lane to summarize", + "rrc-verify": "the remote repo contents cache's nightly check: --nobuild cold fetches, no test lane to summarize", } // ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index d9b6e5fbc..77236fb98 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -580,8 +580,8 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) { for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} { ctx := map[string]string{"needs.rbe.outputs.mode": mode} for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // rrc jobs: push/schedule only, never a fork (literal label) } want := "ubuntu-latest" if mode == "remote" { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5775326b9..24b8d8272 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1779,7 +1779,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} +var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -2148,6 +2148,9 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } + if isBazelRRCJob(name) { + continue // runner, if and setup-bazel: bazel_rrc_test.go + } // F3: the package gates use larger runners (bazelPackageRunsOn) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). @@ -2711,8 +2714,8 @@ func TestBazelLaneIsGated(t *testing.T) { "rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}", } for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // the rrc jobs never run in a PR call (bazel_rrc_test.go) } _, gated := bazelLaneGateIDs[name] _, advisory := bazelAdvisoryLanes[name] @@ -2756,14 +2759,15 @@ func TestBazelLaneIsGated(t *testing.T) { } // One caller on PR events: pr.yml's bazel job, with the four RBE secrets - // only, read-only contents, and no if (the rbe job decides). + // only, bazelCallPermissions (contents: read, plus what the rrc jobs, + // which never run on a PR, ask for), and no if (the rbe job decides). pr := readCIWorkflow(t, "pr.yml") bazel := pr.job(t, "bazel") if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 { t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName) } - if !reflect.DeepEqual(bazel.Permissions, map[string]any{"contents": "read"}) { - t.Errorf("pr.yml bazel job permissions = %v, want contents: read", bazel.Permissions) + if !reflect.DeepEqual(bazel.Permissions, bazelCallPermissions) { + t.Errorf("pr.yml bazel job permissions = %v, want %v", bazel.Permissions, bazelCallPermissions) } if !reflect.DeepEqual(bazel.With, bazelPRCallWith) { t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith) @@ -3039,6 +3043,11 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string // bazelAdvisoryLanes' name-specific exception in // TestBazelGateSimulation below). return map[string]bool{"remote": true} + case bazelRRCSeedJobIf, bazelRRCVerifyJobIf: + // The remote repo contents cache's writer (push to main) and + // nightly check (schedule, dispatch): never in a pull_request, + // merge_group or pull_request_target call, in any mode. + return map[string]bool{} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -4182,12 +4191,13 @@ func TestBazelDoltServerTiers(t *testing.T) { }{ // +2 over the earlier counts: the CI analytics summary/upload steps // (S3; see bazel-test's "CI analytics summary" comment), added to - // every lane job ahead of the result recorder. - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 10}, + // every lane job ahead of the result recorder. +1: the remote repo + // contents cache reader (TestBazelRRCReadSteps). + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 11}, // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- // latency-study.md recommendation 4), since this is the tier whose // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 11}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 12}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { @@ -4195,7 +4205,7 @@ func TestBazelDoltServerTiers(t *testing.T) { } assertBazelTierStep(t, job, c.job, c.config) if n := len(job.Steps); n != c.wantSteps { - t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) + t.Errorf("%s has %d steps, want %d (checkout, the rrc reader, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -4787,12 +4797,15 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "enabled": "${{ steps.decide.outputs.enabled }}", "mode": "${{ steps.decide.outputs.mode }}", "tier": "${{ steps.decide.outputs.tier }}", + // The remote repo contents cache's switch, from decide's mode + // (TestBazelRRCModeStep). + "rrc": "${{ steps.rrc.outputs.rrc }}", } if !reflect.DeepEqual(job.Outputs, wantOutputs) { t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) } - if len(job.Steps) != 3 { - t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) + if len(job.Steps) != 4 { + t.Fatalf("%s has %d steps, want the decision step, the rrc step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) } step := job.Steps[0] wantEnv := map[string]string{ diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index a9d849f24..8958859f2 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -367,21 +367,21 @@ func TestRBEWorkerEnvSync(t *testing.T) { // mode the lanes read is decide's alone. func TestBazelRBEWorkerEnvPreflight(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) - if len(job.Steps) != 3 || job.Steps[0].ID != "decide" { - t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) + if len(job.Steps) != 4 || job.Steps[0].ID != "decide" || job.Steps[1].ID != bazelRRCModeStepID { + t.Fatalf("%s steps = %d (first %q), want decide, rrc (TestBazelRRCModeStep), the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) } if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) { t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName) } - checkout, check := job.Steps[1], job.Steps[2] + checkout, check := job.Steps[2], job.Steps[3] if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout { - t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", + t.Errorf("%s step 3: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf) } wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"} if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) || !strings.Contains(check.Run, "tools/rbe/worker-env-sync main") { - t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", + t.Errorf("%s step 4: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv) } @@ -427,8 +427,11 @@ func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) { } for name, job := range nightly.Jobs { want := any(nil) - if name == nightlyWorkerEnvJob { + switch name { + case nightlyWorkerEnvJob: want = map[string]any{"contents": "read", "issues": "write"} + case "bazel": // the bazel.yml call: what its rrc jobs ask for + want = bazelCallPermissions } if !reflect.DeepEqual(job.Permissions, want) { t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want) diff --git a/scripts/tracked_files_test.go b/scripts/tracked_files_test.go index 1fd09c3af..fddeeea12 100644 --- a/scripts/tracked_files_test.go +++ b/scripts/tracked_files_test.go @@ -67,6 +67,16 @@ const ( publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` ) +// publicRRCWriterPin: rrc-writer-credential.sh (a byte copy of gascity's) +// pins the one endpoint rbe-west's mint may hand the remote repo contents +// cache writer (rbe-west, :443), so a compromised mint cannot point the +// writer's uploads elsewhere. Public, no credential: allowed as exactly +// this line, in exactly that file. +const ( + publicRRCWriterPinFile = ".github/scripts/rrc-writer-credential.sh" + publicRRCWriterPinLine = `ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-west\.ops\.gascity\.com(:443)?$'}` +) + type endpointHit struct { path string line int @@ -121,7 +131,8 @@ func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { continue } - if path == publicRBEForkPinFile && line == publicRBEForkPinLine { + if path == publicRBEForkPinFile && line == publicRBEForkPinLine || + path == publicRRCWriterPinFile && line == publicRRCWriterPinLine { continue } for _, re := range flagRes { diff --git a/tools/bazel/rrc_verify.py b/tools/bazel/rrc_verify.py new file mode 100644 index 000000000..779b49483 --- /dev/null +++ b/tools/bazel/rrc_verify.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Verify rbe-west's remote repo contents cache against a cold fetch (ga-vnycm2.26). + +Bazel's --experimental_remote_repo_contents_cache stores each reproducible +external repository as the ActionResult of a synthetic, never executed action +keyed by the repository rule's predeclared-inputs hash +(RemoteRepoContentsCacheImpl, engdocs/design/bazel-remote-repo-contents-cache.md). +Nothing on the reader checks that the cached tree is what the rule produces: +that is the work the cache skips. So, nightly, this job fetches every lane's +repositories cold, without the cache, and compares each repository's tree and +marker file with the entry a lane would read for the same key. A mismatch is +a poisoned entry or a repository rule falsely marked reproducible (design +R1/R2); both must be looked at by a human. + +Inputs: the cold output base (setup-bazel's rc: --repo_contents_cache= keeps +each repository in external/ with its marker external/@.marker) +and read access to the cache (rbe-west's trusted edge with the CI client +certificate, or any REAPI endpoint). For each marker, it walks the action +cache as the reader does: the final entry under the predeclared hash, or +intermediate entries whose stdout lists batches of recorded inputs, whose +values it takes from the cold marker and rolls into the next key +(Fingerprint: sha256 over length-prefixed UTF-8 strings). The final entry's +marker must equal the cold marker byte for byte, and its tree (the Tree +proto, read from the CAS) must hold the same names, file digests, executable +bits and symlink targets as the cold directory. + +Transport: unary and server-streaming gRPC over curl's HTTP/2 with the +standard library's protobuf-free wire decoding, as tools/rbe/cache-zstd-probe.sh +does: no grpc package on a job that holds a farm credential. + +Exit status: 0 every cached repository matches; 1 a mismatch or an invalid +entry; 2 a usage error or too many transport failures to tell. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import dataclasses +import hashlib +import json +import os +import stat +import subprocess +import sys +import tempfile +from typing import Callable, Iterable, Optional + +# RemoteRepoContentsCacheImpl.GUID per Bazel release: the synthetic Command's +# only argument, so every key depends on it. A .bazelversion bump adds its +# release here (scripts/bazel_rrc_test.go checks .bazelversion has one). +BAZEL_RRC_GUIDS = { + "9.3.0": "06a53d89-9f52-46ed-8064-f7af6ba27769", +} +MARKER_FILE_PATH = ".recorded_inputs" +REPO_DIRECTORY_PATH = "repo_contents" +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +GRPC_NOT_FOUND = 5 +MAX_CHAIN = 64 # intermediate hops per repository +MAX_DIFFS = 20 # per repository, in the report + + +# --- protobuf wire format --------------------------------------------------- + + +def _varint(n: int) -> bytes: + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + if n: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def _field(num: int, payload: bytes) -> bytes: + return _varint(num << 3 | 2) + _varint(len(payload)) + payload + + +def _field_varint(num: int, v: int) -> bytes: + return _varint(num << 3) + _varint(v) + + +def _read_varint(b: bytes, i: int) -> tuple[int, int]: + v = s = 0 + while True: + if i >= len(b) or s > 63: + raise ValueError("truncated varint") + c = b[i] + i += 1 + v |= (c & 0x7F) << s + s += 7 + if c < 0x80: + return v, i + + +def fields(b: bytes) -> Iterable[tuple[int, int, object]]: + """Yields (field number, wire type, value) for one message.""" + i = 0 + while i < len(b): + key, i = _read_varint(b, i) + num, wire = key >> 3, key & 7 + if wire == 0: + v, i = _read_varint(b, i) + elif wire == 1: + v, i = b[i : i + 8], i + 8 + elif wire == 2: + n, i = _read_varint(b, i) + v, i = b[i : i + n], i + n + elif wire == 5: + v, i = b[i : i + 4], i + 4 + else: + raise ValueError("wire type %d" % wire) + if i > len(b): + raise ValueError("truncated field %d" % num) + yield num, wire, v + + +@dataclasses.dataclass(frozen=True) +class Digest: + hash: str + size: int + + def encode(self) -> bytes: + out = _field(1, self.hash.encode()) + if self.size: + out += _field_varint(2, self.size) + return out + + @staticmethod + def decode(b: bytes) -> "Digest": + h, size = "", 0 + for num, _, v in fields(b): + if num == 1: + h = bytes(v).decode() + elif num == 2: + size = int(v) + return Digest(h, size) + + @staticmethod + def of(data: bytes) -> "Digest": + return Digest(hashlib.sha256(data).hexdigest(), len(data)) + + +def command_bytes(guid: str) -> bytes: + """RemoteRepoContentsCacheImpl.COMMAND, serialized in field order.""" + return ( + _field(1, guid.encode()) # arguments + + _field(3, MARKER_FILE_PATH.encode()) # output_files + + _field(4, REPO_DIRECTORY_PATH.encode()) # output_directories + + _field(5, b"") # platform: the default instance, set + + _field(7, MARKER_FILE_PATH.encode()) # output_paths + + _field(7, REPO_DIRECTORY_PATH.encode()) + ) + + +def action_digest(guid: str, input_hash: str) -> Digest: + """The AC key the cache uses for input_hash (buildAction).""" + action = ( + _field(1, Digest.of(command_bytes(guid)).encode()) # command_digest + + _field(2, Digest(EMPTY_SHA256, 0).encode()) # input_root_digest + + _field(9, input_hash.encode("latin-1")) # salt + + _field(10, b"") # platform + ) + return Digest.of(action) + + +def _bazel_string(s: str) -> bytes: + # Bazel holds file contents read as ISO-8859-1 in Java Strings and + # Fingerprint.addString writes those Strings as UTF-8. + return s.encode("utf-8") + + +def roll_forward(hash_: str, recorded_input_with_value: str) -> str: + """RemoteRepoContentsCacheImpl.rollForwardHash.""" + out = hashlib.sha256() + for s in (hash_, recorded_input_with_value): + b = _bazel_string(s) + out.update(_varint(len(b)) + b) + return out.hexdigest() + + +# --- REAPI messages --------------------------------------------------------- + + +@dataclasses.dataclass +class ActionResult: + exit_code: int = 0 + output_files: list = dataclasses.field(default_factory=list) # (path, Digest, is_executable, contents) + output_directories: list = dataclasses.field(default_factory=list) # (path, tree Digest) + symlinks: int = 0 + stdout_raw: bytes = b"" + stdout_digest: Optional[Digest] = None + stderr_size: int = 0 + + @staticmethod + def decode(b: bytes) -> "ActionResult": + r = ActionResult() + for num, _, v in fields(b): + if num == 2: + path, dg, exe, contents = "", None, False, b"" + for n2, _, v2 in fields(v): + if n2 == 1: + path = bytes(v2).decode() + elif n2 == 2: + dg = Digest.decode(v2) + elif n2 == 4: + exe = bool(v2) + elif n2 == 5: + contents = bytes(v2) + r.output_files.append((path, dg, exe, contents)) + elif num == 3: + path, tree = "", None + for n2, _, v2 in fields(v): + if n2 == 1: + path = bytes(v2).decode() + elif n2 == 3: + tree = Digest.decode(v2) + r.output_directories.append((path, tree)) + elif num == 4: + r.exit_code = int(v) + elif num == 5: + r.stdout_raw = bytes(v) + elif num == 6: + r.stdout_digest = Digest.decode(v) + elif num in (7, 8): + r.stderr_size += len(v) + elif num in (10, 11, 12): + r.symlinks += 1 + return r + + +@dataclasses.dataclass +class Directory: + files: dict # name -> (Digest, is_executable) + dirs: dict # name -> Digest + symlinks: dict # name -> target + + @staticmethod + def decode(b: bytes) -> "Directory": + d = Directory({}, {}, {}) + for num, _, v in fields(b): + name, dg, exe, target = "", None, False, "" + for n2, _, v2 in fields(v): + if n2 == 1: + name = bytes(v2).decode("utf-8", "surrogateescape") + elif n2 == 2 and num in (1, 2): + dg = Digest.decode(v2) + elif n2 == 2 and num == 3: + target = bytes(v2).decode("utf-8", "surrogateescape") + elif n2 == 4 and num == 1: + exe = bool(v2) + if num == 1: + d.files[name] = (dg, exe) + elif num == 2: + d.dirs[name] = dg + elif num == 3: + d.symlinks[name] = target + return d + + +def decode_tree(b: bytes) -> tuple[Directory, dict]: + """Tree -> (root, {Directory digest hash: Directory}).""" + root, children = None, {} + for num, _, v in fields(b): + if num == 1: + root = Directory.decode(v) + elif num == 2: + children[hashlib.sha256(v).hexdigest()] = Directory.decode(v) + if root is None: + raise ValueError("Tree without a root") + return root, children + + +# --- transport ---------------------------------------------------------------- + + +class RPCError(Exception): + def __init__(self, code: int, message: str): + super().__init__("gRPC status %d: %s" % (code, message)) + self.code = code + + +def _frame(msg: bytes) -> bytes: + return b"\0" + len(msg).to_bytes(4, "big") + msg + + +def _unframe(body: bytes) -> list[bytes]: + out, i = [], 0 + while i < len(body): + if i + 5 > len(body) or body[i] != 0: + raise ValueError("not an uncompressed gRPC frame") + n = int.from_bytes(body[i + 1 : i + 5], "big") + out.append(body[i + 5 : i + 5 + n]) + i += 5 + n + if i != len(body): + raise ValueError("truncated gRPC frame") + return out + + +class CurlTransport: + """gRPC over curl's HTTP/2: grpcs:// with optional mTLS, or grpc://.""" + + def __init__(self, endpoint: str, cert: str = "", key: str = "", ca: str = "", timeout: int = 60): + if endpoint.startswith("grpcs://"): + self.base = "https://" + endpoint[len("grpcs://") :] + self.tls = True + elif endpoint.startswith("grpc://"): + self.base = "http://" + endpoint[len("grpc://") :] + self.tls = False + else: + raise ValueError("endpoint must be grpcs:// or grpc://, got %r" % endpoint) + self.base = self.base.rstrip("/") + self.cert, self.key, self.ca, self.timeout = cert, key, ca, timeout + + def call(self, method: str, request: bytes) -> list[bytes]: + with tempfile.TemporaryDirectory() as tmp: + req, head, body = (os.path.join(tmp, n) for n in ("req", "head", "body")) + with open(req, "wb") as f: + f.write(_frame(request)) + cmd = ["curl", "-sS", "--connect-timeout", "10", "--max-time", str(self.timeout)] + cmd += ["--http2"] if self.tls else ["--http2-prior-knowledge"] + if self.cert: + cmd += ["--cert", self.cert, "--key", self.key] + if self.ca: + cmd += ["--cacert", self.ca] + cmd += [ + "-H", "content-type: application/grpc", "-H", "te: trailers", + "--data-binary", "@" + req, "-D", head, "-o", body, "-w", "%{http_code}", + self.base + "/" + method, + ] # fmt: skip + p = subprocess.run(cmd, capture_output=True, text=True, check=False) + if p.returncode != 0: + raise RPCError(-1, "curl exit %d: %s" % (p.returncode, p.stderr.strip())) + if p.stdout != "200": + raise RPCError(-1, "HTTP %s" % p.stdout) + headers = read_bytes(head).decode("latin-1").replace("\r", "") + status, message = None, "" + for line in headers.split("\n"): + k, _, v = line.partition(":") + if k.strip().lower() == "grpc-status": + status = int(v.strip()) + elif k.strip().lower() == "grpc-message": + message = v.strip() + if status is None: + raise RPCError(-1, "no grpc-status") + if status != 0: + raise RPCError(status, message) + return _unframe(read_bytes(body)) + + +class Cache: + """The few REAPI reads the reader makes.""" + + def __init__(self, rpc: Callable[[str, bytes], list[bytes]], instance: str): + self.rpc, self.instance = rpc, instance + + def action_result(self, key: Digest) -> Optional[ActionResult]: + req = ( + _field(1, self.instance.encode()) + + _field(2, key.encode()) + + _field_varint(3, 1) # inline_stdout + + _field(5, MARKER_FILE_PATH.encode()) # inline_output_files + ) + try: + msgs = self.rpc("build.bazel.remote.execution.v2.ActionCache/GetActionResult", req) + except RPCError as e: + if e.code == GRPC_NOT_FOUND: + return None + raise + if len(msgs) != 1: + raise RPCError(-1, "GetActionResult: %d messages" % len(msgs)) + return ActionResult.decode(msgs[0]) + + def blob(self, d: Digest) -> bytes: + if d.size == 0: + return b"" + name = "%s/blobs/%s/%d" % (self.instance, d.hash, d.size) if self.instance else "blobs/%s/%d" % (d.hash, d.size) + msgs = self.rpc("google.bytestream.ByteStream/Read", _field(1, name.encode())) + data = b"".join(bytes(v) for m in msgs for num, _, v in fields(m) if num == 10) + if Digest.of(data) != d: + raise RPCError(-1, "blob %s/%d: got %d bytes that do not match" % (d.hash, d.size, len(data))) + return data + + +# --- verification --------------------------------------------------------------- + + +@dataclasses.dataclass +class Result: + repo: str + status: str # match, mismatch, invalid, not_cached, error + detail: list = dataclasses.field(default_factory=list) + + +def read_marker(path: str) -> tuple[str, dict]: + """(predeclared hash, {recorded input: its 'input value' line}).""" + text = read_bytes(path).decode("latin-1") + lines = text.split("\n") + recorded = {} + for line in lines[1:]: + if not line: + continue + inp, sep, _ = line.partition(" ") + if sep: + recorded[inp] = line + return lines[0].strip(), recorded + + +def find_final(cache: Cache, guid: str, predeclared: str, recorded: dict) -> tuple[Optional[ActionResult], list]: + """Walks the AC from predeclared like fetchFinalCacheEntry; returns (final entry or None, invalid reasons).""" + invalid, current, seen = [], [predeclared], set() + for _ in range(MAX_CHAIN): + nxt = [] + for h in current: + if h in seen: + continue + seen.add(h) + ar = cache.action_result(action_digest(guid, h)) + if ar is None: + continue + if ar.exit_code != 0: + invalid.append("entry %s: exit code %d" % (h, ar.exit_code)) + continue + if len(ar.output_files) == 1 and len(ar.output_directories) == 1 and ar.symlinks == 0: + return ar, invalid + if ar.output_files or ar.output_directories or ar.symlinks or not (ar.stdout_raw or (ar.stdout_digest and ar.stdout_digest.size)): + invalid.append("entry %s: neither a final nor an intermediate result" % h) + continue + stdout = ar.stdout_raw if ar.stdout_raw else cache.blob(ar.stdout_digest) + for batch in stdout.decode("latin-1").splitlines(): + rolling = h + for inp in batch.split(" "): + line = recorded.get(inp) + if line is None: + break # the cold fetch recorded other inputs: not this branch + rolling = roll_forward(rolling, line) + else: + nxt.append(rolling) + if not nxt: + return None, invalid + current = nxt + invalid.append("more than %d intermediate hops" % MAX_CHAIN) + return None, invalid + + +def read_bytes(path: str) -> bytes: + with open(path, "rb") as f: + return f.read() + + +def _sha256_file(path: str) -> tuple[str, int]: + h, n = hashlib.sha256(), 0 + with open(path, "rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + n += len(chunk) + return h.hexdigest(), n + + +def compare_tree(root: Directory, children: dict, local: str) -> list: + """Differences between the cached tree and the cold directory local.""" + diffs = [] + + def walk(d: Directory, path: str, rel: str) -> None: + if len(diffs) >= MAX_DIFFS: + return + try: + names = set(os.listdir(path)) + except OSError as e: + diffs.append("%s: cannot list the cold directory (%s)" % (rel or ".", e)) + return + remote = set(d.files) | set(d.dirs) | set(d.symlinks) + for n in sorted(remote - names): + diffs.append("%s: only in the cache" % os.path.join(rel, n)) + for n in sorted(names - remote): + diffs.append("%s: only in the cold fetch" % os.path.join(rel, n)) + for n in sorted(remote & names): + if len(diffs) >= MAX_DIFFS: + return + p, r = os.path.join(path, n), os.path.join(rel, n) + st = os.lstat(p) + if n in d.symlinks: + if not stat.S_ISLNK(st.st_mode): + diffs.append("%s: a symlink in the cache, not in the cold fetch" % r) + elif os.readlink(p) != d.symlinks[n]: + diffs.append("%s: symlink to %r in the cache, %r cold" % (r, d.symlinks[n], os.readlink(p))) + continue + try: + st = os.stat(p) # the upload follows symlinks it does not keep + except OSError as e: + diffs.append("%s: dangling in the cold fetch (%s)" % (r, e)) + continue + if n in d.files: + dg, exe = d.files[n] + if not stat.S_ISREG(st.st_mode): + diffs.append("%s: a file in the cache, not in the cold fetch" % r) + continue + h, size = _sha256_file(p) + if dg is None or (h, size) != (dg.hash, dg.size): + diffs.append("%s: content differs (cache %s/%s, cold %s/%d)" % (r, dg and dg.hash, dg and dg.size, h, size)) + if exe != bool(st.st_mode & stat.S_IXUSR): + diffs.append("%s: executable bit differs (cache %s)" % (r, exe)) + else: + if not stat.S_ISDIR(st.st_mode): + diffs.append("%s: a directory in the cache, not in the cold fetch" % r) + continue + child = children.get(d.dirs[n].hash) + if child is None: + diffs.append("%s: the Tree lacks directory %s" % (r, d.dirs[n].hash)) + continue + walk(child, p, r) + + walk(root, local, "") + return diffs + + +def verify_repo(cache: Cache, guid: str, external: str, name: str) -> Result: + marker = os.path.join(external, "@%s.marker" % name) + try: + predeclared, recorded = read_marker(marker) + final, invalid = find_final(cache, guid, predeclared, recorded) + if invalid: + return Result(name, "invalid", invalid) + if final is None: + return Result(name, "not_cached") + diffs = [] + path, mdg, _, contents = final.output_files[0] + dpath, tree = final.output_directories[0] + if path != MARKER_FILE_PATH or dpath != REPO_DIRECTORY_PATH or tree is None or mdg is None: + return Result(name, "invalid", ["final entry outputs %r and %r" % (path, dpath)]) + cached_marker = contents if contents else cache.blob(mdg) + if cached_marker != read_bytes(marker): + diffs.append("%s: marker file differs" % MARKER_FILE_PATH) + root, children = decode_tree(cache.blob(tree)) + diffs += compare_tree(root, children, os.path.join(external, name)) + return Result(name, "mismatch" if diffs else "match", diffs[:MAX_DIFFS]) + except (RPCError, ValueError, OSError) as e: + return Result(name, "error", [str(e)]) + + +def repos(external: str) -> list: + out = [] + for f in sorted(os.listdir(external)): + if f.startswith("@") and f.endswith(".marker") and os.path.isdir(os.path.join(external, f[1:-7])): + out.append(f[1:-7]) + return out + + +def verify(cache: Cache, guid: str, output_base: str, jobs: int) -> list: + external = os.path.join(output_base, "external") + names = repos(external) + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + return list(pool.map(lambda n: verify_repo(cache, guid, external, n), names)) + + +def summarize(results: list) -> dict: + counts = {} + for r in results: + counts[r.status] = counts.get(r.status, 0) + 1 + return counts + + +def exit_status(results: list, max_error_fraction: float) -> int: + counts = summarize(results) + if counts.get("mismatch") or counts.get("invalid"): + return 1 + if not results or counts.get("error", 0) > max_error_fraction * len(results): + return 2 + return 0 + + +def main(argv: Optional[list] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--output-base", required=True, help="the cold fetch's output base") + ap.add_argument("--endpoint", required=True, help="grpcs://host:port or grpc://host:port") + ap.add_argument("--instance", default="oss") + ap.add_argument("--cert", default="") + ap.add_argument("--key", default="") + ap.add_argument("--ca", default="") + ap.add_argument("--bazel-version", required=True, help="the release that wrote the entries (.bazelversion)") + ap.add_argument("--jobs", type=int, default=16) + ap.add_argument("--max-error-fraction", type=float, default=0.01, help="transport errors tolerated before exit 2") + ap.add_argument("--json", default="", help="write the full report here") + ap.add_argument("--summary", default="", help="append a Markdown summary here (GITHUB_STEP_SUMMARY)") + a = ap.parse_args(argv) + guid = BAZEL_RRC_GUIDS.get(a.bazel_version) + if guid is None: + print("rrc-verify: no GUID for Bazel %s; add RemoteRepoContentsCacheImpl.GUID to BAZEL_RRC_GUIDS" % a.bazel_version, file=sys.stderr) + return 2 + if bool(a.cert) != bool(a.key): + print("rrc-verify: --cert and --key go together", file=sys.stderr) + return 2 + cache = Cache(CurlTransport(a.endpoint, a.cert, a.key, a.ca).call, a.instance) + results = verify(cache, guid, a.output_base, a.jobs) + counts = summarize(results) + bad = [r for r in results if r.status in ("mismatch", "invalid", "error")] + lines = ["rrc-verify: %d repositories: %s" % (len(results), ", ".join("%s %d" % kv for kv in sorted(counts.items())))] + for r in bad: + lines.append(" %s %s: %s" % (r.status, r.repo, "; ".join(r.detail))) + print("\n".join(lines)) + if a.json: + with open(a.json, "w") as f: + json.dump({"counts": counts, "results": [dataclasses.asdict(r) for r in results if r.status != "match"]}, f, indent=1) + if a.summary: + with open(a.summary, "a") as f: + f.write("### Remote repo contents cache verification\n\n") + f.write("| status | repositories |\n|---|---|\n") + for k, v in sorted(counts.items()): + f.write("| %s | %d |\n" % (k, v)) + for r in bad[:50]: + f.write("\n- **%s** `%s`: %s" % (r.status, r.repo, "; ".join(r.detail)[:500])) + f.write("\n") + return exit_status(results, a.max_error_fraction) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/rrc_verify_test.py b/tools/bazel/rrc_verify_test.py new file mode 100644 index 000000000..ac5ef9eef --- /dev/null +++ b/tools/bazel/rrc_verify_test.py @@ -0,0 +1,323 @@ +"""Tests for tools/bazel/rrc_verify.py (ga-vnycm2.26). + +The golden keys below come from Bazel 9.3.0 itself: it wrote the entries +(`bazel test --nobuild --experimental_remote_repo_contents_cache +--remote_upload_local_results` into a throwaway NativeLink 1.7.1), and +rrc_verify found 1124 of 1124 cached repositories under the keys it computes. +Everything else runs against an in-memory cache. +""" + +import hashlib +import os +import stat +import tempfile +import unittest + +import rrc_verify as v + +GUID = v.BAZEL_RRC_GUIDS["9.3.0"] + +# gazelle++go_deps+com_github_aleksi_pointer, a go_repository: its marker +# (predeclared hash, then recorded inputs) and the chain of intermediate +# entries Bazel 9.3.0 wrote for it, one batch per hop. +POINTER_MARKER = ( + "d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b\n" + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n" + "REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache gazelle++non_module_deps+bazel_gazelle_go_repository_cache\n" + "FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n" + "REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools +go_repository_tools+gazelle_go_repository_tools\n" + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo 8b1288089311dc63c3d4b04c433d9ba11df5b09b5e8fdb1f8db0804f360ca6c0\n" + "ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE \\0\n" + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle 1b764f6d9ad08e324bdcab429c3105815b8914f289d594acfc791bdac2bc2b2c\n" + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE cc988347dd32755bac20223ea21cd982da6da90d9113dee8ccd4c4b88b3983db\n" +) +# (input hash, its AC key, the intermediate entry's stdout) +POINTER_CHAIN = [ + ("d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b", + "add188cbb3ec90ac802738e80e42516a17ba75e7563efffa80f64fee38edf866", + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache\n"), + ("8bc57512298f35dd223a75991454f01b20e0581fc5b3cb9ad7e26521a3b89db2", + "5a07101d8884c3f86496450121a048660f511b33076205395149ef6c369ef719", + "FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools\n"), + ("07b34597f53e981474b3e4952e51c5caa2ba3877f61ad0cc2fe39c84f51acc8f", + "f7eac27fd1a860cc4563bf18f42f0672d533e171341e7293aae8b5f5d8cdebc3", + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE\n"), + ("1ef1aa4b4cb73c42f39465f8feba3ae846c412e5a0a7f2790264264c1941df6e", + "531c6579bec5634c2744fdd5b90ea123739d431991472877bffe3548c86697b1", + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle\n"), + ("edc467b59551f6058310c401062c51315e9a1c3b2bc8ec0a57d7411d7ca4b6dc", + "94a6fc01bd63b6c1506919e676603784eb73e9049b121c4580aaf6e75893d276", + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE\n"), +] # fmt: skip +POINTER_FINAL_HASH = "3ceee7642907f9b23db2d1944d05314384151ca509d432eca651bdee2d61997c" +POINTER_FINAL_KEY = v.Digest("b0ca33d505c031b1ee5aa17138c14135803b0989ef76f3203999a316a933e2f2", 206) + + +# --- an in-memory cache --------------------------------------------------------- + + +def enc_directory(files=(), dirs=(), symlinks=()): + out = b"" + for name, dg, exe in sorted(files): + node = v._field(1, name.encode()) + v._field(2, dg.encode()) + if exe: + node += v._field_varint(4, 1) + out += v._field(1, node) + for name, dg in sorted(dirs): + out += v._field(2, v._field(1, name.encode()) + v._field(2, dg.encode())) + for name, target in sorted(symlinks): + out += v._field(3, v._field(1, name.encode()) + v._field(2, target.encode())) + return out + + +class FakeCache: + """An AC (input hash -> ActionResult bytes) and a CAS, behind Cache's rpc.""" + + def __init__(self): + self.ac, self.cas = {}, {} + + def put_blob(self, data): + d = v.Digest.of(data) + self.cas[d.hash] = data + return d + + def tree_of(self, path): + """Uploads path's files and returns the Tree digest (as Bazel would).""" + children = [] + + def directory(p): + files, dirs, links = [], [], [] + for n in sorted(os.listdir(p)): + q = os.path.join(p, n) + st = os.lstat(q) + if stat.S_ISLNK(st.st_mode): + links.append((n, os.readlink(q))) + elif stat.S_ISDIR(st.st_mode): + raw = directory(q) + children.append(raw) + dirs.append((n, v.Digest.of(raw))) + else: + data = v.read_bytes(q) + files.append((n, self.put_blob(data), bool(st.st_mode & stat.S_IXUSR))) + return enc_directory(files, dirs, links) + + root = directory(path) + tree = v._field(1, root) + b"".join(v._field(2, c) for c in children) + return self.put_blob(tree) + + def put_final(self, input_hash, marker, tree, exit_code=0): + md = self.put_blob(marker) + ar = ( + v._field(2, v._field(1, v.MARKER_FILE_PATH.encode()) + v._field(2, md.encode())) + + v._field(3, v._field(1, v.REPO_DIRECTORY_PATH.encode()) + v._field(3, tree.encode())) + ) + if exit_code: + ar += v._field_varint(4, exit_code) + self.ac[input_hash] = ar + + def put_intermediate(self, input_hash, stdout, raw=True): + if raw: + self.ac[input_hash] = v._field(5, stdout.encode()) + v._field(6, v.Digest.of(stdout.encode()).encode()) + else: + self.ac[input_hash] = v._field(6, self.put_blob(stdout.encode()).encode()) + + def rpc(self, method, request): + if method.endswith("ActionCache/GetActionResult"): + key = None + for num, _, val in v.fields(request): + if num == 2: + key = v.Digest.decode(val) + for h, ar in self.ac.items(): + if v.action_digest(GUID, h) == key: + return [ar] + raise v.RPCError(v.GRPC_NOT_FOUND, "not found") + if method.endswith("ByteStream/Read"): + name = bytes(next(val for num, _, val in v.fields(request) if num == 1)).decode() + _, _, h, size = name.split("/") + data = self.cas.get(h) + if data is None: + raise v.RPCError(v.GRPC_NOT_FOUND, "no blob") + return [v._field(10, data[: len(data) // 2]), v._field(10, data[len(data) // 2 :])] + raise AssertionError(method) + + +def write(path, data, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + os.chmod(path, mode) + + +class Fixture: + """A cold output base with one repository, cached as fetched.""" + + def __init__(self, test, name="repo", marker=b"h0\n"): + self.tmp = tempfile.TemporaryDirectory() + test.addCleanup(self.tmp.cleanup) + self.ext = os.path.join(self.tmp.name, "external") + self.name = name + self.dir = os.path.join(self.ext, name) + write(os.path.join(self.dir, "BUILD.bazel"), b"filegroup(name = 'x')\n") + write(os.path.join(self.dir, "bin", "tool"), b"#!/bin/sh\n", 0o755) + write(os.path.join(self.dir, "sub", "deep", "data.txt"), b"data\n") + os.makedirs(os.path.join(self.dir, "empty")) + os.symlink("bin/tool", os.path.join(self.dir, "link")) + self.marker = marker + write(os.path.join(self.ext, "@%s.marker" % name), marker) + self.cache = FakeCache() + + def seed(self, input_hash="h0"): + self.cache.put_final(input_hash, self.marker, self.cache.tree_of(self.dir)) + + def verify(self): + return v.verify_repo(v.Cache(self.cache.rpc, "oss"), GUID, self.ext, self.name) + + +class KeysTest(unittest.TestCase): + def test_action_digest_matches_bazel_9_3_0(self): + for h, key, _ in POINTER_CHAIN: + self.assertEqual(v.action_digest(GUID, h).hash, key) + self.assertEqual(v.action_digest(GUID, POINTER_FINAL_HASH), POINTER_FINAL_KEY) + + def test_roll_forward_matches_bazel_9_3_0(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + path = os.path.join(tmp.name, "m") + write(path, POINTER_MARKER.encode()) + predeclared, recorded = v.read_marker(path) + self.assertEqual(predeclared, POINTER_CHAIN[0][0]) + h = predeclared + for i, (want, _, stdout) in enumerate(POINTER_CHAIN): + self.assertEqual(h, want, "hop %d" % i) + for inp in stdout.strip().split(" "): + h = v.roll_forward(h, recorded[inp]) + self.assertEqual(h, POINTER_FINAL_HASH) + + def test_guid_for_every_release_is_a_uuid(self): + for release, guid in v.BAZEL_RRC_GUIDS.items(): + self.assertRegex(guid, r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", release) + + def test_bazelversion_has_a_guid(self): + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(v.__file__)))) + path = os.path.join(root, ".bazelversion") + version = v.read_bytes(path).decode().strip() + self.assertIn(version, v.BAZEL_RRC_GUIDS, "a .bazelversion bump must add RemoteRepoContentsCacheImpl.GUID") + + +class VerifyRepoTest(unittest.TestCase): + def test_match(self): + f = Fixture(self) + f.seed() + r = f.verify() + self.assertEqual((r.status, r.detail), ("match", [])) + + def test_not_cached(self): + f = Fixture(self) + self.assertEqual(f.verify().status, "not_cached") + + def test_content_poisoned(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.dir, "sub", "deep", "data.txt"), b"other\n") + r = f.verify() + self.assertEqual(r.status, "mismatch") + self.assertIn("sub/deep/data.txt: content differs", " ".join(r.detail)) + + def test_extra_and_missing_files(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.dir, "init.go"), b"package x\n") + os.remove(os.path.join(f.dir, "BUILD.bazel")) + detail = " ".join(f.verify().detail) + self.assertIn("init.go: only in the cold fetch", detail) + self.assertIn("BUILD.bazel: only in the cache", detail) + + def test_executable_bit(self): + f = Fixture(self) + f.seed() + os.chmod(os.path.join(f.dir, "bin", "tool"), 0o644) + self.assertIn("bin/tool: executable bit differs", " ".join(f.verify().detail)) + + def test_symlink_target(self): + f = Fixture(self) + f.seed() + os.remove(os.path.join(f.dir, "link")) + os.symlink("sub", os.path.join(f.dir, "link")) + self.assertIn("link: symlink to", " ".join(f.verify().detail)) + + def test_empty_directory(self): + f = Fixture(self) + f.seed() + os.rmdir(os.path.join(f.dir, "empty")) + self.assertIn("empty: only in the cache", " ".join(f.verify().detail)) + + def test_marker_differs(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.ext, "@repo.marker"), b"h0\nENV:X 1\n") + r = f.verify() + self.assertEqual(r.status, "mismatch") + self.assertIn("marker file differs", " ".join(r.detail)) + + def test_intermediate_chain(self): + marker = b"p0\nENV:A 1\nFILE:@@x//y abc\nENV:B \\0\n" + f = Fixture(self, marker=marker) + h1 = v.roll_forward(v.roll_forward("p0", "ENV:A 1"), "FILE:@@x//y abc") + h2 = v.roll_forward(h1, "ENV:B \\0") + f.cache.put_intermediate("p0", "ENV:Z\nENV:A FILE:@@x//y\n") # the first batch is another fetch's + f.cache.put_intermediate(h1, "ENV:B\n", raw=False) + f.seed(h2) + r = f.verify() + self.assertEqual((r.status, r.detail), ("match", [])) + + def test_intermediate_without_a_matching_batch(self): + f = Fixture(self, marker=b"p0\nENV:A 1\n") + f.cache.put_intermediate("p0", "ENV:Q\n") + f.seed(v.roll_forward("p0", "ENV:A 2")) # another value of A: not this fetch's + self.assertEqual(f.verify().status, "not_cached") + + def test_invalid_exit_code(self): + f = Fixture(self) + f.cache.put_final("h0", f.marker, f.cache.tree_of(f.dir), exit_code=1) + self.assertEqual(f.verify().status, "invalid") + + def test_invalid_shape(self): + f = Fixture(self) + f.cache.ac["h0"] = v._field_varint(4, 0) + v._field(7, b"stderr") + self.assertEqual(f.verify().status, "invalid") + + def test_tampered_blob_is_an_error(self): + f = Fixture(self) + f.seed() + tree = next(d for d in f.cache.cas if f.cache.cas[d].startswith(b"\x0a")) + f.cache.cas[tree] = f.cache.cas[tree] + b"\x00" + self.assertEqual(f.verify().status, "error") + + +class ReportTest(unittest.TestCase): + def test_exit_status(self): + R = v.Result + self.assertEqual(v.exit_status([R("a", "match"), R("b", "not_cached")], 0.01), 0) + self.assertEqual(v.exit_status([R("a", "match"), R("b", "mismatch")], 0.01), 1) + self.assertEqual(v.exit_status([R("a", "invalid")], 0.01), 1) + self.assertEqual(v.exit_status([R("a", "error")] + [R(str(i), "match") for i in range(200)], 0.01), 0) + self.assertEqual(v.exit_status([R("a", "error"), R("b", "match")], 0.01), 2) + self.assertEqual(v.exit_status([], 0.01), 2) + + def test_repos_lists_marked_repositories(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + ext = os.path.join(tmp.name, "external") + for n in ("a", "b+c"): + os.makedirs(os.path.join(ext, n)) + write(os.path.join(ext, "@%s.marker" % n), b"h\n") + os.makedirs(os.path.join(ext, "unmarked")) + write(os.path.join(ext, "@gone.marker"), b"h\n") + self.assertEqual(v.repos(ext), ["a", "b+c"]) + + def test_unknown_release_is_a_usage_error(self): + self.assertEqual(v.main(["--output-base", "/x", "--endpoint", "grpc://x:1", "--bazel-version", "0.0.0"]), 2) + + +if __name__ == "__main__": + unittest.main()