diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 90651c146..92dc96635 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -8,9 +8,13 @@ description: > BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY in the step env, that rc enables --config=remote-exec for every bazel command; with none of them the job runs locally; with BAZEL_FORK_CACHE=true (and no secrets) it enables - the read-only fork cache (--config=fork-cache, fork PRs); a partial set, - or the fork cache together with the secrets, fails. `bazel` on PATH is a - wrapper that always passes the generated rc. + the read-only fork cache (--config=fork-cache); with BAZEL_FORK_REMOTE=true + (fork and Dependabot PRs, bazel.yml modes fork-ro/fork-rw; needs + RBE_FORK_PR and RBE_FORK_TIER) it gets a short-lived certificate from + rbe-fork-mint (fork-credential.sh, proving the run with a CSR artifact) and + enables remote execution on rbe-fork; a partial set, or two of these + together, fails. `bazel` on PATH is a wrapper that always passes the + generated rc. outputs: remote: description: '"true" when remote execution is configured' @@ -77,12 +81,49 @@ runs: echo "BAZEL_CI_BAZEL_SHA256=$bazel_sha" } >> "$GITHUB_ENV" + # Fork and Dependabot runs (no secrets, no OIDC): a key and a CSR, the CSR + # as an artifact of this run (the mint's proof that the request comes + # from a running job of it), then the certificate. The key stays in the + # secret dir; the CSR and the certificate are public. + # fork-credential.sh key does nothing (no outputs) unless the caller set + # BAZEL_FORK_REMOTE=true; the next two steps follow its csr output. + - name: rbe-fork key and CSR + id: fork-key + shell: bash + env: + BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + run: bash "$GITHUB_ACTION_PATH/fork-credential.sh" key + + - name: rbe-fork CSR artifact (proof of this run) + id: fork-csr + if: steps.fork-key.outputs.csr != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.fork-key.outputs.artifact-name }} + path: ${{ steps.fork-key.outputs.csr }} + retention-days: 1 + if-no-files-found: error + compression-level: 0 + + - name: rbe-fork certificate (rbe-fork-mint) + id: fork-cert + if: steps.fork-key.outputs.csr != '' + shell: bash + env: + BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + ARTIFACT_ID: ${{ steps.fork-csr.outputs.artifact-id }} + run: bash "$GITHUB_ACTION_PATH/fork-credential.sh" cert + - name: Write CI bazelrc id: rc shell: bash env: BAZEL_CI_CACHE_DIR: ${{ runner.temp }}/bazel-ci-cache BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + RBE_FORK_CERT_FILE: ${{ steps.fork-cert.outputs.cert }} + RBE_FORK_KEY_FILE: ${{ steps.fork-cert.outputs.key }} + RBE_FORK_ENDPOINT: ${{ steps.fork-cert.outputs.endpoint }} + RBE_FORK_INSTANCE: ${{ steps.fork-cert.outputs.instance }} run: bash "$GITHUB_ACTION_PATH/write-bazelrc.sh" - name: Install bazel wrapper diff --git a/.github/actions/setup-bazel/fork-credential.sh b/.github/actions/setup-bazel/fork-credential.sh new file mode 100644 index 000000000..442bd7641 --- /dev/null +++ b/.github/actions/setup-bazel/fork-credential.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# setup-bazel: a short-lived rbe-west client certificate for fork and +# Dependabot pull_request runs, which get no secrets and no OIDC token +# (infra nativelink-cas/west README "rbe-fork"). Two steps around the CSR +# artifact upload (action.yml): +# +# fork-credential.sh key an EC P-256 key in BAZEL_CI_SECRET_DIR (outside the +# workspace; PKCS#8: Bazel's Netty TLS refuses a +# SEC1 "EC PRIVATE KEY"), and a CSR for the upload. +# Outputs: csr (path), artifact-name. +# fork-credential.sh cert asks rbe-fork-mint for a certificate naming the +# uploaded artifact (ARTIFACT_ID). Outputs: cert, +# key, endpoint, instance, tier. +# +# The mint signs only for an in-progress pull_request run of this repository +# whose artifact holds this CSR (only code running in the run can upload one), +# at the head of RBE_FORK_PR. It decides the tier from the PR's author and the +# run's triggering actor; RBE_FORK_TIER is what the rbe job's status probe +# said, and a different answer fails the job (re-run all jobs). The key never +# leaves the runner; the certificate is public. Nothing here is secret enough +# to mask, but the key file is 0600 like the CI key. +set -euo pipefail +MINT=${RBE_FORK_MINT:-https://rbe-mint.ops.gascity.com:8444} +ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^grpcs://rbe-fork\.ops\.gascity\.com:8444$'} +dir=${BAZEL_CI_SECRET_DIR:?BAZEL_CI_SECRET_DIR is required} +out=${GITHUB_OUTPUT:-/dev/stdout} + +case "${1:-}" in +key) + # Only fork and Dependabot runs (bazel.yml modes fork-ro/fork-rw). + [ "${BAZEL_FORK_REMOTE:-}" = true ] || exit 0 + install -d -m 0700 "$dir" + csr_dir="${RUNNER_TEMP:?}/rbe-csr" + rm -rf "$csr_dir" && install -d -m 0700 "$csr_dir" + umask 077 + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/fork.key" 2>/dev/null + openssl req -new -key "$dir/fork.key" -subj "/CN=rbe-fork-request" -out "$csr_dir/csr.pem" + job=${GITHUB_JOB:?} + [[ $job =~ ^[a-z0-9][a-z0-9_-]{0,39}$ ]] || { echo "setup-bazel: job id '$job' does not fit the mint's job pattern" >&2; exit 1; } + { + echo "csr=$csr_dir/csr.pem" + echo "artifact-name=rbe-csr-$job-${GITHUB_RUN_ATTEMPT:?}-$(openssl rand -hex 8)" + } >>"$out" + ;; +cert) + : "${ARTIFACT_ID:?}" "${RBE_FORK_PR:?}" "${RBE_FORK_TIER:?}" "${GITHUB_REPOSITORY:?}" "${GITHUB_RUN_ID:?}" + body=$(jq -cn --arg repo "${GITHUB_REPOSITORY#*/}" --argjson run "$GITHUB_RUN_ID" \ + --argjson attempt "${GITHUB_RUN_ATTEMPT:?}" --argjson pr "$RBE_FORK_PR" --arg job "${GITHUB_JOB:?}" \ + --argjson artifact "$ARTIFACT_ID" \ + '{repo: $repo, run_id: $run, run_attempt: $attempt, pr: $pr, job: $job, artifact_id: $artifact}') + reply="$dir/mint.json" + code=000 + for attempt in 1 2 3 4; do + # --connect-timeout: a closed gate drops the SYN; fail fast, not in 60 s. + code=$(curl -sS -o "$reply" -w '%{http_code}' --connect-timeout 5 --max-time 60 \ + -H 'content-type: application/json' --data "$body" "$MINT/v1/cert") || code=000 + # Retry what may pass on its own: a rate or live-certificate limit (429; + # the rates are per minute, the backoff sums to one), the mint could not + # reach GitHub (502), or the network (000). Everything else is an answer. + case "$code" in 429 | 502 | 000) ;; *) break ;; esac + [ "$attempt" -eq 4 ] || sleep $((attempt * 10)) + done + if [ "$code" != 200 ]; then + echo "::error title=rbe-fork mint refused (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 1 + fi + endpoint=$(jq -r .endpoint "$reply") + instance=$(jq -r .instance "$reply") + tier=$(jq -r .tier "$reply") + [[ $endpoint =~ $ENDPOINT_RE ]] || { echo "::error::rbe-fork mint returned endpoint '$endpoint'" >&2; exit 1; } + case "$tier/$instance" in ro/oss-fork | rw/oss) ;; *) echo "::error::rbe-fork mint returned tier '$tier' instance '$instance'" >&2; exit 1 ;; esac + if [ "$tier" != "$RBE_FORK_TIER" ]; then + echo "::error title=rbe-fork tier changed::the mint says '$tier', this run's mode says '$RBE_FORK_TIER' (allowlist changed?); re-run all jobs" + exit 1 + fi + umask 077 + jq -r .cert_pem "$reply" >"$dir/fork.crt" + openssl x509 -in "$dir/fork.crt" -noout -subject -enddate + { + echo "cert=$dir/fork.crt" + echo "key=$dir/fork.key" + echo "endpoint=$endpoint" + echo "instance=$instance" + echo "tier=$tier" + } >>"$out" + ;; +*) + echo "usage: $0 key|cert" >&2 + exit 2 + ;; +esac diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh index 70a5721bc..5b9f26228 100644 --- a/.github/actions/setup-bazel/write-bazelrc.sh +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -25,9 +25,16 @@ # cert path) and outside BAZEL_CI_CACHE_DIR (the runner cache must never store # credentials). The endpoint is masked in the job log. # -# BAZEL_FORK_CACHE=true (bazel.yml mode "cache": fork PRs and rbe=cache -# dispatches) appends --config=fork-cache: .bazelrc's credential-free, -# read-only rbe-west cache. It is mutually exclusive with the RBE secrets. +# BAZEL_FORK_CACHE=true (bazel.yml mode "cache": rbe=cache dispatches, and +# fork/Dependabot runs while rbe-fork is closed) appends --config=fork-cache: +# .bazelrc's credential-free, read-only rbe-west cache. +# +# RBE_FORK_CERT_FILE + RBE_FORK_KEY_FILE + RBE_FORK_ENDPOINT + RBE_FORK_INSTANCE +# (bazel.yml modes fork-ro/fork-rw: fork-credential.sh's outputs, a +# short-lived rbe-fork-mint certificate) enable --config=remote-exec against +# rbe-fork: instance oss-fork (the fork pool, nothing cached) or oss +# (allowlisted authors: the OSS pool, results cached by the workers). The +# three modes are mutually exclusive. # # Outputs (to $GITHUB_OUTPUT when set, else stdout): rc=, # remote=true|false, cache=true|false. @@ -82,6 +89,21 @@ key="${RBE_TLS_KEY:-}" ca="${RBE_TLS_CA:-}" instance="${RBE_INSTANCE:-}" fork_cache="${BAZEL_FORK_CACHE:-}" +fork_cert="${RBE_FORK_CERT_FILE:-}" +fork_key="${RBE_FORK_KEY_FILE:-}" +fork_endpoint="${RBE_FORK_ENDPOINT:-}" +fork_instance="${RBE_FORK_INSTANCE:-}" +fork_fields=0 +for v in "$fork_cert" "$fork_key" "$fork_endpoint" "$fork_instance"; do + [[ -n "$v" ]] && fork_fields=$((fork_fields + 1)) +done +case "$fork_fields/$fork_instance" in +0/ | 4/oss | 4/oss-fork) ;; +*) + echo "setup-bazel: RBE_FORK_CERT_FILE, RBE_FORK_KEY_FILE, RBE_FORK_ENDPOINT and RBE_FORK_INSTANCE (oss or oss-fork) go together" >&2 + exit 1 + ;; +esac case "$fork_cache" in "" | true) ;; *) @@ -142,7 +164,25 @@ done remote=false cache=false -if [[ "$set_fields" -eq 3 ]]; then +if [[ "$fork_fields" -eq 4 ]]; then + if [[ "$set_fields" -ne 0 || "$fork_cache" == true ]]; then + echo "setup-bazel: the rbe-fork certificate, the remote execution secrets and BAZEL_FORK_CACHE are mutually exclusive" >&2 + exit 1 + fi + { + echo "build:remote-exec --remote_executor=$fork_endpoint" + echo "build:remote-exec --tls_client_certificate=$fork_cert" + echo "build:remote-exec --tls_client_key=$fork_key" + echo "build:remote-exec --remote_instance_name=$fork_instance" + echo "build:remote-exec --noremote_upload_local_results" + # The farm admits 128 connections per source IP (Blacksmith runners + # share addresses). Key-neutral, like everything in this rc. + echo "build:remote-exec --remote_max_connections=8" + echo "build --config=remote-exec" + } >>"$rc" + remote=true + echo "setup-bazel: remote execution on rbe-fork (instance $fork_instance, short-lived certificate)" +elif [[ "$set_fields" -eq 3 ]]; then if [[ "$fork_cache" == true ]]; then echo "setup-bazel: BAZEL_FORK_CACHE and the remote execution secrets are mutually exclusive" >&2 exit 1 diff --git a/.github/bazel-farm-allowlist.txt b/.github/bazel-farm-allowlist.txt index e0bc51afe..0f70a5807 100644 --- a/.github/bazel-farm-allowlist.txt +++ b/.github/bazel-farm-allowlist.txt @@ -10,6 +10,11 @@ # must be listed, and the PR must come from the author's own fork. A listed # user gets same-repo-PR-level trust: its PR code runs with the RBE client # certificate. Same people as gascity's .github/blacksmith-allowlist.txt. +# rbe-west's rbe-fork-mint reads this file too, from the default branch, by +# the same rules: a listed author's own-fork PR, triggered by a listed user, +# gets rbe-fork tier rw (bazel.yml mode fork-rw: the trusted OSS pool) when +# the farm has rw switched on; everyone else gets tier ro (fork pool). A +# line that is not a numeric id makes the mint treat the list as empty. 8082291 # julianknutsen 91582 # quad341 36544495 # sjarmak diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh index ffb34d6d4..40f58be47 100755 --- a/.github/scripts/bazel-gate.sh +++ b/.github/scripts/bazel-gate.sh @@ -13,7 +13,7 @@ # cache: BAZEL_EMBEDDED, BAZEL_PROXIED, # BAZEL_SERVER_STORAGE (integration # runs with the read-only cache) -# remote: none +# remote, fork-ro, fork-rw: none # bazel-gate.sh aggregate the value to gate on for BAZEL: the call's # aggregate result, or, when the mode is missing # or invalid (the rbe job failed, the call never @@ -32,7 +32,7 @@ mode="${BAZEL_RBE_MODE:-}" enabled="${BAZEL_RBE_ENABLED:-}" valid=false case "$mode/$enabled" in - remote/true | local/false | cache/false | skip/false) valid=true ;; + remote/true | fork-ro/true | fork-rw/true | local/false | cache/false | skip/false) valid=true ;; esac case "${1:-}" in diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 67213dba8..4894b4068 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -31,8 +31,9 @@ # gate. bazel-integration runs in the PR call (pr.yml passes integration: # "on") although its legacy counterparts, main.yml's integration jobs, run # only on push to main: remotely it is minutes, and it catches on the PR what -# main.yml would catch after the merge. Unlike the remote-only lanes it also -# runs for fork and Dependabot PRs (mode cache): with the read-only cache +# main.yml would catch after the merge. Fork and Dependabot PRs run it +# remotely in modes fork-ro/fork-rw like every lane, and in mode cache +# (rbe-fork closed) unlike the remote-only lanes: with the read-only cache # only what the PR changed is built and tested on the runner, and it is # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, @@ -41,35 +42,54 @@ # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to # .github/actions/setup-bazel: -# remote: repo variable RBE_WEST_WORKERS=true, rbe not off, not a fork PR -# (or a fork PR bazel-farm.yml authorized, below), and the secret -# RBE_WEST_EXECUTOR (grpcs:// endpoint) present, plus +# remote: repo variable RBE_WEST_WORKERS=true, rbe not off, not a fork or +# Dependabot PR (or a fork PR bazel-farm.yml authorized, below), and the +# secret RBE_WEST_EXECUTOR (grpcs:// endpoint) present, plus # RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): remote # execution on rbe-west, instance "oss" (shared with gascity), from the # Blacksmith runner. -# cache: fork PRs (unless bazel-farm.yml authorized them), rbe=cache -# dispatches and same-repo runs without the secret (Dependabot PRs: -# same-repo, but no Actions secrets): like local (ubuntu-latest, -# bazel-embedded, bazel-proxied and bazel-server-storage skipped), plus -# rbe-west's anonymous read-only cache (.bazelrc --config=fork-cache): -# hits for every action trusted CI already executed, no credentials, no -# uploads; and bazel-integration runs too (a warm run hits all but what -# the PR changed). The farm can close the cache at any time; Bazel then -# runs everything locally. +# fork-ro, fork-rw: fork and Dependabot pull_request runs while rbe-west's +# certificate mint (rbe-fork-mint, infra nativelink-cas/west README +# "rbe-fork") says open for this run. Such runs get no secrets, no OIDC +# token and no repository variables, so the farm decides: the rbe job +# asks the mint's /v1/status, and every lane then proves it is a +# running job of this run (a CSR uploaded as an artifact of it) and gets +# its own certificate of about two hours (setup-bazel, +# fork-credential.sh). fork-ro: instance "oss-fork", the fork pool, +# nothing cached for anyone; fork-rw (PRs from an allowlisted author's +# own fork, triggered by an allowlisted user; the mint reads the +# default branch's .github/bazel-farm-allowlist.txt): instance "oss", +# the trusted OSS pool, results cached by its workers. Every lane runs, +# remotely, as in mode remote, from a GitHub-hosted runner (the client +# only). A lane whose certificate says another tier than this job fails +# ("re-run all jobs"). +# cache: rbe=cache dispatches, fork and Dependabot PRs while rbe-fork is +# closed, unreachable or refuses the run (the mint's canary list, its +# switches, the farm's gate), and same-repo runs without the secret: +# like local (ubuntu-latest, bazel-embedded, bazel-proxied and +# bazel-server-storage skipped), plus rbe-west's anonymous read-only +# cache (.bazelrc --config=fork-cache): hits for every action trusted CI +# already executed, no credentials, no uploads; and bazel-integration +# runs too (a warm run hits all but what the PR changed). The farm can +# close the cache at any time; Bazel then runs everything locally. # local: rbe=off dispatches (fork simulation without the cache): local # execution on ubuntu-latest, slower but the same results, for every job # but the remote-only ones (bazel-embedded, bazel-proxied, # bazel-server-storage) and bazel-integration, which are skipped. # skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch), -# and authorized fork-farm runs without it or without the secret. +# and authorized fork-farm runs without it or without the secret. Fork +# and Dependabot pull_request runs cannot read the variable; rbe-fork's +# own switches decide for them. # Trusted-author fork PRs: bazel-farm.yml (pull_request_target, advisory) # calls this workflow for fork PRs whose author and triggering user are on # the base branch's .github/bazel-farm-allowlist.txt, with fork-farm: # authorized and checkout-sha: the event's pinned head SHA. Only that # combination, on a pull_request_target event, lets a fork run remotely; a # fork's pull_request run (pr.yml, which the fork may edit) stays in mode -# cache (or local) whatever it passes, and has no secrets anyway. See -# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". +# cache (or local, or fork-ro/fork-rw as the mint decides) whatever it +# passes, and has no secrets anyway. See engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, +# "Trusted-author fork PRs". (rbe-fork's tier rw replaces this workflow +# once allowlisted forks have run it.) # workflow_dispatch with rbe=cache simulates a fork run; rbe=off also drops # the cache. The decision is also a workflow_call output (rbe-mode, # rbe-enabled), so pr.yml's gate can tell an expected skip from a missing @@ -87,7 +107,12 @@ # trust; that workflow's header lists what it adds): pull_request from a # fork receives no secrets, bazel-farm.yml is the only pull_request_target # caller (policy-tested), and forks cannot write the runner cache. Nor do -# Dependabot PRs, which run as same-repo but get no Actions secrets. In place: secrets are referenced by the setup step +# Dependabot PRs, which run as same-repo but get no Actions secrets. Fork +# and Dependabot runs in modes fork-ro/fork-rw hold a certificate of their +# own instead, worth what their run gets anyway (remote execution under the +# farm's quotas, AC_OSS reads, hash-verified CAS uploads; no action-cache +# writes) for about two hours: anyone who copies it from the run gets no +# more than the PR's author already has. In place: secrets are referenced by the setup step # alone (plus the rbe job's emptiness test of RBE_WEST_EXECUTOR, which yields # a boolean) and never by a workflow- or job-level env (policy-tested), decoded key # material is masked in logs, and CI uploads only remotely executed results @@ -220,7 +245,7 @@ on: description: "'true' when the lanes executed remotely on rbe-west, else 'false'" value: ${{ jobs.rbe.outputs.enabled }} rbe-mode: - description: "remote, cache (no remote-only jobs but integration), local (no remote-only jobs), or skip (no jobs): the rbe job's decision" + description: "remote, fork-ro or fork-rw (every lane runs remotely), cache (no remote-only jobs but integration), local (no remote-only jobs), or skip (no jobs): the rbe job's decision" value: ${{ jobs.rbe.outputs.mode }} bazel-test: description: "bazel-test's job.status" @@ -271,8 +296,9 @@ jobs: # The one place the execution mode is decided (see the header), so every # job sees the same answer for the whole run. Its step reads one secret, and # only to test it for emptiness: the step's env holds a boolean, never the - # value, and the step runs no repository code (no checkout). Dependabot - # runs read vars but get no secrets, so they come out cache, not remote. + # value, and the step runs no repository code (no checkout). Fork and + # Dependabot pull_request runs get no secrets: rbe-fork-mint decides + # theirs (fork-ro, fork-rw, or cache while it is closed). rbe: name: execution mode # Same-repo PRs (and merge_group/push/dispatch/schedule runs, which are @@ -285,31 +311,60 @@ jobs: outputs: enabled: ${{ steps.decide.outputs.enabled }} mode: ${{ steps.decide.outputs.mode }} + tier: ${{ steps.decide.outputs.tier }} steps: # The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts). # A push, merge_group or dispatch has no pull_request, so FORK is false. + # Fork and Dependabot pull_request runs see no secrets and no vars: for + # them rbe-west decides (rbe-fork-mint's status: open, and the tier from + # the PR's author and this run's triggering actor). Its answer is the + # whole run's mode; every lane then gets its own certificate + # (setup-bazel, BAZEL_FORK_REMOTE) and fails if the tier changed. A + # mint that is closed, refuses or cannot be reached leaves mode cache, + # as before rbe-fork. - name: Decide remote execution id: decide env: RBE_VAR_ON: ${{ vars.RBE_WEST_WORKERS == 'true' }} RBE_INPUT_OFF: ${{ inputs.rbe == 'off' }} RBE_INPUT_CACHE: ${{ inputs.rbe == 'cache' }} + PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} + DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} FORK_FARM: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' && inputs.checkout-sha != '' }} HAS_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR != '' }} + PR_NUMBER: ${{ github.event.pull_request.number }} run: | - for v in RBE_VAR_ON RBE_INPUT_OFF RBE_INPUT_CACHE FORK FORK_FARM HAS_EXECUTOR; do + for v in RBE_VAR_ON RBE_INPUT_OFF RBE_INPUT_CACHE PULL_REQUEST FORK DEPENDABOT FORK_FARM HAS_EXECUTOR; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - # A fork runs locally with the read-only cache unless - # bazel-farm.yml authorized it; an authorized fork run is remote or - # nothing (pr.yml's call already runs it locally). + # A fork or Dependabot pull_request run executes remotely only with + # rbe-fork's certificate (mint open for this run), else locally with + # the read-only cache. Any other fork run (pull_request_target) + # stays local unless bazel-farm.yml authorized it; an authorized + # fork run is remote or nothing (pr.yml's call already runs it). + tier= if [ "$RBE_INPUT_OFF" = true ]; then mode=local - elif [ "$RBE_INPUT_CACHE" = true ] || { [ "$FORK" = true ] && [ "$FORK_FARM" != true ]; }; then + elif [ "$RBE_INPUT_CACHE" = true ]; then + mode=cache + elif [ "$PULL_REQUEST" = true ] && { [ "$FORK" = true ] || [ "$DEPENDABOT" = true ]; }; then + [[ "$PR_NUMBER" =~ ^[1-9][0-9]{0,8}$ ]] || { echo "::error::PR number '$PR_NUMBER'"; exit 1; } + url="https://rbe-mint.ops.gascity.com:8444/v1/status?repo=${GITHUB_REPOSITORY#*/}&run=$GITHUB_RUN_ID&attempt=$GITHUB_RUN_ATTEMPT&pr=$PR_NUMBER" + st=$(curl -sS --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors "$url" || echo '{}') + tier=$(jq -r 'if .open == true then .tier else "" end' <<<"$st" 2>/dev/null || true) + case "$tier" in + ro|rw) mode="fork-$tier" ;; + *) + tier= + mode=cache + echo "::warning title=rbe-fork closed for this run::$(jq -r '.reason // .error // "rbe-fork-mint unreachable"' <<<"$st" 2>/dev/null || echo unreachable)" + ;; + esac + elif [ "$FORK" = true ] && [ "$FORK_FARM" != true ]; then mode=cache elif [ "$RBE_VAR_ON" != true ]; then mode=skip @@ -321,10 +376,13 @@ jobs: mode=cache fi enabled=false - if [ "$mode" = remote ]; then enabled=true; fi - echo "mode=$mode" >> "$GITHUB_OUTPUT" - echo "enabled=$enabled" >> "$GITHUB_OUTPUT" - echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac + { + echo "mode=$mode" + echo "enabled=$enabled" + echo "tier=$tier" + } >> "$GITHUB_OUTPUT" + echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" bazel-test: name: test @@ -333,8 +391,10 @@ jobs: # allowlist) and do not count against GitHub's concurrent-job cap. Local # and cache runs (forks, rbe=off/cache, no secrets) build everything # they miss on the runner, so they get the larger GitHub-hosted one. + # Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted, + # like every fork job: the runner is only a client. needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed # worker pool shared with gascity). Skipped only in mode skip (same-repo # runs while RBE_WEST_WORKERS is unset). @@ -347,9 +407,10 @@ jobs: # pre-merge run of pr.yml's legacy PR Core job, which ran every test # every time and never retried: --config=sole-run then executes every # test (no cached results) and fails on a remote cache eviction - # instead of re-running the whole invocation. Local and cache runs - # (forks, Dependabot: the legacy jobs still run) keep cached results. - BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }} + # instead of re-running the whole invocation. Fork modes do the same + # (with BAZEL_COVERS_FORKS "true" it is their only run too). Local and + # cache runs (the legacy jobs still run) keep cached results. + BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -386,12 +447,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} # T1: remote + restored repository cache; T2: remote, cold runner; local: # no remote execution (reported, never budgeted). @@ -579,7 +643,7 @@ jobs: name: pure-Go and js/wasm # Same runner and skip rule as bazel-test. needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -590,9 +654,10 @@ jobs: # pre-merge run of pr.yml's legacy pure-Go and js/wasm job, which ran every test # every time and never retried: --config=sole-run then executes every # test (no cached results) and fails on a remote cache eviction - # instead of re-running the whole invocation. Local and cache runs - # (forks, Dependabot: the legacy jobs still run) keep cached results. - BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }} + # instead of re-running the whole invocation. Fork modes do the same + # (with BAZEL_COVERS_FORKS "true" it is their only run too). Local and + # cache runs (the legacy jobs still run) keep cached results. + BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -620,12 +685,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} - name: Run js/wasm hook boundary run: | @@ -727,18 +795,19 @@ jobs: # does not (same race configuration, shared through the remote cache). bazel-embedded: name: embedded Dolt tier - # Remote execution only: skipped unless the rbe job chose remote (no - # fork, rbe=off or secret-less run). Locally the tier is 27 race test + # Remote execution only: skipped unless the rbe job chose remote, + # fork-ro or fork-rw (enabled; not in modes cache, local or skip). + # Locally the tier is 27 race test # processes (~7-10k test-seconds) on one runner, an hour or more per run # on the pool whose job cap this track exists to relieve, and it proves # nothing pr-risk.yml's embedded jobs (which do run for forks) do not. # On same-repo pull_requests those jobs stand down (pr-risk.yml's # bazel-coverage), so this lane is then the tier's only PR run # and pr.yml's gate requires it to have run here and passed. - # The runs-on expression is the shared one and always picks the - # Blacksmith runner under this if. + # The runs-on expression is the shared one: Blacksmith in mode remote, + # GitHub-hosted in the fork modes (a client only). needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 60 outputs: @@ -757,12 +826,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} # Bazel retries a flaky = True test (up to three attempts) and reports a # later pass as PASSED. The policy test greps BUILD files for it; this @@ -852,16 +924,17 @@ jobs: # on same-repo PRs; see the header. bazel-integration: name: integration - # Remote, or local with the read-only cache (mode cache: fork and - # Dependabot PRs, rbe=cache dispatches), which serves every action + # Remote (modes remote, fork-ro, fork-rw), or local with the read-only + # cache (mode cache: fork and Dependabot PRs while rbe-fork is closed, + # rbe=cache dispatches), which serves every action # trusted CI already ran, so only what the PR changed builds and tests # on the GitHub-hosted runner. Never plain local (rbe=off): the tagged # race build of //... plus 14 jobs' worth of tests on one runner, cold. # A caller may turn it off with integration: "off"; pr.yml and # bazel-farm.yml pass "on". needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} timeout-minutes: 85 outputs: result: ${{ steps.result.outputs.result }} @@ -879,12 +952,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} # Remote: a cold compile (a PR touching a widely imported package; the # first cold run took 17 minutes end to end, warm ones ~3.5) plus the @@ -949,7 +1025,7 @@ jobs: name: dolt-server lane # Same runner and skip rule as bazel-test. needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -960,9 +1036,10 @@ jobs: # pre-merge run of pr.yml's legacy domain+uow and contract corpus jobs, which ran every test # every time and never retried: --config=sole-run then executes every # test (no cached results) and fails on a remote cache eviction - # instead of re-running the whole invocation. Local and cache runs - # (forks, Dependabot: the legacy jobs still run) keep cached results. - BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }} + # instead of re-running the whole invocation. Fork modes do the same + # (with BAZEL_COVERS_FORKS "true" it is their only run too). Local and + # cache runs (the legacy jobs still run) keep cached results. + BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -977,12 +1054,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} # Docker lane only (the A/B control), as in the mirrored jobs; its # tests fail rather than skip without the image. @@ -1045,7 +1125,7 @@ jobs: # bazel-coverage; D2 step 2), so this lane is then the tier's only PR # run and pr.yml's gate requires it to have run here and passed. needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 outputs: @@ -1064,12 +1144,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} # Bazel retries a flaky = True test (up to three attempts) and reports a # later pass as PASSED. The policy test greps BUILD files for it; this @@ -1181,7 +1264,7 @@ jobs: # bazel-coverage; D2 step 2), so this lane is then the tier's only PR # run and pr.yml's gate requires it to have run here and passed. needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 outputs: @@ -1200,12 +1283,15 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} # Bazel retries a flaky = True test (up to three attempts) and reports a # later pass as PASSED. The policy test greps BUILD files for it; this @@ -1279,15 +1365,17 @@ jobs: # bazel-farm.yml and nightly.yml keep the default "off" and this job does # not run for them. Same-repo PRs get a Bazel-built bd (no --config=ci: # only test:ci is defined in .bazelrc, so bazel build //cmd/bd:bd_for_tests - # is run directly); forks, Dependabot and rbe-off/skip runs fall back to + # is run directly); forks, Dependabot (whatever rbe-fork says: these gates + # take no fork certificate) and rbe-off/skip/cache runs fall back to # `go build`, same as package-mcp.sh's own prepare_bd_binary() does when - # BEADS_TEST_BD_BINARY is unset. + # BEADS_TEST_BD_BINARY is unset. Hence mode == 'remote' below, not enabled + # (the fork modes are enabled too). package-mcp: name: Package Gate (MCP) needs: rbe # 4 vCPU on purpose: pytest-xdist -n 8 measured ~2.7-3x on 4 CPUs # (tools/f3 timing, 2026-10), well past 2 vCPU's ceiling. - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ inputs.package-gates == 'on' }} timeout-minutes: 30 outputs: @@ -1311,14 +1399,14 @@ jobs: - name: Set up Bazel id: bazel - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode == 'remote' }} uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # No --config=ci: only test:ci is defined in .bazelrc, and `bazel @@ -1331,11 +1419,11 @@ jobs: # bd (verified with cquery: both configs cquery to the identical # bazel-out/k8-fastbuild-ST-.../bin/cmd/bd/bd_for_tests/bd path). - name: bazel build //cmd/bd:bd_for_tests - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode == 'remote' }} run: bazel build --@rules_go//go/config:race //cmd/bd:bd_for_tests --remote_download_regex='.*/bin/cmd/bd/bd_for_tests/bd$' - name: Package the Bazel-built bd - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode == 'remote' }} run: | ./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts" echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" @@ -1348,14 +1436,14 @@ jobs: - name: Set up Go id: setup-go - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode != 'remote' }} uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' cache: false - name: Restore Go module cache - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode != 'remote' }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/go/pkg/mod @@ -1363,11 +1451,11 @@ jobs: restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - name: Verify Go modules - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode != 'remote' }} run: go mod verify - name: Restore non-race Go build cache - if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.mode != 'remote' }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/go-cache/non-race @@ -1406,7 +1494,7 @@ jobs: package-npm: name: Package Gate (npm) needs: rbe - runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ inputs.package-gates == 'on' }} timeout-minutes: 30 outputs: @@ -1430,14 +1518,14 @@ jobs: - name: Set up Bazel id: bazel - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode == 'remote' }} uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # No --config=ci: only test:ci is defined in .bazelrc, and `bazel @@ -1450,11 +1538,11 @@ jobs: # bd (verified with cquery: both configs cquery to the identical # bazel-out/k8-fastbuild-ST-.../bin/cmd/bd/bd_for_tests/bd path). - name: bazel build //cmd/bd:bd_for_tests - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode == 'remote' }} run: bazel build --@rules_go//go/config:race //cmd/bd:bd_for_tests --remote_download_regex='.*/bin/cmd/bd/bd_for_tests/bd$' - name: Package the Bazel-built bd - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode == 'remote' }} run: | ./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts" echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" @@ -1465,14 +1553,14 @@ jobs: - name: Set up Go id: setup-go - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode != 'remote' }} uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' cache: false - name: Restore Go module cache - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode != 'remote' }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/go/pkg/mod @@ -1480,11 +1568,11 @@ jobs: restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - name: Verify Go modules - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode != 'remote' }} run: go mod verify - name: Restore non-race Go build cache - if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.mode != 'remote' }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/go-cache/non-race diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index f65b665d8..9ec3048a3 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -30,6 +30,14 @@ env: # lane`). pr-risk.yml retires nothing for it; its copy only keeps the # shared bazel-coverage job identical. BAZEL_RETIRES_LEGACY_PR_LANES: "true" + # Fork and Dependabot PRs: "true" covers them like same-repo PRs (the + # flags above then retire their legacy tiers too): rbe-west's mint + # (rbe-fork) gives their Bazel lanes remote execution, instance oss-fork + # or, for allowlisted authors, oss, and a run the mint does not serve + # (mode cache) is red. "false": they keep the legacy tiers, and the Bazel + # lanes run remotely beside them whenever rbe-fork is open (bazel.yml + # modes fork-ro/fork-rw), else locally with the read-only cache as before. + BAZEL_COVERS_FORKS: "false" jobs: detect-ci-tier: @@ -113,8 +121,11 @@ jobs: # and not the executor secret (either, evaluated per run, would let one # workflow's re-run disagree with the other's). A same-repo PR whose run # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # cache) instead of a tier silently moving back. Dependabot runs get - # no Actions secrets by design and keep the legacy tiers; github.actor + # cache) instead of a tier silently moving back. Fork and Dependabot + # runs get no Actions secrets by design: they are covered only while + # BAZEL_COVERS_FORKS is "true" (rbe-fork gives their lanes remote + # execution), and then a closed rbe-fork turns them red the same way; + # it never moves them back to a legacy tier silently. github.actor # (unlike triggering_actor) stays the PR's actor on a re-run. - name: Decide which legacy tiers the Bazel lanes cover id: decide @@ -122,30 +133,38 @@ jobs: RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} + COVERS_FORKS: ${{ env.BAZEL_COVERS_FORKS == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - same_repo_pr=false - if [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then - same_repo_pr=true + # Covered: the Bazel lanes are this run's only run of the retired + # tiers. Same-repo PRs (the CI secrets); fork and Dependabot PRs + # too while BAZEL_COVERS_FORKS is "true". + covered=false + if [ "$PULL_REQUEST" = true ]; then + if [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + covered=true + elif [ "$COVERS_FORKS" = true ]; then + covered=true + fi fi embedded=false - if [ "$same_repo_pr" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then + if [ "$covered" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then embedded=true fi dolt_server=false - if [ "$same_repo_pr" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then + if [ "$covered" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then dolt_server=true fi pr_lanes=false - if [ "$same_repo_pr" = true ] && [ "$RETIRED_PR_LANES" = true ]; then + if [ "$covered" = true ] && [ "$RETIRED_PR_LANES" = true ]; then pr_lanes=true fi { @@ -153,7 +172,7 @@ jobs: echo "dolt_server=$dolt_server" echo "pr_lanes=$pr_lanes" } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" # End-to-end coverage for proxied-server CLI subcommands, sharded across the # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 55a93e662..fef162248 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -43,6 +43,14 @@ env: # step's run script. Committed, not a repo variable, for the same reason as # the BAZEL_RETIRES_LEGACY_* flags above: a re-run can never change it. WINDOWS_PREBUILT_REQUIRED: "false" + # Fork and Dependabot PRs: "true" covers them like same-repo PRs (the + # flags above then retire their legacy tiers too): rbe-west's mint + # (rbe-fork) gives their Bazel lanes remote execution, instance oss-fork + # or, for allowlisted authors, oss, and a run the mint does not serve + # (mode cache) is red. "false": they keep the legacy tiers, and the Bazel + # lanes run remotely beside them whenever rbe-fork is open (bazel.yml + # modes fork-ro/fork-rw), else locally with the read-only cache as before. + BAZEL_COVERS_FORKS: "false" jobs: @@ -2019,8 +2027,11 @@ jobs: # and not the executor secret (either, evaluated per run, would let one # workflow's re-run disagree with the other's). A same-repo PR whose run # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # cache) instead of a tier silently moving back. Dependabot runs get - # no Actions secrets by design and keep the legacy tiers; github.actor + # cache) instead of a tier silently moving back. Fork and Dependabot + # runs get no Actions secrets by design: they are covered only while + # BAZEL_COVERS_FORKS is "true" (rbe-fork gives their lanes remote + # execution), and then a closed rbe-fork turns them red the same way; + # it never moves them back to a legacy tier silently. github.actor # (unlike triggering_actor) stays the PR's actor on a re-run. - name: Decide which legacy tiers the Bazel lanes cover id: decide @@ -2028,30 +2039,38 @@ jobs: RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} + COVERS_FORKS: ${{ env.BAZEL_COVERS_FORKS == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - same_repo_pr=false - if [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then - same_repo_pr=true + # Covered: the Bazel lanes are this run's only run of the retired + # tiers. Same-repo PRs (the CI secrets); fork and Dependabot PRs + # too while BAZEL_COVERS_FORKS is "true". + covered=false + if [ "$PULL_REQUEST" = true ]; then + if [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + covered=true + elif [ "$COVERS_FORKS" = true ]; then + covered=true + fi fi embedded=false - if [ "$same_repo_pr" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then + if [ "$covered" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then embedded=true fi dolt_server=false - if [ "$same_repo_pr" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then + if [ "$covered" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then dolt_server=true fi pr_lanes=false - if [ "$same_repo_pr" = true ] && [ "$RETIRED_PR_LANES" = true ]; then + if [ "$covered" = true ] && [ "$RETIRED_PR_LANES" = true ]; then pr_lanes=true fi { @@ -2059,7 +2078,7 @@ jobs: echo "dolt_server=$dolt_server" echo "pr_lanes=$pr_lanes" } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" ci-gate: name: CI Gate / Required @@ -2233,20 +2252,23 @@ jobs: fi # Where a legacy tier is retired (by PR Risk or by this # workflow), this run's Bazel lanes for it are the tier's only - # pre-merge run: each must have run remotely and passed. Mode skip - # (the farm switch off), or any other result, is red whatever - # bazel-gate.sh allows. + # pre-merge run: each must have run remotely (mode remote, or + # fork-ro/fork-rw with an rbe-fork certificate where + # BAZEL_COVERS_FORKS covers a PR without the CI secrets) and + # passed. Mode skip (the farm switch off), cache (no secret, or + # rbe-fork closed or refusing), or any other result, is red + # whatever bazel-gate.sh allows. # retired ID FLAG COVERED TIER LANE...: set and export ID. retired() { local id="$1" flag="$2" covered="$3" tier="$4" result=success lane shift 4 if [[ "$covered" == "true" ]]; then - [[ "$BAZEL_RBE_MODE" == "remote" ]] || result="not-run-remotely" + [[ "$BAZEL_RBE_MODE" =~ ^(remote|fork-ro|fork-rw)$ ]] || result="not-run-remotely" for lane in "$@"; do [[ "${!lane:-}" == "success" ]] || result="not-run-remotely" done if [[ "$result" != "success" ]]; then - echo "::error::The legacy $tier is retired ($flag) and its Bazel lanes ($*) did not all run remotely and pass (mode '${BAZEL_RBE_MODE:-none}'). If remote execution is off on purpose, commit $flag: \"false\" in pr.yml and pr-risk.yml and push this PR again." + echo "::error::The legacy $tier is retired ($flag) and its Bazel lanes ($*) did not all run remotely and pass (mode '${BAZEL_RBE_MODE:-none}'). In mode cache on a PR without the CI secrets, rbe-fork was closed or refused this run: re-run all jobs later. If remote execution is off on purpose, commit $flag: \"false\" (or BAZEL_COVERS_FORKS: \"false\" for forks) in pr.yml and pr-risk.yml and push this PR again." fi fi printf -v "$id" '%s' "$result" diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 7501c20b7..6927ccc7d 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -42,11 +42,18 @@ Current PR-related workflow names: - `.github/workflows/bazel.yml`: `Bazel` Runs on `push` to `main`, manual dispatch, and `workflow_call` only. PRs and merge groups run it once, through `pr.yml`'s `bazel` job. Its `rbe` job - decides the execution mode once (`remote`; `cache` for fork and Dependabot - PRs: local execution plus rbe-west's anonymous read-only action cache, - `.bazelrc`'s `fork-cache` config; `local` for `rbe=off` dispatches; or - `skip` while the `RBE_WEST_WORKERS` repo variable is unset) and - exports it as the `rbe-mode` / `rbe-enabled` outputs; every lane exports its + decides the execution mode once (`remote`; for fork and Dependabot PRs, + which get no secrets or variables, rbe-west's certificate mint decides + (`rbe-mint.ops.gascity.com:8444/v1/status`, infra README "rbe-fork"): + `fork-ro` or `fork-rw` while it is open for the run, every lane executing + remotely on `rbe-fork.ops.gascity.com:8444` with its own short-lived + certificate (instance `oss-fork`, the fork pool; or `oss`, the OSS pool, + for allowlisted authors), else `cache`: local execution plus rbe-west's + anonymous read-only action cache, `.bazelrc`'s `fork-cache` config; + `local` for `rbe=off` dispatches; or `skip` while the `RBE_WEST_WORKERS` + repo variable is unset) and exports it as the `rbe-mode` / `rbe-enabled` + outputs (`rbe-enabled` is `true` in `remote`, `fork-ro` and `fork-rw`); + every lane exports its `job.status` as an output named after the job. `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and @@ -60,14 +67,17 @@ Current PR-related workflow names: allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s legacy tiers for them), mode `local` (`rbe=off` dispatches) those and - `BAZEL_INTEGRATION`, and mode `remote` allows none. + `BAZEL_INTEGRATION`, and modes `remote`, `fork-ro` and `fork-rw` allow + none. A lane that should run and fails, is cancelled, or reports no result fails the gate, and so does a missing or invalid mode. `bazel-integration` (`bazel test //... --config=integration`, the Bazel twin of `main.yml`'s Linux integration shards) is required on every PR: - `pr.yml` passes `integration: "on"` (policy-pinned), so it runs in mode - `remote` (same-repo PRs) and in mode `cache` (fork and Dependabot PRs, - locally on the GitHub-hosted runner with the read-only cache), and a + `pr.yml` passes `integration: "on"` (policy-pinned), so it runs in modes + `remote` (same-repo PRs) and `fork-ro`/`fork-rw` (fork and Dependabot PRs + while rbe-fork is open), and in mode `cache` (fork and Dependabot PRs + while it is closed, locally on the GitHub-hosted runner with the + read-only cache), and a failure, cancellation or missing result turns the gate red. Its legacy twins still run only on push to `main`, so this lane is the only PR-time run of the full integration-tagged suite. In mode `cache` every action @@ -800,10 +810,11 @@ manifests. On those PRs they are the tiers' only pre-merge run, and check. - `--config=sole-run` (`--nocache_test_results`, `--experimental_remote_cache_eviction_retries=0`, the step 1 and 2 - hardening) is added to every `bazel test` of the three lanes in mode - `remote` only (`BAZEL_SOLE_RUN`), which every covered PR runs in. Fork - and Dependabot runs (modes `cache` and `local`), whose legacy jobs still - run, keep cached results, which keeps their local runs short. Measured + hardening) is added to every `bazel test` of the three lanes wherever + they execute remotely (`BAZEL_SOLE_RUN`: modes `remote`, `fork-ro` and + `fork-rw`), which every covered PR runs in. Runs in modes `cache` and + `local`, whose legacy jobs still run, keep cached results, which keeps + their local runs short. Measured 2026-10-02: `bazel test //... --config=ci --nocache_test_results` remotely took 132 s (113 targets). - Pinned for step 3 (`scripts/pr_lanes_bazel_coverage_test.go`): the @@ -830,9 +841,17 @@ manifests. On those PRs they are the tiers' only pre-merge run, and lane) and so turns `CI Gate / Required` red, instead of quietly moving back to a legacy tier that one of its runs may already have skipped. + Fork and Dependabot PRs too, but only while the committed + `BAZEL_COVERS_FORKS` flag (the same literal in both workflows, + policy-tested) is `"true"`: their lanes then run remotely through + rbe-fork (modes `fork-ro`/`fork-rw`), and a run rbe-fork does not serve + (mode `cache`) turns `CI Gate / Required` red rather than falling back. + It ships `"false"`. - Everyone else keeps the legacy tiers unchanged: - - fork PRs; - - Dependabot PRs (no Actions secrets; `github.actor`, unlike + - fork PRs, while `BAZEL_COVERS_FORKS` is `"false"` (their Bazel lanes + run beside the legacy tiers: remotely while rbe-fork is open, else in + mode `cache`); + - Dependabot PRs, likewise (no Actions secrets; `github.actor`, unlike `github.triggering_actor`, stays `dependabot[bot]` when someone else re-runs them); - every `merge_group` run (there is no merge queue today, so this is not diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 95b8df683..2bc8336db 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -20,6 +20,7 @@ go_test( "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", "bazel_farm_workflow_test.go", + "bazel_fork_mode_test.go", "bazel_policy_test.go", "bazel_skip_allowlist_test.go", "check_build_tags_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 56befa6c7..83c09e70b 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -14,13 +14,20 @@ import ( // Mode cache, end to end, for every way a bazel.yml run can start: the rbe // job's real decision step, each lane's `if:` and setup-bazel env (GitHub's // expressions, evaluated) and the rc write-bazelrc.sh then writes. A cache -// run (fork and Dependabot PRs, rbe=cache, a missing executor secret) must -// reach the rc with --config=fork-cache and nothing else remote: no -// executor, no TLS material, no instance, no upload. That holds even where -// the run could read the secrets (an rbe=cache dispatch on the base repo, -// and, adversarially, a fork PR whose secrets GitHub would never pass). Its +// run (fork and Dependabot PRs while rbe-fork is closed or unreachable, +// rbe=cache, a missing executor secret) must reach the rc with +// --config=fork-cache and nothing else remote: no executor, no TLS +// material, no instance, no upload. That holds even where the run could +// read the secrets (an rbe=cache dispatch on the base repo, and, +// adversarially, a fork PR whose secrets GitHub would never pass). Its // lanes are the local ones plus bazel-integration; the remote-only tiers // skip, and pr.yml's gate (bazel-gate.sh) accepts exactly those skips. +// +// The fork modes too (rbe-fork open, the mint answering ro or rw): every +// lane runs, setup-bazel's fork steps (setupBazelForkSim: key, CSR +// artifact, certificate from a stand-in mint) hand write-bazelrc.sh the +// minted certificate, and the rc executes on rbe-fork with instance +// oss-fork (ro) or oss (rw) and that certificate, never the CI secrets. func TestBazelCacheModeReachesTheRC(t *testing.T) { bash := requireHostTool(t, "bash") root := sourceRepoRoot(t) @@ -42,8 +49,19 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // RHS a secret or a literal. Anything else fails the test, so the // simulation cannot drift from the workflow. setupExpr := regexp.MustCompile(`^\$\{\{ needs\.rbe\.outputs\.(enabled|mode) == '([a-z]+)' && (?:secrets\.([A-Z_]+)|'([^']*)') \|\| '' \}\}$`) - evalSetup := func(t *testing.T, expr, mode, enabled string, haveSecrets bool) string { + evalSetup := func(t *testing.T, expr, mode, enabled, tier string, haveSecrets bool) string { t.Helper() + switch expr { + case bazelForkRemoteValue: + if strings.HasPrefix(mode, "fork-") { + return "true" + } + return "" + case "${{ needs.rbe.outputs.tier }}": + return tier + case "${{ github.event.pull_request.number }}": + return "7123" // every start that reaches a fork mode is a pull_request + } m := setupExpr.FindStringSubmatch(expr) if m == nil { t.Fatalf("setup-bazel env %q: teach TestBazelCacheModeReachesTheRC how GitHub evaluates it", expr) @@ -65,21 +83,22 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { return m[4] } - writeRC := func(t *testing.T, env map[string]string) (outputs, rc string, files []string, logs string, err error) { + // writeRC: write-bazelrc.sh as the rc step runs it, in runner.temp (the + // fork steps put the key and certificate in the same secret dir). + writeRC := func(t *testing.T, dir string, env map[string]string) (outputs, rc string, files []string, logs string, err error) { t.Helper() - dir := t.TempDir() ws := filepath.Join(dir, "ws") if err := os.MkdirAll(ws, 0o755); err != nil { t.Fatal(err) } - secretDir := filepath.Join(dir, "secret") + secretDir := filepath.Join(dir, "bazel-ci-secret") cmd := exec.Command(bash, script) cmd.Dir = ws cmd.Env = []string{ "PATH=" + os.Getenv("PATH"), "GITHUB_WORKSPACE=" + ws, "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), - "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), + "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "bazel-ci-cache"), "BAZEL_CI_SECRET_DIR=" + secretDir, } for k, v := range env { @@ -101,6 +120,7 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { with map[string]string secret bool // the run can read the RBE secrets } + mint := newForkMint(t) var starts []start farm := map[string]string{"fork-farm": "authorized", "checkout-sha": "0123456789abcdef0123456789abcdef01234567"} for _, rbeVar := range []string{"true", ""} { @@ -112,6 +132,12 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { } return rbeFacts{event: "pull_request", rbeVar: rbeVar, secret: s, fork: fork, dependabot: dependabot} } + // prMint: the same, with rbe-fork-mint's /v1/status answer. + prMint := func(fork, dependabot, secret bool, answer string) rbeFacts { + f := pr(fork, dependabot, secret) + f.mint = answer + return f + } starts = append(starts, start{"same-repo PR, " + v, pr(false, false, true), nil, true}, start{"same-repo PR without the secret, " + v, pr(false, false, false), nil, false}, @@ -124,6 +150,15 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { start{"dispatch rbe=off, " + v, rbeFacts{event: "workflow_dispatch", rbeVar: rbeVar, secret: "x"}, map[string]string{"rbe": "off"}, true}, start{"bazel-farm authorized fork, " + v, rbeFacts{event: "pull_request_target", rbeVar: rbeVar, secret: "x", fork: true}, farm, true}, start{"bazel-farm authorized fork without the secret, " + v, rbeFacts{event: "pull_request_target", rbeVar: rbeVar, fork: true}, farm, false}, + // rbe-fork: the mint's answer decides fork and Dependabot runs. + start{"fork PR, rbe-fork ro, " + v, prMint(true, false, false, "ro"), nil, false}, + start{"fork PR, rbe-fork rw, " + v, prMint(true, false, false, "rw"), nil, false}, + start{"fork PR, rbe-fork closed, " + v, prMint(true, false, false, "closed"), nil, false}, + start{"fork PR, rbe-fork rw tier closed, " + v, prMint(true, false, false, "rw-closed"), nil, false}, + start{"fork PR, rbe-fork canary refuses, " + v, prMint(true, false, false, "canary"), nil, false}, + start{"Dependabot PR, rbe-fork ro, " + v, prMint(false, true, false, "ro"), nil, false}, + start{"fork PR that somehow has the secrets, rbe-fork ro, " + v, prMint(true, false, true, "ro"), nil, true}, + start{"fork PR, rbe-fork ro, dispatch-style rbe=cache, " + v, prMint(true, false, false, "ro"), map[string]string{"rbe": "cache"}, false}, ) } @@ -134,18 +169,33 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { if err != nil { t.Fatal(err) } - mode, enabled := out["mode"], out["enabled"] + mode, enabled, tier := out["mode"], out["enabled"], out["tier"] seen[mode] = true - // Forks outside bazel-farm.yml's authorized call and rbe=cache - // are mode cache whatever else holds; Dependabot (no secrets) - // too unless the farm switch is off (mode skip). + // Fork and Dependabot pull_request runs: fork- when the + // mint says open with tier ro or rw, else cache, whatever else + // holds (rbe=off: local). Other forks outside bazel-farm.yml's + // authorized call, and rbe=cache, are mode cache. authorized := s.f.event == "pull_request_target" && s.with["fork-farm"] == "authorized" - dependabot := s.f.dependabot && s.f.rbeVar == "true" - if s.with["rbe"] != "off" && !authorized && (s.f.fork || dependabot || s.with["rbe"] == "cache") && mode != "cache" { - t.Errorf("mode = %s, want cache", mode) + forkPR := s.f.event == "pull_request" && (s.f.fork || s.f.dependabot) + want := "" + switch { + case s.with["rbe"] == "off": + want = "local" + case s.with["rbe"] == "cache": + want = "cache" + case forkPR && (s.f.mint == "ro" || s.f.mint == "rw"): + want = "fork-" + s.f.mint + case forkPR, s.f.fork && !authorized: + want = "cache" } - if mode == "remote" && (!s.secret || s.f.fork && !authorized) { - t.Errorf("mode remote without the secret or for an unauthorized fork") + if want != "" && mode != want { + t.Errorf("mode = %s, want %s", mode, want) + } + if wantTier := strings.TrimPrefix(want, "fork-"); strings.HasPrefix(want, "fork-") && tier != wantTier || !strings.HasPrefix(mode, "fork-") && tier != "" { + t.Errorf("mode %s with tier %q", mode, tier) + } + if mode == "remote" && (!s.secret || s.f.fork && !authorized || forkPR) { + t.Errorf("mode remote without the secret, for an unauthorized fork, or for a fork or Dependabot pull_request") } var lanes []string @@ -155,6 +205,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { } runs := bazelLaneRunModes(t, name, job.If, s.with)[mode] switch { + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name]: + t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && name == bazelIntegJobName && !runs: t.Errorf("%s does not run in mode cache", name) case mode == "cache" && bazelRemoteOnlyJobs[name] && runs: @@ -172,13 +224,26 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { } env := map[string]string{} for k, expr := range step.Env { - env[k] = evalSetup(t, expr, mode, enabled, s.secret) + env[k] = evalSetup(t, expr, mode, enabled, tier, s.secret) } - outputs, rc, files, logs, err := writeRC(t, env) + runnerTemp := t.TempDir() + env = setupBazelForkSim(t, mint, runnerTemp, name, env, tier) + outputs, rc, files, logs, err := writeRC(t, runnerTemp, env) if err != nil { t.Fatalf("%s: write-bazelrc.sh failed in mode %s: %v\n%s", name, mode, err, logs) } checkModeRC(t, name, mode, outputs, rc, files, logs) + if strings.HasPrefix(mode, "fork-") { + secretDir := filepath.Join(runnerTemp, "bazel-ci-secret") + for _, want := range []string{ + "\nbuild:remote-exec --tls_client_certificate=" + filepath.Join(secretDir, "fork.crt") + "\n", + "\nbuild:remote-exec --tls_client_key=" + filepath.Join(secretDir, "fork.key") + "\n", + } { + if !strings.Contains(rc, want) { + t.Errorf("%s (mode %s): rc lacks the minted certificate %q:\n%s", name, mode, strings.TrimSpace(want), rc) + } + } + } } } if mode == "skip" && len(lanes) != 0 { @@ -270,6 +335,33 @@ func checkModeRC(t *testing.T, lane, mode, outputs, rc string, files []string, l if !has("\nbuild --config=remote-exec\n") || has("fork-cache") || !strings.Contains(outputs, "remote=true") { t.Errorf("%s (mode remote): outputs %q rc %q; want remote-exec and no fork-cache", lane, outputs, rc) } + case "fork-ro", "fork-rw": + // rbe-fork with the minted certificate: the mint's endpoint and + // instance, no upload from the runner, never the CI secrets (their + // endpoint is farm.invalid here) or the fork cache. + instance := rbeForkInstance[strings.TrimPrefix(mode, "fork-")] + for _, want := range []string{ + "\nbuild:remote-exec --remote_executor=" + rbeForkEndpoint + "\n", + "\nbuild:remote-exec --remote_instance_name=" + instance + "\n", + "\nbuild:remote-exec --noremote_upload_local_results\n", + "\nbuild --config=remote-exec\n", + } { + if !has(want) { + t.Errorf("%s (mode %s): rc lacks %q:\n%s", lane, mode, strings.TrimSpace(want), rc) + } + } + for _, bad := range []string{"fork-cache", "farm.invalid", "client.crt", "client.key", "--tls_certificate", "--remote_upload_local_results", "--remote_cache", "--remote_header", "--bes_"} { + if has(bad) { + t.Errorf("%s (mode %s): rc carries %q:\n%s", lane, mode, bad, rc) + } + } + sort.Strings(files) + if strings.Join(files, " ") != "ci.bazelrc fork.crt fork.key mint.json" { + t.Errorf("%s (mode %s): secret dir holds %v, want the rc, the minted certificate, its key and the mint's reply", lane, mode, files) + } + if !strings.Contains(outputs, "remote=true") || !strings.Contains(outputs, "cache=false") || strings.Contains(logs, "::add-mask::") { + t.Errorf("%s (mode %s): outputs %q logs %q; want remote=true, cache=false and nothing masked (no secret seen)", lane, mode, outputs, logs) + } default: t.Errorf("%s runs in mode %q", lane, mode) } diff --git a/scripts/bazel_fork_mode_test.go b/scripts/bazel_fork_mode_test.go new file mode 100644 index 000000000..0ee2bf7fa --- /dev/null +++ b/scripts/bazel_fork_mode_test.go @@ -0,0 +1,583 @@ +package scripts_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// rbe-fork (bazel.yml modes fork-ro and fork-rw): fork and Dependabot PR +// lanes get a short-lived client certificate from rbe-west's rbe-fork-mint +// (infra nativelink-cas/west README "rbe-fork"). setup-bazel makes a key and +// a CSR (fork-credential.sh key), uploads the CSR as an artifact of the run, +// asks the mint for a certificate naming that artifact (fork-credential.sh +// cert) and writes an rc for rbe-fork (write-bazelrc.sh). These tests run +// the real scripts against a stand-in mint (bazelTestMintCertStub signs the +// CSR with a throwaway CA); nothing reaches the network. + +const ( + forkCredentialScript = setupBazelActionDir + "/fork-credential.sh" + // The only endpoint fork-credential.sh accepts from the mint, and the + // instances per tier (the mint's INSTANCE map). + rbeForkEndpoint = "grpcs://rbe-fork.ops.gascity.com:8444" + rbeForkMintURL = "https://rbe-mint.ops.gascity.com:8444" +) + +var rbeForkInstance = map[string]string{"ro": "oss-fork", "rw": "oss"} + +// bazelTestMintCertStub stands in for curl in fork-credential.sh cert: it +// parses the arguments that script passes, logs the URL and body (and the +// timeouts on a "timeouts" line), and answers as BAZEL_TEST_MINT_CERT says: ro / rw (sign the CSR at +// BAZEL_TEST_CSR, as the mint would after downloading the artifact), +// ro-as-rw (tier rw with instance oss-fork), evil-endpoint, an HTTP status +// with an error body, 000 (connection refused), or "-then-" +// (the first call answers ). +const bazelTestMintCertStub = `#!/usr/bin/env bash +set -euo pipefail +d=$BAZEL_TEST_MINT_DIR +out= data= url= connect= max= +while [ $# -gt 0 ]; do + case "$1" in + -o) out=$2; shift 2 ;; + --data) data=$2; shift 2 ;; + --connect-timeout) connect=$2; shift 2 ;; + --max-time) max=$2; shift 2 ;; + -w | -H) shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done +n=$(($(cat "$d/calls" 2>/dev/null || echo 0) + 1)) +echo "$n" >"$d/calls" +printf 'curl %s %s\n' "$url" "$data" >>"$d/log" +printf 'timeouts connect=%s max=%s\n' "$connect" "$max" >>"$d/log" +answer=$BAZEL_TEST_MINT_CERT +case "$answer" in +*-then-*) if [ "$n" -eq 1 ]; then answer=${answer%%-then-*}; else answer=${answer#*-then-}; fi ;; +esac +reply() { printf '%s' "$2" >"$out"; printf '%s' "$1"; } +sign() { + openssl x509 -req -in "$BAZEL_TEST_CSR" -CA "$d/ca.pem" -CAkey "$d/ca.key" -days 1 -set_serial 7 -out "$d/cert.pem" 2>/dev/null + reply 200 "$(jq -cn --arg tier "$1" --arg instance "$2" --arg endpoint "${3:-grpcs://rbe-fork.ops.gascity.com:8444}" \ + --rawfile pem "$d/cert.pem" '{tier: $tier, instance: $instance, endpoint: $endpoint, cert_pem: $pem, serial: "7", not_after: "2026-10-03T12:00:00+00:00"}')" +} +case "$answer" in +ro) sign ro oss-fork ;; +rw) sign rw oss ;; +ro-as-rw) sign rw oss-fork ;; +evil-endpoint) sign ro oss-fork grpcs://elsewhere.example:443 ;; +000) echo "curl: (7) Failed to connect to rbe-mint.ops.gascity.com port 8444" >&2; exit 7 ;; +*) reply "$answer" "{\"error\": \"stub answer $answer\"}" ;; +esac +` + +// forkMint: a throwaway CA and the stub's state directory. +type forkMint struct { + dir, bin string +} + +func newForkMint(t *testing.T) forkMint { + t.Helper() + requireHostTool(t, "openssl") + requireHostTool(t, "jq") + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range map[string]string{ + "curl": bazelTestMintCertStub, + // fork-credential.sh backs off between retries; the stub logs it. + "sleep": "#!/bin/sh\necho \"sleep $*\" >>\"$BAZEL_TEST_MINT_DIR/log\"\n", + } { + if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command("openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", + "-keyout", filepath.Join(dir, "ca.key"), "-out", filepath.Join(dir, "ca.pem"), "-days", "1", "-subj", "/CN=test fork CA") + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("test CA: %v\n%s", err, b) + } + return forkMint{dir: dir, bin: bin} +} + +func (m forkMint) log(t *testing.T) string { + t.Helper() + b, _ := os.ReadFile(filepath.Join(m.dir, "log")) + return string(b) +} + +func (m forkMint) reset(t *testing.T) { + t.Helper() + for _, f := range []string{"log", "calls"} { + if err := os.RemoveAll(filepath.Join(m.dir, f)); err != nil { + t.Fatal(err) + } + } +} + +// runForkCredential runs fork-credential.sh with env (plus PATH with the +// stub first) and returns its $GITHUB_OUTPUT and combined log. +func runForkCredential(t *testing.T, m forkMint, arg string, env map[string]string) (map[string]string, string, error) { + t.Helper() + dir := t.TempDir() + out := filepath.Join(dir, "out") + cmd := exec.Command("bash", filepath.Join(sourceRepoRoot(t), forkCredentialScript), arg) + cmd.Dir = dir + cmd.Env = []string{"PATH=" + m.bin + string(os.PathListSeparator) + os.Getenv("PATH"), "GITHUB_OUTPUT=" + out, "BAZEL_TEST_MINT_DIR=" + m.dir} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + b, err := cmd.CombinedOutput() + data, _ := os.ReadFile(out) + return parseGitHubOutput(string(data)), string(b), err +} + +// forkMintSleeps: the stub sleep's arguments in the mint log, in order. +func forkMintSleeps(log string) string { + var got []string + for _, line := range strings.Split(log, "\n") { + if n, ok := strings.CutPrefix(line, "sleep "); ok { + got = append(got, n) + } + } + return strings.Join(got, " ") +} + +func parseGitHubOutput(data string) map[string]string { + got := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(data), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + got[k] = v + } + } + return got +} + +// The CSR artifact's name is the mint's source for the CN's job and attempt +// (infra rbe-fork-mint.py ARTIFACT_RE); the job id must fit its JOB_RE. +var ( + rbeForkArtifactRE = regexp.MustCompile(`^rbe-csr-(?P[a-z0-9][a-z0-9_-]{0,39})-(?P[0-9]{1,3})-[0-9a-f]{16,32}$`) + rbeForkJobRE = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,39}$`) +) + +// fork-credential.sh, both halves, against the mint's client contract: the +// key (EC P-256, PKCS#8, 0600, outside the workspace), the CSR artifact's +// name, the request body (exactly the mint's six keys), the endpoint pin, the +// tier/instance pairing, the tier check against the rbe job's, the retries +// (429, 502 and connection failures only; backoff 10, 20, 30 s, none after +// the last attempt), the timeouts (a closed gate drops the connection) and +// the refusals. +func TestSetupBazelForkCredential(t *testing.T) { + requireHostTool(t, "bash") + m := newForkMint(t) + base := func(secret string) map[string]string { + return map[string]string{ + "BAZEL_CI_SECRET_DIR": secret, "RUNNER_TEMP": filepath.Dir(secret), "BAZEL_FORK_REMOTE": "true", + "GITHUB_JOB": "bazel-test", "GITHUB_RUN_ATTEMPT": "2", "GITHUB_RUN_ID": "4242", + "GITHUB_REPOSITORY": "gastownhall/beads", "RBE_FORK_PR": "7123", + } + } + + // key: nothing at all unless BAZEL_FORK_REMOTE=true. + for _, v := range []string{"", "false", "TRUE"} { + secret := filepath.Join(t.TempDir(), "secret") + env := base(secret) + env["BAZEL_FORK_REMOTE"] = v + out, logs, err := runForkCredential(t, m, "key", env) + if err != nil || len(out) != 0 { + t.Errorf("key with BAZEL_FORK_REMOTE=%q: outputs %v, err %v\n%s", v, out, err, logs) + } + if _, err := os.Stat(secret); !os.IsNotExist(err) { + t.Errorf("key with BAZEL_FORK_REMOTE=%q created %s", v, secret) + } + } + + key := func(t *testing.T, env map[string]string) map[string]string { + t.Helper() + out, logs, err := runForkCredential(t, m, "key", env) + if err != nil { + t.Fatalf("key: %v\n%s", err, logs) + } + return out + } + secret := filepath.Join(t.TempDir(), "secret") + out := key(t, base(secret)) + if want := []string{"artifact-name", "csr"}; !sameKeys(out, want) { + t.Fatalf("key outputs %v, want exactly %v", out, want) + } + name := rbeForkArtifactRE.FindStringSubmatch(out["artifact-name"]) + if name == nil || name[1] != "bazel-test" || name[2] != "2" { + t.Errorf("artifact name %q: want rbe-csr--- (the mint's ARTIFACT_RE)", out["artifact-name"]) + } + if filepath.Base(out["csr"]) != "csr.pem" || strings.HasPrefix(out["csr"], secret) { + t.Errorf("csr %q: want csr.pem (the one file the mint accepts in the archive) outside the secret dir", out["csr"]) + } + keyFile := filepath.Join(secret, "fork.key") + if st, err := os.Stat(keyFile); err != nil || st.Mode().Perm() != 0o600 { + t.Errorf("key file %s: %v, mode %v; want 0600", keyFile, err, st) + } + if b, _ := os.ReadFile(keyFile); !strings.HasPrefix(string(b), "-----BEGIN PRIVATE KEY-----") { + t.Errorf("key file is not PKCS#8 (Bazel's Netty TLS refuses SEC1 keys):\n%.40s", b) + } + if st, err := os.Stat(secret); err != nil || st.Mode().Perm() != 0o700 { + t.Errorf("secret dir %s: %v, mode %v; want 0700", secret, err, st) + } + csrText, err := exec.Command("openssl", "req", "-in", out["csr"], "-noout", "-text", "-verify").CombinedOutput() + if err != nil || !strings.Contains(string(csrText), "prime256v1") && !strings.Contains(string(csrText), "P-256") { + t.Errorf("CSR is not a self-signed EC P-256 request: %v\n%s", err, csrText) + } + // Two runs never share an artifact name (an attempt re-runs failed + // jobs with fresh CSRs). + if again := key(t, base(filepath.Join(t.TempDir(), "secret"))); again["artifact-name"] == out["artifact-name"] { + t.Errorf("two keys share the artifact name %q", out["artifact-name"]) + } + // A job id the mint's pattern refuses fails here, before any upload. + for _, job := range []string{"Bazel_Test", "-x", strings.Repeat("a", 41), ""} { + env := base(filepath.Join(t.TempDir(), "secret")) + env["GITHUB_JOB"] = job + if rbeForkJobRE.MatchString(job) { + t.Fatalf("test job %q fits the mint's pattern", job) + } + if out, logs, err := runForkCredential(t, m, "key", env); err == nil { + t.Errorf("key with GITHUB_JOB %q succeeded: %v\n%s", job, out, logs) + } + } + // Every bazel.yml job that runs setup-bazel has an id the mint accepts. + for jobName := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if !rbeForkJobRE.MatchString(jobName) { + t.Errorf("%s job id %q does not fit the mint's job pattern", bazelWorkflowName, jobName) + } + } + + cert := func(t *testing.T, answer, tier string) (map[string]string, string, error) { + t.Helper() + m.reset(t) + env := base(secret) + env["ARTIFACT_ID"] = "99" + env["RBE_FORK_TIER"] = tier + env["BAZEL_TEST_MINT_CERT"] = answer + env["BAZEL_TEST_CSR"] = out["csr"] + return runForkCredential(t, m, "cert", env) + } + wantBody := map[string]any{"repo": "beads", "run_id": 4242.0, "run_attempt": 2.0, "pr": 7123.0, "job": "bazel-test", "artifact_id": 99.0} + for _, tier := range []string{"ro", "rw"} { + for _, answer := range []string{tier, "429-then-" + tier, "502-then-" + tier, "000-then-" + tier} { + got, logs, err := cert(t, answer, tier) + if err != nil { + t.Errorf("cert %s (mint %s): %v\n%s", tier, answer, err, logs) + continue + } + want := map[string]string{"cert": filepath.Join(secret, "fork.crt"), "key": keyFile, "endpoint": rbeForkEndpoint, + "instance": rbeForkInstance[tier], "tier": tier} + if !equalMaps(got, want) { + t.Errorf("cert %s (mint %s): outputs %v, want %v", tier, answer, got, want) + } + if st, err := os.Stat(want["cert"]); err != nil || st.Mode().Perm() != 0o600 { + t.Errorf("cert file: %v %v; want 0600", err, st) + } + calls := strings.Count(m.log(t), "curl ") + if wantCalls := map[bool]int{false: 1, true: 2}[strings.Contains(answer, "-then-")]; calls != wantCalls { + t.Errorf("mint %s: %d requests, want %d:\n%s", answer, calls, wantCalls, m.log(t)) + } + if got, want := forkMintSleeps(m.log(t)), map[bool]string{false: "", true: "10"}[strings.Contains(answer, "-then-")]; got != want { + t.Errorf("mint %s: slept %q, want %q:\n%s", answer, got, want, m.log(t)) + } + if got := strings.Count(m.log(t), "timeouts connect=5 max=60\n"); got != calls { + t.Errorf("mint %s: %d of %d requests with --connect-timeout 5 --max-time 60:\n%s", answer, got, calls, m.log(t)) + } + for _, line := range strings.Split(strings.TrimSpace(m.log(t)), "\n") { + if !strings.HasPrefix(line, "curl ") { + continue + } + url, body, _ := strings.Cut(strings.TrimPrefix(line, "curl "), " ") + var gotBody map[string]any + if url != rbeForkMintURL+"/v1/cert" || json.Unmarshal([]byte(body), &gotBody) != nil || !equalAny(gotBody, wantBody) { + t.Errorf("request %s %s; want POST %s/v1/cert with %v", url, body, rbeForkMintURL, wantBody) + } + } + } + } + // Answers that are not a certificate for this run's tier: the step fails + // (a fork lane in a fork mode is required to run remotely), after + // retrying only what may pass on its own. + for _, c := range []struct { + answer, tier string + calls int + why string + }{ + {"403", "ro", 1, "rbe-fork mint refused (HTTP 403)"}, + {"409", "ro", 1, "rbe-fork mint refused (HTTP 409)"}, + {"429", "ro", 4, "rbe-fork mint refused (HTTP 429)"}, + {"503", "rw", 1, "rbe-fork mint refused (HTTP 503)"}, + {"502", "ro", 4, "rbe-fork mint refused (HTTP 502)"}, + {"000", "ro", 4, "rbe-fork mint refused (HTTP 000)"}, + {"rw", "ro", 1, "rbe-fork tier changed"}, + {"ro", "rw", 1, "rbe-fork tier changed"}, + {"ro-as-rw", "rw", 1, "returned tier 'rw' instance 'oss-fork'"}, + {"evil-endpoint", "ro", 1, "returned endpoint 'grpcs://elsewhere.example:443'"}, + } { + got, logs, err := cert(t, c.answer, c.tier) + if err == nil || !strings.Contains(logs, c.why) { + t.Errorf("mint %s for tier %s: outputs %v, err %v; want a failure saying %q\n%s", c.answer, c.tier, got, err, c.why, logs) + } + if len(got) != 0 { + t.Errorf("mint %s for tier %s: outputs %v, want none (write-bazelrc.sh must not see a partial set)", c.answer, c.tier, got) + } + if calls := strings.Count(m.log(t), "curl "); calls != c.calls { + t.Errorf("mint %s: %d requests, want %d", c.answer, calls, c.calls) + } + // Backoff between attempts only: nothing after the last one. + if got, want := forkMintSleeps(m.log(t)), map[int]string{1: "", 4: "10 20 30"}[c.calls]; got != want { + t.Errorf("mint %s: slept %q, want %q", c.answer, got, want) + } + } + // cert needs every fact it sends. + for _, k := range []string{"ARTIFACT_ID", "RBE_FORK_PR", "RBE_FORK_TIER", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_JOB", "GITHUB_REPOSITORY"} { + m.reset(t) + env := base(secret) + env["ARTIFACT_ID"], env["RBE_FORK_TIER"], env["BAZEL_TEST_MINT_CERT"], env["BAZEL_TEST_CSR"] = "99", "ro", "ro", out["csr"] + delete(env, k) + if got, logs, err := runForkCredential(t, m, "cert", env); err == nil { + t.Errorf("cert without %s succeeded: %v\n%s", k, got, logs) + } + } +} + +// write-bazelrc.sh in a fork mode: exactly the rbe-fork lines (the mint's +// endpoint, instance and short-lived certificate, nothing uploaded from the +// runner), and the three remote modes are mutually exclusive. +func TestSetupBazelRCWriterForkMode(t *testing.T) { + bash := requireHostTool(t, "bash") + script := filepath.Join(sourceRepoRoot(t), setupBazelActionDir, "write-bazelrc.sh") + run := func(t *testing.T, env map[string]string) (map[string]string, string, string, error) { + t.Helper() + dir := t.TempDir() + ws := filepath.Join(dir, "ws") + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + secret := filepath.Join(dir, "secret") + cmd := exec.Command(bash, script) + cmd.Dir = ws + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_WORKSPACE=" + ws, "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), + "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), "BAZEL_CI_SECRET_DIR=" + secret} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + b, err := cmd.CombinedOutput() + o, _ := os.ReadFile(filepath.Join(dir, "out")) + rc, _ := os.ReadFile(filepath.Join(secret, "ci.bazelrc")) + return parseGitHubOutput(string(o)), string(rc), string(b), err + } + fork := func(instance string) map[string]string { + return map[string]string{"RBE_FORK_CERT_FILE": "/tmp/s/fork.crt", "RBE_FORK_KEY_FILE": "/tmp/s/fork.key", + "RBE_FORK_ENDPOINT": rbeForkEndpoint, "RBE_FORK_INSTANCE": instance} + } + for _, instance := range []string{"oss-fork", "oss"} { + out, rc, logs, err := run(t, fork(instance)) + if err != nil { + t.Fatalf("%s: %v\n%s", instance, err, logs) + } + var remoteLines []string + for _, line := range strings.Split(rc, "\n") { + if strings.HasPrefix(line, "build") { + remoteLines = append(remoteLines, line) + } + } + want := []string{ + "build:remote-exec --remote_executor=" + rbeForkEndpoint, + "build:remote-exec --tls_client_certificate=/tmp/s/fork.crt", + "build:remote-exec --tls_client_key=/tmp/s/fork.key", + "build:remote-exec --remote_instance_name=" + instance, + "build:remote-exec --noremote_upload_local_results", + "build:remote-exec --remote_max_connections=8", + "build --config=remote-exec", + } + if strings.Join(remoteLines, "\n") != strings.Join(want, "\n") || out["remote"] != "true" || out["cache"] != "false" { + t.Errorf("%s: outputs %v, build lines:\n%s\nwant:\n%s", instance, out, strings.Join(remoteLines, "\n"), strings.Join(want, "\n")) + } + if strings.Contains(logs, "::add-mask::") || !strings.Contains(logs, "setup-bazel: remote execution on rbe-fork (instance "+instance) { + t.Errorf("%s: log %q", instance, logs) + } + } + // Exclusive with the secrets and the fork cache; all four or none; only + // the two instances the mint hands out. + secrets := map[string]string{"BAZEL_REMOTE_EXECUTOR": "grpcs://farm.invalid:443", "RBE_TLS_CERT": "x", "RBE_TLS_KEY": "y"} + bad := map[string]map[string]string{ + "fork + secrets": mergeMaps(fork("oss-fork"), secrets), + "fork + one secret": mergeMaps(fork("oss-fork"), map[string]string{"BAZEL_REMOTE_EXECUTOR": "grpcs://farm.invalid:443"}), + "fork + fork cache": mergeMaps(fork("oss"), map[string]string{"BAZEL_FORK_CACHE": "true"}), + "instance other": fork("oss-private"), + "instance empty": mergeMaps(fork("oss"), map[string]string{"RBE_FORK_INSTANCE": ""}), + "no endpoint": mergeMaps(fork("oss"), map[string]string{"RBE_FORK_ENDPOINT": ""}), + "no key": mergeMaps(fork("oss"), map[string]string{"RBE_FORK_KEY_FILE": ""}), + "instance alone": {"RBE_FORK_INSTANCE": "oss-fork"}, + "cert alone": {"RBE_FORK_CERT_FILE": "/tmp/s/fork.crt"}, + "instance + cache": {"RBE_FORK_INSTANCE": "oss-fork", "BAZEL_FORK_CACHE": "true"}, + "endpoint + secrets": mergeMaps(secrets, map[string]string{"RBE_FORK_ENDPOINT": rbeForkEndpoint}), + } + var names []string + for n := range bad { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + if out, rc, logs, err := run(t, bad[n]); err == nil { + t.Errorf("%s: write-bazelrc.sh succeeded: %v\n%s\n%s", n, out, rc, logs) + } + } +} + +// setupBazelForkSim runs setup-bazel's fork steps (action.yml's fork-key, +// fork-csr and fork-cert, by their ids, ifs and env) for one lane, the way +// the composite action runs them with the caller's step env, and returns +// the env its rc step then gets (caller env plus the evaluated RBE_FORK_*). +// The CSR upload is simulated (artifact id 99); the mint is the stub, +// answering mintAnswer. Every mode runs it: fork-credential.sh key must do +// nothing outside the fork modes. +func setupBazelForkSim(t *testing.T, m forkMint, runnerTemp, lane string, caller map[string]string, mintAnswer string) map[string]string { + t.Helper() + root := sourceRepoRoot(t) + steps := map[string]ciWorkflowStep{} + for _, s := range readSetupBazelAction(t).Runs.Steps { + if s.ID != "" { + steps[s.ID] = s + } + } + outputs := map[string]map[string]string{} + ref := regexp.MustCompile(`^\$\{\{ steps\.([a-z-]+)\.outputs\.([a-z-]+) \}\}$`) + eval := func(expr string) string { + if expr == "${{ runner.temp }}/bazel-ci-secret" { + return filepath.Join(runnerTemp, "bazel-ci-secret") + } + if expr == "${{ runner.temp }}/bazel-ci-cache" { + return filepath.Join(runnerTemp, "bazel-ci-cache") + } + if m := ref.FindStringSubmatch(expr); m != nil { + return outputs[m[1]][m[2]] + } + if !strings.Contains(expr, "${{") { + return expr + } + t.Fatalf("setup-bazel expression %q: teach setupBazelForkSim how GitHub evaluates it", expr) + return "" + } + cond := regexp.MustCompile(`^steps\.([a-z-]+)\.outputs\.([a-z-]+) != ''$`) + runs := func(id string) bool { + s := steps[id] + if s.If == "" { + return true + } + c := cond.FindStringSubmatch(s.If) + if c == nil { + t.Fatalf("setup-bazel %s if %q: teach setupBazelForkSim how GitHub evaluates it", id, s.If) + } + return outputs[c[1]][c[2]] != "" + } + run := func(id string, extra map[string]string) { + t.Helper() + s, ok := steps[id] + if !ok { + t.Fatalf("setup-bazel has no step %s", id) + } + dir := t.TempDir() + out := filepath.Join(dir, "out") + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", s.Run) + cmd.Dir = dir + cmd.Env = []string{"PATH=" + m.bin + string(os.PathListSeparator) + os.Getenv("PATH"), "GITHUB_OUTPUT=" + out, + "GITHUB_ACTION_PATH=" + filepath.Join(root, setupBazelActionDir), "RUNNER_TEMP=" + runnerTemp, + "GITHUB_JOB=" + lane, "GITHUB_RUN_ID=4242", "GITHUB_RUN_ATTEMPT=1", "GITHUB_REPOSITORY=gastownhall/beads", + "BAZEL_TEST_MINT_DIR=" + m.dir} + for k, v := range caller { + cmd.Env = append(cmd.Env, k+"="+v) + } + for k, v := range s.Env { + cmd.Env = append(cmd.Env, k+"="+eval(v)) + } + for k, v := range extra { + cmd.Env = append(cmd.Env, k+"="+v) + } + b, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s: setup-bazel step %s failed: %v\n%s", lane, id, err, b) + } + data, _ := os.ReadFile(out) + outputs[id] = parseGitHubOutput(string(data)) + } + + run("fork-key", nil) + fork := caller["BAZEL_FORK_REMOTE"] == "true" + if got := runs("fork-csr"); got != fork || runs("fork-cert") != fork { + t.Fatalf("%s: BAZEL_FORK_REMOTE %q, but the CSR upload / certificate steps run = %v / %v", lane, caller["BAZEL_FORK_REMOTE"], got, runs("fork-cert")) + } + if fork { + csr := steps["fork-csr"] + if !strings.HasPrefix(csr.Uses, "actions/upload-artifact@") || eval(csr.With["path"]) != outputs["fork-key"]["csr"] || + eval(csr.With["name"]) != outputs["fork-key"]["artifact-name"] || csr.With["compression-level"] != "0" { + t.Fatalf("%s: the CSR upload %+v does not upload fork-key's csr under its artifact name, stored", lane, csr) + } + outputs["fork-csr"] = map[string]string{"artifact-id": "99"} + run("fork-cert", map[string]string{"BAZEL_TEST_MINT_CERT": mintAnswer, "BAZEL_TEST_CSR": outputs["fork-key"]["csr"]}) + } + env := map[string]string{} + for k, v := range caller { + env[k] = v + } + for k, v := range steps["rc"].Env { + if strings.HasPrefix(k, "RBE_FORK_") { + env[k] = eval(v) + } + } + return env +} + +func sameKeys(m map[string]string, want []string) bool { + var got []string + for k := range m { + got = append(got, k) + } + sort.Strings(got) + sort.Strings(want) + return strings.Join(got, ",") == strings.Join(want, ",") +} + +func equalMaps(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if bv, ok := b[k]; !ok || bv != v { + return false + } + } + return true +} + +func equalAny(a, b map[string]any) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func mergeMaps(ms ...map[string]string) map[string]string { + out := map[string]string{} + for _, m := range ms { + for k, v := range m { + out[k] = v + } + } + return out +} diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 6be3f86fb..71d7dc2f5 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -344,6 +344,16 @@ const ( forkCacheInstance = "oss" ) +// publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint +// rbe-fork-mint may hand a fork run (rbe-fork, :8444), so a compromised +// mint cannot point fork builds elsewhere. It is public and carries no +// credential: allowed as exactly this line, in exactly that file (a byte +// copy of gascity's tools/rbe/fork-credential.sh). +const ( + publicRBEForkPinFile = ".github/actions/setup-bazel/fork-credential.sh" + publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` +) + type endpointHit struct { path string line int @@ -398,6 +408,9 @@ func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { continue } + if path == publicRBEForkPinFile && line == publicRBEForkPinLine { + continue + } for _, re := range flagRes { for _, m := range re.FindAllStringSubmatch(line, -1) { if !allowedRemoteValue(m[2]) { @@ -486,6 +499,26 @@ func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad) } } + // rbe-fork's endpoint pin: that line in that file only; anything else + // naming the endpoint, there or elsewhere, is flagged. + if hits := findRemoteEndpoints(publicRBEForkPinFile, []byte("set -eu\n"+publicRBEForkPinLine+"\n"), true); len(hits) != 0 { + t.Errorf("rbe-fork endpoint pin flagged in %s: %v", publicRBEForkPinFile, hits) + } + for _, path := range []string{".github/actions/setup-bazel/write-bazelrc.sh", ".github/workflows/bazel.yml", ".bazelrc", "tools/rbe/fork-credential.sh"} { + if len(findRemoteEndpoints(path, []byte(publicRBEForkPinLine+"\n"), true)) == 0 { + t.Errorf("rbe-fork endpoint pin not flagged in %s (allowlisted in %s only)", path, publicRBEForkPinFile) + } + } + for _, bad := range []string{ + " " + publicRBEForkPinLine, + strings.Replace(publicRBEForkPinLine, "8444", "443", 1), + `ENDPOINT=` + "grpc" + `s://rbe-fork.ops.gascity.com:8444`, + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe-fork.ops.gascity.com:8444", + } { + if len(findRemoteEndpoints(publicRBEForkPinFile, []byte(bad+"\n"), true)) == 0 { + t.Errorf("non-allowlisted rbe-fork line not detected in %s: %q", publicRBEForkPinFile, bad) + } + } // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; // a literal flag value is not. for _, ok := range []string{ diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 8c9a893ca..63ec661f6 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2294,6 +2294,13 @@ const ( bazelRBESecretValue = "${{ secrets.RBE_WEST_EXECUTOR != '' }}" ) +// rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes +// fork-ro and fork-rw only), and the mint's status URL the rbe job probes. +const ( + bazelForkRemoteValue = "${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }}" + bazelMintStatusURL = "https://rbe-mint.ops.gascity.com:8444/v1/status?repo=" +) + // The only triggers bazel.yml may have. pull_request_target (and // workflow_run) would run with secrets in the context of fork PRs. PRs and // merge groups reach it only through pr.yml's call, so it runs once per PR. @@ -2328,14 +2335,15 @@ var bazelLaneGateIDs = map[string]string{ // turns off has a place to record why. var bazelAdvisoryLanes = map[string]string{} -// bazel-integration's if: remote, or local with the read-only cache (mode -// cache: fork and Dependabot PRs, whose only PR-time integration run it is; -// a warm run builds and tests only what the PR changed), never plain local +// bazel-integration's if: remote (modes remote, fork-ro, fork-rw: enabled), +// or local with the read-only cache (mode cache: fork and Dependabot PRs +// while rbe-fork is closed, whose only PR-time integration run it is; a +// warm run builds and tests only what the PR changed), never plain local // (rbe=off: a cold tagged race build of //... on one runner); and off when a // caller passes integration: "off" (pr.yml and bazel-farm.yml pass "on"). A // string input: on push and dispatch it is null, and null != 'off', so the // lane keeps running on main. -const bazelIntegIf = "${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }}" +const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }}" // F3: package-mcp and package-npm's if. Unlike every other lane's, it does // not read the rbe job's outputs at all - only the caller's package-gates @@ -2344,8 +2352,10 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outpu const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" // F3: the package gates' runner. 4 vCPU, not bazel.yml's usual 2: pytest-xdist -// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). -const bazelPackageRunsOn = "${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" +// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). Mode +// remote only: the gates take no rbe-fork certificate, so fork modes +// (enabled too) build bd with go build on a GitHub-hosted runner, as before. +const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // F3: the rbe job's own runner (not gated by its own outputs - it decides // them). Same-repo PRs and merge_group/push/dispatch/schedule (never forks) @@ -2397,7 +2407,15 @@ const bazelAggregateGateID = "BAZEL" // The gate script and the rbe job's execution modes. const bazelGateScript = ".github/scripts/bazel-gate.sh" -var bazelRBEModes = []string{"remote", "cache", "local", "skip"} +var bazelRBEModes = []string{"remote", "fork-ro", "fork-rw", "cache", "local", "skip"} + +// bazelRemoteModes: the modes whose lanes execute on rbe-west (the rbe job's +// enabled output is "true" in exactly these): remote with the CI secrets, +// fork-ro and fork-rw with an rbe-fork-mint certificate. +var bazelRemoteModes = map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true} + +// bazelModeEnabled: the rbe job's enabled output for a mode. +func bazelModeEnabled(mode string) string { return strconv.FormatBool(bazelRemoteModes[mode]) } // The four RBE secrets, the only ones a caller may hand bazel.yml. var bazelCallSecrets = map[string]string{ @@ -2452,18 +2470,23 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { } // Every lane needs the rbe job and takes its runner, skip rule and // setup-bazel env from that job's outputs alone. Remote runs use the - // Blacksmith pool; local and cache runs (forks, rbe=off/cache, - // secret-less Dependabot runs) the GitHub-hosted runner. Cache runs get - // BAZEL_FORK_CACHE and never a secret. - const wantRunsOn = "${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + // Blacksmith pool; fork modes (fork-ro/fork-rw: remote execution with an + // rbe-fork-mint certificate, the runner only a client), local and cache + // runs (forks while rbe-fork is closed, rbe=off/cache) the GitHub-hosted + // runner. Only mode remote gets the secrets; fork modes get + // BAZEL_FORK_REMOTE (setup-bazel mints the certificate), cache runs + // BAZEL_FORK_CACHE. + const wantRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS // unset); remote-only lanes (27 race processes and more, an hour or more // on a GitHub-hosted runner, for tests the Go jobs already run there) // skip unless remote. const wantIf = "${{ needs.rbe.outputs.mode != 'skip' }}" const wantRemoteOnlyIf = "${{ needs.rbe.outputs.enabled == 'true' }}" - gate := "needs.rbe.outputs.enabled == 'true' && " - wantSetupEnv := map[string]string{ + gate := "needs.rbe.outputs.mode == 'remote' && " + // The package gates' setup-bazel (mode remote only, see + // bazelPackageRunsOn): the secrets, never a fork certificate. + wantPackageSetupEnv := map[string]string{ "BAZEL_REMOTE_EXECUTOR": "${{ " + gate + "secrets.RBE_WEST_EXECUTOR || '' }}", "RBE_TLS_CERT": "${{ " + gate + "secrets.RBE_TLS_CERT || '' }}", "RBE_TLS_KEY": "${{ " + gate + "secrets.RBE_TLS_KEY || '' }}", @@ -2471,6 +2494,10 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { "RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}", "BAZEL_FORK_CACHE": "${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }}", } + wantSetupEnv := copyMap(wantPackageSetupEnv) + wantSetupEnv["BAZEL_FORK_REMOTE"] = bazelForkRemoteValue + wantSetupEnv["RBE_FORK_TIER"] = "${{ needs.rbe.outputs.tier }}" + wantSetupEnv["RBE_FORK_PR"] = "${{ github.event.pull_request.number }}" // F3: the rbe job itself runs on Blacksmith for same-repo PRs (and // merge_group/push/dispatch/schedule, which are never forks); a // pull_request_target farm run and any fork or Dependabot PR stay @@ -2490,9 +2517,9 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // F3: the package gates use a 4 vCPU runner (pytest-xdist -n 8) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). - wantJobRunsOn, wantJobIf := wantRunsOn, wantIf + wantJobRunsOn, wantJobIf, wantJobSetupEnv := wantRunsOn, wantIf, wantSetupEnv if bazelPackageJobs[name] { - wantJobRunsOn, wantJobIf = bazelPackageRunsOn, bazelPackageGatesIf + wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn, bazelPackageGatesIf, wantPackageSetupEnv } else if bazelRemoteOnlyJobs[name] { wantJobIf = wantRemoteOnlyIf } else if name == bazelIntegJobName { @@ -2505,8 +2532,8 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { t.Errorf("%s if = %q, want %q", name, job.If, wantJobIf) } for _, step := range job.Steps { - if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantSetupEnv) { - t.Errorf("%s setup-bazel env = %v, want %v", name, step.Env, wantSetupEnv) + if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantJobSetupEnv) { + t.Errorf("%s setup-bazel env = %v, want %v", name, step.Env, wantJobSetupEnv) } } } @@ -2949,20 +2976,20 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string t.Helper() switch ifExpr { case "${{ needs.rbe.outputs.mode != 'skip' }}": - return map[string]bool{"remote": true, "cache": true, "local": true} + return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true, "local": true} case "${{ needs.rbe.outputs.enabled == 'true' }}": - return map[string]bool{"remote": true} + return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true} case bazelIntegIf: if strings.EqualFold(with["integration"], "off") { return map[string]bool{} } - return map[string]bool{"remote": true, "cache": true} + return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true} case bazelPackageGatesIf: // F3: unlike every other lane, these two never skip because of the // rbe job's decision - only because the caller turned them off // entirely (pr.yml always passes "on"). if strings.EqualFold(with["package-gates"], "on") { - return map[string]bool{"remote": true, "cache": true, "local": true, "skip": true} + return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true, "local": true, "skip": true} } return map[string]bool{} } @@ -3078,7 +3105,7 @@ func TestBazelGateSimulation(t *testing.T) { t.Errorf("gated lane %s does not run in pr.yml's call even in remote mode", name) } } - enabledFor := func(mode string) string { return strconv.FormatBool(mode == "remote") } + enabledFor := bazelModeEnabled runScript := func(mode, enabled, arg string) string { t.Helper() @@ -3180,6 +3207,7 @@ func TestBazelGateSimulation(t *testing.T) { // started, or the outputs disagree) allows no skip and fails the gate. for _, bad := range []struct{ mode, enabled string }{ {"", ""}, {"remote", "false"}, {"local", "true"}, {"cache", "true"}, {"skip", "true"}, {"remote", ""}, {"bogus", "false"}, {"REMOTE", "true"}, {"CACHE", "false"}, + {"fork-ro", "false"}, {"fork-rw", "false"}, {"fork-ro", ""}, {"FORK-RO", "true"}, {"fork", "true"}, {"fork-", "true"}, {"fork-rx", "true"}, } { if got := runScript(bad.mode, bad.enabled, "skips"); got != "" { t.Errorf("mode %q enabled %q: skips = %q, want none", bad.mode, bad.enabled, got) @@ -4730,6 +4758,7 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { wantOutputs := map[string]string{ "enabled": "${{ steps.decide.outputs.enabled }}", "mode": "${{ steps.decide.outputs.mode }}", + "tier": "${{ steps.decide.outputs.tier }}", } if !reflect.DeepEqual(job.Outputs, wantOutputs) { t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) @@ -4742,9 +4771,12 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}", "RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}", "RBE_INPUT_CACHE": "${{ inputs.rbe == 'cache' }}", + "PULL_REQUEST": prRiskPullRequestValue, "FORK": "${{ github.event.pull_request.head.repo.fork == true }}", + "DEPENDABOT": prRiskDependabotValue, "FORK_FARM": bazelForkFarmValue, "HAS_EXECUTOR": bazelRBESecretValue, + "PR_NUMBER": "${{ github.event.pull_request.number }}", } if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || !reflect.DeepEqual(step.Env, wantEnv) { t.Errorf("%s step: id %q, uses %q, shell %q, with %v, env %v; want id decide, a run step with env %v", @@ -4777,7 +4809,10 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { // Nothing outside the decision step re-derives the condition. rederive := regexp.MustCompile(`(?i)vars\.RBE_WEST_WORKERS|inputs\.rbe\b|inputs\.fork-farm|head\.repo\.fork|github\.actor|dependabot`) walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) { - if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") { + // The decision step itself (its env and, since rbe-fork, its run: + // the DEPENDABOT fact) is the one place that may. + if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") || + path == ".jobs."+bazelRBEJobName+".steps[0].run" { return } // F3: the rbe job's own runs-on picks a runner venue (Blacksmith vs @@ -4814,44 +4849,65 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { } // The decision, for the facts GitHub evaluates the env expressions on - // (its == is case-insensitive; a missing secret reads as ''). + // (its == is case-insensitive; a missing secret reads as ''), and for + // fork and Dependabot pull_request runs what rbe-fork-mint's /v1/status + // answers (runBazelRBEDecision's curl stub; "" = unreachable). type facts struct { rbeVar, rbeInput, secret string fork, farm bool + pr, dependabot bool // event pull_request; github.actor dependabot[bot] + mint string } cases := []struct { - name string - in facts - mode, enabled string + name string + in facts + mode, enabled, tier string }{ - {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false, false}, "remote", "true"}, - {"push to main", facts{"true", "", "grpcs://x", false, false}, "remote", "true"}, - {"var in other case", facts{"True", "on", "grpcs://x", false, false}, "remote", "true"}, - // Fork and secret-less runs: local execution with the read-only - // cache (rbe=cache simulates them); rbe=off drops the cache too. - {"Dependabot PR (no secrets)", facts{"true", "", "", false, false}, "cache", "false"}, - {"fork PR", facts{"true", "", "", true, false}, "cache", "false"}, - {"fork PR, var unset", facts{"", "", "", true, false}, "cache", "false"}, - {"fork PR, var false", facts{"false", "", "", true, false}, "cache", "false"}, - {"dispatch rbe=cache", facts{"true", "cache", "grpcs://x", false, false}, "cache", "false"}, - {"dispatch rbe=cache, var unset", facts{"", "cache", "", false, false}, "cache", "false"}, - {"call rbe=CACHE", facts{"true", "CACHE", "grpcs://x", false, false}, "cache", "false"}, - {"fork PR rbe=cache", facts{"true", "cache", "", true, false}, "cache", "false"}, - {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false, false}, "local", "false"}, - {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false, false}, "local", "false"}, - {"fork PR rbe=off", facts{"true", "off", "", true, false}, "local", "false"}, - {"same-repo, var unset", facts{"", "", "grpcs://x", false, false}, "skip", "false"}, - {"Dependabot, var unset", facts{"", "", "", false, false}, "skip", "false"}, - {"var false", facts{"false", "on", "grpcs://x", false, false}, "skip", "false"}, + {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false, false, true, false, ""}, "remote", "true", ""}, + {"push to main", facts{"true", "", "grpcs://x", false, false, false, false, ""}, "remote", "true", ""}, + {"var in other case", facts{"True", "on", "grpcs://x", false, false, false, false, ""}, "remote", "true", ""}, + // Fork and Dependabot pull_request runs: rbe-fork decides. Open: + // remote execution with a mint certificate, tier ro or rw. Closed, + // refusing, unreachable or nonsense: local execution with the + // read-only cache, as before rbe-fork (rbe=cache simulates them); + // rbe=off drops the cache too. + {"fork PR, mint ro", facts{"true", "", "", true, false, true, false, "ro"}, "fork-ro", "true", "ro"}, + {"fork PR, mint rw", facts{"true", "", "", true, false, true, false, "rw"}, "fork-rw", "true", "rw"}, + {"fork PR, mint ro, var unset (forks cannot read it)", facts{"", "", "", true, false, true, false, "ro"}, "fork-ro", "true", "ro"}, + {"fork PR somehow with the secret, mint ro", facts{"true", "", "grpcs://x", true, false, true, false, "ro"}, "fork-ro", "true", "ro"}, + {"Dependabot PR, mint ro", facts{"true", "", "", false, false, true, true, "ro"}, "fork-ro", "true", "ro"}, + {"Dependabot PR, var unset, mint ro", facts{"", "", "", false, false, true, true, "ro"}, "fork-ro", "true", "ro"}, + {"fork PR, mint unreachable", facts{"true", "", "", true, false, true, false, ""}, "cache", "false", ""}, + {"fork PR, mint closed", facts{"true", "", "", true, false, true, false, "closed"}, "cache", "false", ""}, + {"fork PR, rw tier closed", facts{"true", "", "", true, false, true, false, "rw-closed"}, "cache", "false", ""}, + {"fork PR, canary refuses", facts{"true", "", "", true, false, true, false, "canary"}, "cache", "false", ""}, + {"fork PR, mint busy", facts{"true", "", "", true, false, true, false, "busy"}, "cache", "false", ""}, + {"fork PR, mint answers garbage", facts{"true", "", "", true, false, true, false, "garbage"}, "cache", "false", ""}, + {"fork PR, mint answers an unknown tier", facts{"true", "", "", true, false, true, false, "evil"}, "cache", "false", ""}, + {"fork PR, var unset, mint unreachable", facts{"", "", "", true, false, true, false, ""}, "cache", "false", ""}, + {"fork PR, var false, mint unreachable", facts{"false", "", "", true, false, true, false, ""}, "cache", "false", ""}, + {"Dependabot PR, mint unreachable", facts{"true", "", "", false, false, true, true, ""}, "cache", "false", ""}, + {"Dependabot, var unset, mint unreachable", facts{"", "", "", false, false, true, true, ""}, "cache", "false", ""}, + {"dispatch rbe=cache", facts{"true", "cache", "grpcs://x", false, false, false, false, ""}, "cache", "false", ""}, + {"dispatch rbe=cache, var unset", facts{"", "cache", "", false, false, false, false, ""}, "cache", "false", ""}, + {"call rbe=CACHE", facts{"true", "CACHE", "grpcs://x", false, false, true, false, ""}, "cache", "false", ""}, + {"fork PR rbe=cache, mint ro", facts{"true", "cache", "", true, false, true, false, "ro"}, "cache", "false", ""}, + {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false, false, false, false, ""}, "local", "false", ""}, + {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false, false, true, false, ""}, "local", "false", ""}, + {"fork PR rbe=off, mint ro", facts{"true", "off", "", true, false, true, false, "ro"}, "local", "false", ""}, + {"same-repo, var unset", facts{"", "", "grpcs://x", false, false, true, false, ""}, "skip", "false", ""}, + {"same-repo without the secret", facts{"true", "", "", false, false, true, false, ""}, "cache", "false", ""}, + {"var false", facts{"false", "on", "grpcs://x", false, false, false, false, ""}, "skip", "false", ""}, // bazel-farm.yml's authorized fork runs: remote or nothing, never - // a second local run (pr.yml already runs one). - {"authorized fork farm", facts{"true", "", "grpcs://x", true, true}, "remote", "true"}, - {"authorized fork farm, var unset", facts{"", "", "grpcs://x", true, true}, "skip", "false"}, - {"authorized fork farm, no secret", facts{"true", "", "", true, true}, "skip", "false"}, - {"authorized fork farm, rbe=off", facts{"true", "off", "grpcs://x", true, true}, "local", "false"}, - // FORK_FARM without the secret (a fork's own pull_request run - // cannot make it true: it needs event pull_request_target). - {"fork, not authorized, secret", facts{"true", "", "grpcs://x", true, false}, "cache", "false"}, + // a second local run (pr.yml already runs one). Not a pull_request + // event, so the mint is never asked. + {"authorized fork farm", facts{"true", "", "grpcs://x", true, true, false, false, "ro"}, "remote", "true", ""}, + {"authorized fork farm, var unset", facts{"", "", "grpcs://x", true, true, false, false, ""}, "skip", "false", ""}, + {"authorized fork farm, no secret", facts{"true", "", "", true, true, false, false, ""}, "skip", "false", ""}, + {"authorized fork farm, rbe=off", facts{"true", "off", "grpcs://x", true, true, false, false, ""}, "local", "false", ""}, + // A fork run that is neither a pull_request nor authorized + // (pull_request_target without bazel-farm.yml's inputs) stays local. + {"fork, not authorized, secret", facts{"true", "", "grpcs://x", true, false, false, false, "ro"}, "cache", "false", ""}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -4859,57 +4915,133 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")), "RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")), "RBE_INPUT_CACHE": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "cache")), + "PULL_REQUEST": strconv.FormatBool(c.in.pr), "FORK": strconv.FormatBool(c.in.fork), + "DEPENDABOT": strconv.FormatBool(c.in.dependabot), "FORK_FARM": strconv.FormatBool(c.in.farm), "HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""), + "PR_NUMBER": "", + bazelTestMintEnv: c.in.mint, } - out, err := runBazelRBEDecision(t, step.Run, env) + if c.in.pr || c.in.farm { + env["PR_NUMBER"] = "7123" + } + out, log, err := runBazelRBEDecisionLog(t, step.Run, env) if err != nil { t.Fatal(err) } - want := map[string]string{"mode": c.mode, "enabled": c.enabled} + want := map[string]string{"mode": c.mode, "enabled": c.enabled, "tier": c.tier} if !reflect.DeepEqual(out, want) { t.Errorf("outputs = %v, want %v", out, want) } + // The mint is asked exactly for fork and Dependabot + // pull_request runs not forced local or cache, about this run. + asked := c.in.pr && (c.in.fork || c.in.dependabot) && c.in.rbeInput == "" + if got := strings.Count(log, bazelMintStatusURL); got != map[bool]int{false: 0, true: 1}[asked] { + t.Errorf("mint status asked %d times, want %v:\n%s", got, asked, log) + } + if asked && !strings.Contains(log, bazelMintStatusURL+"beads&run=4242&attempt=2&pr=7123") { + t.Errorf("mint status URL does not name this run:\n%s", log) + } + // A closed gate drops the connection: each try gives up in 5 s. + if asked && !strings.Contains(log, "curl -sS --connect-timeout 5 --max-time 30 ") { + t.Errorf("mint status asked without --connect-timeout 5 --max-time 30:\n%s", log) + } }) } // A value that is not a boolean fails the job rather than picking a mode. if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ - "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "RBE_INPUT_CACHE": "false", "FORK": "", "FORK_FARM": "false", "HAS_EXECUTOR": "true", + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "RBE_INPUT_CACHE": "false", "PULL_REQUEST": "false", "FORK": "", "DEPENDABOT": "false", "FORK_FARM": "false", "HAS_EXECUTOR": "true", }); err == nil { t.Errorf("decision with FORK='' succeeded with %v; want failure", out) } if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ - "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "false", "FORK_FARM": "false", "HAS_EXECUTOR": "true", + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "PULL_REQUEST": "false", "FORK": "false", "DEPENDABOT": "false", "FORK_FARM": "false", "HAS_EXECUTOR": "true", }); err == nil { t.Errorf("decision without RBE_INPUT_CACHE succeeded with %v; want failure", out) } + // A fork pull_request run without a sane PR number fails rather than + // asking the mint about something else. + for _, n := range []string{"", "0", "12a", "1&repo=gascity", "1234567890"} { + if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ + "RBE_VAR_ON": "false", "RBE_INPUT_OFF": "false", "RBE_INPUT_CACHE": "false", "PULL_REQUEST": "true", "FORK": "true", "DEPENDABOT": "false", + "FORK_FARM": "false", "HAS_EXECUTOR": "false", "PR_NUMBER": n, bazelTestMintEnv: "ro", + }); err == nil { + t.Errorf("decision with PR_NUMBER %q succeeded with %v; want failure", n, out) + } + } } +// bazelTestMintEnv selects runBazelRBEDecision's curl stub's answer for +// rbe-fork-mint's /v1/status: ro, rw (open, that tier), closed, rw-closed +// (open false; today's mint answers ro instead while rw is off), canary (403), busy (502), garbage, evil (open with a tier +// that is neither), or anything else (connection refused, as while the +// farm's gate is closed or DNS has no rbe-mint yet). +const bazelTestMintEnv = "BAZEL_TEST_MINT" + +// bazelTestCurlStub stands in for curl on the decision step's PATH: it logs +// its arguments and prints what the mint would. Its exit status is curl's +// (no -f: HTTP errors still print the body and exit 0). +const bazelTestCurlStub = `#!/usr/bin/env bash +printf 'curl %s\n' "$*" >>"$BAZEL_TEST_CURL_LOG" +open='"instance": "oss-fork", "endpoint": "grpcs://rbe-fork.ops.gascity.com:8444"' +case "${BAZEL_TEST_MINT:-}" in +ro) echo "{\"open\": true, \"tier\": \"ro\", $open, \"reason\": \"eligible\"}" ;; +rw) echo '{"open": true, "tier": "rw", "instance": "oss", "endpoint": "grpcs://rbe-fork.ops.gascity.com:8444", "reason": "eligible"}' ;; +closed) echo "{\"open\": false, \"tier\": \"ro\", $open, \"reason\": \"rbe-fork is closed\"}" ;; +rw-closed) echo '{"open": false, "tier": "rw", "instance": "oss", "endpoint": "grpcs://rbe-fork.ops.gascity.com:8444", "reason": "the rw tier is closed"}' ;; +canary) echo '{"error": "rbe-fork canary: this PR is not enabled yet"}' ;; +busy) echo '{"error": "mint busy; retry later"}' ;; +garbage) echo 'bad gateway' ;; +evil) echo '{"open": true, "tier": "admin", "instance": "", "endpoint": "grpcs://elsewhere:1"}' ;; +*) echo "curl: (7) Failed to connect to rbe-mint.ops.gascity.com port 8444" >&2; exit 7 ;; +esac +` + // runBazelRBEDecision runs the rbe job's step script under bash -e (as -// GitHub's default shell) and returns its $GITHUB_OUTPUT. +// GitHub's default shell) and returns its $GITHUB_OUTPUT. curl is +// bazelTestCurlStub (env bazelTestMintEnv picks the mint's answer), so no +// test reaches the network. func runBazelRBEDecision(t *testing.T, script string, env map[string]string) (map[string]string, error) { + t.Helper() + out, _, err := runBazelRBEDecisionLog(t, script, env) + return out, err +} + +// runBazelRBEDecisionLog: runBazelRBEDecision, plus the stub's curl log. +func runBazelRBEDecisionLog(t *testing.T, script string, env map[string]string) (map[string]string, string, error) { t.Helper() dir := t.TempDir() output := filepath.Join(dir, "output") + bin := filepath.Join(dir, "bin") + curlLog := filepath.Join(dir, "curl.log") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "curl"), []byte(bazelTestCurlStub), 0o755); err != nil { + t.Fatal(err) + } cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) - cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_OUTPUT=" + output, "GITHUB_STEP_SUMMARY=" + filepath.Join(dir, "summary")} + cmd.Env = []string{"PATH=" + bin + string(os.PathListSeparator) + os.Getenv("PATH"), "GITHUB_OUTPUT=" + output, "GITHUB_STEP_SUMMARY=" + filepath.Join(dir, "summary"), + "GITHUB_REPOSITORY=gastownhall/beads", "GITHUB_RUN_ID=4242", "GITHUB_RUN_ATTEMPT=2", "BAZEL_TEST_CURL_LOG=" + curlLog} for k, v := range env { cmd.Env = append(cmd.Env, k+"="+v) } - if b, err := cmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("%v: %s", err, b) + b, err := cmd.CombinedOutput() + logData, _ := os.ReadFile(curlLog) + if err != nil { + return nil, string(logData), fmt.Errorf("%v: %s", err, b) } data, err := os.ReadFile(output) if err != nil { - return nil, err + return nil, string(logData), err } out := map[string]string{} for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { k, v, _ := strings.Cut(line, "=") out[k] = v } - return out, nil + return out, string(logData), nil } // write-bazelrc.sh: no secrets means a local-only rc, plus --config=fork-cache diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index a7fdc8051..fb1c5ed87 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -32,8 +32,11 @@ const ( prAllowlistedStep = "Run the Go tests the Bazel lane skips" prScriptsChecksJob = "scripts-go-checks" prScriptsChecksID = "SCRIPTS_GO_CHECKS" - // bazel.yml's lanes for the step add --config=sole-run in mode remote. - bazelSoleRunEnv = "${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }}" + // bazel.yml's lanes for the step add --config=sole-run whenever they + // execute remotely (enabled: mode remote, and the rbe-fork modes + // fork-ro/fork-rw, whose lanes are a fork's only run once + // BAZEL_COVERS_FORKS covers it). + bazelSoleRunEnv = "${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }}" bazelSoleRunArg = `${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"}` bazelSoleRunNoCacheLine = "test:sole-run --nocache_test_results" bazelSoleRunEvictionLine = "test:sole-run --experimental_remote_cache_eviction_retries=0" @@ -284,6 +287,12 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { add("covered, remote, lanes passed, legacy skipped", prGateFor(t, lanes, "pull_request", "remote", cov("true")), true, "") add("not covered, remote, legacy ran", prGateFor(t, lanes, "pull_request", "remote", cov("false")), true, "") add("merge_group, not covered, remote", prGateFor(t, lanes, "merge_group", "remote", cov("false")), true, "") + // rbe-fork: a covered fork or Dependabot PR (BAZEL_COVERS_FORKS) whose + // lanes ran remotely with a mint certificate. + for _, mode := range []string{"fork-ro", "fork-rw"} { + add("covered, mode "+mode+", lanes passed, legacy skipped", prGateFor(t, lanes, "pull_request", mode, cov("true")), true, "") + add("not covered, mode "+mode+", legacy ran", prGateFor(t, lanes, "pull_request", mode, cov("false")), true, "") + } for _, mode := range []string{"local", "cache"} { add("not covered, mode "+mode+", legacy ran", prGateFor(t, lanes, "pull_request", mode, cov("false")), true, "") } @@ -404,8 +413,10 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { detectCond := "steps.detect.outputs." + lane.detectOutput + " == 'true'" detectIf := "${{ " + detectCond + " }}" - bazelPathIf := "${{ " + detectCond + " && needs." + bazelRBEJobName + ".outputs.enabled == 'true' }}" - fallbackIf := "${{ " + detectCond + " && needs." + bazelRBEJobName + ".outputs.enabled != 'true' }}" + // Mode remote, not enabled: the package gates take no rbe-fork + // certificate, so fork modes keep the go build fallback. + bazelPathIf := "${{ " + detectCond + " && needs." + bazelRBEJobName + ".outputs.mode == 'remote' }}" + fallbackIf := "${{ " + detectCond + " && needs." + bazelRBEJobName + ".outputs.mode != 'remote' }}" wantNames := []string{ "", // checkout (no name) "Decide applicability", diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 4dfbbe975..69735c196 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -128,9 +128,16 @@ func retiredJobs() map[string]retiredTier { var prRiskSharedDecisionEnv = []string{"FORK"} // The decision's Dependabot test: Dependabot runs get no Actions secrets, so -// bazel.yml runs them locally and they keep the legacy tiers. +// they are covered (like fork PRs) only while BAZEL_COVERS_FORKS is "true". const prRiskDependabotValue = "${{ github.actor == 'dependabot[bot]' }}" +// The committed flag that covers fork and Dependabot PRs too (rbe-fork), and +// its expression in both decision steps. +const ( + prCoversForksFlag = "BAZEL_COVERS_FORKS" + prCoversForksValue = "${{ env." + prCoversForksFlag + " == 'true' }}" +) + // rbeFacts: what GitHub evaluates the decision steps' env expressions on. type rbeFacts struct { event string // github.event_name @@ -141,18 +148,33 @@ type rbeFacts struct { retired [3]string // github.actor is dependabot[bot] (fixed for a PR's runs and re-runs). dependabot bool + // The committed env.BAZEL_COVERS_FORKS. + coversForks string + // What rbe-fork-mint's /v1/status answers this run (bazelTestMintEnv; + // "" = unreachable). Only fork and Dependabot pull_request runs ask. + mint string } func (f rbeFacts) String() string { - return fmt.Sprintf("event=%s var=%q secret=%v fork=%v retired=%q dependabot=%v", f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot) + return fmt.Sprintf("event=%s var=%q secret=%v fork=%v retired=%q dependabot=%v covers-forks=%q mint=%q", + f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot, f.coversForks, f.mint) } -// covers: whether a same-repo, non-Dependabot pull_request with tier i's -// flag set, the decision both workflows must take for tier i. +// covers: whether a pull_request with tier i's flag set is covered, the +// decision both workflows must take for tier i: same-repo, non-Dependabot +// PRs always; fork and Dependabot PRs while BAZEL_COVERS_FORKS is "true". +// Never what the mint says: a covered fork run it does not serve is red. func (f rbeFacts) covers(i int) bool { - return strings.EqualFold(f.retired[i], "true") && f.event == "pull_request" && !f.fork && !f.dependabot + return strings.EqualFold(f.retired[i], "true") && f.event == "pull_request" && + (!f.fork && !f.dependabot || strings.EqualFold(f.coversForks, "true")) } +// forkPR: a run that asks rbe-fork-mint (bazel.yml's rbe job). +func (f rbeFacts) forkPR() bool { return f.event == "pull_request" && (f.fork || f.dependabot) } + +// rbeMintAnswers: the mint states every forkPR fact is tried with. +var rbeMintAnswers = []string{"", "closed", "ro", "rw"} + var ( rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} @@ -166,15 +188,37 @@ var ( ) // rbeFactsMatrix: every combination of the facts the decisions read. +// BAZEL_COVERS_FORKS takes both values (and other spellings) where it can +// matter, fork and Dependabot facts, and "true" elsewhere only with every +// tier retired (to show it changes nothing there); the mint answers only +// matter to forkPR facts. func rbeFactsMatrix() []rbeFacts { var out []rbeFacts + allRetired := [3]string{"true", "true", "true"} for _, event := range rbeEvents { for _, v := range rbeVarValues { for _, secret := range rbeSecrets { for _, fork := range []bool{false, true} { for _, retired := range retiredValues { for _, dependabot := range []bool{false, true} { - out = append(out, rbeFacts{event, v, secret, fork, retired, dependabot}) + covers := []string{"false"} + switch { + case (fork || dependabot) && retired == allRetired: + covers = []string{"false", "true", "True", "", "yes"} + case fork || dependabot, retired == allRetired: + covers = []string{"false", "true"} + } + for _, c := range covers { + f := rbeFacts{event, v, secret, fork, retired, dependabot, c, ""} + if !f.forkPR() { + out = append(out, f) + continue + } + for _, m := range rbeMintAnswers { + f.mint = m + out = append(out, f) + } + } } } } @@ -206,6 +250,13 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) } } switch expr { + case prCoversForksValue: + return strconv.FormatBool(strings.EqualFold(f.coversForks, "true")) + case "${{ github.event.pull_request.number }}": + if f.event == "pull_request" || f.event == "pull_request_target" { + return "7123" + } + return "" case prRiskPullRequestValue: return strconv.FormatBool(f.event == "pull_request") case prRiskDependabotValue: @@ -232,7 +283,7 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) // with its env evaluated for f, and returns its $GITHUB_OUTPUT. func runDecisionStep(t *testing.T, step ciWorkflowStep, f rbeFacts, with map[string]string) (map[string]string, error) { t.Helper() - env := map[string]string{} + env := map[string]string{bazelTestMintEnv: f.mint} for k, v := range step.Env { env[k] = evalRBEExpr(t, v, f, with) } @@ -279,7 +330,7 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes, sameRepoBlacksmith2vcpu) } wantOutputs := map[string]string{} - wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "DEPENDABOT": prRiskDependabotValue} + wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "DEPENDABOT": prRiskDependabotValue, "COVERS_FORKS": prCoversForksValue} for _, r := range retiredTiers { wantOutputs[r.output] = "${{ steps.decide.outputs." + r.output + " }}" wantEnv[r.envKey] = r.retiredValue() @@ -291,9 +342,13 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { t.Errorf("pr.yml's %s differs from pr-risk.yml's:\n%+v\n%+v", prRiskCoverageJobName, prJob, job) } riskEnv, prEnv := workflowEnv(t, prRiskWorkflowName), workflowEnv(t, "pr.yml") + committed := []string{prCoversForksFlag} for _, r := range retiredTiers { - if riskFlag, prFlag := riskEnv[r.flag], prEnv[r.flag]; riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { - t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", r.flag, riskFlag, prFlag) + committed = append(committed, r.flag) + } + for _, flag := range committed { + if riskFlag, prFlag := riskEnv[flag], prEnv[flag]; riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { + t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", flag, riskFlag, prFlag) } } step := prRiskCoverageStep(t) @@ -318,8 +373,8 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { // repository variable; only the workflow env sets a flag. stepEnv := ".jobs." + prRiskCoverageJobName + ".steps[0].env." secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) - flags := map[string]bool{} - var flagAlt []string + flags := map[string]bool{prCoversForksFlag: true} + flagAlt := []string{prCoversForksFlag} for _, r := range retiredTiers { flags[r.flag] = true flagAlt = append(flagAlt, r.flag) @@ -357,8 +412,8 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { } return } - if regexp.MustCompile(`env\.BAZEL_RETIRES_`).MatchString(value) && !strings.HasPrefix(path, stepEnv) { - t.Errorf("pr.yml: %s reads a BAZEL_RETIRES_* flag (%q); read needs.%s.outputs", path, value, prRiskCoverageJobName) + if regexp.MustCompile(`env\.BAZEL_(RETIRES_|COVERS_FORKS\b)`).MatchString(value) && !strings.HasPrefix(path, stepEnv) { + t.Errorf("pr.yml: %s reads a BAZEL_RETIRES_* or BAZEL_COVERS_FORKS flag (%q); read needs.%s.outputs", path, value, prRiskCoverageJobName) } }) // Every BAZEL_RETIRES_* flag either workflow commits is one of retiredTiers. @@ -384,7 +439,7 @@ func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode strin } return bazelGateScenario{ name: fmt.Sprintf("%s mode %s covered %v", event, mode, covered), event: event, - mode: mode, enabled: strconv.FormatBool(mode == "remote"), call: "success", + mode: mode, enabled: bazelModeEnabled(mode), call: "success", outputs: outputs, covered: covered, } } @@ -501,6 +556,7 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { mode string } modeMemo := map[rbeFacts]string{} + coverMemo := map[rbeFacts][2]map[string]string{} decideMemo := map[rbeFacts]decided{} decide := func(t *testing.T, f rbeFacts) decided { t.Helper() @@ -509,6 +565,7 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { } noFlags := f noFlags.retired = [3]string{} + noFlags.coversForks = "" mode, ok := modeMemo[noFlags] if !ok { bazel, err := runDecisionStep(t, rbeStep, f, call.With) @@ -518,15 +575,25 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { mode = bazel["mode"] modeMemo[noFlags] = mode } - risk, err := runDecisionStep(t, riskStep, f, nil) - if err != nil || len(risk) != len(retiredTiers) { - t.Fatalf("%s decision step: %v %v", prRiskCoverageJobName, risk, err) + // The coverage steps read no variable, secret or mint (their env is + // pinned exactly by TestPRRiskBazelCoverageJob): one run per fact + // without them. + noMint := f + noMint.rbeVar, noMint.secret, noMint.mint = "", "", "" + cov, ok := coverMemo[noMint] + if !ok { + risk, err := runDecisionStep(t, riskStep, noMint, nil) + if err != nil || len(risk) != len(retiredTiers) { + t.Fatalf("%s decision step: %v %v", prRiskCoverageJobName, risk, err) + } + prd, err := runDecisionStep(t, prStep, noMint, nil) + if err != nil { + t.Fatalf("pr.yml %s step: %v", prRiskCoverageJobName, err) + } + cov = [2]map[string]string{risk, prd} + coverMemo[noMint] = cov } - prd, err := runDecisionStep(t, prStep, f, nil) - if err != nil { - t.Fatalf("pr.yml %s step: %v", prRiskCoverageJobName, err) - } - d := decided{risk, prd, mode} + d := decided{cov[0], cov[1], mode} decideMemo[f] = d return d } @@ -568,31 +635,40 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { continue // the events both workflows run on } risk := decide(t, f) + mints := []string{f.mint} + if f.forkPR() { + mints = rbeMintAnswers // rbe-fork may open or close between the runs + } for _, v := range rbeVarValues { for _, secret := range rbeSecrets { - g := f - g.rbeVar, g.secret = v, secret - prRun := decide(t, g) - bazelRan := prRun.mode == "remote" // and passed: every lane succeeds here - for _, r := range retiredTiers { - // PR Risk's tiers stand down in PR Risk's run; pr.yml's - // own (step 3) in this same pr.yml run. - legacyRan := risk.covered[r.output] != "true" - if r.workflow == "pr.yml" { - legacyRan = prRun.prCovered[r.output] != "true" - } - if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { - t.Errorf("PR Risk run %v skipped the legacy %s tier and pr.yml run (var %q, secret %v, mode %s, covered %v) is green without its Bazel lanes", - f, r.output, v, secret != "", prRun.mode, prRun.prCovered) + for _, m := range mints { + g := f + g.rbeVar, g.secret, g.mint = v, secret, m + prRun := decide(t, g) + bazelRan := bazelRemoteModes[prRun.mode] // and passed: every lane succeeds here + for _, r := range retiredTiers { + // PR Risk's tiers stand down in PR Risk's run; pr.yml's + // own (step 3) in this same pr.yml run. + legacyRan := risk.covered[r.output] != "true" + if r.workflow == "pr.yml" { + legacyRan = prRun.prCovered[r.output] != "true" + } + if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { + t.Errorf("PR Risk run %v skipped the legacy %s tier and pr.yml run (var %q, secret %v, mint %q, mode %s, covered %v) is green without its Bazel lanes", + f, r.output, v, secret != "", m, prRun.mode, prRun.prCovered) + } } } } } } - // Named cases, for the record. Forks and Dependabot run in mode cache, - // which, like local, skips the remote-only lanes, so they keep the - // legacy tiers. reds: the retired ids a red gate must name. + // Named cases, for the record. Forks and Dependabot run in mode cache + // while rbe-fork is closed, which, like local, skips the remote-only + // lanes, or remotely in fork-ro/fork-rw; while BAZEL_COVERS_FORKS is + // "false" they keep the legacy tiers either way. While it is "true" they + // are covered: green only with their lanes run remotely, red in mode + // cache. reds: the retired ids a red gate must name. both, embOnly, dsOnly, prOnly, none := [3]string{"true", "true", "true"}, [3]string{"true", "false", "false"}, [3]string{"false", "true", "false"}, [3]string{"false", "false", "true"}, [3]string{"false", "false", "false"} allRetired := []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED", "BAZEL_PR_LANES_RETIRED"} @@ -604,23 +680,36 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { prPasses bool reds []string }{ - {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, both, false}, "remote", coveredAll("true"), true, nil}, - {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, both, false}, "skip", coveredAll("true"), false, allRetired}, - {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, both, false}, "cache", coveredAll("true"), false, allRetired}, - {"same-repo PR, only embedded retired, var unset", rbeFacts{"pull_request", "", "x", false, embOnly, false}, "skip", map[string]string{"embedded": "true", "dolt_server": "false", "pr_lanes": "false"}, false, []string{"BAZEL_EMBEDDED_RETIRED"}}, - {"same-repo PR, only proxied/server retired, var unset", rbeFacts{"pull_request", "", "x", false, dsOnly, false}, "skip", map[string]string{"embedded": "false", "dolt_server": "true", "pr_lanes": "false"}, false, []string{"BAZEL_DOLT_SERVER_RETIRED"}}, - {"same-repo PR, only pr.yml's jobs retired, var unset", rbeFacts{"pull_request", "", "x", false, prOnly, false}, "skip", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, - {"same-repo PR, only pr.yml's jobs retired, secret missing", rbeFacts{"pull_request", "true", "", false, prOnly, false}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, - {"same-repo PR, only pr.yml's jobs retired, farm on", rbeFacts{"pull_request", "true", "x", false, prOnly, false}, "remote", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, true, nil}, - {"same-repo PR, flags reverted, var unset", rbeFacts{"pull_request", "", "x", false, none, false}, "skip", coveredAll("false"), true, nil}, - {"same-repo PR, flags reverted, secret missing", rbeFacts{"pull_request", "true", "", false, none, false}, "cache", coveredAll("false"), true, nil}, - {"fork PR", rbeFacts{"pull_request", "true", "", true, both, false}, "cache", coveredAll("false"), true, nil}, - {"fork PR, var unset", rbeFacts{"pull_request", "", "", true, both, false}, "cache", coveredAll("false"), true, nil}, - {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, both, false}, "cache", coveredAll("false"), true, nil}, - {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, both, true}, "cache", coveredAll("false"), true, nil}, - {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, both, true}, "skip", coveredAll("false"), true, nil}, - {"merge_group", rbeFacts{"merge_group", "true", "x", false, both, false}, "remote", coveredAll("false"), true, nil}, - {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, both, false}, "skip", coveredAll("false"), true, nil}, + {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, both, false, "false", ""}, "remote", coveredAll("true"), true, nil}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, both, false, "false", ""}, "skip", coveredAll("true"), false, allRetired}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, both, false, "false", ""}, "cache", coveredAll("true"), false, allRetired}, + {"same-repo PR, only embedded retired, var unset", rbeFacts{"pull_request", "", "x", false, embOnly, false, "false", ""}, "skip", map[string]string{"embedded": "true", "dolt_server": "false", "pr_lanes": "false"}, false, []string{"BAZEL_EMBEDDED_RETIRED"}}, + {"same-repo PR, only proxied/server retired, var unset", rbeFacts{"pull_request", "", "x", false, dsOnly, false, "false", ""}, "skip", map[string]string{"embedded": "false", "dolt_server": "true", "pr_lanes": "false"}, false, []string{"BAZEL_DOLT_SERVER_RETIRED"}}, + {"same-repo PR, only pr.yml's jobs retired, var unset", rbeFacts{"pull_request", "", "x", false, prOnly, false, "false", ""}, "skip", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, + {"same-repo PR, only pr.yml's jobs retired, secret missing", rbeFacts{"pull_request", "true", "", false, prOnly, false, "false", ""}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, + {"same-repo PR, only pr.yml's jobs retired, farm on", rbeFacts{"pull_request", "true", "x", false, prOnly, false, "false", ""}, "remote", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, true, nil}, + {"same-repo PR, flags reverted, var unset", rbeFacts{"pull_request", "", "x", false, none, false, "false", ""}, "skip", coveredAll("false"), true, nil}, + {"same-repo PR, flags reverted, secret missing", rbeFacts{"pull_request", "true", "", false, none, false, "false", ""}, "cache", coveredAll("false"), true, nil}, + {"fork PR", rbeFacts{"pull_request", "true", "", true, both, false, "false", ""}, "cache", coveredAll("false"), true, nil}, + {"fork PR, var unset", rbeFacts{"pull_request", "", "", true, both, false, "false", ""}, "cache", coveredAll("false"), true, nil}, + {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, both, false, "false", ""}, "cache", coveredAll("false"), true, nil}, + {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, both, true, "false", ""}, "cache", coveredAll("false"), true, nil}, + // rbe-fork decides for Dependabot like for forks (the mint, not the + // farm switch): unreachable here, so cache. + {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, both, true, "false", ""}, "cache", coveredAll("false"), true, nil}, + {"fork PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", true, both, false, "false", "ro"}, "fork-ro", coveredAll("false"), true, nil}, + {"fork PR, rbe-fork rw", rbeFacts{"pull_request", "", "", true, both, false, "false", "rw"}, "fork-rw", coveredAll("false"), true, nil}, + {"fork PR, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, both, false, "false", "closed"}, "cache", coveredAll("false"), true, nil}, + {"Dependabot PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", false, both, true, "false", "ro"}, "fork-ro", coveredAll("false"), true, nil}, + {"covered fork PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", true, both, false, "true", "ro"}, "fork-ro", coveredAll("true"), true, nil}, + {"covered fork PR, rbe-fork rw", rbeFacts{"pull_request", "", "", true, both, false, "true", "rw"}, "fork-rw", coveredAll("true"), true, nil}, + {"covered fork PR, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, both, false, "true", "closed"}, "cache", coveredAll("true"), false, allRetired}, + {"covered fork PR, mint unreachable", rbeFacts{"pull_request", "true", "", true, both, false, "true", ""}, "cache", coveredAll("true"), false, allRetired}, + {"covered Dependabot PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", false, both, true, "true", "ro"}, "fork-ro", coveredAll("true"), true, nil}, + {"covered Dependabot PR, rbe-fork closed", rbeFacts{"pull_request", "", "", false, both, true, "true", "closed"}, "cache", coveredAll("true"), false, allRetired}, + {"covered fork PR, only pr.yml's jobs retired, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, prOnly, false, "true", "closed"}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, + {"merge_group", rbeFacts{"merge_group", "true", "x", false, both, false, "false", ""}, "remote", coveredAll("false"), true, nil}, + {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, both, false, "false", ""}, "skip", coveredAll("false"), true, nil}, } { d := decide(t, c.f) if !reflect.DeepEqual(d.covered, c.covered) || d.mode != c.mode { @@ -679,7 +768,7 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { } } // A value that is not a boolean fails the job rather than deciding. - base := map[string]string{"PULL_REQUEST": "true", "FORK": "false", "DEPENDABOT": "false"} + base := map[string]string{"PULL_REQUEST": "true", "FORK": "false", "DEPENDABOT": "false", "COVERS_FORKS": "false"} for _, r := range retiredTiers { base[r.envKey] = "true" }