diff --git a/.github/scripts/rbe-prewarm.sh b/.github/scripts/rbe-prewarm.sh new file mode 100755 index 000000000..1b4d705f8 --- /dev/null +++ b/.github/scripts/rbe-prewarm.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# bazel.yml's rbe-prewarm job: best-effort pre-warm of gastownhall/gascity's +# rbe-west OSS worker pool (one Blacksmith-hosted NativeLink worker, +# rbe-worker-pool.yml) so capacity is already booting instead of beads' +# Bazel lanes waiting ~120s at "N actions, 0 running" while the pool scales +# from zero. gascity pre-warms its own pool from inside its bazel-test.yml +# job (same-repo dispatch, github.token); beads cannot use its GITHUB_TOKEN +# against gascity, so the rbe-prewarm job mints a short-lived installation +# token for the "bazel-allocator" GitHub App (installed on +# gastownhall/gascity only, Actions read/write + Metadata read, nothing +# else; app id and private key are secrets.RBE_POOL_APP_ID / +# secrets.RBE_POOL_APP_PRIVATE_KEY) and passes it to this script as +# GH_TOKEN. See engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm". +# +# Best effort by design: every failure path (missing token, gascity +# unreachable, gh rate-limited, dispatch rejected) prints a ::warning:: and +# this script still exits 0, so a pre-warm problem never fails the +# rbe-prewarm job, let alone the bazel.yml run it is part of or pr.yml's +# gate (the job is advisory: not a `needs` of any lane, not in ci-gate's +# required list). +# +# Inputs (environment): +# GH_TOKEN gastownhall/gascity-scoped installation token, +# minted by the job's own +# actions/create-github-app-token step (empty +# when the app secrets are not provisioned, or +# when the mint step itself fails). Required for +# every gh call below; empty or invalid fails +# closed into a warning, never a non-zero exit. +# RBE_PREWARM_WORKERS desired worker count N (repo variable; default +# "1" when unset or non-numeric, clamped to +# 1..4). "0" (or any value <= 0 after clamping) +# is the kill switch: pre-warm is disabled. +# RBE_PREWARM_IDLE_MINUTES optional idle_minutes passed to the dispatched +# workflow_dispatch; omitted (gascity's own +# default applies) when unset or empty. +# +# Exits 0 unconditionally. +set -u + +POOL_REPO="gastownhall/gascity" +POOL_WORKFLOW="rbe-worker-pool.yml" +POOL_REF="main" + +n="${RBE_PREWARM_WORKERS:-1}" +case "$n" in + '') n=1 ;; + *[!0-9]*) n=1 ;; +esac +if [ "$n" -gt 4 ]; then + n=4 +fi + +if [ "$n" -le 0 ]; then + echo "RBE_PREWARM_WORKERS=0: rbe-west pre-warm disabled (kill switch)" + exit 0 +fi + +if [ -z "${GH_TOKEN:-}" ]; then + echo "::warning title=rbe-west pre-warm::no dispatch token (bazel-allocator app not provisioned, or the mint step failed); skipping" + exit 0 +fi + +active="$(gh run list -R "$POOL_REPO" --workflow "$POOL_WORKFLOW" --limit 50 \ + --json status --jq '[.[] | select(.status != "completed")] | length' 2>/dev/null)" +case "$active" in + '' | *[!0-9]*) + echo "::warning title=rbe-west pre-warm::could not read $POOL_REPO/$POOL_WORKFLOW run status; skipping dispatch" + exit 0 + ;; +esac + +missing=$((n - active)) +if [ "$missing" -le 0 ]; then + echo "${active} $POOL_REPO/$POOL_WORKFLOW worker(s) already active (want ${n}); nothing to dispatch" + exit 0 +fi + +args=(workflow run "$POOL_WORKFLOW" -R "$POOL_REPO" --ref "$POOL_REF") +if [ -n "${RBE_PREWARM_IDLE_MINUTES:-}" ]; then + args+=(-f "idle_minutes=${RBE_PREWARM_IDLE_MINUTES}") +fi + +dispatched=0 +i=1 +while [ "$i" -le "$missing" ]; do + if gh "${args[@]}"; then + dispatched=$((dispatched + 1)) + else + echo "::warning title=rbe-west pre-warm::gh workflow run $POOL_WORKFLOW -R $POOL_REPO failed (attempt $i/$missing)" + fi + i=$((i + 1)) +done + +echo "dispatched ${dispatched}/${missing} $POOL_REPO/$POOL_WORKFLOW worker(s) (active before: ${active}, want: ${n})" +exit 0 diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index 7828c5465..4aa8efc78 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -128,6 +128,8 @@ jobs: RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} + RBE_POOL_APP_ID: ${{ secrets.RBE_POOL_APP_ID }} + RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }} with: checkout-sha: ${{ github.event.pull_request.head.sha }} fork-farm: authorized diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 4894b4068..890d47222 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -237,6 +237,18 @@ on: required: false RBE_TLS_CA: required: false + # rbe-prewarm only (never read by setup-bazel or the rbe job): the + # "bazel-allocator" GitHub App's id and private key. The app is + # installed on gastownhall/gascity alone with Actions (read/write) and + # Metadata (read) permissions, nothing else; rbe-prewarm mints a + # short-lived installation token from these two and never sees a + # long-lived credential. Unset (either one empty) disables + # pre-warming. See engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west + # Pre-warm". + RBE_POOL_APP_ID: + required: false + RBE_POOL_APP_PRIVATE_KEY: + required: false # The rbe job's decision, and each lane's job.status (success, failure or # cancelled; empty when the lane was skipped or never reported). pr.yml's # ci-gate reads them through .github/scripts/bazel-gate.sh. @@ -274,6 +286,9 @@ on: package-npm: description: "package-npm's job.status (empty unless package-gates is 'on')" value: ${{ jobs.package-npm.outputs.result }} + rbe-prewarm: + description: "rbe-prewarm's job.status (advisory: continue-on-error, never gates or blocks a lane)" + value: ${{ jobs.rbe-prewarm.outputs.result }} # Main runs always finish (they may save the runner cache); PR runs (called # from pr.yml or bazel-farm.yml, whose own groups cancel the calling run too) @@ -384,6 +399,113 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + # Best-effort pre-warm of gastownhall/gascity's rbe-west OSS worker pool + # (one Blacksmith-hosted NativeLink worker, rbe-worker-pool.yml) so it is + # already booting instead of the lanes below waiting ~120s at "N actions, 0 + # running" while the pool scales from zero. A job of its own (not a step in + # `rbe`), so it never delays the mode decision every other job needs, and + # it is never a `needs` of any lane and never in pr.yml's ci-gate required + # list (advisory; .github/scripts/bazel-gate.sh does not read it). See + # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm". + # + # Only when the run's lanes will actually execute on instance "oss": mode + # remote (the CI secrets) or fork-rw (rbe-fork-mint's trusted-author tier, + # cached by the same OSS workers per the header above). Mode fork-ro targets + # instance "oss-fork", a separate, uncached pool rbe-worker-pool.yml does + # not serve, so it is excluded (TestBazelCacheModeReachesTheRC's one named + # exception to "every lane runs remotely" in a fork mode). Like every other + # lane, the `if:` reads only needs.rbe.outputs - TestBazelRBEJobDecidesOnce + # forbids re-deriving fork/event facts outside the rbe job itself - and + # runs-on is the same mode-ternary every other lane uses, so a genuine fork + # or Dependabot PR that reaches mode fork-rw still never touches Blacksmith: + # the runner is only a client there, like every other fork-mode lane (see + # bazel-test's comment above). That PR also carries no repository secrets + # regardless of runs-on (GitHub strips them from a fork's pull_request + # event), so the credential-check step below finds nothing to mint even if + # it runs. Neither app secret (secrets.RBE_POOL_APP_ID, + # secrets.RBE_POOL_APP_PRIVATE_KEY) is ever read in this `if:` or anywhere + # else in this file - only inside this job's own step env/with, same as the + # rbe job's HAS_EXECUTOR pattern - so a missing credential (the app not + # provisioned yet, or the kill switch) fails soft (no mint attempt, then + # the script's own ::warning::, exit 0) rather than changing whether the + # job runs. + rbe-prewarm: + name: pre-warm rbe-west pool + needs: rbe + if: ${{ needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'fork-rw' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 5 + # Job-level, not step-level: the only continue-on-error in this workflow + # (TestBazelWorkflowSecretsAndFailureSurface and + # TestBazelWorkflowJobsAndExecutionMode both carve out this one path by + # name). Every other job here is a gated lane whose real failure must + # reach pr.yml's ci-gate; this one is advisory and must never fail the + # call bazel.yml's own run represents (a job's failure fails the whole + # reusable-workflow run regardless of `needs`, which would otherwise turn + # pr.yml's ci-gate red for a pre-warm hiccup nothing requires). + continue-on-error: true + outputs: + result: ${{ steps.result.outputs.result }} + steps: + # Only the dispatch script, never the rest of the PR (same pinned-ref + # and allow-unsafe-pr-checkout pattern as every other job; see the + # header). No token on disk: persist-credentials is false. + - name: Checkout the pre-warm script + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + sparse-checkout: | + .github/scripts/rbe-prewarm.sh + sparse-checkout-cone-mode: false + + # Same env-boolean pattern as the rbe job's HAS_EXECUTOR: the secret is + # tested for emptiness, never referenced in an `if:` directly, and the + # next step's `if:` reads this step's output (a job-level env can't be + # read from another step's `if:`, only its own `env:`). + - name: Check for the pre-warm app credential + id: has-app + env: + HAS_POOL_APP: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }} + run: echo "has-app=$HAS_POOL_APP" >> "$GITHUB_OUTPUT" + + # Mints a short-lived installation token for the "bazel-allocator" + # GitHub App (installed on gastownhall/gascity only, Actions + # read/write + Metadata read): skipped entirely while the app secrets + # are not provisioned, leaving steps.mint.outputs.token empty, which + # the dispatch step and script below treat the same as any other + # missing-token case (::warning::, exit 0). permission-actions narrows + # the minted token to Actions even if the app is ever granted more. + - name: Mint gastownhall/gascity installation token + id: mint + if: ${{ steps.has-app.outputs.has-app == 'true' }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.RBE_POOL_APP_ID }} + private-key: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }} + owner: gastownhall + repositories: gascity + permission-actions: write + + - name: Pre-warm rbe-west OSS worker pool (gastownhall/gascity) + env: + GH_TOKEN: ${{ steps.mint.outputs.token }} + RBE_PREWARM_WORKERS: ${{ vars.RBE_PREWARM_WORKERS }} + run: bash .github/scripts/rbe-prewarm.sh + + # Every lane reports its job.status from an always() last step + # (TestBazelLaneIsGatedAlongsideLegacy). The script above always exits + # 0, so this normally reads success regardless of what it found; the + # job-level continue-on-error above is the backstop for a step that + # fails outside the script (checkout, the mint action). + - name: Record result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + bazel-test: name: test # Remote-execution runs use the org's Blacksmith pool (the farm's edge diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a1e1355ad..f6d403e4b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -109,6 +109,8 @@ jobs: RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} + RBE_POOL_APP_ID: ${{ secrets.RBE_POOL_APP_ID }} + RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }} with: details: true go-test-json-artifact: bazel-equivalence-go-test-json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ca162c7e6..acf7ab4f6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -2268,6 +2268,8 @@ jobs: RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} + RBE_POOL_APP_ID: ${{ secrets.RBE_POOL_APP_ID }} + RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }} with: build-artifact-name: bazel-ci-build-artifacts integration: "on" diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index b5811b291..1c506f545 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -468,6 +468,69 @@ Only a GitHub run can verify these: it doesn't). - An unlisted author, or a push by an unlisted collaborator, skips `farm`. +## rbe-west Pre-warm + +`bazel.yml`'s `rbe-prewarm` job is a best-effort attempt to have +gastownhall/gascity's rbe-west OSS worker pool (one Blacksmith-hosted +NativeLink worker, driven by gascity's own `rbe-worker-pool.yml` +`workflow_dispatch`) already booting by the time this run's Bazel lanes need +it, instead of each of them separately waiting out the ~120s it takes +NativeLink to notice demand and scale the pool up from zero. gascity dispatches +its own pool from inside its own `bazel-test.yml` job, same-repo, with +`github.token`; beads cannot use its own `GITHUB_TOKEN` against gascity's +repository, so this job needs its own credential into gascity instead. + +**What it does.** A job of its own (not a step inside `rbe`, so it never +delays the mode decision every lane waits on), gated on +`needs.rbe.outputs.mode` being `remote` or `fork-rw` - the two modes whose +lanes actually target rbe-west's `oss` instance, which is what +`rbe-worker-pool.yml` serves. (Mode `fork-ro` targets a separate, uncached +`oss-fork` instance this pool does not serve, so it is excluded.) When +scheduled, it mints a short-lived GitHub App installation token and uses it to +check gastownhall/gascity's current `rbe-worker-pool.yml` run count and, if +under the desired worker count, dispatch enough new runs to reach it. Every +failure path - no credential, gascity unreachable, `gh` rate-limited, the +dispatch itself rejected - prints a `::warning::` and the script still exits +0: this job never gates anything (it is not a `needs` of any lane, and it is +never in `.github/scripts/bazel-gate.sh`'s vocabulary, so pr.yml's `ci-gate` +never looks at it), and it cannot fail the run either, since a called reusable +workflow's overall conclusion is "failure" if any job inside it fails +regardless of `needs` - the job itself therefore carries bazel.yml's one +deliberate `continue-on-error: true`. + +**Credential scope.** A GitHub App named "bazel-allocator" is installed on +gastownhall/gascity alone, granted exactly two permissions: Actions +(read/write) and Metadata (read). bazel.yml's own secrets never include a +long-lived credential for it - only the app's id and private key +(`RBE_POOL_APP_ID`, `RBE_POOL_APP_PRIVATE_KEY`), from which the job mints a +short-lived installation token (`actions/create-github-app-token`, pinned to a +released commit SHA) scoped with `owner: gastownhall` and +`repositories: gascity`, so the minted token can act on gascity and nothing +else in the gastownhall org. That token is used as `GH_TOKEN` for the +`gh run list` / `gh workflow run` calls against gastownhall/gascity's +`rbe-worker-pool.yml` on `main`, and nowhere else. Every caller of bazel.yml +(pr.yml, bazel-farm.yml, nightly.yml) passes these two secrets straight +through like the four RBE secrets; a fork or Dependabot `pull_request` run +gets neither (GitHub never forwards repository secrets to a fork's +`pull_request` event), so even a trusted-author fork PR that reaches mode +`fork-rw` finds no credential to mint and the job no-ops. + +**Secret names.** + +- `RBE_POOL_APP_ID` - the bazel-allocator App's id. +- `RBE_POOL_APP_PRIVATE_KEY` - the App's private key (PEM). + +**Kill switch.** Pre-warming disables itself cleanly, without touching the +job's `if:`, in either of two ways: + +- Leave `RBE_POOL_APP_PRIVATE_KEY` unset (or clear it): the job's own + `HAS_POOL_APP` check (the same env-boolean pattern as the `rbe` job's + `HAS_EXECUTOR`) skips the mint step, the dispatch script finds no + `GH_TOKEN`, warns, and exits 0. This is also the job's natural state before + the App is provisioned at all. +- Set the repository variable `RBE_PREWARM_WORKERS` to `0`: the script's own + kill switch, checked before any network call. + ## Required Check Contract After the aggregate checks are verified on the branch, branch protection or the diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 357d5af87..0e7660037 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -22,6 +22,7 @@ go_test( "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", "bazel_policy_test.go", + "bazel_rbe_prewarm_test.go", "bazel_skip_allowlist_test.go", "check_build_tags_test.go", "check_doc_freshness_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 83c09e70b..188f6ba6b 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -205,7 +205,11 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { } runs := bazelLaneRunModes(t, name, job.If, s.with)[mode] switch { - case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name]: + // rbe-prewarm targets instance "oss" only (modes remote and + // fork-rw); fork-ro targets the separate "oss-fork" instance + // rbe-worker-pool.yml does not serve, so it is excluded by + // design (see bazel.yml's comment on the job). + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && mode == "fork-ro"): t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && name == bazelIntegJobName && !runs: t.Errorf("%s does not run in mode cache", name) diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index a36d9fb99..0efe0bc2d 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -251,6 +251,13 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { if bazelPackageJobs[name] { want["fetch-depth"] = "0" } + // rbe-prewarm only needs its own dispatch script, never the rest + // of the PR tree; a sparse, non-cone checkout of exactly that one + // path keeps it as cheap as every other lane's shallow checkout. + if name == bazelRBEPrewarmJobName { + want["sparse-checkout"] = ".github/scripts/rbe-prewarm.sh\n" + want["sparse-checkout-cone-mode"] = "false" + } if !reflect.DeepEqual(step.With, want) { t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) } diff --git a/scripts/bazel_rbe_prewarm_test.go b/scripts/bazel_rbe_prewarm_test.go new file mode 100644 index 000000000..292da8993 --- /dev/null +++ b/scripts/bazel_rbe_prewarm_test.go @@ -0,0 +1,266 @@ +package scripts_test + +import ( + "path/filepath" + "regexp" + "strings" + "testing" +) + +// This file exists alongside ci_workflow_test.go's broader structural checks +// (TestBazelWorkflowJobsAndExecutionMode, TestBazelWorkflowSecretsAndFailureSurface, +// TestBazelWorkflowActionsArePinned, TestBazelRBEJobDecidesOnce, +// TestBazelGateSimulation, TestBazelCacheModeReachesTheRC) to pin, by name, +// the specific invariants engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md's "rbe-west +// Pre-warm" section promises a reader: the job never runs for an untrusted +// fork or on a mode rbe-worker-pool.yml cannot serve, its credentials are +// never readable outside its own steps, it is never required by pr.yml's +// gate or a `needs` of any lane, it cannot fail the run, and its dispatch is +// pinned to exactly gastownhall/gascity's rbe-worker-pool.yml on main with a +// token scoped to that one repository. +// +// Each test below was mutation-tested by hand while this job was written: +// reverting the fix it pins (restoring the old event-gated `if:`, widening +// the mint's `repositories:`, dropping the job-level continue-on-error, +// adding rbe-prewarm to bazel-gate.sh's skip/aggregate logic or to another +// job's `needs:`) reliably fails the corresponding test here. See this +// task's notes file for the transcript. + +// TestRBEPrewarmIfOnlyRemoteAndForkRW runs the job's real, pinned `if:` +// expression (bazelRBEPrewarmIf, via the shared evalGHExpr from +// ci_blacksmith_runner_test.go, not a hand-written mirror) against every mode +// the rbe job can produce. Only remote and fork-rw must schedule the job: +// fork-ro targets rbe-west's separate, uncached "oss-fork" instance (never +// served by rbe-worker-pool.yml), and cache/local/skip mean nothing to +// pre-warm. fork-rw is deliberately included - a trusted fork author's PR, +// per rbe-fork-mint's tier decision, not just bazel-farm.yml's +// pull_request_target path - and a fork or Dependabot pull_request is exactly +// how that mode is reached, so this doubles as the fork/Dependabot +// reachability check the original spec asked for: such a run only matters +// here through needs.rbe.outputs.mode, never through re-inspecting +// github.actor or head.repo.fork in this job's own `if:` (that ban is +// TestBazelRBEJobDecidesOnce's job). +func TestRBEPrewarmIfOnlyRemoteAndForkRW(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName) + if job.If != bazelRBEPrewarmIf { + t.Fatalf("rbe-prewarm if = %q, want the pinned %q", job.If, bazelRBEPrewarmIf) + } + for _, tc := range []struct { + mode string + want bool + }{ + {"remote", true}, + {"fork-rw", true}, + {"fork-ro", false}, + {"cache", false}, + {"local", false}, + {"skip", false}, + } { + v, err := evalGHExpr(job.If, map[string]string{"needs.rbe.outputs.mode": tc.mode}) + if err != nil { + t.Fatalf("evalGHExpr(%q) mode=%s: %v", job.If, tc.mode, err) + } + if ghTruthy(v) != tc.want { + t.Errorf("rbe-prewarm if, mode=%s: evaluated to %v, want %v", tc.mode, v, tc.want) + } + } +} + +// TestRBEPrewarmNeverGatedOrNeeded: no lane depends on rbe-prewarm (a +// `needs:` cycle or ordering dependency would turn "advisory" into "gating"), +// and neither of pr.yml's ci-gate paths - the required-checks list it reads +// from .github/scripts/ci-gate.sh/pr-policy, nor bazel-gate.sh's own +// skip/aggregate vocabulary - ever mentions it by name. +func TestRBEPrewarmNeverGatedOrNeeded(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + for name, job := range workflow.Jobs { + if name == bazelRBEPrewarmJobName { + continue + } + for _, need := range job.Needs { + if need == bazelRBEPrewarmJobName { + t.Errorf("%s needs %s; rbe-prewarm is advisory and must never gate another lane", name, bazelRBEPrewarmJobName) + } + } + } + + root := sourceRepoRoot(t) + gate := readPolicyFile(t, root, filepath.Join(".github", "scripts", "bazel-gate.sh")) + if strings.Contains(gate, bazelRBEPrewarmJobName) { + t.Errorf("bazel-gate.sh mentions %s; it must stay outside the gate's skip/aggregate vocabulary", bazelRBEPrewarmJobName) + } + + // pr.yml's ci-gate required-check id list: BAZEL* ids come from + // bazel-gate.sh's own vocabulary (BAZEL, BAZEL_TEST, ...), never a + // per-job id for rbe-prewarm. + ciGate := readPolicyFile(t, root, filepath.Join(".github", "scripts", "ci-gate.sh")) + if regexp.MustCompile(`(?i)rbe.prewarm`).MatchString(ciGate) { + t.Errorf("ci-gate.sh mentions rbe-prewarm; it must never be a required check") + } +} + +// TestRBEPrewarmNeverFailsTheRun: the job carries the one deliberate +// continue-on-error in bazel.yml (every other occurrence is banned by +// TestBazelWorkflowSecretsAndFailureSurface), and the dispatch script it +// runs has no path that exits non-zero. +func TestRBEPrewarmNeverFailsTheRun(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName) + if !job.ContinueOnError { + t.Errorf("rbe-prewarm job.continue-on-error = false; a real step failure would fail bazel.yml's own run and cascade into pr.yml's ci-gate") + } + for _, step := range job.Steps { + if v, ok := step.ContinueOnError.(bool); ok && v { + t.Errorf("rbe-prewarm step %q has its own continue-on-error; only the job level should (TestBazelWorkflowSecretsAndFailureSurface bans the key everywhere else)", step.Name) + } + } + + root := sourceRepoRoot(t) + script := readPolicyFile(t, root, filepath.Join(".github", "scripts", "rbe-prewarm.sh")) + if !strings.Contains(script, "set -u") || strings.Contains(script, "set -e") { + t.Errorf("rbe-prewarm.sh must run under set -u only (no set -e/pipefail): every gh/curl failure path is handled explicitly and must fall through to the trailing exit 0") + } + exits := regexp.MustCompile(`(?m)^\s*exit\s+(\S+)`).FindAllStringSubmatch(script, -1) + if len(exits) == 0 { + t.Fatalf("rbe-prewarm.sh has no exit statement; expected only exit 0") + } + for _, m := range exits { + if m[1] != "0" { + t.Errorf("rbe-prewarm.sh has %q; every exit must be exit 0 (best-effort by design)", strings.TrimSpace(m[0])) + } + } +} + +// TestRBEPrewarmSecretsOnlyInItsOwnJob walks the raw YAML of bazel.yml and +// confirms the two app-credential secrets are referenced only inside the +// rbe-prewarm job (its HAS_POOL_APP env check and the mint step's `with:`), +// never anywhere else in the file - not another job's env, not a workflow +// top-level env, not an `if:`. +func TestRBEPrewarmSecretsOnlyInItsOwnJob(t *testing.T) { + secretRef := regexp.MustCompile(`\bsecrets\.RBE_POOL_APP_(ID|PRIVATE_KEY)\b`) + jobPrefix := ".jobs." + bazelRBEPrewarmJobName + "." + found := 0 + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) { + if key || !secretRef.MatchString(value) { + return + } + found++ + if !strings.HasPrefix(path, jobPrefix) { + t.Errorf("%s: %s references an rbe-prewarm app credential outside the job (prefix %q)", bazelWorkflowName, path, jobPrefix) + } + }) + if found == 0 { + t.Fatalf("found no secrets.RBE_POOL_APP_* reference in %s; the test fixture or the job moved", bazelWorkflowName) + } +} + +// TestRBEPrewarmAppTokenScoped pins the mint step to the exact "bazel- +// allocator" installation-token shape the docs promise: the action pinned to +// a full commit SHA with its version comment (same SHA this repo already +// uses for the same action in update-flake-lock.yml), scoped to +// gastownhall/gascity alone (never, say, the whole gastownhall org or an +// unrelated repo), and narrowed to the Actions permission. +func TestRBEPrewarmAppTokenScoped(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName) + mint := job.step(t, "Mint gastownhall/gascity installation token") + + family, sha, found := strings.Cut(mint.Uses, "@") + if family != "actions/create-github-app-token" { + t.Fatalf("mint step uses %q, want actions/create-github-app-token", mint.Uses) + } + if !found || !actionPin.MatchString(sha) { + t.Errorf("mint step action %q is not pinned to a 40-hex commit SHA", mint.Uses) + } + const wantSHA = "bcd2ba49218906704ab6c1aa796996da409d3eb1" // v3, same as update-flake-lock.yml + if sha != wantSHA { + t.Errorf("mint step action SHA = %q, want %q (update-flake-lock.yml's pin for the same action)", sha, wantSHA) + } + + if got := mint.With["owner"]; got != "gastownhall" { + t.Errorf("mint step owner = %q, want %q", got, "gastownhall") + } + if got := mint.With["repositories"]; got != "gascity" { + t.Errorf("mint step repositories = %q, want exactly %q (not a list, not the whole org)", got, "gascity") + } + if got := mint.With["app-id"]; got != "${{ secrets.RBE_POOL_APP_ID }}" { + t.Errorf("mint step app-id = %q, want the RBE_POOL_APP_ID secret", got) + } + if got := mint.With["private-key"]; got != "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}" { + t.Errorf("mint step private-key = %q, want the RBE_POOL_APP_PRIVATE_KEY secret", got) + } + if got := mint.With["permission-actions"]; got != "write" { + t.Errorf("mint step permission-actions = %q, want %q (narrow even if the app is ever granted more)", got, "write") + } + // Gated on the credential check, not unconditional: TestRBEPrewarmGatedOnAppSecret. + if mint.If != "${{ steps.has-app.outputs.has-app == 'true' }}" { + t.Errorf("mint step if = %q, want it gated on the has-app credential check", mint.If) + } +} + +// TestRBEPrewarmGatedOnAppSecret: the HAS_POOL_APP pattern (mirroring the rbe +// job's own HAS_EXECUTOR) - the mint step is skipped, not failed, whenever +// either app secret is unset, which is also the kill switch's main path +// (before the app is provisioned at all). +func TestRBEPrewarmGatedOnAppSecret(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName) + check := job.step(t, "Check for the pre-warm app credential") + if check.Env["HAS_POOL_APP"] != "${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }}" { + t.Errorf("has-app step env HAS_POOL_APP = %q, want an emptiness test on RBE_POOL_APP_PRIVATE_KEY", check.Env["HAS_POOL_APP"]) + } + if check.ID != "has-app" { + t.Errorf("has-app step id = %q, want %q", check.ID, "has-app") + } +} + +// TestRBEPrewarmDispatchTargetPinned: the script's three target constants are +// literal (never a variable the caller or a repo var could redirect), and the +// only gh workflow/run subcommands in the script reference them, never a +// hardcoded alternative. +func TestRBEPrewarmDispatchTargetPinned(t *testing.T) { + root := sourceRepoRoot(t) + script := readPolicyFile(t, root, filepath.Join(".github", "scripts", "rbe-prewarm.sh")) + + for _, want := range []string{ + `POOL_REPO="gastownhall/gascity"`, + `POOL_WORKFLOW="rbe-worker-pool.yml"`, + `POOL_REF="main"`, + } { + if !strings.Contains(script, want) { + t.Errorf("rbe-prewarm.sh does not contain %q", want) + } + } + + ghCalls := regexp.MustCompile(`(?m)^\s*(gh |args=\()`).FindAllString(script, -1) + if len(ghCalls) == 0 { + t.Fatalf("rbe-prewarm.sh has no gh invocation; the dispatch logic moved") + } + // Every gh run-list/workflow-run invocation must use $POOL_REPO and + // $POOL_WORKFLOW, never a literal repo or workflow file. + ghLine := regexp.MustCompile(`(?m)^.*\bgh\b.*$`) + for _, line := range ghLine.FindAllString(script, -1) { + if strings.Contains(line, "run list") || strings.Contains(line, "workflow run") || strings.Contains(line, `args=(workflow`) { + if !strings.Contains(line, "POOL_REPO") && !strings.Contains(line, `"${args[@]}"`) { + t.Errorf("rbe-prewarm.sh gh invocation does not reference $POOL_REPO: %q", strings.TrimSpace(line)) + } + } + } +} + +// TestRBEPrewarmTokenIsGHToken: the dispatch step hands the script the +// mint step's own output, never a long-lived or ambient credential +// (secrets.GITHUB_TOKEN, a PAT, or the mint step's app-id/private-key +// directly). +func TestRBEPrewarmTokenIsGHToken(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName) + dispatch := job.step(t, "Pre-warm rbe-west OSS worker pool (gastownhall/gascity)") + if got := dispatch.Env["GH_TOKEN"]; got != "${{ steps.mint.outputs.token }}" { + t.Errorf("dispatch step env GH_TOKEN = %q, want the mint step's own token output", got) + } + if strings.Contains(dispatch.Env["GH_TOKEN"], "secrets.") { + t.Errorf("dispatch step GH_TOKEN reads a secret directly; it must use the minted installation token") + } +} + +// actionPin and readYAMLNode/walkYAML/evalGHExpr/ghTruthy/readCIWorkflow/ +// readPolicyFile/sourceRepoRoot are shared helpers already defined in +// ci_workflow_test.go and ci_blacksmith_runner_test.go; this file adds no new +// infrastructure of its own. diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 68de8ef6f..19a34ff68 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2471,6 +2471,10 @@ const ( // config; pr.yml opts them in with package-gates: "on". bazelPackageMCPJobName = "package-mcp" bazelPackageNPMJobName = "package-npm" + // Best-effort, advisory pre-warm of gastownhall/gascity's rbe-west OSS + // worker pool; see bazel.yml's own comment and + // engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm". + bazelRBEPrewarmJobName = "rbe-prewarm" setupBazelActionDir = ".github/actions/setup-bazel" ) @@ -2495,7 +2499,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName} +var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -2509,6 +2513,25 @@ const ( bazelRBESecretValue = "${{ secrets.RBE_WEST_EXECUTOR != '' }}" ) +// rbe-prewarm reads its two app secrets in exactly three places, mirroring +// the rbe job's own emptiness-test/mint pattern above: an env boolean +// (never an `if:`) gating the mint step, and the mint step's own `with:` +// (the only step in bazel.yml allowed to read a secret outside a +// setup-bazel env, alongside bazelRBESecretPath/Value). Step indices are +// steps[0]=checkout, [1]=credential check, [2]=mint, [3]=dispatch, +// [4]=record result. +const ( + bazelRBEPrewarmHasAppPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[1].env.HAS_POOL_APP" + bazelRBEPrewarmHasAppValue = "${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }}" + bazelRBEPrewarmAppIDPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[2].with.app-id" + bazelRBEPrewarmAppIDValue = "${{ secrets.RBE_POOL_APP_ID }}" + bazelRBEPrewarmKeyPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[2].with.private-key" + bazelRBEPrewarmKeyValue = "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}" + // The job-level continue-on-error path itself: see + // TestBazelWorkflowJobsAndExecutionMode's comment on the same exception. + bazelRBEPrewarmContinueOnErrorPath = ".jobs." + bazelRBEPrewarmJobName + ".continue-on-error" +) + // rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes // fork-ro and fork-rw only), and the mint's status URL the rbe job probes. const ( @@ -2546,9 +2569,13 @@ var bazelLaneGateIDs = map[string]string{ bazelPackageNPMJobName: "PACKAGE_NPM", } -// None today: every lane is gated. Kept so a future lane that pr.yml's call -// turns off has a place to record why. -var bazelAdvisoryLanes = map[string]string{} +// rbe-prewarm: not a build/test lane at all (no Bazel config mirrors it), so +// it has nothing for ci-gate to require; it is advisory by design +// (continue-on-error, best-effort pre-warm dispatch) and reports only so +// TestBazelLaneIsGatedAlongsideLegacy's generic shape check still applies. +var bazelAdvisoryLanes = map[string]string{ + bazelRBEPrewarmJobName: "best-effort pre-warm dispatch to gastownhall/gascity; never gates or blocks a lane", +} // bazel-integration's if: remote (modes remote, fork-ro, fork-rw: enabled), // or local with the read-only cache (mode cache: fork and Dependabot PRs @@ -2572,6 +2599,18 @@ const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" // (enabled too) build bd with go build on a GitHub-hosted runner, as before. const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" +// rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): unlike +// every other lane, it runs only when the lanes below will actually target +// instance "oss" (modes remote and fork-rw; fork-ro targets the separate, +// uncached "oss-fork" instance rbe-worker-pool.yml does not serve - see +// bazel.yml's own comment above the job). Its runs-on reuses the exact same +// mode-ternary every other lane uses (wantRunsOn): TestBazelRBEJobDecidesOnce +// forbids re-deriving fork/event facts outside the rbe job, and a fork-rw run +// (a genuine fork/Dependabot pull_request, whose workflow file content is the +// fork's own) must never be hardcoded onto the trusted same-repo Blacksmith +// runner. +const bazelRBEPrewarmIf = "${{ needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'fork-rw' }}" + // F3: the rbe job's own runner (not gated by its own outputs - it decides // them). Same-repo PRs and merge_group/push/dispatch/schedule (never forks) // get Blacksmith; a pull_request_target farm run and any fork or Dependabot @@ -2653,12 +2692,17 @@ var bazelRemoteModes = map[string]bool{"remote": true, "fork-ro": true, "fork-rw // bazelModeEnabled: the rbe job's enabled output for a mode. func bazelModeEnabled(mode string) string { return strconv.FormatBool(bazelRemoteModes[mode]) } -// The four RBE secrets, the only ones a caller may hand bazel.yml. +// The four RBE secrets plus the rbe-prewarm job's "bazel-allocator" GitHub +// App id and private key (it mints its own gastownhall/gascity installation +// token from these; see rbe-prewarm.sh), the only ones a caller may hand +// bazel.yml. var bazelCallSecrets = map[string]string{ - "RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}", - "RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}", - "RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}", - "RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}", + "RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}", + "RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}", + "RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}", + "RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}", + "RBE_POOL_APP_ID": "${{ secrets.RBE_POOL_APP_ID }}", + "RBE_POOL_APP_PRIVATE_KEY": "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}", } type ciCompositeAction struct { @@ -2693,7 +2737,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { var names []string for name, job := range workflow.Jobs { names = append(names, name) - if job.ContinueOnError { + // rbe-prewarm is the one deliberate exception: it is advisory (never + // a `needs` of any lane, never in ci-gate's required list - see + // bazelAdvisoryLanes and TestBazelLaneIsGatedAlongsideLegacy), and a + // called reusable workflow's overall conclusion is "failure" if ANY + // job within it fails regardless of `needs`, so without job-level + // continue-on-error here a pre-warm hiccup would still turn + // bazel.yml's own run - and thus pr.yml's ci-gate - red for a job + // nothing requires. + if job.ContinueOnError && name != bazelRBEPrewarmJobName { t.Errorf("%s continue-on-error hides failures from pr.yml's ci-gate", name) } if job.TimeoutMinutes == 0 { @@ -2760,6 +2812,12 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { wantJobIf = wantRemoteOnlyIf } else if name == bazelIntegJobName { wantJobIf = bazelIntegIf + } else if name == bazelRBEPrewarmJobName { + // Same runs-on ternary as every other lane (wantRunsOn, set + // above); only the if differs. Not a lane: never uses + // setup-bazel, so wantJobSetupEnv is moot (the per-step check + // below only fires for a setup-bazel step). + wantJobIf = bazelRBEPrewarmIf } if job.RunsOn != wantJobRunsOn { t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantJobRunsOn) @@ -3560,6 +3618,13 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true, "local": true, "skip": true} } return map[string]bool{} + case bazelRBEPrewarmIf: + // Same two modes as every lane that targets instance "oss" + // (TestBazelWorkflowJobsAndExecutionMode's comment on the job): + // remote and fork-rw. Unlike the historical advisory lanes, this one + // legitimately runs in pr.yml's own call (see bazelAdvisoryLanes' + // name-specific exception in TestBazelGateSimulation below). + return map[string]bool{"remote": true, "fork-rw": true} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -3688,9 +3753,13 @@ func TestBazelGateSimulation(t *testing.T) { continue } lanes[name] = bazelLaneRunModes(t, name, job.If, callWith) - // Advisory lanes are not in the PR call; gated lanes run at least - // when remote. - if _, advisory := bazelAdvisoryLanes[name]; advisory && len(lanes[name]) != 0 { + // Advisory lanes are normally not in the PR call at all; the one + // exception is rbe-prewarm, which is advisory (never gated, never a + // `needs`: TestBazelLaneIsGatedAlongsideLegacy) but, unlike every + // other advisory lane this map has ever held, is deliberately part + // of every pr.yml call so it can dispatch before the lanes below + // need capacity - see bazel.yml's own comment on the job. + if _, advisory := bazelAdvisoryLanes[name]; advisory && name != bazelRBEPrewarmJobName && len(lanes[name]) != 0 { t.Errorf("advisory lane %s runs in pr.yml's call in modes %v; turn it off there (%s)", name, lanes[name], bazelAdvisoryLanes[name]) } if _, gated := bazelLaneGateIDs[name]; gated && !lanes[name]["remote"] { @@ -4005,6 +4074,9 @@ func TestBazelWorkflowActionsArePinned(t *testing.T) { cacheSaveActionFamily: cacheSHA, "actions/upload-artifact": uploadArtifactSHA, "actions/download-artifact": downloadArtifactSHA, + // rbe-prewarm's mint step: same SHA/comment already used in this repo + // for the same action, in update-flake-lock.yml. + "actions/create-github-app-token": "bcd2ba49218906704ab6c1aa796996da409d3eb1", } for file, list := range steps { for _, step := range list { @@ -5285,8 +5357,8 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { setupSteps := map[string]bool{} for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { - if name == bazelRBEJobName { - continue // no setup-bazel; its one secret read is pinned below + if name == bazelRBEJobName || name == bazelRBEPrewarmJobName { + continue // no setup-bazel; rbe-prewarm's secret reads are pinned below } n := 0 for i, step := range job.Steps { @@ -5302,7 +5374,14 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { - t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) + // rbe-prewarm's job-level continue-on-error is the one + // deliberate exception (TestBazelWorkflowJobsAndExecutionMode's + // comment on the job has the full reasoning): without it, this + // advisory job's failure would still fail bazel.yml's own run + // and cascade into pr.yml's ci-gate. + if path != bazelRBEPrewarmContinueOnErrorPath { + t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) + } } if key && (value == "secrets" && strings.HasPrefix(path, ".jobs.")) { t.Errorf("%s: %s passes secrets to a called workflow or container", bazelWorkflowName, path) @@ -5313,6 +5392,12 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { if path == bazelRBESecretPath && value == bazelRBESecretValue { return // the rbe job's emptiness test (TestBazelRBEJobDecidesOnce) } + switch { + case path == bazelRBEPrewarmHasAppPath && value == bazelRBEPrewarmHasAppValue, + path == bazelRBEPrewarmAppIDPath && value == bazelRBEPrewarmAppIDValue, + path == bazelRBEPrewarmKeyPath && value == bazelRBEPrewarmKeyValue: + return // rbe-prewarm's own credential-check/mint step (see their doc comment) + } prefix := path[:strings.LastIndex(path, ".")+1] if !setupSteps[prefix] { t.Errorf("%s: %s reads secrets (%q); only the setup-bazel step's env may", bazelWorkflowName, path, value)