ci(bazel): best-effort pre-warm rbe-west's OSS worker pool on PR open
bazel.yml's Bazel lanes each separately wait ~120s at "N actions, 0
running" while gastownhall/gascity's rbe-west NativeLink worker pool
scales up from zero. Add an advisory rbe-prewarm job (never a `needs`
of any lane, never in bazel-gate.sh's vocabulary, job-level
continue-on-error so it can never fail bazel.yml's own run) that
mints a short-lived installation token for a "bazel-allocator" GitHub
App (installed on gastownhall/gascity only, Actions read/write +
Metadata read) and dispatches gastownhall/gascity's
rbe-worker-pool.yml ahead of time, only when this run's lanes will
actually target rbe-west's "oss" instance (modes remote and fork-rw;
fork-ro targets a separate pool this doesn't serve).
Adds RBE_POOL_APP_ID / RBE_POOL_APP_PRIVATE_KEY as new optional
workflow_call secrets, threaded through every caller (pr.yml,
bazel-farm.yml, nightly.yml); either secret missing (the app not yet
provisioned) or repo var RBE_PREWARM_WORKERS=0 disables pre-warming
cleanly. Policy tests (scripts/bazel_rbe_prewarm_test.go, plus updates
to the existing bazel.yml structural tests) pin the mode gate, the
credential scope, the dispatch target, and that the job can never gate
or fail a run; each invariant was hand mutation-tested. Docs added to
engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md ("rbe-west Pre-warm").
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
84ce244857
commit
bdef342d59
11 changed files with 667 additions and 17 deletions
96
.github/scripts/rbe-prewarm.sh
vendored
Executable file
96
.github/scripts/rbe-prewarm.sh
vendored
Executable file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env bash
|
||||
# bazel.yml's rbe-prewarm job: best-effort pre-warm of gastownhall/gascity's
|
||||
# rbe-west OSS worker pool (one Blacksmith-hosted NativeLink worker,
|
||||
# rbe-worker-pool.yml) so capacity is already booting instead of beads'
|
||||
# Bazel lanes waiting ~120s at "N actions, 0 running" while the pool scales
|
||||
# from zero. gascity pre-warms its own pool from inside its bazel-test.yml
|
||||
# job (same-repo dispatch, github.token); beads cannot use its GITHUB_TOKEN
|
||||
# against gascity, so the rbe-prewarm job mints a short-lived installation
|
||||
# token for the "bazel-allocator" GitHub App (installed on
|
||||
# gastownhall/gascity only, Actions read/write + Metadata read, nothing
|
||||
# else; app id and private key are secrets.RBE_POOL_APP_ID /
|
||||
# secrets.RBE_POOL_APP_PRIVATE_KEY) and passes it to this script as
|
||||
# GH_TOKEN. See engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm".
|
||||
#
|
||||
# Best effort by design: every failure path (missing token, gascity
|
||||
# unreachable, gh rate-limited, dispatch rejected) prints a ::warning:: and
|
||||
# this script still exits 0, so a pre-warm problem never fails the
|
||||
# rbe-prewarm job, let alone the bazel.yml run it is part of or pr.yml's
|
||||
# gate (the job is advisory: not a `needs` of any lane, not in ci-gate's
|
||||
# required list).
|
||||
#
|
||||
# Inputs (environment):
|
||||
# GH_TOKEN gastownhall/gascity-scoped installation token,
|
||||
# minted by the job's own
|
||||
# actions/create-github-app-token step (empty
|
||||
# when the app secrets are not provisioned, or
|
||||
# when the mint step itself fails). Required for
|
||||
# every gh call below; empty or invalid fails
|
||||
# closed into a warning, never a non-zero exit.
|
||||
# RBE_PREWARM_WORKERS desired worker count N (repo variable; default
|
||||
# "1" when unset or non-numeric, clamped to
|
||||
# 1..4). "0" (or any value <= 0 after clamping)
|
||||
# is the kill switch: pre-warm is disabled.
|
||||
# RBE_PREWARM_IDLE_MINUTES optional idle_minutes passed to the dispatched
|
||||
# workflow_dispatch; omitted (gascity's own
|
||||
# default applies) when unset or empty.
|
||||
#
|
||||
# Exits 0 unconditionally.
|
||||
set -u
|
||||
|
||||
POOL_REPO="gastownhall/gascity"
|
||||
POOL_WORKFLOW="rbe-worker-pool.yml"
|
||||
POOL_REF="main"
|
||||
|
||||
n="${RBE_PREWARM_WORKERS:-1}"
|
||||
case "$n" in
|
||||
'') n=1 ;;
|
||||
*[!0-9]*) n=1 ;;
|
||||
esac
|
||||
if [ "$n" -gt 4 ]; then
|
||||
n=4
|
||||
fi
|
||||
|
||||
if [ "$n" -le 0 ]; then
|
||||
echo "RBE_PREWARM_WORKERS=0: rbe-west pre-warm disabled (kill switch)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "::warning title=rbe-west pre-warm::no dispatch token (bazel-allocator app not provisioned, or the mint step failed); skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
active="$(gh run list -R "$POOL_REPO" --workflow "$POOL_WORKFLOW" --limit 50 \
|
||||
--json status --jq '[.[] | select(.status != "completed")] | length' 2>/dev/null)"
|
||||
case "$active" in
|
||||
'' | *[!0-9]*)
|
||||
echo "::warning title=rbe-west pre-warm::could not read $POOL_REPO/$POOL_WORKFLOW run status; skipping dispatch"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
missing=$((n - active))
|
||||
if [ "$missing" -le 0 ]; then
|
||||
echo "${active} $POOL_REPO/$POOL_WORKFLOW worker(s) already active (want ${n}); nothing to dispatch"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
args=(workflow run "$POOL_WORKFLOW" -R "$POOL_REPO" --ref "$POOL_REF")
|
||||
if [ -n "${RBE_PREWARM_IDLE_MINUTES:-}" ]; then
|
||||
args+=(-f "idle_minutes=${RBE_PREWARM_IDLE_MINUTES}")
|
||||
fi
|
||||
|
||||
dispatched=0
|
||||
i=1
|
||||
while [ "$i" -le "$missing" ]; do
|
||||
if gh "${args[@]}"; then
|
||||
dispatched=$((dispatched + 1))
|
||||
else
|
||||
echo "::warning title=rbe-west pre-warm::gh workflow run $POOL_WORKFLOW -R $POOL_REPO failed (attempt $i/$missing)"
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
echo "dispatched ${dispatched}/${missing} $POOL_REPO/$POOL_WORKFLOW worker(s) (active before: ${active}, want: ${n})"
|
||||
exit 0
|
||||
2
.github/workflows/bazel-farm.yml
vendored
2
.github/workflows/bazel-farm.yml
vendored
|
|
@ -128,6 +128,8 @@ jobs:
|
|||
RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }}
|
||||
RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }}
|
||||
RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }}
|
||||
RBE_POOL_APP_ID: ${{ secrets.RBE_POOL_APP_ID }}
|
||||
RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}
|
||||
with:
|
||||
checkout-sha: ${{ github.event.pull_request.head.sha }}
|
||||
fork-farm: authorized
|
||||
|
|
|
|||
122
.github/workflows/bazel.yml
vendored
122
.github/workflows/bazel.yml
vendored
|
|
@ -237,6 +237,18 @@ on:
|
|||
required: false
|
||||
RBE_TLS_CA:
|
||||
required: false
|
||||
# rbe-prewarm only (never read by setup-bazel or the rbe job): the
|
||||
# "bazel-allocator" GitHub App's id and private key. The app is
|
||||
# installed on gastownhall/gascity alone with Actions (read/write) and
|
||||
# Metadata (read) permissions, nothing else; rbe-prewarm mints a
|
||||
# short-lived installation token from these two and never sees a
|
||||
# long-lived credential. Unset (either one empty) disables
|
||||
# pre-warming. See engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west
|
||||
# Pre-warm".
|
||||
RBE_POOL_APP_ID:
|
||||
required: false
|
||||
RBE_POOL_APP_PRIVATE_KEY:
|
||||
required: false
|
||||
# The rbe job's decision, and each lane's job.status (success, failure or
|
||||
# cancelled; empty when the lane was skipped or never reported). pr.yml's
|
||||
# ci-gate reads them through .github/scripts/bazel-gate.sh.
|
||||
|
|
@ -274,6 +286,9 @@ on:
|
|||
package-npm:
|
||||
description: "package-npm's job.status (empty unless package-gates is 'on')"
|
||||
value: ${{ jobs.package-npm.outputs.result }}
|
||||
rbe-prewarm:
|
||||
description: "rbe-prewarm's job.status (advisory: continue-on-error, never gates or blocks a lane)"
|
||||
value: ${{ jobs.rbe-prewarm.outputs.result }}
|
||||
|
||||
# Main runs always finish (they may save the runner cache); PR runs (called
|
||||
# from pr.yml or bazel-farm.yml, whose own groups cancel the calling run too)
|
||||
|
|
@ -384,6 +399,113 @@ jobs:
|
|||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Best-effort pre-warm of gastownhall/gascity's rbe-west OSS worker pool
|
||||
# (one Blacksmith-hosted NativeLink worker, rbe-worker-pool.yml) so it is
|
||||
# already booting instead of the lanes below waiting ~120s at "N actions, 0
|
||||
# running" while the pool scales from zero. A job of its own (not a step in
|
||||
# `rbe`), so it never delays the mode decision every other job needs, and
|
||||
# it is never a `needs` of any lane and never in pr.yml's ci-gate required
|
||||
# list (advisory; .github/scripts/bazel-gate.sh does not read it). See
|
||||
# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm".
|
||||
#
|
||||
# Only when the run's lanes will actually execute on instance "oss": mode
|
||||
# remote (the CI secrets) or fork-rw (rbe-fork-mint's trusted-author tier,
|
||||
# cached by the same OSS workers per the header above). Mode fork-ro targets
|
||||
# instance "oss-fork", a separate, uncached pool rbe-worker-pool.yml does
|
||||
# not serve, so it is excluded (TestBazelCacheModeReachesTheRC's one named
|
||||
# exception to "every lane runs remotely" in a fork mode). Like every other
|
||||
# lane, the `if:` reads only needs.rbe.outputs - TestBazelRBEJobDecidesOnce
|
||||
# forbids re-deriving fork/event facts outside the rbe job itself - and
|
||||
# runs-on is the same mode-ternary every other lane uses, so a genuine fork
|
||||
# or Dependabot PR that reaches mode fork-rw still never touches Blacksmith:
|
||||
# the runner is only a client there, like every other fork-mode lane (see
|
||||
# bazel-test's comment above). That PR also carries no repository secrets
|
||||
# regardless of runs-on (GitHub strips them from a fork's pull_request
|
||||
# event), so the credential-check step below finds nothing to mint even if
|
||||
# it runs. Neither app secret (secrets.RBE_POOL_APP_ID,
|
||||
# secrets.RBE_POOL_APP_PRIVATE_KEY) is ever read in this `if:` or anywhere
|
||||
# else in this file - only inside this job's own step env/with, same as the
|
||||
# rbe job's HAS_EXECUTOR pattern - so a missing credential (the app not
|
||||
# provisioned yet, or the kill switch) fails soft (no mint attempt, then
|
||||
# the script's own ::warning::, exit 0) rather than changing whether the
|
||||
# job runs.
|
||||
rbe-prewarm:
|
||||
name: pre-warm rbe-west pool
|
||||
needs: rbe
|
||||
if: ${{ needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'fork-rw' }}
|
||||
runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
# Job-level, not step-level: the only continue-on-error in this workflow
|
||||
# (TestBazelWorkflowSecretsAndFailureSurface and
|
||||
# TestBazelWorkflowJobsAndExecutionMode both carve out this one path by
|
||||
# name). Every other job here is a gated lane whose real failure must
|
||||
# reach pr.yml's ci-gate; this one is advisory and must never fail the
|
||||
# call bazel.yml's own run represents (a job's failure fails the whole
|
||||
# reusable-workflow run regardless of `needs`, which would otherwise turn
|
||||
# pr.yml's ci-gate red for a pre-warm hiccup nothing requires).
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
result: ${{ steps.result.outputs.result }}
|
||||
steps:
|
||||
# Only the dispatch script, never the rest of the PR (same pinned-ref
|
||||
# and allow-unsafe-pr-checkout pattern as every other job; see the
|
||||
# header). No token on disk: persist-credentials is false.
|
||||
- name: Checkout the pre-warm script
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.checkout-sha }}
|
||||
persist-credentials: false
|
||||
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
|
||||
sparse-checkout: |
|
||||
.github/scripts/rbe-prewarm.sh
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
# Same env-boolean pattern as the rbe job's HAS_EXECUTOR: the secret is
|
||||
# tested for emptiness, never referenced in an `if:` directly, and the
|
||||
# next step's `if:` reads this step's output (a job-level env can't be
|
||||
# read from another step's `if:`, only its own `env:`).
|
||||
- name: Check for the pre-warm app credential
|
||||
id: has-app
|
||||
env:
|
||||
HAS_POOL_APP: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }}
|
||||
run: echo "has-app=$HAS_POOL_APP" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Mints a short-lived installation token for the "bazel-allocator"
|
||||
# GitHub App (installed on gastownhall/gascity only, Actions
|
||||
# read/write + Metadata read): skipped entirely while the app secrets
|
||||
# are not provisioned, leaving steps.mint.outputs.token empty, which
|
||||
# the dispatch step and script below treat the same as any other
|
||||
# missing-token case (::warning::, exit 0). permission-actions narrows
|
||||
# the minted token to Actions even if the app is ever granted more.
|
||||
- name: Mint gastownhall/gascity installation token
|
||||
id: mint
|
||||
if: ${{ steps.has-app.outputs.has-app == 'true' }}
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
|
||||
with:
|
||||
app-id: ${{ secrets.RBE_POOL_APP_ID }}
|
||||
private-key: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}
|
||||
owner: gastownhall
|
||||
repositories: gascity
|
||||
permission-actions: write
|
||||
|
||||
- name: Pre-warm rbe-west OSS worker pool (gastownhall/gascity)
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.mint.outputs.token }}
|
||||
RBE_PREWARM_WORKERS: ${{ vars.RBE_PREWARM_WORKERS }}
|
||||
run: bash .github/scripts/rbe-prewarm.sh
|
||||
|
||||
# Every lane reports its job.status from an always() last step
|
||||
# (TestBazelLaneIsGatedAlongsideLegacy). The script above always exits
|
||||
# 0, so this normally reads success regardless of what it found; the
|
||||
# job-level continue-on-error above is the backstop for a step that
|
||||
# fails outside the script (checkout, the mint action).
|
||||
- name: Record result
|
||||
id: result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
JOB_STATUS: ${{ job.status }}
|
||||
run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
bazel-test:
|
||||
name: test
|
||||
# Remote-execution runs use the org's Blacksmith pool (the farm's edge
|
||||
|
|
|
|||
2
.github/workflows/nightly.yml
vendored
2
.github/workflows/nightly.yml
vendored
|
|
@ -109,6 +109,8 @@ jobs:
|
|||
RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }}
|
||||
RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }}
|
||||
RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }}
|
||||
RBE_POOL_APP_ID: ${{ secrets.RBE_POOL_APP_ID }}
|
||||
RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}
|
||||
with:
|
||||
details: true
|
||||
go-test-json-artifact: bazel-equivalence-go-test-json
|
||||
|
|
|
|||
2
.github/workflows/pr.yml
vendored
2
.github/workflows/pr.yml
vendored
|
|
@ -2268,6 +2268,8 @@ jobs:
|
|||
RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }}
|
||||
RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }}
|
||||
RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }}
|
||||
RBE_POOL_APP_ID: ${{ secrets.RBE_POOL_APP_ID }}
|
||||
RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}
|
||||
with:
|
||||
build-artifact-name: bazel-ci-build-artifacts
|
||||
integration: "on"
|
||||
|
|
|
|||
|
|
@ -468,6 +468,69 @@ Only a GitHub run can verify these:
|
|||
it doesn't).
|
||||
- An unlisted author, or a push by an unlisted collaborator, skips `farm`.
|
||||
|
||||
## rbe-west Pre-warm
|
||||
|
||||
`bazel.yml`'s `rbe-prewarm` job is a best-effort attempt to have
|
||||
gastownhall/gascity's rbe-west OSS worker pool (one Blacksmith-hosted
|
||||
NativeLink worker, driven by gascity's own `rbe-worker-pool.yml`
|
||||
`workflow_dispatch`) already booting by the time this run's Bazel lanes need
|
||||
it, instead of each of them separately waiting out the ~120s it takes
|
||||
NativeLink to notice demand and scale the pool up from zero. gascity dispatches
|
||||
its own pool from inside its own `bazel-test.yml` job, same-repo, with
|
||||
`github.token`; beads cannot use its own `GITHUB_TOKEN` against gascity's
|
||||
repository, so this job needs its own credential into gascity instead.
|
||||
|
||||
**What it does.** A job of its own (not a step inside `rbe`, so it never
|
||||
delays the mode decision every lane waits on), gated on
|
||||
`needs.rbe.outputs.mode` being `remote` or `fork-rw` - the two modes whose
|
||||
lanes actually target rbe-west's `oss` instance, which is what
|
||||
`rbe-worker-pool.yml` serves. (Mode `fork-ro` targets a separate, uncached
|
||||
`oss-fork` instance this pool does not serve, so it is excluded.) When
|
||||
scheduled, it mints a short-lived GitHub App installation token and uses it to
|
||||
check gastownhall/gascity's current `rbe-worker-pool.yml` run count and, if
|
||||
under the desired worker count, dispatch enough new runs to reach it. Every
|
||||
failure path - no credential, gascity unreachable, `gh` rate-limited, the
|
||||
dispatch itself rejected - prints a `::warning::` and the script still exits
|
||||
0: this job never gates anything (it is not a `needs` of any lane, and it is
|
||||
never in `.github/scripts/bazel-gate.sh`'s vocabulary, so pr.yml's `ci-gate`
|
||||
never looks at it), and it cannot fail the run either, since a called reusable
|
||||
workflow's overall conclusion is "failure" if any job inside it fails
|
||||
regardless of `needs` - the job itself therefore carries bazel.yml's one
|
||||
deliberate `continue-on-error: true`.
|
||||
|
||||
**Credential scope.** A GitHub App named "bazel-allocator" is installed on
|
||||
gastownhall/gascity alone, granted exactly two permissions: Actions
|
||||
(read/write) and Metadata (read). bazel.yml's own secrets never include a
|
||||
long-lived credential for it - only the app's id and private key
|
||||
(`RBE_POOL_APP_ID`, `RBE_POOL_APP_PRIVATE_KEY`), from which the job mints a
|
||||
short-lived installation token (`actions/create-github-app-token`, pinned to a
|
||||
released commit SHA) scoped with `owner: gastownhall` and
|
||||
`repositories: gascity`, so the minted token can act on gascity and nothing
|
||||
else in the gastownhall org. That token is used as `GH_TOKEN` for the
|
||||
`gh run list` / `gh workflow run` calls against gastownhall/gascity's
|
||||
`rbe-worker-pool.yml` on `main`, and nowhere else. Every caller of bazel.yml
|
||||
(pr.yml, bazel-farm.yml, nightly.yml) passes these two secrets straight
|
||||
through like the four RBE secrets; a fork or Dependabot `pull_request` run
|
||||
gets neither (GitHub never forwards repository secrets to a fork's
|
||||
`pull_request` event), so even a trusted-author fork PR that reaches mode
|
||||
`fork-rw` finds no credential to mint and the job no-ops.
|
||||
|
||||
**Secret names.**
|
||||
|
||||
- `RBE_POOL_APP_ID` - the bazel-allocator App's id.
|
||||
- `RBE_POOL_APP_PRIVATE_KEY` - the App's private key (PEM).
|
||||
|
||||
**Kill switch.** Pre-warming disables itself cleanly, without touching the
|
||||
job's `if:`, in either of two ways:
|
||||
|
||||
- Leave `RBE_POOL_APP_PRIVATE_KEY` unset (or clear it): the job's own
|
||||
`HAS_POOL_APP` check (the same env-boolean pattern as the `rbe` job's
|
||||
`HAS_EXECUTOR`) skips the mint step, the dispatch script finds no
|
||||
`GH_TOKEN`, warns, and exits 0. This is also the job's natural state before
|
||||
the App is provisioned at all.
|
||||
- Set the repository variable `RBE_PREWARM_WORKERS` to `0`: the script's own
|
||||
kill switch, checked before any network call.
|
||||
|
||||
## Required Check Contract
|
||||
|
||||
After the aggregate checks are verified on the branch, branch protection or the
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ go_test(
|
|||
"bazel_farm_workflow_test.go",
|
||||
"bazel_fork_mode_test.go",
|
||||
"bazel_policy_test.go",
|
||||
"bazel_rbe_prewarm_test.go",
|
||||
"bazel_skip_allowlist_test.go",
|
||||
"check_build_tags_test.go",
|
||||
"check_doc_freshness_test.go",
|
||||
|
|
|
|||
|
|
@ -205,7 +205,11 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) {
|
|||
}
|
||||
runs := bazelLaneRunModes(t, name, job.If, s.with)[mode]
|
||||
switch {
|
||||
case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name]:
|
||||
// rbe-prewarm targets instance "oss" only (modes remote and
|
||||
// fork-rw); fork-ro targets the separate "oss-fork" instance
|
||||
// rbe-worker-pool.yml does not serve, so it is excluded by
|
||||
// design (see bazel.yml's comment on the job).
|
||||
case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && mode == "fork-ro"):
|
||||
t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode)
|
||||
case mode == "cache" && name == bazelIntegJobName && !runs:
|
||||
t.Errorf("%s does not run in mode cache", name)
|
||||
|
|
|
|||
|
|
@ -251,6 +251,13 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) {
|
|||
if bazelPackageJobs[name] {
|
||||
want["fetch-depth"] = "0"
|
||||
}
|
||||
// rbe-prewarm only needs its own dispatch script, never the rest
|
||||
// of the PR tree; a sparse, non-cone checkout of exactly that one
|
||||
// path keeps it as cheap as every other lane's shallow checkout.
|
||||
if name == bazelRBEPrewarmJobName {
|
||||
want["sparse-checkout"] = ".github/scripts/rbe-prewarm.sh\n"
|
||||
want["sparse-checkout-cone-mode"] = "false"
|
||||
}
|
||||
if !reflect.DeepEqual(step.With, want) {
|
||||
t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want)
|
||||
}
|
||||
|
|
|
|||
266
scripts/bazel_rbe_prewarm_test.go
Normal file
266
scripts/bazel_rbe_prewarm_test.go
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
package scripts_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This file exists alongside ci_workflow_test.go's broader structural checks
|
||||
// (TestBazelWorkflowJobsAndExecutionMode, TestBazelWorkflowSecretsAndFailureSurface,
|
||||
// TestBazelWorkflowActionsArePinned, TestBazelRBEJobDecidesOnce,
|
||||
// TestBazelGateSimulation, TestBazelCacheModeReachesTheRC) to pin, by name,
|
||||
// the specific invariants engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md's "rbe-west
|
||||
// Pre-warm" section promises a reader: the job never runs for an untrusted
|
||||
// fork or on a mode rbe-worker-pool.yml cannot serve, its credentials are
|
||||
// never readable outside its own steps, it is never required by pr.yml's
|
||||
// gate or a `needs` of any lane, it cannot fail the run, and its dispatch is
|
||||
// pinned to exactly gastownhall/gascity's rbe-worker-pool.yml on main with a
|
||||
// token scoped to that one repository.
|
||||
//
|
||||
// Each test below was mutation-tested by hand while this job was written:
|
||||
// reverting the fix it pins (restoring the old event-gated `if:`, widening
|
||||
// the mint's `repositories:`, dropping the job-level continue-on-error,
|
||||
// adding rbe-prewarm to bazel-gate.sh's skip/aggregate logic or to another
|
||||
// job's `needs:`) reliably fails the corresponding test here. See this
|
||||
// task's notes file for the transcript.
|
||||
|
||||
// TestRBEPrewarmIfOnlyRemoteAndForkRW runs the job's real, pinned `if:`
|
||||
// expression (bazelRBEPrewarmIf, via the shared evalGHExpr from
|
||||
// ci_blacksmith_runner_test.go, not a hand-written mirror) against every mode
|
||||
// the rbe job can produce. Only remote and fork-rw must schedule the job:
|
||||
// fork-ro targets rbe-west's separate, uncached "oss-fork" instance (never
|
||||
// served by rbe-worker-pool.yml), and cache/local/skip mean nothing to
|
||||
// pre-warm. fork-rw is deliberately included - a trusted fork author's PR,
|
||||
// per rbe-fork-mint's tier decision, not just bazel-farm.yml's
|
||||
// pull_request_target path - and a fork or Dependabot pull_request is exactly
|
||||
// how that mode is reached, so this doubles as the fork/Dependabot
|
||||
// reachability check the original spec asked for: such a run only matters
|
||||
// here through needs.rbe.outputs.mode, never through re-inspecting
|
||||
// github.actor or head.repo.fork in this job's own `if:` (that ban is
|
||||
// TestBazelRBEJobDecidesOnce's job).
|
||||
func TestRBEPrewarmIfOnlyRemoteAndForkRW(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName)
|
||||
if job.If != bazelRBEPrewarmIf {
|
||||
t.Fatalf("rbe-prewarm if = %q, want the pinned %q", job.If, bazelRBEPrewarmIf)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
mode string
|
||||
want bool
|
||||
}{
|
||||
{"remote", true},
|
||||
{"fork-rw", true},
|
||||
{"fork-ro", false},
|
||||
{"cache", false},
|
||||
{"local", false},
|
||||
{"skip", false},
|
||||
} {
|
||||
v, err := evalGHExpr(job.If, map[string]string{"needs.rbe.outputs.mode": tc.mode})
|
||||
if err != nil {
|
||||
t.Fatalf("evalGHExpr(%q) mode=%s: %v", job.If, tc.mode, err)
|
||||
}
|
||||
if ghTruthy(v) != tc.want {
|
||||
t.Errorf("rbe-prewarm if, mode=%s: evaluated to %v, want %v", tc.mode, v, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmNeverGatedOrNeeded: no lane depends on rbe-prewarm (a
|
||||
// `needs:` cycle or ordering dependency would turn "advisory" into "gating"),
|
||||
// and neither of pr.yml's ci-gate paths - the required-checks list it reads
|
||||
// from .github/scripts/ci-gate.sh/pr-policy, nor bazel-gate.sh's own
|
||||
// skip/aggregate vocabulary - ever mentions it by name.
|
||||
func TestRBEPrewarmNeverGatedOrNeeded(t *testing.T) {
|
||||
workflow := readCIWorkflow(t, bazelWorkflowName)
|
||||
for name, job := range workflow.Jobs {
|
||||
if name == bazelRBEPrewarmJobName {
|
||||
continue
|
||||
}
|
||||
for _, need := range job.Needs {
|
||||
if need == bazelRBEPrewarmJobName {
|
||||
t.Errorf("%s needs %s; rbe-prewarm is advisory and must never gate another lane", name, bazelRBEPrewarmJobName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
root := sourceRepoRoot(t)
|
||||
gate := readPolicyFile(t, root, filepath.Join(".github", "scripts", "bazel-gate.sh"))
|
||||
if strings.Contains(gate, bazelRBEPrewarmJobName) {
|
||||
t.Errorf("bazel-gate.sh mentions %s; it must stay outside the gate's skip/aggregate vocabulary", bazelRBEPrewarmJobName)
|
||||
}
|
||||
|
||||
// pr.yml's ci-gate required-check id list: BAZEL* ids come from
|
||||
// bazel-gate.sh's own vocabulary (BAZEL, BAZEL_TEST, ...), never a
|
||||
// per-job id for rbe-prewarm.
|
||||
ciGate := readPolicyFile(t, root, filepath.Join(".github", "scripts", "ci-gate.sh"))
|
||||
if regexp.MustCompile(`(?i)rbe.prewarm`).MatchString(ciGate) {
|
||||
t.Errorf("ci-gate.sh mentions rbe-prewarm; it must never be a required check")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmNeverFailsTheRun: the job carries the one deliberate
|
||||
// continue-on-error in bazel.yml (every other occurrence is banned by
|
||||
// TestBazelWorkflowSecretsAndFailureSurface), and the dispatch script it
|
||||
// runs has no path that exits non-zero.
|
||||
func TestRBEPrewarmNeverFailsTheRun(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName)
|
||||
if !job.ContinueOnError {
|
||||
t.Errorf("rbe-prewarm job.continue-on-error = false; a real step failure would fail bazel.yml's own run and cascade into pr.yml's ci-gate")
|
||||
}
|
||||
for _, step := range job.Steps {
|
||||
if v, ok := step.ContinueOnError.(bool); ok && v {
|
||||
t.Errorf("rbe-prewarm step %q has its own continue-on-error; only the job level should (TestBazelWorkflowSecretsAndFailureSurface bans the key everywhere else)", step.Name)
|
||||
}
|
||||
}
|
||||
|
||||
root := sourceRepoRoot(t)
|
||||
script := readPolicyFile(t, root, filepath.Join(".github", "scripts", "rbe-prewarm.sh"))
|
||||
if !strings.Contains(script, "set -u") || strings.Contains(script, "set -e") {
|
||||
t.Errorf("rbe-prewarm.sh must run under set -u only (no set -e/pipefail): every gh/curl failure path is handled explicitly and must fall through to the trailing exit 0")
|
||||
}
|
||||
exits := regexp.MustCompile(`(?m)^\s*exit\s+(\S+)`).FindAllStringSubmatch(script, -1)
|
||||
if len(exits) == 0 {
|
||||
t.Fatalf("rbe-prewarm.sh has no exit statement; expected only exit 0")
|
||||
}
|
||||
for _, m := range exits {
|
||||
if m[1] != "0" {
|
||||
t.Errorf("rbe-prewarm.sh has %q; every exit must be exit 0 (best-effort by design)", strings.TrimSpace(m[0]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmSecretsOnlyInItsOwnJob walks the raw YAML of bazel.yml and
|
||||
// confirms the two app-credential secrets are referenced only inside the
|
||||
// rbe-prewarm job (its HAS_POOL_APP env check and the mint step's `with:`),
|
||||
// never anywhere else in the file - not another job's env, not a workflow
|
||||
// top-level env, not an `if:`.
|
||||
func TestRBEPrewarmSecretsOnlyInItsOwnJob(t *testing.T) {
|
||||
secretRef := regexp.MustCompile(`\bsecrets\.RBE_POOL_APP_(ID|PRIVATE_KEY)\b`)
|
||||
jobPrefix := ".jobs." + bazelRBEPrewarmJobName + "."
|
||||
found := 0
|
||||
walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) {
|
||||
if key || !secretRef.MatchString(value) {
|
||||
return
|
||||
}
|
||||
found++
|
||||
if !strings.HasPrefix(path, jobPrefix) {
|
||||
t.Errorf("%s: %s references an rbe-prewarm app credential outside the job (prefix %q)", bazelWorkflowName, path, jobPrefix)
|
||||
}
|
||||
})
|
||||
if found == 0 {
|
||||
t.Fatalf("found no secrets.RBE_POOL_APP_* reference in %s; the test fixture or the job moved", bazelWorkflowName)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmAppTokenScoped pins the mint step to the exact "bazel-
|
||||
// allocator" installation-token shape the docs promise: the action pinned to
|
||||
// a full commit SHA with its version comment (same SHA this repo already
|
||||
// uses for the same action in update-flake-lock.yml), scoped to
|
||||
// gastownhall/gascity alone (never, say, the whole gastownhall org or an
|
||||
// unrelated repo), and narrowed to the Actions permission.
|
||||
func TestRBEPrewarmAppTokenScoped(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName)
|
||||
mint := job.step(t, "Mint gastownhall/gascity installation token")
|
||||
|
||||
family, sha, found := strings.Cut(mint.Uses, "@")
|
||||
if family != "actions/create-github-app-token" {
|
||||
t.Fatalf("mint step uses %q, want actions/create-github-app-token", mint.Uses)
|
||||
}
|
||||
if !found || !actionPin.MatchString(sha) {
|
||||
t.Errorf("mint step action %q is not pinned to a 40-hex commit SHA", mint.Uses)
|
||||
}
|
||||
const wantSHA = "bcd2ba49218906704ab6c1aa796996da409d3eb1" // v3, same as update-flake-lock.yml
|
||||
if sha != wantSHA {
|
||||
t.Errorf("mint step action SHA = %q, want %q (update-flake-lock.yml's pin for the same action)", sha, wantSHA)
|
||||
}
|
||||
|
||||
if got := mint.With["owner"]; got != "gastownhall" {
|
||||
t.Errorf("mint step owner = %q, want %q", got, "gastownhall")
|
||||
}
|
||||
if got := mint.With["repositories"]; got != "gascity" {
|
||||
t.Errorf("mint step repositories = %q, want exactly %q (not a list, not the whole org)", got, "gascity")
|
||||
}
|
||||
if got := mint.With["app-id"]; got != "${{ secrets.RBE_POOL_APP_ID }}" {
|
||||
t.Errorf("mint step app-id = %q, want the RBE_POOL_APP_ID secret", got)
|
||||
}
|
||||
if got := mint.With["private-key"]; got != "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}" {
|
||||
t.Errorf("mint step private-key = %q, want the RBE_POOL_APP_PRIVATE_KEY secret", got)
|
||||
}
|
||||
if got := mint.With["permission-actions"]; got != "write" {
|
||||
t.Errorf("mint step permission-actions = %q, want %q (narrow even if the app is ever granted more)", got, "write")
|
||||
}
|
||||
// Gated on the credential check, not unconditional: TestRBEPrewarmGatedOnAppSecret.
|
||||
if mint.If != "${{ steps.has-app.outputs.has-app == 'true' }}" {
|
||||
t.Errorf("mint step if = %q, want it gated on the has-app credential check", mint.If)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmGatedOnAppSecret: the HAS_POOL_APP pattern (mirroring the rbe
|
||||
// job's own HAS_EXECUTOR) - the mint step is skipped, not failed, whenever
|
||||
// either app secret is unset, which is also the kill switch's main path
|
||||
// (before the app is provisioned at all).
|
||||
func TestRBEPrewarmGatedOnAppSecret(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName)
|
||||
check := job.step(t, "Check for the pre-warm app credential")
|
||||
if check.Env["HAS_POOL_APP"] != "${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }}" {
|
||||
t.Errorf("has-app step env HAS_POOL_APP = %q, want an emptiness test on RBE_POOL_APP_PRIVATE_KEY", check.Env["HAS_POOL_APP"])
|
||||
}
|
||||
if check.ID != "has-app" {
|
||||
t.Errorf("has-app step id = %q, want %q", check.ID, "has-app")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmDispatchTargetPinned: the script's three target constants are
|
||||
// literal (never a variable the caller or a repo var could redirect), and the
|
||||
// only gh workflow/run subcommands in the script reference them, never a
|
||||
// hardcoded alternative.
|
||||
func TestRBEPrewarmDispatchTargetPinned(t *testing.T) {
|
||||
root := sourceRepoRoot(t)
|
||||
script := readPolicyFile(t, root, filepath.Join(".github", "scripts", "rbe-prewarm.sh"))
|
||||
|
||||
for _, want := range []string{
|
||||
`POOL_REPO="gastownhall/gascity"`,
|
||||
`POOL_WORKFLOW="rbe-worker-pool.yml"`,
|
||||
`POOL_REF="main"`,
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Errorf("rbe-prewarm.sh does not contain %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
ghCalls := regexp.MustCompile(`(?m)^\s*(gh |args=\()`).FindAllString(script, -1)
|
||||
if len(ghCalls) == 0 {
|
||||
t.Fatalf("rbe-prewarm.sh has no gh invocation; the dispatch logic moved")
|
||||
}
|
||||
// Every gh run-list/workflow-run invocation must use $POOL_REPO and
|
||||
// $POOL_WORKFLOW, never a literal repo or workflow file.
|
||||
ghLine := regexp.MustCompile(`(?m)^.*\bgh\b.*$`)
|
||||
for _, line := range ghLine.FindAllString(script, -1) {
|
||||
if strings.Contains(line, "run list") || strings.Contains(line, "workflow run") || strings.Contains(line, `args=(workflow`) {
|
||||
if !strings.Contains(line, "POOL_REPO") && !strings.Contains(line, `"${args[@]}"`) {
|
||||
t.Errorf("rbe-prewarm.sh gh invocation does not reference $POOL_REPO: %q", strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEPrewarmTokenIsGHToken: the dispatch step hands the script the
|
||||
// mint step's own output, never a long-lived or ambient credential
|
||||
// (secrets.GITHUB_TOKEN, a PAT, or the mint step's app-id/private-key
|
||||
// directly).
|
||||
func TestRBEPrewarmTokenIsGHToken(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEPrewarmJobName)
|
||||
dispatch := job.step(t, "Pre-warm rbe-west OSS worker pool (gastownhall/gascity)")
|
||||
if got := dispatch.Env["GH_TOKEN"]; got != "${{ steps.mint.outputs.token }}" {
|
||||
t.Errorf("dispatch step env GH_TOKEN = %q, want the mint step's own token output", got)
|
||||
}
|
||||
if strings.Contains(dispatch.Env["GH_TOKEN"], "secrets.") {
|
||||
t.Errorf("dispatch step GH_TOKEN reads a secret directly; it must use the minted installation token")
|
||||
}
|
||||
}
|
||||
|
||||
// actionPin and readYAMLNode/walkYAML/evalGHExpr/ghTruthy/readCIWorkflow/
|
||||
// readPolicyFile/sourceRepoRoot are shared helpers already defined in
|
||||
// ci_workflow_test.go and ci_blacksmith_runner_test.go; this file adds no new
|
||||
// infrastructure of its own.
|
||||
|
|
@ -2471,6 +2471,10 @@ const (
|
|||
// config; pr.yml opts them in with package-gates: "on".
|
||||
bazelPackageMCPJobName = "package-mcp"
|
||||
bazelPackageNPMJobName = "package-npm"
|
||||
// Best-effort, advisory pre-warm of gastownhall/gascity's rbe-west OSS
|
||||
// worker pool; see bazel.yml's own comment and
|
||||
// engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm".
|
||||
bazelRBEPrewarmJobName = "rbe-prewarm"
|
||||
setupBazelActionDir = ".github/actions/setup-bazel"
|
||||
)
|
||||
|
||||
|
|
@ -2495,7 +2499,7 @@ const (
|
|||
// bazel.yml's jobs: the rbe job that decides the execution mode, the
|
||||
// --config=ci lane, one job per CI job a Bazel config mirrors, and the two
|
||||
// package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings.
|
||||
var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName}
|
||||
var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName}
|
||||
|
||||
// The lanes that only run remotely (skipped unless the rbe job chose remote);
|
||||
// bazel-integration runs remotely or with the read-only cache (bazelIntegIf);
|
||||
|
|
@ -2509,6 +2513,25 @@ const (
|
|||
bazelRBESecretValue = "${{ secrets.RBE_WEST_EXECUTOR != '' }}"
|
||||
)
|
||||
|
||||
// rbe-prewarm reads its two app secrets in exactly three places, mirroring
|
||||
// the rbe job's own emptiness-test/mint pattern above: an env boolean
|
||||
// (never an `if:`) gating the mint step, and the mint step's own `with:`
|
||||
// (the only step in bazel.yml allowed to read a secret outside a
|
||||
// setup-bazel env, alongside bazelRBESecretPath/Value). Step indices are
|
||||
// steps[0]=checkout, [1]=credential check, [2]=mint, [3]=dispatch,
|
||||
// [4]=record result.
|
||||
const (
|
||||
bazelRBEPrewarmHasAppPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[1].env.HAS_POOL_APP"
|
||||
bazelRBEPrewarmHasAppValue = "${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }}"
|
||||
bazelRBEPrewarmAppIDPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[2].with.app-id"
|
||||
bazelRBEPrewarmAppIDValue = "${{ secrets.RBE_POOL_APP_ID }}"
|
||||
bazelRBEPrewarmKeyPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[2].with.private-key"
|
||||
bazelRBEPrewarmKeyValue = "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}"
|
||||
// The job-level continue-on-error path itself: see
|
||||
// TestBazelWorkflowJobsAndExecutionMode's comment on the same exception.
|
||||
bazelRBEPrewarmContinueOnErrorPath = ".jobs." + bazelRBEPrewarmJobName + ".continue-on-error"
|
||||
)
|
||||
|
||||
// rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes
|
||||
// fork-ro and fork-rw only), and the mint's status URL the rbe job probes.
|
||||
const (
|
||||
|
|
@ -2546,9 +2569,13 @@ var bazelLaneGateIDs = map[string]string{
|
|||
bazelPackageNPMJobName: "PACKAGE_NPM",
|
||||
}
|
||||
|
||||
// None today: every lane is gated. Kept so a future lane that pr.yml's call
|
||||
// turns off has a place to record why.
|
||||
var bazelAdvisoryLanes = map[string]string{}
|
||||
// rbe-prewarm: not a build/test lane at all (no Bazel config mirrors it), so
|
||||
// it has nothing for ci-gate to require; it is advisory by design
|
||||
// (continue-on-error, best-effort pre-warm dispatch) and reports only so
|
||||
// TestBazelLaneIsGatedAlongsideLegacy's generic shape check still applies.
|
||||
var bazelAdvisoryLanes = map[string]string{
|
||||
bazelRBEPrewarmJobName: "best-effort pre-warm dispatch to gastownhall/gascity; never gates or blocks a lane",
|
||||
}
|
||||
|
||||
// bazel-integration's if: remote (modes remote, fork-ro, fork-rw: enabled),
|
||||
// or local with the read-only cache (mode cache: fork and Dependabot PRs
|
||||
|
|
@ -2572,6 +2599,18 @@ const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}"
|
|||
// (enabled too) build bd with go build on a GitHub-hosted runner, as before.
|
||||
const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}"
|
||||
|
||||
// rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): unlike
|
||||
// every other lane, it runs only when the lanes below will actually target
|
||||
// instance "oss" (modes remote and fork-rw; fork-ro targets the separate,
|
||||
// uncached "oss-fork" instance rbe-worker-pool.yml does not serve - see
|
||||
// bazel.yml's own comment above the job). Its runs-on reuses the exact same
|
||||
// mode-ternary every other lane uses (wantRunsOn): TestBazelRBEJobDecidesOnce
|
||||
// forbids re-deriving fork/event facts outside the rbe job, and a fork-rw run
|
||||
// (a genuine fork/Dependabot pull_request, whose workflow file content is the
|
||||
// fork's own) must never be hardcoded onto the trusted same-repo Blacksmith
|
||||
// runner.
|
||||
const bazelRBEPrewarmIf = "${{ needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'fork-rw' }}"
|
||||
|
||||
// F3: the rbe job's own runner (not gated by its own outputs - it decides
|
||||
// them). Same-repo PRs and merge_group/push/dispatch/schedule (never forks)
|
||||
// get Blacksmith; a pull_request_target farm run and any fork or Dependabot
|
||||
|
|
@ -2653,12 +2692,17 @@ var bazelRemoteModes = map[string]bool{"remote": true, "fork-ro": true, "fork-rw
|
|||
// bazelModeEnabled: the rbe job's enabled output for a mode.
|
||||
func bazelModeEnabled(mode string) string { return strconv.FormatBool(bazelRemoteModes[mode]) }
|
||||
|
||||
// The four RBE secrets, the only ones a caller may hand bazel.yml.
|
||||
// The four RBE secrets plus the rbe-prewarm job's "bazel-allocator" GitHub
|
||||
// App id and private key (it mints its own gastownhall/gascity installation
|
||||
// token from these; see rbe-prewarm.sh), the only ones a caller may hand
|
||||
// bazel.yml.
|
||||
var bazelCallSecrets = map[string]string{
|
||||
"RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}",
|
||||
"RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}",
|
||||
"RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}",
|
||||
"RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}",
|
||||
"RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}",
|
||||
"RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}",
|
||||
"RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}",
|
||||
"RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}",
|
||||
"RBE_POOL_APP_ID": "${{ secrets.RBE_POOL_APP_ID }}",
|
||||
"RBE_POOL_APP_PRIVATE_KEY": "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}",
|
||||
}
|
||||
|
||||
type ciCompositeAction struct {
|
||||
|
|
@ -2693,7 +2737,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
|
|||
var names []string
|
||||
for name, job := range workflow.Jobs {
|
||||
names = append(names, name)
|
||||
if job.ContinueOnError {
|
||||
// rbe-prewarm is the one deliberate exception: it is advisory (never
|
||||
// a `needs` of any lane, never in ci-gate's required list - see
|
||||
// bazelAdvisoryLanes and TestBazelLaneIsGatedAlongsideLegacy), and a
|
||||
// called reusable workflow's overall conclusion is "failure" if ANY
|
||||
// job within it fails regardless of `needs`, so without job-level
|
||||
// continue-on-error here a pre-warm hiccup would still turn
|
||||
// bazel.yml's own run - and thus pr.yml's ci-gate - red for a job
|
||||
// nothing requires.
|
||||
if job.ContinueOnError && name != bazelRBEPrewarmJobName {
|
||||
t.Errorf("%s continue-on-error hides failures from pr.yml's ci-gate", name)
|
||||
}
|
||||
if job.TimeoutMinutes == 0 {
|
||||
|
|
@ -2760,6 +2812,12 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
|
|||
wantJobIf = wantRemoteOnlyIf
|
||||
} else if name == bazelIntegJobName {
|
||||
wantJobIf = bazelIntegIf
|
||||
} else if name == bazelRBEPrewarmJobName {
|
||||
// Same runs-on ternary as every other lane (wantRunsOn, set
|
||||
// above); only the if differs. Not a lane: never uses
|
||||
// setup-bazel, so wantJobSetupEnv is moot (the per-step check
|
||||
// below only fires for a setup-bazel step).
|
||||
wantJobIf = bazelRBEPrewarmIf
|
||||
}
|
||||
if job.RunsOn != wantJobRunsOn {
|
||||
t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantJobRunsOn)
|
||||
|
|
@ -3560,6 +3618,13 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string
|
|||
return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true, "local": true, "skip": true}
|
||||
}
|
||||
return map[string]bool{}
|
||||
case bazelRBEPrewarmIf:
|
||||
// Same two modes as every lane that targets instance "oss"
|
||||
// (TestBazelWorkflowJobsAndExecutionMode's comment on the job):
|
||||
// remote and fork-rw. Unlike the historical advisory lanes, this one
|
||||
// legitimately runs in pr.yml's own call (see bazelAdvisoryLanes'
|
||||
// name-specific exception in TestBazelGateSimulation below).
|
||||
return map[string]bool{"remote": true, "fork-rw": true}
|
||||
}
|
||||
t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr)
|
||||
return nil
|
||||
|
|
@ -3688,9 +3753,13 @@ func TestBazelGateSimulation(t *testing.T) {
|
|||
continue
|
||||
}
|
||||
lanes[name] = bazelLaneRunModes(t, name, job.If, callWith)
|
||||
// Advisory lanes are not in the PR call; gated lanes run at least
|
||||
// when remote.
|
||||
if _, advisory := bazelAdvisoryLanes[name]; advisory && len(lanes[name]) != 0 {
|
||||
// Advisory lanes are normally not in the PR call at all; the one
|
||||
// exception is rbe-prewarm, which is advisory (never gated, never a
|
||||
// `needs`: TestBazelLaneIsGatedAlongsideLegacy) but, unlike every
|
||||
// other advisory lane this map has ever held, is deliberately part
|
||||
// of every pr.yml call so it can dispatch before the lanes below
|
||||
// need capacity - see bazel.yml's own comment on the job.
|
||||
if _, advisory := bazelAdvisoryLanes[name]; advisory && name != bazelRBEPrewarmJobName && len(lanes[name]) != 0 {
|
||||
t.Errorf("advisory lane %s runs in pr.yml's call in modes %v; turn it off there (%s)", name, lanes[name], bazelAdvisoryLanes[name])
|
||||
}
|
||||
if _, gated := bazelLaneGateIDs[name]; gated && !lanes[name]["remote"] {
|
||||
|
|
@ -4005,6 +4074,9 @@ func TestBazelWorkflowActionsArePinned(t *testing.T) {
|
|||
cacheSaveActionFamily: cacheSHA,
|
||||
"actions/upload-artifact": uploadArtifactSHA,
|
||||
"actions/download-artifact": downloadArtifactSHA,
|
||||
// rbe-prewarm's mint step: same SHA/comment already used in this repo
|
||||
// for the same action, in update-flake-lock.yml.
|
||||
"actions/create-github-app-token": "bcd2ba49218906704ab6c1aa796996da409d3eb1",
|
||||
}
|
||||
for file, list := range steps {
|
||||
for _, step := range list {
|
||||
|
|
@ -5285,8 +5357,8 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) {
|
|||
|
||||
setupSteps := map[string]bool{}
|
||||
for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs {
|
||||
if name == bazelRBEJobName {
|
||||
continue // no setup-bazel; its one secret read is pinned below
|
||||
if name == bazelRBEJobName || name == bazelRBEPrewarmJobName {
|
||||
continue // no setup-bazel; rbe-prewarm's secret reads are pinned below
|
||||
}
|
||||
n := 0
|
||||
for i, step := range job.Steps {
|
||||
|
|
@ -5302,7 +5374,14 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) {
|
|||
secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`)
|
||||
walkYAML(root, "", func(path string, key bool, value string) {
|
||||
if key && value == "continue-on-error" {
|
||||
t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path)
|
||||
// rbe-prewarm's job-level continue-on-error is the one
|
||||
// deliberate exception (TestBazelWorkflowJobsAndExecutionMode's
|
||||
// comment on the job has the full reasoning): without it, this
|
||||
// advisory job's failure would still fail bazel.yml's own run
|
||||
// and cascade into pr.yml's ci-gate.
|
||||
if path != bazelRBEPrewarmContinueOnErrorPath {
|
||||
t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path)
|
||||
}
|
||||
}
|
||||
if key && (value == "secrets" && strings.HasPrefix(path, ".jobs.")) {
|
||||
t.Errorf("%s: %s passes secrets to a called workflow or container", bazelWorkflowName, path)
|
||||
|
|
@ -5313,6 +5392,12 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) {
|
|||
if path == bazelRBESecretPath && value == bazelRBESecretValue {
|
||||
return // the rbe job's emptiness test (TestBazelRBEJobDecidesOnce)
|
||||
}
|
||||
switch {
|
||||
case path == bazelRBEPrewarmHasAppPath && value == bazelRBEPrewarmHasAppValue,
|
||||
path == bazelRBEPrewarmAppIDPath && value == bazelRBEPrewarmAppIDValue,
|
||||
path == bazelRBEPrewarmKeyPath && value == bazelRBEPrewarmKeyValue:
|
||||
return // rbe-prewarm's own credential-check/mint step (see their doc comment)
|
||||
}
|
||||
prefix := path[:strings.LastIndex(path, ".")+1]
|
||||
if !setupSteps[prefix] {
|
||||
t.Errorf("%s: %s reads secrets (%q); only the setup-bazel step's env may", bazelWorkflowName, path, value)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue