Compare commits
1 commit
main
...
ci/worker-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cc84303d40 |
7 changed files with 472 additions and 79 deletions
59
.github/workflows/bazel.yml
vendored
59
.github/workflows/bazel.yml
vendored
|
|
@ -382,11 +382,30 @@ env:
|
|||
|
||||
jobs:
|
||||
# The one place the execution mode is decided (see the header), so every
|
||||
# job sees the same answer for the whole run. Its step reads one secret, and
|
||||
# only to test it for emptiness: the step's env holds a boolean, never the
|
||||
# value, and the step runs no repository code (no checkout). Fork and
|
||||
# Dependabot pull_request runs get no secrets: rbe-fork-mint decides
|
||||
# theirs (fork-ro, fork-rw, or cache while it is closed).
|
||||
# job sees the same answer for the whole run. Its decide step reads one
|
||||
# secret, and only to test it for emptiness: the step's env holds a
|
||||
# boolean, never the value, and the step runs no repository code (it comes
|
||||
# before the job's checkout). Fork and Dependabot pull_request runs get no
|
||||
# secrets: rbe-fork-mint decides theirs (fork-ro, fork-rw, or cache while
|
||||
# it is closed).
|
||||
#
|
||||
# Then, only when the lanes will execute remotely (enabled: modes remote,
|
||||
# fork-ro, fork-rw; merge groups are mode remote), the worker-env preflight:
|
||||
# rbe-west's pool workers run gascity main's tools/rbe/worker-env and
|
||||
# advertise its pin, and the schedulers match //platforms:rbe_worker's pin
|
||||
# exactly, so when gascity re-pins and beads' copy (tools/rbe/worker-env.txt
|
||||
# and the pin in platforms/BUILD.bazel) falls behind, every remote action
|
||||
# queues until the lanes time out. tools/rbe/worker-env-sync compares the
|
||||
# copy with gascity main and, out of step, fails this job: every lane needs
|
||||
# it, so they all skip and pr.yml's gate is red within a minute, with the
|
||||
# diff and the pin to commit in this job's summary. A fallback to mode
|
||||
# cache would not keep the PR green (pr.yml's ci-gate requires a remote
|
||||
# run, BAZEL_REMOTE) and would only make it slower, so fail fast. gascity
|
||||
# unreachable (exit 2: unknown) warns and proceeds: GitHub's availability
|
||||
# never blocks a PR. The steps after decide run repository code (the PR's
|
||||
# script, sparse-checked-out) but hold no secret and set no output: the
|
||||
# job's outputs are decide's alone. nightly.yml runs the same check every
|
||||
# night and opens a ci/infra issue.
|
||||
rbe:
|
||||
name: execution mode
|
||||
# Same-repo PRs (and merge_group/push/dispatch/schedule runs, which are
|
||||
|
|
@ -472,6 +491,36 @@ jobs:
|
|||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# The worker-env preflight (see the job's comment): the PR's own copy,
|
||||
# so a PR that syncs it passes. Only the two paths it compares.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
if: ${{ steps.decide.outputs.enabled == 'true' }}
|
||||
with:
|
||||
ref: ${{ inputs.checkout-sha }}
|
||||
persist-credentials: false
|
||||
allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}
|
||||
sparse-checkout: |
|
||||
tools/rbe
|
||||
platforms
|
||||
|
||||
- name: worker-env in step with gascity main
|
||||
if: ${{ steps.decide.outputs.enabled == 'true' }}
|
||||
env:
|
||||
MODE: ${{ steps.decide.outputs.mode }}
|
||||
run: |
|
||||
rc=0
|
||||
tools/rbe/worker-env-sync main || rc=$?
|
||||
case "$rc" in
|
||||
0) ;;
|
||||
1)
|
||||
echo "::error title=Bazel lanes stopped before queueing::beads' worker-env pin is out of step with gascity main, so rbe-west has no worker for this run's remote actions (mode $MODE): copy tools/rbe/worker-env.txt and the worker-env pin in platforms/BUILD.bazel from gastownhall/gascity main (this job's summary has the diff and the line to commit; see tools/rbe/worker-env-sync)."
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "::warning title=rbe worker-env not checked::could not compare beads' worker-env pin with gascity main (exit $rc); proceeding in mode $MODE. If the lanes sit at 0 running actions, run tools/rbe/worker-env-sync."
|
||||
;;
|
||||
esac
|
||||
|
||||
# Best-effort pre-warm of gastownhall/gascity's rbe-west OSS worker pool
|
||||
# (one Blacksmith-hosted NativeLink worker, rbe-worker-pool.yml) so it is
|
||||
# already booting instead of the lanes below waiting ~120s at "N actions, 0
|
||||
|
|
|
|||
30
.github/workflows/nightly.yml
vendored
30
.github/workflows/nightly.yml
vendored
|
|
@ -9,6 +9,9 @@ on:
|
|||
- cron: '0 2 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
BD_DISABLE_METRICS: "1"
|
||||
BD_DISABLE_EVENT_FLUSH: "1"
|
||||
|
|
@ -39,15 +42,38 @@ jobs:
|
|||
# rbe-west's pool workers are gascity's and advertise the sha256 of
|
||||
# gascity's worker-env manifest; beads' tools/rbe/worker-env.txt is a
|
||||
# byte-for-byte copy and platforms/BUILD.bazel carries the same pin. A copy
|
||||
# that falls behind queues every remote action, so say so every night.
|
||||
# that falls behind queues every remote action (bazel.yml's rbe job runs
|
||||
# the same check before each remote run and fails it fast), so check every
|
||||
# night and, out of step, open or update a ci/infra issue with the diff and
|
||||
# the pin to commit (tools/rbe/worker-env-sync-issue). The only job here
|
||||
# with issues: write, and only its last step uses the token. gascity
|
||||
# unreachable (exit 2) fails the job without an issue.
|
||||
rbe-worker-env-sync:
|
||||
name: rbe worker-env in step with gascity
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Compare the worker-env manifest and pin with gascity's main
|
||||
run: tools/rbe/worker-env-sync main
|
||||
id: sync
|
||||
env:
|
||||
WORKER_ENV_SYNC_REPORT: ${{ runner.temp }}/worker-env-sync.md
|
||||
run: |
|
||||
rc=0
|
||||
tools/rbe/worker-env-sync main || rc=$?
|
||||
echo "rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
exit "$rc"
|
||||
|
||||
- name: Open or update the out-of-step issue
|
||||
if: ${{ failure() && steps.sync.outputs.rc == '1' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPORT: ${{ runner.temp }}/worker-env-sync.md
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: tools/rbe/worker-env-sync-issue "$REPORT" "$RUN_URL"
|
||||
|
|
|
|||
|
|
@ -255,18 +255,24 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) {
|
|||
if bazelPackageJobs[name] {
|
||||
want["fetch-depth"] = "0"
|
||||
}
|
||||
// rbe's worker-env preflight reads two paths of the same ref.
|
||||
if name == bazelRBEJobName {
|
||||
want["sparse-checkout"] = bazelRBESparseCheckout
|
||||
}
|
||||
if !reflect.DeepEqual(step.With, want) {
|
||||
t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every lane checks out the PR except rbe (decides the mode before any
|
||||
// checkout) and rbe-prewarm (no checkout at all, by design: B1, security
|
||||
// review of bdef342d5 - its dispatch logic is inlined into the job's own
|
||||
// `run:` instead of a checked-out script file, so no step in this job
|
||||
// ever reads repository content under the shared gascity credential).
|
||||
if checkouts != len(workflow.Jobs)-2 {
|
||||
t.Errorf("%d checkouts in %s, want one per lane excluding %s and %s (%d)", checkouts, bazelWorkflowName, bazelRBEJobName, bazelRBEPrewarmJobName, len(workflow.Jobs)-2)
|
||||
// Every job checks out the PR once except rbe-prewarm (no checkout at
|
||||
// all, by design: B1, security review of bdef342d5 - its dispatch logic
|
||||
// is inlined into the job's own `run:` instead of a checked-out script
|
||||
// file, so no step in this job ever reads repository content under the
|
||||
// shared gascity credential). rbe's checkout (sparse) comes after its
|
||||
// decide step, for the worker-env preflight
|
||||
// (TestBazelRBEWorkerEnvPreflight).
|
||||
if checkouts != len(workflow.Jobs)-1 {
|
||||
t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1)
|
||||
}
|
||||
if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue {
|
||||
t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue)
|
||||
|
|
|
|||
|
|
@ -4736,8 +4736,10 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) {
|
|||
// so a run cannot mix modes and no lane can drift from the others (review D1
|
||||
// F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition
|
||||
// on vars and fork alone sent them to the remote-only path with nothing to
|
||||
// run it. The job's one step reads nothing but booleans, runs no
|
||||
// repository code, and its outputs are the workflow_call outputs.
|
||||
// run it. The job's decide step (its first) reads nothing but booleans, runs
|
||||
// no repository code, and its outputs are the job's and the workflow_call
|
||||
// outputs; the steps after it are the worker-env preflight
|
||||
// (TestBazelRBEWorkerEnvPreflight), which set no output.
|
||||
func TestBazelRBEJobDecidesOnce(t *testing.T) {
|
||||
workflow := readCIWorkflow(t, bazelWorkflowName)
|
||||
job := workflow.job(t, bazelRBEJobName)
|
||||
|
|
@ -4757,8 +4759,8 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) {
|
|||
if !reflect.DeepEqual(job.Outputs, wantOutputs) {
|
||||
t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs)
|
||||
}
|
||||
if len(job.Steps) != 1 {
|
||||
t.Fatalf("%s has %d steps, want exactly the decision step", bazelRBEJobName, len(job.Steps))
|
||||
if len(job.Steps) != 3 {
|
||||
t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps))
|
||||
}
|
||||
step := job.Steps[0]
|
||||
wantEnv := map[string]string{
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@ import (
|
|||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Test actions exec host tools and load the shared libraries of every cgo
|
||||
|
|
@ -33,6 +37,12 @@ const (
|
|||
rbeWorkerEnvManifest = "tools/rbe/worker-env.txt"
|
||||
rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker"
|
||||
rbeWorkerEnvSync = "tools/rbe/worker-env-sync"
|
||||
rbeWorkerEnvSyncIssue = "tools/rbe/worker-env-sync-issue"
|
||||
// bazel.yml's rbe job checks out only what worker-env-sync compares.
|
||||
bazelRBESparseCheckout = "tools/rbe\nplatforms\n"
|
||||
rbeWorkerEnvEnabledIf = "${{ steps.decide.outputs.enabled == 'true' }}"
|
||||
nightlyWorkerEnvJob = "rbe-worker-env-sync"
|
||||
nightlyWorkerEnvReport = "${{ runner.temp }}/worker-env-sync.md"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -205,86 +215,327 @@ func TestRBEWorkerEnvManifestGuards(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local
|
||||
// gascity (file:// URLs): quiet success when the manifest and pin match
|
||||
// gascity's at REF, and otherwise exit 1 with the diff and the pin to commit
|
||||
// in the output and step summary, for a manifest or a pin that differs.
|
||||
func TestRBEWorkerEnvSync(t *testing.T) {
|
||||
// rbeWorkerEnvFixture is a local gascity (served to worker-env-sync through
|
||||
// file:// URLs, WORKER_ENV_SYNC_URL) whose main pins manifest at pin, and a
|
||||
// way to lay out a beads checkout with the given manifest and pin plus the
|
||||
// real tools/rbe scripts, as the workflows run them.
|
||||
type rbeWorkerEnvFixture struct {
|
||||
gascity, manifest, pin string
|
||||
}
|
||||
|
||||
func rbeTestPin(manifest string) string {
|
||||
sum := sha256.Sum256([]byte(manifest))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func rbeTestPlatformBuild(pin string) string {
|
||||
return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n"
|
||||
}
|
||||
|
||||
func rbeTestWrite(t *testing.T, path, body string, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func newRBEWorkerEnvFixture(t *testing.T) rbeWorkerEnvFixture {
|
||||
t.Helper()
|
||||
requireHostTool(t, "curl")
|
||||
requireHostTool(t, "bash")
|
||||
script := filepath.Join(bazelPolicyRoot(t), rbeWorkerEnvSync)
|
||||
manifest := "arch x86_64\npkg git 2\n"
|
||||
sum := sha256.Sum256([]byte(manifest))
|
||||
pin := "sha256:" + hex.EncodeToString(sum[:])
|
||||
build := func(pin string) string {
|
||||
return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n"
|
||||
}
|
||||
write := func(path, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gascity := t.TempDir()
|
||||
write(filepath.Join(gascity, "main", "tools/rbe/worker-env.txt"), manifest)
|
||||
write(filepath.Join(gascity, "main", "platforms/BUILD.bazel"), build(pin))
|
||||
f := rbeWorkerEnvFixture{gascity: t.TempDir(), manifest: "arch x86_64\npkg git 2\n"}
|
||||
f.pin = rbeTestPin(f.manifest)
|
||||
rbeTestWrite(t, filepath.Join(f.gascity, "main", rbeWorkerEnvManifest), f.manifest, 0o644)
|
||||
rbeTestWrite(t, filepath.Join(f.gascity, "main", rbeWorkerPlatformBuild), rbeTestPlatformBuild(f.pin), 0o644)
|
||||
return f
|
||||
}
|
||||
|
||||
run := func(ourManifest, ourPin string) (string, string, error) {
|
||||
// beads lays out a beads checkout pinning manifest at pin, with the real
|
||||
// worker-env-sync scripts, and returns its root.
|
||||
func (f rbeWorkerEnvFixture) beads(t *testing.T, manifest, pin string) string {
|
||||
t.Helper()
|
||||
root := bazelPolicyRoot(t)
|
||||
dir := t.TempDir()
|
||||
rbeTestWrite(t, filepath.Join(dir, rbeWorkerEnvManifest), manifest, 0o644)
|
||||
rbeTestWrite(t, filepath.Join(dir, rbeWorkerPlatformBuild), rbeTestPlatformBuild(pin), 0o644)
|
||||
for _, script := range []string{rbeWorkerEnvSync, rbeWorkerEnvSyncIssue} {
|
||||
rbeTestWrite(t, filepath.Join(dir, script), readPolicyFile(t, root, script), 0o755)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// run runs a workflow step's script (bash -e, as GitHub does) in dir with
|
||||
// gascity served from the fixture plus env, and returns its exit code and
|
||||
// output.
|
||||
func (f rbeWorkerEnvFixture) run(t *testing.T, dir, script string, env ...string) (int, string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), append([]string{"WORKER_ENV_SYNC_URL=file://" + f.gascity}, env...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case err == nil:
|
||||
return 0, string(out)
|
||||
case errors.As(err, &exitErr):
|
||||
return exitErr.ExitCode(), string(out)
|
||||
default:
|
||||
t.Fatalf("run step: %v\n%s", err, out)
|
||||
return -1, ""
|
||||
}
|
||||
}
|
||||
|
||||
const rbeTestStaleManifest = "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n"
|
||||
|
||||
// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local
|
||||
// gascity: quiet success when the manifest and pin match gascity's at REF;
|
||||
// exit 1 with the diff and the pin to commit in the output, step summary and
|
||||
// WORKER_ENV_SYNC_REPORT for a manifest or a pin that differs; and exit 2
|
||||
// ("unknown", a warning) when gascity cannot be read, never "in step".
|
||||
func TestRBEWorkerEnvSync(t *testing.T) {
|
||||
f := newRBEWorkerEnvFixture(t)
|
||||
run := func(ourManifest, ourPin string) (int, string, string, string) {
|
||||
t.Helper()
|
||||
beads := t.TempDir()
|
||||
write(filepath.Join(beads, rbeWorkerEnvManifest), ourManifest)
|
||||
write(filepath.Join(beads, rbeWorkerPlatformBuild), build(ourPin))
|
||||
beads := f.beads(t, ourManifest, ourPin)
|
||||
summary := filepath.Join(beads, "summary.md")
|
||||
cmd := exec.Command("bash", script)
|
||||
cmd.Dir = beads
|
||||
cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity, "GITHUB_STEP_SUMMARY="+summary)
|
||||
out, err := cmd.CombinedOutput()
|
||||
b, _ := os.ReadFile(summary)
|
||||
return string(out), string(b), err
|
||||
report := filepath.Join(beads, "report.md")
|
||||
code, out := f.run(t, beads, "tools/rbe/worker-env-sync\n", "GITHUB_STEP_SUMMARY="+summary, "WORKER_ENV_SYNC_REPORT="+report)
|
||||
s, _ := os.ReadFile(summary)
|
||||
r, _ := os.ReadFile(report)
|
||||
return code, out, string(s), string(r)
|
||||
}
|
||||
|
||||
out, summary, err := run(manifest, pin)
|
||||
if err != nil || !strings.Contains(out, "worker-env: in step with gascity main ("+pin+")") || summary != "" {
|
||||
t.Fatalf("in step: %v\n%s\nsummary:\n%s", err, out, summary)
|
||||
code, out, summary, report := run(f.manifest, f.pin)
|
||||
if code != 0 || !strings.Contains(out, "worker-env: in step with gascity main ("+f.pin+")") || summary != "" || report != "" {
|
||||
t.Fatalf("in step: exit %d\n%s\nsummary:\n%s\nreport:\n%s", code, out, summary, report)
|
||||
}
|
||||
|
||||
stale := "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n"
|
||||
staleSum := sha256.Sum256([]byte(stale))
|
||||
stalePin := "sha256:" + hex.EncodeToString(staleSum[:])
|
||||
stalePin := rbeTestPin(rbeTestStaleManifest)
|
||||
for name, c := range map[string][2]string{
|
||||
"manifest and pin": {stale, stalePin},
|
||||
"pin only": {manifest, stalePin},
|
||||
"manifest only": {stale, pin},
|
||||
"manifest and pin": {rbeTestStaleManifest, stalePin},
|
||||
"pin only": {f.manifest, stalePin},
|
||||
"manifest only": {rbeTestStaleManifest, f.pin},
|
||||
} {
|
||||
out, summary, err := run(c[0], c[1])
|
||||
if err == nil {
|
||||
t.Errorf("%s out of step: succeeded\n%s", name, out)
|
||||
code, out, summary, report := run(c[0], c[1])
|
||||
if code != 1 {
|
||||
t.Errorf("%s out of step: exit %d, want 1\n%s", name, code, out)
|
||||
continue
|
||||
}
|
||||
for _, want := range []string{
|
||||
"::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + pin,
|
||||
" \"worker-env\": \"" + pin + "\",\n",
|
||||
"::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + f.pin,
|
||||
"beads' worker-env pin is out of step with gascity main",
|
||||
" \"worker-env\": \"" + f.pin + "\",\n",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("%s: output lacks %q:\n%s", name, want, out)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(summary, "### rbe worker-env out of step with gascity") {
|
||||
t.Errorf("%s: no step summary:\n%s", name, summary)
|
||||
for what, md := range map[string]string{"step summary": summary, "report": report} {
|
||||
if !strings.Contains(md, "### rbe worker-env out of step with gascity") || !strings.Contains(md, "\"worker-env\": \""+f.pin+"\"") {
|
||||
t.Errorf("%s: no %s:\n%s", name, what, md)
|
||||
}
|
||||
}
|
||||
if c[0] != manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") {
|
||||
if c[0] != f.manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") {
|
||||
t.Errorf("%s: no manifest diff:\n%s", name, out)
|
||||
}
|
||||
}
|
||||
|
||||
// gascity unreachable (a REF that does not exist): an error, never "in step".
|
||||
cmd := exec.Command("bash", script, "no-such-ref")
|
||||
cmd.Dir = t.TempDir()
|
||||
cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity)
|
||||
if out, err := cmd.CombinedOutput(); err == nil {
|
||||
t.Errorf("worker-env-sync of a missing ref succeeded:\n%s", out)
|
||||
// gascity unreachable (a REF that does not exist, or no gascity at all):
|
||||
// exit 2 with a warning, never "in step" and never "out of step".
|
||||
beads := f.beads(t, f.manifest, f.pin)
|
||||
for name, env := range map[string][]string{
|
||||
"missing ref": {"REF=no-such-ref"},
|
||||
"unreachable": {"REF=main", "WORKER_ENV_SYNC_URL=file:///nonexistent-gascity"},
|
||||
} {
|
||||
code, out := f.run(t, beads, `tools/rbe/worker-env-sync "$REF"`+"\n", env...)
|
||||
if code != 2 || !strings.Contains(out, "::warning title=rbe worker-env unknown::") || strings.Contains(out, "in step") || strings.Contains(out, "::error") {
|
||||
t.Errorf("worker-env-sync, %s: exit %d, want 2 with an unknown warning:\n%s", name, code, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBazelRBEWorkerEnvPreflight: when gascity re-pins and beads' copy falls
|
||||
// behind, a remote-execution run fails in bazel.yml's rbe job (every lane
|
||||
// needs it, so none starts and pr.yml's gate is red at once) with the
|
||||
// actionable message, instead of every lane queueing on rbe-west until it
|
||||
// times out. Only runs whose lanes execute remotely (enabled: remote,
|
||||
// fork-ro, fork-rw; merge groups are remote) check; gascity unreachable
|
||||
// warns and proceeds. The steps run after decide and set no output, so the
|
||||
// mode the lanes read is decide's alone.
|
||||
func TestBazelRBEWorkerEnvPreflight(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName)
|
||||
if len(job.Steps) != 3 || job.Steps[0].ID != "decide" {
|
||||
t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID)
|
||||
}
|
||||
if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) {
|
||||
t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName)
|
||||
}
|
||||
checkout, check := job.Steps[1], job.Steps[2]
|
||||
if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout {
|
||||
t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s",
|
||||
bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf)
|
||||
}
|
||||
wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"}
|
||||
if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) ||
|
||||
!strings.Contains(check.Run, "tools/rbe/worker-env-sync main") {
|
||||
t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main",
|
||||
bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv)
|
||||
}
|
||||
|
||||
f := newRBEWorkerEnvFixture(t)
|
||||
stalePin := rbeTestPin(rbeTestStaleManifest)
|
||||
for _, c := range []struct {
|
||||
name, manifest, pin, url string
|
||||
code int
|
||||
want string
|
||||
}{
|
||||
{"in step", f.manifest, f.pin, "", 0, "worker-env: in step with gascity main"},
|
||||
{"gascity re-pinned", rbeTestStaleManifest, stalePin, "", 1,
|
||||
"::error title=Bazel lanes stopped before queueing::beads' worker-env pin is out of step with gascity main, so rbe-west has no worker for this run's remote actions (mode remote): copy tools/rbe/worker-env.txt and the worker-env pin in platforms/BUILD.bazel from gastownhall/gascity main"},
|
||||
{"gascity unreachable", rbeTestStaleManifest, stalePin, "file:///nonexistent-gascity", 0,
|
||||
"::warning title=rbe worker-env not checked::could not compare beads' worker-env pin with gascity main (exit 2); proceeding in mode remote."},
|
||||
} {
|
||||
env := []string{"MODE=remote", "GITHUB_STEP_SUMMARY=" + filepath.Join(t.TempDir(), "summary")}
|
||||
if c.url != "" {
|
||||
env = append(env, "WORKER_ENV_SYNC_URL="+c.url)
|
||||
}
|
||||
code, out := f.run(t, f.beads(t, c.manifest, c.pin), check.Run, env...)
|
||||
if code != c.code || !strings.Contains(out, c.want) {
|
||||
t.Errorf("rbe preflight, %s: exit %d, want %d with %q:\n%s", c.name, code, c.code, c.want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNightlyWorkerEnvSyncOpensIssue: nightly's check, out of step, opens a
|
||||
// ci/infra issue with the diff (or, with one open, updates its body and
|
||||
// comments with the run), with GITHUB_TOKEN's issues: write scoped to that
|
||||
// job and the token in that one step; the rest of nightly.yml reads only.
|
||||
// gascity unreachable fails the job without an issue.
|
||||
func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) {
|
||||
nightly := readCIWorkflow(t, "nightly.yml")
|
||||
var top struct {
|
||||
Permissions map[string]string `yaml:"permissions"`
|
||||
}
|
||||
if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/nightly.yml")), &top); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(top.Permissions, map[string]string{"contents": "read"}) {
|
||||
t.Errorf("nightly.yml permissions = %v, want contents: read", top.Permissions)
|
||||
}
|
||||
for name, job := range nightly.Jobs {
|
||||
want := any(nil)
|
||||
if name == nightlyWorkerEnvJob {
|
||||
want = map[string]any{"contents": "read", "issues": "write"}
|
||||
}
|
||||
if !reflect.DeepEqual(job.Permissions, want) {
|
||||
t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want)
|
||||
}
|
||||
}
|
||||
job := nightly.job(t, nightlyWorkerEnvJob)
|
||||
if len(job.Steps) != 3 {
|
||||
t.Fatalf("nightly.yml %s has %d steps, want checkout, check, issue", nightlyWorkerEnvJob, len(job.Steps))
|
||||
}
|
||||
check, issue := job.Steps[1], job.Steps[2]
|
||||
if check.ID != "sync" || check.If != "" || !reflect.DeepEqual(check.Env, map[string]string{"WORKER_ENV_SYNC_REPORT": nightlyWorkerEnvReport}) {
|
||||
t.Errorf("nightly check step: id %q, if %q, env %v; want id sync, no if, WORKER_ENV_SYNC_REPORT=%s", check.ID, check.If, check.Env, nightlyWorkerEnvReport)
|
||||
}
|
||||
wantIssueEnv := map[string]string{
|
||||
"GH_TOKEN": "${{ github.token }}",
|
||||
"REPORT": nightlyWorkerEnvReport,
|
||||
"RUN_URL": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
|
||||
}
|
||||
if issue.If != "${{ failure() && steps.sync.outputs.rc == '1' }}" || !reflect.DeepEqual(issue.Env, wantIssueEnv) || !strings.Contains(issue.Run, rbeWorkerEnvSyncIssue) {
|
||||
t.Errorf("nightly issue step: if %q, env %v, run %q; want only on rc 1, env %v, running %s", issue.If, issue.Env, issue.Run, wantIssueEnv, rbeWorkerEnvSyncIssue)
|
||||
}
|
||||
for name, j := range nightly.Jobs {
|
||||
for i, step := range j.Steps {
|
||||
if (name != nightlyWorkerEnvJob || i != 2) && strings.Contains(fmt.Sprint(step.Env, step.With, step.Run), "github.token") {
|
||||
t.Errorf("nightly.yml job %s step %d uses github.token; only the issue step may", name, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The check step's rc output: 1 out of step (the issue step runs), 2
|
||||
// unknown (it does not), 0 in step.
|
||||
f := newRBEWorkerEnvFixture(t)
|
||||
stalePin := rbeTestPin(rbeTestStaleManifest)
|
||||
for _, c := range []struct {
|
||||
name, manifest, pin, url, rc string
|
||||
}{
|
||||
{"in step", f.manifest, f.pin, "", "0"},
|
||||
{"out of step", rbeTestStaleManifest, stalePin, "", "1"},
|
||||
{"gascity unreachable", f.manifest, f.pin, "file:///nonexistent-gascity", "2"},
|
||||
} {
|
||||
dir := t.TempDir()
|
||||
output := filepath.Join(dir, "output")
|
||||
report := filepath.Join(dir, "report.md")
|
||||
env := []string{"GITHUB_OUTPUT=" + output, "WORKER_ENV_SYNC_REPORT=" + report}
|
||||
if c.url != "" {
|
||||
env = append(env, "WORKER_ENV_SYNC_URL="+c.url)
|
||||
}
|
||||
code, out := f.run(t, f.beads(t, c.manifest, c.pin), check.Run, env...)
|
||||
got, _ := os.ReadFile(output)
|
||||
if fmt.Sprint(code) != c.rc || string(got) != "rc="+c.rc+"\n" {
|
||||
t.Errorf("nightly check, %s: exit %d, output %q; want exit and rc %s\n%s", c.name, code, got, c.rc, out)
|
||||
}
|
||||
if r, _ := os.ReadFile(report); (c.rc == "1") != strings.Contains(string(r), "### rbe worker-env out of step with gascity") {
|
||||
t.Errorf("nightly check, %s: report %q", c.name, r)
|
||||
}
|
||||
}
|
||||
|
||||
// The issue step, against a stand-in gh that logs its calls and lists
|
||||
// the open ci/infra issues it is given.
|
||||
const ghStub = `#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$GH_LOG"
|
||||
for a in "$@"; do
|
||||
case "$prev" in --body-file) cp "$a" "$GH_LOG.body" ;; esac
|
||||
prev=$a
|
||||
done
|
||||
case "$1 $2" in
|
||||
"issue list") jq -r "$(for a in "$@"; do [ "$p" = --jq ] && echo "$a"; p=$a; done)" <<<"$GH_ISSUES" ;;
|
||||
esac
|
||||
`
|
||||
requireHostTool(t, "jq")
|
||||
title := "rbe worker-env out of step with gascity main"
|
||||
for _, c := range []struct {
|
||||
name, issues string
|
||||
want []string
|
||||
}{
|
||||
{"no open issue", `[{"number":7,"title":"something else"}]`, []string{
|
||||
"issue create --repo gastownhall/beads --title " + title + " --label ci/infra --body-file ",
|
||||
}},
|
||||
{"open issue", `[{"number":7,"title":"something else"},{"number":42,"title":"` + title + `"}]`, []string{
|
||||
"issue edit 42 --repo gastownhall/beads --body-file ",
|
||||
"issue comment 42 --repo gastownhall/beads --body Still out of step with gascity main: https://github.com/gastownhall/beads/actions/runs/9",
|
||||
}},
|
||||
} {
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "bin")
|
||||
rbeTestWrite(t, filepath.Join(bin, "gh"), ghStub, 0o755)
|
||||
report := filepath.Join(dir, "report.md")
|
||||
rbeTestWrite(t, report, "### rbe worker-env out of step with gascity\n\nthe diff\n", 0o644)
|
||||
log := filepath.Join(dir, "gh.log")
|
||||
code, out := f.run(t, f.beads(t, f.manifest, f.pin), issue.Run,
|
||||
"PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), "GH_LOG="+log, "GH_ISSUES="+c.issues,
|
||||
"GITHUB_REPOSITORY=gastownhall/beads", "REPORT="+report, "RUN_URL=https://github.com/gastownhall/beads/actions/runs/9")
|
||||
calls, _ := os.ReadFile(log)
|
||||
body, _ := os.ReadFile(log + ".body")
|
||||
if code != 0 {
|
||||
t.Errorf("issue step, %s: exit %d\n%s", c.name, code, out)
|
||||
}
|
||||
for _, want := range c.want {
|
||||
if !strings.Contains(string(calls), want) {
|
||||
t.Errorf("issue step, %s: gh calls lack %q:\n%s", c.name, want, calls)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(calls), "issue create") == (c.name == "open issue") {
|
||||
t.Errorf("issue step, %s: gh calls:\n%s", c.name, calls)
|
||||
}
|
||||
if !strings.Contains(string(body), "the diff") || !strings.Contains(string(body), "https://github.com/gastownhall/beads/actions/runs/9") {
|
||||
t.Errorf("issue step, %s: body:\n%s", c.name, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,16 +11,28 @@
|
|||
# queue on a pin no worker serves. This compares the manifest and the pin
|
||||
# with gascity's at REF (default main) and, if either differs, prints the
|
||||
# diff and what to commit (stdout and $GITHUB_STEP_SUMMARY) and exits 1.
|
||||
# If it cannot compare (gascity unreachable, a missing REF, any other
|
||||
# failure) it warns and exits 2: callers that must not block on GitHub's
|
||||
# availability (bazel.yml's rbe job) proceed on 2 and fail only on 1.
|
||||
# Run from the repository root.
|
||||
#
|
||||
# Env (tests only): WORKER_ENV_SYNC_URL, gascity's raw-content base (default
|
||||
# https://raw.githubusercontent.com/gastownhall/gascity).
|
||||
set -euo pipefail
|
||||
# Callers: bazel.yml's rbe job, before any lane of a remote-execution run
|
||||
# (fails the run fast instead of letting the lanes queue), and nightly.yml's
|
||||
# rbe-worker-env-sync job (opens or updates a ci/infra issue,
|
||||
# tools/rbe/worker-env-sync-issue).
|
||||
#
|
||||
# Env: WORKER_ENV_SYNC_REPORT, a file to also write the out-of-step report
|
||||
# (markdown) to. Tests only: WORKER_ENV_SYNC_URL, gascity's raw-content base
|
||||
# (default https://raw.githubusercontent.com/gastownhall/gascity).
|
||||
set -Eeuo pipefail
|
||||
|
||||
ref=${1:-main}
|
||||
base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
# Anything that fails before the comparison is "unknown", never "in step"
|
||||
# and never "out of step".
|
||||
trap 'echo "::warning title=rbe worker-env unknown::worker-env-sync could not compare with gascity $ref ($BASH_COMMAND failed); not checked."; exit 2' ERR
|
||||
|
||||
summary() {
|
||||
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat >>"$GITHUB_STEP_SUMMARY"; else cat >/dev/null; fi
|
||||
|
|
@ -33,8 +45,8 @@ pin_of() {
|
|||
echo "${pin:-none}"
|
||||
}
|
||||
|
||||
curl -fsSL --retry 3 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt"
|
||||
curl -fsSL --retry 3 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel"
|
||||
curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt"
|
||||
curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel"
|
||||
ours=$(pin_of platforms/BUILD.bazel)
|
||||
theirs=$(pin_of "$tmp/BUILD.bazel")
|
||||
if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs" ]; then
|
||||
|
|
@ -42,6 +54,8 @@ if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# Out of step is known from here on: no failure below turns it into "unknown".
|
||||
trap - ERR
|
||||
diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/worker-env.txt" >"$tmp/diff" || true
|
||||
{
|
||||
cat <<-EOF
|
||||
|
|
@ -63,5 +77,6 @@ diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/work
|
|||
} >"$tmp/report.md"
|
||||
cat "$tmp/report.md"
|
||||
summary <"$tmp/report.md"
|
||||
echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: copy gascity's tools/rbe/worker-env.txt and its pin (the step summary has the diff)."
|
||||
if [ -n "${WORKER_ENV_SYNC_REPORT:-}" ]; then cp "$tmp/report.md" "$WORKER_ENV_SYNC_REPORT"; fi
|
||||
echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: beads' worker-env pin is out of step with gascity $ref, so no rbe-west pool worker serves beads' remote actions. Copy gascity's tools/rbe/worker-env.txt byte for byte and its platforms/BUILD.bazel pin (the step summary has the diff; see tools/rbe/worker-env-sync)."
|
||||
exit 1
|
||||
|
|
|
|||
44
tools/rbe/worker-env-sync-issue
Executable file
44
tools/rbe/worker-env-sync-issue
Executable file
|
|
@ -0,0 +1,44 @@
|
|||
#!/usr/bin/env bash
|
||||
# worker-env-sync-issue: open or update the "out of step" issue.
|
||||
#
|
||||
# tools/rbe/worker-env-sync-issue REPORT RUN_URL
|
||||
#
|
||||
# nightly.yml's rbe-worker-env-sync job runs this when tools/rbe/worker-env-sync
|
||||
# exits 1 (beads' worker-env pin differs from gascity main's, so beads' remote
|
||||
# actions queue on rbe-west). REPORT is the markdown report worker-env-sync
|
||||
# wrote (WORKER_ENV_SYNC_REPORT): the diff and the pin to commit. With an open
|
||||
# issue of the same title labelled ci/infra, the issue's body becomes the
|
||||
# latest report and a comment links this run (so watchers hear about it every
|
||||
# night until it is fixed); otherwise a new issue is opened with the label.
|
||||
# Close the issue once the copy is synced.
|
||||
#
|
||||
# Env: GITHUB_REPOSITORY (owner/repo), GH_TOKEN (issues: write).
|
||||
set -euo pipefail
|
||||
|
||||
report=${1:?usage: worker-env-sync-issue REPORT RUN_URL}
|
||||
run_url=${2:?usage: worker-env-sync-issue REPORT RUN_URL}
|
||||
repo=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is unset}
|
||||
title="rbe worker-env out of step with gascity main"
|
||||
label="ci/infra"
|
||||
|
||||
body=$(mktemp)
|
||||
trap 'rm -f "$body"' EXIT
|
||||
{
|
||||
cat "$report"
|
||||
printf '\nReported by %s. Close this issue once tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin match gascity main (tools/rbe/worker-env-sync passes).\n' "$run_url"
|
||||
} >"$body"
|
||||
|
||||
# The label may not exist yet; an existing one is left as it is.
|
||||
gh label create "$label" --repo "$repo" --color d93f0b \
|
||||
--description "CI infrastructure (rbe-west, runners, cross-repo pins)" >/dev/null 2>&1 || true
|
||||
|
||||
# Exact-title match among the open ci/infra issues (no search index lag).
|
||||
number=$(gh issue list --repo "$repo" --state open --label "$label" --limit 100 \
|
||||
--json number,title --jq ".[] | select(.title == \"$title\") | .number" | head -n 1)
|
||||
if [ -n "$number" ]; then
|
||||
gh issue edit "$number" --repo "$repo" --body-file "$body"
|
||||
gh issue comment "$number" --repo "$repo" --body "Still out of step with gascity main: $run_url"
|
||||
echo "worker-env-sync-issue: updated #$number"
|
||||
else
|
||||
gh issue create --repo "$repo" --title "$title" --label "$label" --body-file "$body"
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue