diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index fbf8f2fef..625466eb2 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -382,11 +382,30 @@ env: jobs: # The one place the execution mode is decided (see the header), so every - # job sees the same answer for the whole run. Its step reads one secret, and - # only to test it for emptiness: the step's env holds a boolean, never the - # value, and the step runs no repository code (no checkout). Fork and - # Dependabot pull_request runs get no secrets: rbe-fork-mint decides - # theirs (fork-ro, fork-rw, or cache while it is closed). + # job sees the same answer for the whole run. Its decide step reads one + # secret, and only to test it for emptiness: the step's env holds a + # boolean, never the value, and the step runs no repository code (it comes + # before the job's checkout). Fork and Dependabot pull_request runs get no + # secrets: rbe-fork-mint decides theirs (fork-ro, fork-rw, or cache while + # it is closed). + # + # Then, only when the lanes will execute remotely (enabled: modes remote, + # fork-ro, fork-rw; merge groups are mode remote), the worker-env preflight: + # rbe-west's pool workers run gascity main's tools/rbe/worker-env and + # advertise its pin, and the schedulers match //platforms:rbe_worker's pin + # exactly, so when gascity re-pins and beads' copy (tools/rbe/worker-env.txt + # and the pin in platforms/BUILD.bazel) falls behind, every remote action + # queues until the lanes time out. tools/rbe/worker-env-sync compares the + # copy with gascity main and, out of step, fails this job: every lane needs + # it, so they all skip and pr.yml's gate is red within a minute, with the + # diff and the pin to commit in this job's summary. A fallback to mode + # cache would not keep the PR green (pr.yml's ci-gate requires a remote + # run, BAZEL_REMOTE) and would only make it slower, so fail fast. gascity + # unreachable (exit 2: unknown) warns and proceeds: GitHub's availability + # never blocks a PR. The steps after decide run repository code (the PR's + # script, sparse-checked-out) but hold no secret and set no output: the + # job's outputs are decide's alone. nightly.yml runs the same check every + # night and opens a ci/infra issue. rbe: name: execution mode # Same-repo PRs (and merge_group/push/dispatch/schedule runs, which are @@ -472,6 +491,36 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + # The worker-env preflight (see the job's comment): the PR's own copy, + # so a PR that syncs it passes. Only the two paths it compares. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: ${{ steps.decide.outputs.enabled == 'true' }} + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + sparse-checkout: | + tools/rbe + platforms + + - name: worker-env in step with gascity main + if: ${{ steps.decide.outputs.enabled == 'true' }} + env: + MODE: ${{ steps.decide.outputs.mode }} + run: | + rc=0 + tools/rbe/worker-env-sync main || rc=$? + case "$rc" in + 0) ;; + 1) + echo "::error title=Bazel lanes stopped before queueing::beads' worker-env pin is out of step with gascity main, so rbe-west has no worker for this run's remote actions (mode $MODE): copy tools/rbe/worker-env.txt and the worker-env pin in platforms/BUILD.bazel from gastownhall/gascity main (this job's summary has the diff and the line to commit; see tools/rbe/worker-env-sync)." + exit 1 + ;; + *) + echo "::warning title=rbe worker-env not checked::could not compare beads' worker-env pin with gascity main (exit $rc); proceeding in mode $MODE. If the lanes sit at 0 running actions, run tools/rbe/worker-env-sync." + ;; + esac + # Best-effort pre-warm of gastownhall/gascity's rbe-west OSS worker pool # (one Blacksmith-hosted NativeLink worker, rbe-worker-pool.yml) so it is # already booting instead of the lanes below waiting ~120s at "N actions, 0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 97f7bb42b..56f6a23e7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,6 +9,9 @@ on: - cron: '0 2 * * *' workflow_dispatch: +permissions: + contents: read + env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" @@ -39,15 +42,38 @@ jobs: # rbe-west's pool workers are gascity's and advertise the sha256 of # gascity's worker-env manifest; beads' tools/rbe/worker-env.txt is a # byte-for-byte copy and platforms/BUILD.bazel carries the same pin. A copy - # that falls behind queues every remote action, so say so every night. + # that falls behind queues every remote action (bazel.yml's rbe job runs + # the same check before each remote run and fails it fast), so check every + # night and, out of step, open or update a ci/infra issue with the diff and + # the pin to commit (tools/rbe/worker-env-sync-issue). The only job here + # with issues: write, and only its last step uses the token. gascity + # unreachable (exit 2) fails the job without an issue. rbe-worker-env-sync: name: rbe worker-env in step with gascity runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read + issues: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 with: persist-credentials: false - name: Compare the worker-env manifest and pin with gascity's main - run: tools/rbe/worker-env-sync main + id: sync + env: + WORKER_ENV_SYNC_REPORT: ${{ runner.temp }}/worker-env-sync.md + run: | + rc=0 + tools/rbe/worker-env-sync main || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit "$rc" + + - name: Open or update the out-of-step issue + if: ${{ failure() && steps.sync.outputs.rc == '1' }} + env: + GH_TOKEN: ${{ github.token }} + REPORT: ${{ runner.temp }}/worker-env-sync.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: tools/rbe/worker-env-sync-issue "$REPORT" "$RUN_URL" diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index 2b6fb8e6f..36a40b5b0 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -255,18 +255,24 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { if bazelPackageJobs[name] { want["fetch-depth"] = "0" } + // rbe's worker-env preflight reads two paths of the same ref. + if name == bazelRBEJobName { + want["sparse-checkout"] = bazelRBESparseCheckout + } if !reflect.DeepEqual(step.With, want) { t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) } } } - // Every lane checks out the PR except rbe (decides the mode before any - // checkout) and rbe-prewarm (no checkout at all, by design: B1, security - // review of bdef342d5 - its dispatch logic is inlined into the job's own - // `run:` instead of a checked-out script file, so no step in this job - // ever reads repository content under the shared gascity credential). - if checkouts != len(workflow.Jobs)-2 { - t.Errorf("%d checkouts in %s, want one per lane excluding %s and %s (%d)", checkouts, bazelWorkflowName, bazelRBEJobName, bazelRBEPrewarmJobName, len(workflow.Jobs)-2) + // Every job checks out the PR once except rbe-prewarm (no checkout at + // all, by design: B1, security review of bdef342d5 - its dispatch logic + // is inlined into the job's own `run:` instead of a checked-out script + // file, so no step in this job ever reads repository content under the + // shared gascity credential). rbe's checkout (sparse) comes after its + // decide step, for the worker-env preflight + // (TestBazelRBEWorkerEnvPreflight). + if checkouts != len(workflow.Jobs)-1 { + t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1) } if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue { t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index bf8d8d6b9..4878ba465 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -4736,8 +4736,10 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { // so a run cannot mix modes and no lane can drift from the others (review D1 // F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition // on vars and fork alone sent them to the remote-only path with nothing to -// run it. The job's one step reads nothing but booleans, runs no -// repository code, and its outputs are the workflow_call outputs. +// run it. The job's decide step (its first) reads nothing but booleans, runs +// no repository code, and its outputs are the job's and the workflow_call +// outputs; the steps after it are the worker-env preflight +// (TestBazelRBEWorkerEnvPreflight), which set no output. func TestBazelRBEJobDecidesOnce(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) job := workflow.job(t, bazelRBEJobName) @@ -4757,8 +4759,8 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { if !reflect.DeepEqual(job.Outputs, wantOutputs) { t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) } - if len(job.Steps) != 1 { - t.Fatalf("%s has %d steps, want exactly the decision step", bazelRBEJobName, len(job.Steps)) + if len(job.Steps) != 3 { + t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) } step := job.Steps[0] wantEnv := map[string]string{ diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index ff6f4c2f3..a9d849f24 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -4,13 +4,17 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "os" "os/exec" "path/filepath" + "reflect" "regexp" "slices" "strings" "testing" + + "gopkg.in/yaml.v3" ) // Test actions exec host tools and load the shared libraries of every cgo @@ -33,6 +37,12 @@ const ( rbeWorkerEnvManifest = "tools/rbe/worker-env.txt" rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker" rbeWorkerEnvSync = "tools/rbe/worker-env-sync" + rbeWorkerEnvSyncIssue = "tools/rbe/worker-env-sync-issue" + // bazel.yml's rbe job checks out only what worker-env-sync compares. + bazelRBESparseCheckout = "tools/rbe\nplatforms\n" + rbeWorkerEnvEnabledIf = "${{ steps.decide.outputs.enabled == 'true' }}" + nightlyWorkerEnvJob = "rbe-worker-env-sync" + nightlyWorkerEnvReport = "${{ runner.temp }}/worker-env-sync.md" ) var ( @@ -205,86 +215,327 @@ func TestRBEWorkerEnvManifestGuards(t *testing.T) { } } -// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local -// gascity (file:// URLs): quiet success when the manifest and pin match -// gascity's at REF, and otherwise exit 1 with the diff and the pin to commit -// in the output and step summary, for a manifest or a pin that differs. -func TestRBEWorkerEnvSync(t *testing.T) { +// rbeWorkerEnvFixture is a local gascity (served to worker-env-sync through +// file:// URLs, WORKER_ENV_SYNC_URL) whose main pins manifest at pin, and a +// way to lay out a beads checkout with the given manifest and pin plus the +// real tools/rbe scripts, as the workflows run them. +type rbeWorkerEnvFixture struct { + gascity, manifest, pin string +} + +func rbeTestPin(manifest string) string { + sum := sha256.Sum256([]byte(manifest)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func rbeTestPlatformBuild(pin string) string { + return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n" +} + +func rbeTestWrite(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func newRBEWorkerEnvFixture(t *testing.T) rbeWorkerEnvFixture { + t.Helper() requireHostTool(t, "curl") requireHostTool(t, "bash") - script := filepath.Join(bazelPolicyRoot(t), rbeWorkerEnvSync) - manifest := "arch x86_64\npkg git 2\n" - sum := sha256.Sum256([]byte(manifest)) - pin := "sha256:" + hex.EncodeToString(sum[:]) - build := func(pin string) string { - return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n" - } - write := func(path, body string) { - t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - gascity := t.TempDir() - write(filepath.Join(gascity, "main", "tools/rbe/worker-env.txt"), manifest) - write(filepath.Join(gascity, "main", "platforms/BUILD.bazel"), build(pin)) + f := rbeWorkerEnvFixture{gascity: t.TempDir(), manifest: "arch x86_64\npkg git 2\n"} + f.pin = rbeTestPin(f.manifest) + rbeTestWrite(t, filepath.Join(f.gascity, "main", rbeWorkerEnvManifest), f.manifest, 0o644) + rbeTestWrite(t, filepath.Join(f.gascity, "main", rbeWorkerPlatformBuild), rbeTestPlatformBuild(f.pin), 0o644) + return f +} - run := func(ourManifest, ourPin string) (string, string, error) { +// beads lays out a beads checkout pinning manifest at pin, with the real +// worker-env-sync scripts, and returns its root. +func (f rbeWorkerEnvFixture) beads(t *testing.T, manifest, pin string) string { + t.Helper() + root := bazelPolicyRoot(t) + dir := t.TempDir() + rbeTestWrite(t, filepath.Join(dir, rbeWorkerEnvManifest), manifest, 0o644) + rbeTestWrite(t, filepath.Join(dir, rbeWorkerPlatformBuild), rbeTestPlatformBuild(pin), 0o644) + for _, script := range []string{rbeWorkerEnvSync, rbeWorkerEnvSyncIssue} { + rbeTestWrite(t, filepath.Join(dir, script), readPolicyFile(t, root, script), 0o755) + } + return dir +} + +// run runs a workflow step's script (bash -e, as GitHub does) in dir with +// gascity served from the fixture plus env, and returns its exit code and +// output. +func (f rbeWorkerEnvFixture) run(t *testing.T, dir, script string, env ...string) (int, string) { + t.Helper() + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Dir = dir + cmd.Env = append(os.Environ(), append([]string{"WORKER_ENV_SYNC_URL=file://" + f.gascity}, env...)...) + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + switch { + case err == nil: + return 0, string(out) + case errors.As(err, &exitErr): + return exitErr.ExitCode(), string(out) + default: + t.Fatalf("run step: %v\n%s", err, out) + return -1, "" + } +} + +const rbeTestStaleManifest = "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n" + +// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local +// gascity: quiet success when the manifest and pin match gascity's at REF; +// exit 1 with the diff and the pin to commit in the output, step summary and +// WORKER_ENV_SYNC_REPORT for a manifest or a pin that differs; and exit 2 +// ("unknown", a warning) when gascity cannot be read, never "in step". +func TestRBEWorkerEnvSync(t *testing.T) { + f := newRBEWorkerEnvFixture(t) + run := func(ourManifest, ourPin string) (int, string, string, string) { t.Helper() - beads := t.TempDir() - write(filepath.Join(beads, rbeWorkerEnvManifest), ourManifest) - write(filepath.Join(beads, rbeWorkerPlatformBuild), build(ourPin)) + beads := f.beads(t, ourManifest, ourPin) summary := filepath.Join(beads, "summary.md") - cmd := exec.Command("bash", script) - cmd.Dir = beads - cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity, "GITHUB_STEP_SUMMARY="+summary) - out, err := cmd.CombinedOutput() - b, _ := os.ReadFile(summary) - return string(out), string(b), err + report := filepath.Join(beads, "report.md") + code, out := f.run(t, beads, "tools/rbe/worker-env-sync\n", "GITHUB_STEP_SUMMARY="+summary, "WORKER_ENV_SYNC_REPORT="+report) + s, _ := os.ReadFile(summary) + r, _ := os.ReadFile(report) + return code, out, string(s), string(r) } - out, summary, err := run(manifest, pin) - if err != nil || !strings.Contains(out, "worker-env: in step with gascity main ("+pin+")") || summary != "" { - t.Fatalf("in step: %v\n%s\nsummary:\n%s", err, out, summary) + code, out, summary, report := run(f.manifest, f.pin) + if code != 0 || !strings.Contains(out, "worker-env: in step with gascity main ("+f.pin+")") || summary != "" || report != "" { + t.Fatalf("in step: exit %d\n%s\nsummary:\n%s\nreport:\n%s", code, out, summary, report) } - stale := "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n" - staleSum := sha256.Sum256([]byte(stale)) - stalePin := "sha256:" + hex.EncodeToString(staleSum[:]) + stalePin := rbeTestPin(rbeTestStaleManifest) for name, c := range map[string][2]string{ - "manifest and pin": {stale, stalePin}, - "pin only": {manifest, stalePin}, - "manifest only": {stale, pin}, + "manifest and pin": {rbeTestStaleManifest, stalePin}, + "pin only": {f.manifest, stalePin}, + "manifest only": {rbeTestStaleManifest, f.pin}, } { - out, summary, err := run(c[0], c[1]) - if err == nil { - t.Errorf("%s out of step: succeeded\n%s", name, out) + code, out, summary, report := run(c[0], c[1]) + if code != 1 { + t.Errorf("%s out of step: exit %d, want 1\n%s", name, code, out) continue } for _, want := range []string{ - "::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + pin, - " \"worker-env\": \"" + pin + "\",\n", + "::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + f.pin, + "beads' worker-env pin is out of step with gascity main", + " \"worker-env\": \"" + f.pin + "\",\n", } { if !strings.Contains(out, want) { t.Errorf("%s: output lacks %q:\n%s", name, want, out) } } - if !strings.Contains(summary, "### rbe worker-env out of step with gascity") { - t.Errorf("%s: no step summary:\n%s", name, summary) + for what, md := range map[string]string{"step summary": summary, "report": report} { + if !strings.Contains(md, "### rbe worker-env out of step with gascity") || !strings.Contains(md, "\"worker-env\": \""+f.pin+"\"") { + t.Errorf("%s: no %s:\n%s", name, what, md) + } } - if c[0] != manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") { + if c[0] != f.manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") { t.Errorf("%s: no manifest diff:\n%s", name, out) } } - // gascity unreachable (a REF that does not exist): an error, never "in step". - cmd := exec.Command("bash", script, "no-such-ref") - cmd.Dir = t.TempDir() - cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity) - if out, err := cmd.CombinedOutput(); err == nil { - t.Errorf("worker-env-sync of a missing ref succeeded:\n%s", out) + // gascity unreachable (a REF that does not exist, or no gascity at all): + // exit 2 with a warning, never "in step" and never "out of step". + beads := f.beads(t, f.manifest, f.pin) + for name, env := range map[string][]string{ + "missing ref": {"REF=no-such-ref"}, + "unreachable": {"REF=main", "WORKER_ENV_SYNC_URL=file:///nonexistent-gascity"}, + } { + code, out := f.run(t, beads, `tools/rbe/worker-env-sync "$REF"`+"\n", env...) + if code != 2 || !strings.Contains(out, "::warning title=rbe worker-env unknown::") || strings.Contains(out, "in step") || strings.Contains(out, "::error") { + t.Errorf("worker-env-sync, %s: exit %d, want 2 with an unknown warning:\n%s", name, code, out) + } + } +} + +// TestBazelRBEWorkerEnvPreflight: when gascity re-pins and beads' copy falls +// behind, a remote-execution run fails in bazel.yml's rbe job (every lane +// needs it, so none starts and pr.yml's gate is red at once) with the +// actionable message, instead of every lane queueing on rbe-west until it +// times out. Only runs whose lanes execute remotely (enabled: remote, +// fork-ro, fork-rw; merge groups are remote) check; gascity unreachable +// warns and proceeds. The steps run after decide and set no output, so the +// mode the lanes read is decide's alone. +func TestBazelRBEWorkerEnvPreflight(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) + if len(job.Steps) != 3 || job.Steps[0].ID != "decide" { + t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) + } + if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) { + t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName) + } + checkout, check := job.Steps[1], job.Steps[2] + if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout { + t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", + bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf) + } + wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"} + if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) || + !strings.Contains(check.Run, "tools/rbe/worker-env-sync main") { + t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", + bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv) + } + + f := newRBEWorkerEnvFixture(t) + stalePin := rbeTestPin(rbeTestStaleManifest) + for _, c := range []struct { + name, manifest, pin, url string + code int + want string + }{ + {"in step", f.manifest, f.pin, "", 0, "worker-env: in step with gascity main"}, + {"gascity re-pinned", rbeTestStaleManifest, stalePin, "", 1, + "::error title=Bazel lanes stopped before queueing::beads' worker-env pin is out of step with gascity main, so rbe-west has no worker for this run's remote actions (mode remote): copy tools/rbe/worker-env.txt and the worker-env pin in platforms/BUILD.bazel from gastownhall/gascity main"}, + {"gascity unreachable", rbeTestStaleManifest, stalePin, "file:///nonexistent-gascity", 0, + "::warning title=rbe worker-env not checked::could not compare beads' worker-env pin with gascity main (exit 2); proceeding in mode remote."}, + } { + env := []string{"MODE=remote", "GITHUB_STEP_SUMMARY=" + filepath.Join(t.TempDir(), "summary")} + if c.url != "" { + env = append(env, "WORKER_ENV_SYNC_URL="+c.url) + } + code, out := f.run(t, f.beads(t, c.manifest, c.pin), check.Run, env...) + if code != c.code || !strings.Contains(out, c.want) { + t.Errorf("rbe preflight, %s: exit %d, want %d with %q:\n%s", c.name, code, c.code, c.want, out) + } + } +} + +// TestNightlyWorkerEnvSyncOpensIssue: nightly's check, out of step, opens a +// ci/infra issue with the diff (or, with one open, updates its body and +// comments with the run), with GITHUB_TOKEN's issues: write scoped to that +// job and the token in that one step; the rest of nightly.yml reads only. +// gascity unreachable fails the job without an issue. +func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) { + nightly := readCIWorkflow(t, "nightly.yml") + var top struct { + Permissions map[string]string `yaml:"permissions"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/nightly.yml")), &top); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(top.Permissions, map[string]string{"contents": "read"}) { + t.Errorf("nightly.yml permissions = %v, want contents: read", top.Permissions) + } + for name, job := range nightly.Jobs { + want := any(nil) + if name == nightlyWorkerEnvJob { + want = map[string]any{"contents": "read", "issues": "write"} + } + if !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want) + } + } + job := nightly.job(t, nightlyWorkerEnvJob) + if len(job.Steps) != 3 { + t.Fatalf("nightly.yml %s has %d steps, want checkout, check, issue", nightlyWorkerEnvJob, len(job.Steps)) + } + check, issue := job.Steps[1], job.Steps[2] + if check.ID != "sync" || check.If != "" || !reflect.DeepEqual(check.Env, map[string]string{"WORKER_ENV_SYNC_REPORT": nightlyWorkerEnvReport}) { + t.Errorf("nightly check step: id %q, if %q, env %v; want id sync, no if, WORKER_ENV_SYNC_REPORT=%s", check.ID, check.If, check.Env, nightlyWorkerEnvReport) + } + wantIssueEnv := map[string]string{ + "GH_TOKEN": "${{ github.token }}", + "REPORT": nightlyWorkerEnvReport, + "RUN_URL": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + } + if issue.If != "${{ failure() && steps.sync.outputs.rc == '1' }}" || !reflect.DeepEqual(issue.Env, wantIssueEnv) || !strings.Contains(issue.Run, rbeWorkerEnvSyncIssue) { + t.Errorf("nightly issue step: if %q, env %v, run %q; want only on rc 1, env %v, running %s", issue.If, issue.Env, issue.Run, wantIssueEnv, rbeWorkerEnvSyncIssue) + } + for name, j := range nightly.Jobs { + for i, step := range j.Steps { + if (name != nightlyWorkerEnvJob || i != 2) && strings.Contains(fmt.Sprint(step.Env, step.With, step.Run), "github.token") { + t.Errorf("nightly.yml job %s step %d uses github.token; only the issue step may", name, i) + } + } + } + + // The check step's rc output: 1 out of step (the issue step runs), 2 + // unknown (it does not), 0 in step. + f := newRBEWorkerEnvFixture(t) + stalePin := rbeTestPin(rbeTestStaleManifest) + for _, c := range []struct { + name, manifest, pin, url, rc string + }{ + {"in step", f.manifest, f.pin, "", "0"}, + {"out of step", rbeTestStaleManifest, stalePin, "", "1"}, + {"gascity unreachable", f.manifest, f.pin, "file:///nonexistent-gascity", "2"}, + } { + dir := t.TempDir() + output := filepath.Join(dir, "output") + report := filepath.Join(dir, "report.md") + env := []string{"GITHUB_OUTPUT=" + output, "WORKER_ENV_SYNC_REPORT=" + report} + if c.url != "" { + env = append(env, "WORKER_ENV_SYNC_URL="+c.url) + } + code, out := f.run(t, f.beads(t, c.manifest, c.pin), check.Run, env...) + got, _ := os.ReadFile(output) + if fmt.Sprint(code) != c.rc || string(got) != "rc="+c.rc+"\n" { + t.Errorf("nightly check, %s: exit %d, output %q; want exit and rc %s\n%s", c.name, code, got, c.rc, out) + } + if r, _ := os.ReadFile(report); (c.rc == "1") != strings.Contains(string(r), "### rbe worker-env out of step with gascity") { + t.Errorf("nightly check, %s: report %q", c.name, r) + } + } + + // The issue step, against a stand-in gh that logs its calls and lists + // the open ci/infra issues it is given. + const ghStub = `#!/usr/bin/env bash +printf '%s\n' "$*" >>"$GH_LOG" +for a in "$@"; do + case "$prev" in --body-file) cp "$a" "$GH_LOG.body" ;; esac + prev=$a +done +case "$1 $2" in + "issue list") jq -r "$(for a in "$@"; do [ "$p" = --jq ] && echo "$a"; p=$a; done)" <<<"$GH_ISSUES" ;; +esac +` + requireHostTool(t, "jq") + title := "rbe worker-env out of step with gascity main" + for _, c := range []struct { + name, issues string + want []string + }{ + {"no open issue", `[{"number":7,"title":"something else"}]`, []string{ + "issue create --repo gastownhall/beads --title " + title + " --label ci/infra --body-file ", + }}, + {"open issue", `[{"number":7,"title":"something else"},{"number":42,"title":"` + title + `"}]`, []string{ + "issue edit 42 --repo gastownhall/beads --body-file ", + "issue comment 42 --repo gastownhall/beads --body Still out of step with gascity main: https://github.com/gastownhall/beads/actions/runs/9", + }}, + } { + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + rbeTestWrite(t, filepath.Join(bin, "gh"), ghStub, 0o755) + report := filepath.Join(dir, "report.md") + rbeTestWrite(t, report, "### rbe worker-env out of step with gascity\n\nthe diff\n", 0o644) + log := filepath.Join(dir, "gh.log") + code, out := f.run(t, f.beads(t, f.manifest, f.pin), issue.Run, + "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), "GH_LOG="+log, "GH_ISSUES="+c.issues, + "GITHUB_REPOSITORY=gastownhall/beads", "REPORT="+report, "RUN_URL=https://github.com/gastownhall/beads/actions/runs/9") + calls, _ := os.ReadFile(log) + body, _ := os.ReadFile(log + ".body") + if code != 0 { + t.Errorf("issue step, %s: exit %d\n%s", c.name, code, out) + } + for _, want := range c.want { + if !strings.Contains(string(calls), want) { + t.Errorf("issue step, %s: gh calls lack %q:\n%s", c.name, want, calls) + } + } + if strings.Contains(string(calls), "issue create") == (c.name == "open issue") { + t.Errorf("issue step, %s: gh calls:\n%s", c.name, calls) + } + if !strings.Contains(string(body), "the diff") || !strings.Contains(string(body), "https://github.com/gastownhall/beads/actions/runs/9") { + t.Errorf("issue step, %s: body:\n%s", c.name, body) + } } } diff --git a/tools/rbe/worker-env-sync b/tools/rbe/worker-env-sync index 6152b86f9..792c560e2 100755 --- a/tools/rbe/worker-env-sync +++ b/tools/rbe/worker-env-sync @@ -11,16 +11,28 @@ # queue on a pin no worker serves. This compares the manifest and the pin # with gascity's at REF (default main) and, if either differs, prints the # diff and what to commit (stdout and $GITHUB_STEP_SUMMARY) and exits 1. +# If it cannot compare (gascity unreachable, a missing REF, any other +# failure) it warns and exits 2: callers that must not block on GitHub's +# availability (bazel.yml's rbe job) proceed on 2 and fail only on 1. # Run from the repository root. # -# Env (tests only): WORKER_ENV_SYNC_URL, gascity's raw-content base (default -# https://raw.githubusercontent.com/gastownhall/gascity). -set -euo pipefail +# Callers: bazel.yml's rbe job, before any lane of a remote-execution run +# (fails the run fast instead of letting the lanes queue), and nightly.yml's +# rbe-worker-env-sync job (opens or updates a ci/infra issue, +# tools/rbe/worker-env-sync-issue). +# +# Env: WORKER_ENV_SYNC_REPORT, a file to also write the out-of-step report +# (markdown) to. Tests only: WORKER_ENV_SYNC_URL, gascity's raw-content base +# (default https://raw.githubusercontent.com/gastownhall/gascity). +set -Eeuo pipefail ref=${1:-main} base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT +# Anything that fails before the comparison is "unknown", never "in step" +# and never "out of step". +trap 'echo "::warning title=rbe worker-env unknown::worker-env-sync could not compare with gascity $ref ($BASH_COMMAND failed); not checked."; exit 2' ERR summary() { if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat >>"$GITHUB_STEP_SUMMARY"; else cat >/dev/null; fi @@ -33,8 +45,8 @@ pin_of() { echo "${pin:-none}" } -curl -fsSL --retry 3 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" -curl -fsSL --retry 3 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" ours=$(pin_of platforms/BUILD.bazel) theirs=$(pin_of "$tmp/BUILD.bazel") if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs" ]; then @@ -42,6 +54,8 @@ if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs exit 0 fi +# Out of step is known from here on: no failure below turns it into "unknown". +trap - ERR diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/worker-env.txt" >"$tmp/diff" || true { cat <<-EOF @@ -63,5 +77,6 @@ diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/work } >"$tmp/report.md" cat "$tmp/report.md" summary <"$tmp/report.md" -echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: copy gascity's tools/rbe/worker-env.txt and its pin (the step summary has the diff)." +if [ -n "${WORKER_ENV_SYNC_REPORT:-}" ]; then cp "$tmp/report.md" "$WORKER_ENV_SYNC_REPORT"; fi +echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: beads' worker-env pin is out of step with gascity $ref, so no rbe-west pool worker serves beads' remote actions. Copy gascity's tools/rbe/worker-env.txt byte for byte and its platforms/BUILD.bazel pin (the step summary has the diff; see tools/rbe/worker-env-sync)." exit 1 diff --git a/tools/rbe/worker-env-sync-issue b/tools/rbe/worker-env-sync-issue new file mode 100755 index 000000000..63d7e34b2 --- /dev/null +++ b/tools/rbe/worker-env-sync-issue @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# worker-env-sync-issue: open or update the "out of step" issue. +# +# tools/rbe/worker-env-sync-issue REPORT RUN_URL +# +# nightly.yml's rbe-worker-env-sync job runs this when tools/rbe/worker-env-sync +# exits 1 (beads' worker-env pin differs from gascity main's, so beads' remote +# actions queue on rbe-west). REPORT is the markdown report worker-env-sync +# wrote (WORKER_ENV_SYNC_REPORT): the diff and the pin to commit. With an open +# issue of the same title labelled ci/infra, the issue's body becomes the +# latest report and a comment links this run (so watchers hear about it every +# night until it is fixed); otherwise a new issue is opened with the label. +# Close the issue once the copy is synced. +# +# Env: GITHUB_REPOSITORY (owner/repo), GH_TOKEN (issues: write). +set -euo pipefail + +report=${1:?usage: worker-env-sync-issue REPORT RUN_URL} +run_url=${2:?usage: worker-env-sync-issue REPORT RUN_URL} +repo=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is unset} +title="rbe worker-env out of step with gascity main" +label="ci/infra" + +body=$(mktemp) +trap 'rm -f "$body"' EXIT +{ + cat "$report" + printf '\nReported by %s. Close this issue once tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin match gascity main (tools/rbe/worker-env-sync passes).\n' "$run_url" +} >"$body" + +# The label may not exist yet; an existing one is left as it is. +gh label create "$label" --repo "$repo" --color d93f0b \ + --description "CI infrastructure (rbe-west, runners, cross-repo pins)" >/dev/null 2>&1 || true + +# Exact-title match among the open ci/infra issues (no search index lag). +number=$(gh issue list --repo "$repo" --state open --label "$label" --limit 100 \ + --json number,title --jq ".[] | select(.title == \"$title\") | .number" | head -n 1) +if [ -n "$number" ]; then + gh issue edit "$number" --repo "$repo" --body-file "$body" + gh issue comment "$number" --repo "$repo" --body "Still out of step with gascity main: $run_url" + echo "worker-env-sync-issue: updated #$number" +else + gh issue create --repo "$repo" --title "$title" --label "$label" --body-file "$body" +fi