diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8bca8b8da..025a35ac0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -120,6 +120,23 @@ jobs: # so this keeps sampling flaky and -race failures every night. fresh-test-results: true + # rbe-west's pool workers are gascity's and advertise the sha256 of + # gascity's worker-env manifest; beads' tools/rbe/worker-env.txt is a + # byte-for-byte copy and platforms/BUILD.bazel carries the same pin. A copy + # that falls behind queues every remote action, so say so every night. + rbe-worker-env-sync: + name: rbe worker-env in step with gascity + if: ${{ github.event_name == 'schedule' || inputs.suite == 'bazel' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + - name: Compare the worker-env manifest and pin with gascity's main + run: tools/rbe/worker-env-sync main + full-test: name: Full Test Suite if: ${{ github.event_name == 'schedule' || inputs.suite == 'full-test' }} diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index e9bd07d6c..35cd55d7e 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -2,25 +2,28 @@ # unconditionally with --extra_execution_platforms, so every action and test # executes on it and local, fork-cache and remote runs hash alike). # -# Test actions exec host tools (bash, git, coreutils, ...) and load the -# shared libraries of every cgo binary they link (glibc, libstdc++), so the -# worker host is an input to every cached result. Its exec_properties enter -# each action's key: worker-env is the sha256 of tools/rbe/worker-env.txt, -# the manifest of the pinned worker host. +# Test actions exec host tools (bash, git, python3, coreutils, ...) and load +# the shared libraries of every cgo binary they link (glibc, libstdc++), so +# the worker host is an input to every cached result. Its exec_properties +# enter each action's key: worker-env is the sha256 of +# tools/rbe/worker-env.txt, the toolchain manifest of the pinned worker host +# (arch, OS release, Go, dolt, and the upstream releases of the libraries and +# tools actions reach; not the image's security patch revisions). # # rbe-west's OSS pool is shared with gastownhall/gascity, which owns the # worker image: its tools/rbe/blacksmith-worker.sh advertises the sha256 of # the manifest it measures on its own host (gascity tools/rbe/worker-env), # and the schedulers match worker-env exactly, so these actions run only on -# the pinned host. tools/rbe/worker-env.txt is a byte-for-byte copy of -# gascity's: when gascity moves its pin (a new runner image, a package or Go -# upgrade), copy its manifest here and update this sha256 in the same -# commit, which is a new key for every action. -# scripts/bazel_hermetic_cc_test.go keeps the pin and the manifest in step. +# the pinned toolchain. tools/rbe/worker-env.txt is a byte-for-byte copy of +# gascity's: when gascity moves its pin (a toolchain change: a distribution, +# library or tool release, Go or dolt), copy its manifest here and update +# this sha256 in the same commit, which is a new key for every action. +# scripts/rbe_worker_env_test.go keeps the pin and the manifest in step, and +# tools/rbe/worker-env-sync (nightly) compares both with gascity's main. platform( name = "rbe_worker", exec_properties = { - "worker-env": "sha256:b2c8be1f53ba4c2f1603b1d65f31e0d4c7ddd498110b9928eda37f193c7eb674", + "worker-env": "sha256:6be1381cf08e7fd66684cfce6f26aca57de2561e722c505d1f2f578e9fdeafa7", }, parents = ["@bazel_tools//tools:host_platform"], visibility = ["//visibility:public"], diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index 0a3408672..ff6f4c2f3 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -4,7 +4,11 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "os" + "os/exec" + "path/filepath" "regexp" + "slices" "strings" "testing" ) @@ -15,13 +19,20 @@ import ( // worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and // rbe-west's schedulers run an action only on a worker that advertises that // exact value. The OSS pool and its manifest are gastownhall/gascity's -// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a copy of the -// manifest so the pin is reviewable and moves only with it. +// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a byte-for-byte +// copy of the manifest so the pin is reviewable and moves only with it, and +// tools/rbe/worker-env-sync (nightly) checks the copy against gascity's main. +// +// The manifest is the worker's toolchain, not its image: arch, OS release, +// Go, dolt, and the upstream releases (at most major.minor) of the libraries +// and tools actions reach. The Blacksmith image's Ubuntu security revisions +// are not in it, so they neither re-key every action nor strand the pool. const ( rbeWorkerPlatformBuild = "platforms/BUILD.bazel" rbeWorkerEnvManifest = "tools/rbe/worker-env.txt" rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker" + rbeWorkerEnvSync = "tools/rbe/worker-env-sync" ) var ( @@ -117,3 +128,163 @@ func TestRBEWorkerPlatformGuards(t *testing.T) { } } } + +// rbeWorkerEnvLineRE is every line gascity's tools/rbe/worker-env prints on +// a worker that has what it measures: one arch, dolt, go, os and yq line, +// and each package at its upstream release, at most major.minor. +var rbeWorkerEnvLineRE = map[string]*regexp.Regexp{ + "arch": regexp.MustCompile(`^arch [a-z0-9_]+$`), + "dolt": regexp.MustCompile(`^dolt dolt version \S+$`), + "go": regexp.MustCompile(`^go go version go\S+ linux/\S+$`), + "os": regexp.MustCompile(`^os ubuntu \d+\.\d+$`), + "tool": regexp.MustCompile(`^tool yq \d+$`), + "pkg": regexp.MustCompile(`^pkg [a-z0-9][a-z0-9+.-]* \d+(\.\d+)?$`), +} + +// checkRBEWorkerEnvManifest: manifest is a rendering of gascity's +// tools/rbe/worker-env in its toolchain-only form. A copy of an older +// gascity manifest (dpkg revisions such as 9.4-3ubuntu6.2), a hand edit, or +// a truncated copy fails here rather than queueing every remote action. +func checkRBEWorkerEnvManifest(manifest string) []error { + if !strings.HasSuffix(manifest, "\n") { + return []error{errors.New(rbeWorkerEnvManifest + " must end with a newline, as gascity's tools/rbe/worker-env prints it")} + } + lines := strings.Split(strings.TrimSuffix(manifest, "\n"), "\n") + var errs []error + if !slices.IsSorted(lines) { + errs = append(errs, errors.New(rbeWorkerEnvManifest+" is not sorted (LC_ALL=C), as gascity's tools/rbe/worker-env prints it")) + } + seen := map[string]int{} + for _, line := range lines { + kind, _, _ := strings.Cut(line, " ") + seen[kind]++ + re := rbeWorkerEnvLineRE[kind] + if re == nil || !re.MatchString(line) { + errs = append(errs, errors.New(rbeWorkerEnvManifest+": "+strings.TrimSpace(line)+ + " is not a gascity tools/rbe/worker-env line (packages are measured at their upstream release, at most major.minor, and installed)")) + } + } + for _, kind := range []string{"arch", "dolt", "go", "os", "tool"} { + if seen[kind] != 1 { + errs = append(errs, errors.New(rbeWorkerEnvManifest+" needs exactly one "+kind+" line")) + } + } + if seen["pkg"] == 0 { + errs = append(errs, errors.New(rbeWorkerEnvManifest+" has no packages")) + } + return errs +} + +func TestRBEWorkerEnvManifestIsGascitysRendering(t *testing.T) { + root := bazelPolicyRoot(t) + for _, err := range checkRBEWorkerEnvManifest(readPolicyFile(t, root, rbeWorkerEnvManifest)) { + t.Error(err) + } +} + +func TestRBEWorkerEnvManifestGuards(t *testing.T) { + good := "arch x86_64\ndolt dolt version 2.1.8\ngo go version go1.26.6 linux/amd64\nos ubuntu 24.04\n" + + "pkg coreutils 9.4\npkg git 2\npkg libc6 2.39\ntool yq 4\n" + if errs := checkRBEWorkerEnvManifest(good); len(errs) != 0 { + t.Fatalf("good manifest fixture: %v", errs) + } + for name, bad := range map[string]string{ + "dpkg revision": strings.Replace(good, "pkg coreutils 9.4\n", "pkg coreutils 9.4-3ubuntu6.2\n", 1), + "epoch": strings.Replace(good, "pkg git 2\n", "pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n", 1), + "patch level": strings.Replace(good, "pkg libc6 2.39\n", "pkg libc6 2.39.0\n", 1), + "missing package": strings.Replace(good, "pkg git 2\n", "pkg git missing\n", 1), + "no yq line": strings.Replace(good, "tool yq 4\n", "", 1), + "unsorted": strings.Replace(good, "arch x86_64\n", "", 1) + "arch x86_64\n", + "no newline": strings.TrimSuffix(good, "\n"), + "unknown line": good + "zzz extra\n", + "two os lines": strings.Replace(good, "os ubuntu 24.04\n", "os ubuntu 24.04\nos ubuntu 26.04\n", 1), + } { + if len(checkRBEWorkerEnvManifest(bad)) == 0 { + t.Errorf("%s: expected an error", name) + } + } +} + +// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local +// gascity (file:// URLs): quiet success when the manifest and pin match +// gascity's at REF, and otherwise exit 1 with the diff and the pin to commit +// in the output and step summary, for a manifest or a pin that differs. +func TestRBEWorkerEnvSync(t *testing.T) { + requireHostTool(t, "curl") + requireHostTool(t, "bash") + script := filepath.Join(bazelPolicyRoot(t), rbeWorkerEnvSync) + manifest := "arch x86_64\npkg git 2\n" + sum := sha256.Sum256([]byte(manifest)) + pin := "sha256:" + hex.EncodeToString(sum[:]) + build := func(pin string) string { + return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n" + } + write := func(path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + gascity := t.TempDir() + write(filepath.Join(gascity, "main", "tools/rbe/worker-env.txt"), manifest) + write(filepath.Join(gascity, "main", "platforms/BUILD.bazel"), build(pin)) + + run := func(ourManifest, ourPin string) (string, string, error) { + t.Helper() + beads := t.TempDir() + write(filepath.Join(beads, rbeWorkerEnvManifest), ourManifest) + write(filepath.Join(beads, rbeWorkerPlatformBuild), build(ourPin)) + summary := filepath.Join(beads, "summary.md") + cmd := exec.Command("bash", script) + cmd.Dir = beads + cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity, "GITHUB_STEP_SUMMARY="+summary) + out, err := cmd.CombinedOutput() + b, _ := os.ReadFile(summary) + return string(out), string(b), err + } + + out, summary, err := run(manifest, pin) + if err != nil || !strings.Contains(out, "worker-env: in step with gascity main ("+pin+")") || summary != "" { + t.Fatalf("in step: %v\n%s\nsummary:\n%s", err, out, summary) + } + + stale := "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n" + staleSum := sha256.Sum256([]byte(stale)) + stalePin := "sha256:" + hex.EncodeToString(staleSum[:]) + for name, c := range map[string][2]string{ + "manifest and pin": {stale, stalePin}, + "pin only": {manifest, stalePin}, + "manifest only": {stale, pin}, + } { + out, summary, err := run(c[0], c[1]) + if err == nil { + t.Errorf("%s out of step: succeeded\n%s", name, out) + continue + } + for _, want := range []string{ + "::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + pin, + " \"worker-env\": \"" + pin + "\",\n", + } { + if !strings.Contains(out, want) { + t.Errorf("%s: output lacks %q:\n%s", name, want, out) + } + } + if !strings.Contains(summary, "### rbe worker-env out of step with gascity") { + t.Errorf("%s: no step summary:\n%s", name, summary) + } + if c[0] != manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") { + t.Errorf("%s: no manifest diff:\n%s", name, out) + } + } + + // gascity unreachable (a REF that does not exist): an error, never "in step". + cmd := exec.Command("bash", script, "no-such-ref") + cmd.Dir = t.TempDir() + cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity) + if out, err := cmd.CombinedOutput(); err == nil { + t.Errorf("worker-env-sync of a missing ref succeeded:\n%s", out) + } +} diff --git a/tools/rbe/worker-env-sync b/tools/rbe/worker-env-sync new file mode 100755 index 000000000..6152b86f9 --- /dev/null +++ b/tools/rbe/worker-env-sync @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# worker-env-sync: check that beads' worker-env pin is gascity's. +# +# tools/rbe/worker-env-sync [REF] +# +# rbe-west's OSS pool workers are gastownhall/gascity's: each advertises the +# sha256 of the toolchain manifest it measures (gascity tools/rbe/worker-env), +# and the schedulers match worker-env exactly. beads' actions request +# //platforms:rbe_worker's pin, the sha256 of tools/rbe/worker-env.txt, a +# byte-for-byte copy of gascity's. If the copies part, beads' remote actions +# queue on a pin no worker serves. This compares the manifest and the pin +# with gascity's at REF (default main) and, if either differs, prints the +# diff and what to commit (stdout and $GITHUB_STEP_SUMMARY) and exits 1. +# Run from the repository root. +# +# Env (tests only): WORKER_ENV_SYNC_URL, gascity's raw-content base (default +# https://raw.githubusercontent.com/gastownhall/gascity). +set -euo pipefail + +ref=${1:-main} +base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +summary() { + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat >>"$GITHUB_STEP_SUMMARY"; else cat >/dev/null; fi +} + +# pin_of FILE: the worker-env pin in a platforms/BUILD.bazel ("none" if none). +pin_of() { + local pin + pin=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' "$1") + echo "${pin:-none}" +} + +curl -fsSL --retry 3 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" +curl -fsSL --retry 3 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" +ours=$(pin_of platforms/BUILD.bazel) +theirs=$(pin_of "$tmp/BUILD.bazel") +if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs" ]; then + echo "worker-env: in step with gascity $ref ($ours)" + exit 0 +fi + +diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/worker-env.txt" >"$tmp/diff" || true +{ + cat <<-EOF + ### rbe worker-env out of step with gascity + + beads pins \`worker-env=$ours\`; gascity \`$ref\` pins \`$theirs\`. rbe-west's pool workers advertise gascity's, so beads' remote actions on another pin only queue. Copy gascity's \`tools/rbe/worker-env.txt\` here byte for byte, and in \`platforms/BUILD.bazel\` set: + + \`\`\` + "worker-env": "$theirs", + \`\`\` + +
diff (beads to gascity $ref) + + \`\`\`diff + $(cat "$tmp/diff") + \`\`\` +
+ EOF +} >"$tmp/report.md" +cat "$tmp/report.md" +summary <"$tmp/report.md" +echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: copy gascity's tools/rbe/worker-env.txt and its pin (the step summary has the diff)." +exit 1 diff --git a/tools/rbe/worker-env.txt b/tools/rbe/worker-env.txt index b52c65a48..17ef8ffc9 100644 --- a/tools/rbe/worker-env.txt +++ b/tools/rbe/worker-env.txt @@ -2,37 +2,32 @@ arch x86_64 dolt dolt version 2.1.8 go go version go1.26.6 linux/amd64 os ubuntu 24.04 -pkg bash 5.2.21-2ubuntu4 -pkg cmake 3.28.3-1build7 -pkg coreutils 9.4-3ubuntu6.2 -pkg diffutils 1:3.10-1build1 -pkg findutils 4.9.0-5build1 -pkg git 1:2.55.0-0ppa1~ubuntu24.04.2 -pkg grep 3.11-4build1 -pkg jq 1.7.1-3ubuntu0.24.04.2 -pkg libbz2-dev 1.0.8-5.1ubuntu0.1 -pkg libc6 2.39-0ubuntu8.8 -pkg libexpat1-dev 2.6.1-2ubuntu0.6 -pkg libffi-dev 3.4.6-1build1 -pkg libgcc-s1 14.2.0-4ubuntu2~24.04.1 -pkg libicu74 74.2-1ubuntu3.1 -pkg liblzma-dev 5.6.1+really5.4.5-1ubuntu0.3 -pkg liblzma5 5.6.1+really5.4.5-1ubuntu0.3 -pkg libncurses-dev 6.4+20240113-1ubuntu2.2 -pkg libreadline-dev 8.2-4build1 -pkg libsqlite3-dev 3.45.1-1ubuntu2.8 -pkg libstdc++6 14.2.0-4ubuntu2~24.04.1 -pkg libxml2 2.9.14+dfsg-1.3ubuntu3.9 -pkg libxml2-dev 2.9.14+dfsg-1.3ubuntu3.9 -pkg lsof 4.95.0-1build3 -pkg make 4.3-4.1build2 -pkg procps 2:4.0.4-4ubuntu3.2 -pkg python3-dev 3.12.3-0ubuntu2.1 -pkg sed 4.9-2ubuntu0.24.04.1 -pkg sqlite3 3.45.1-1ubuntu2.8 -pkg tar 1.35+dfsg-3ubuntu0.4 -pkg tmux 3.4-1ubuntu0.1 -pkg util-linux 2.39.3-9ubuntu6.5 -pkg xz-utils 5.6.1+really5.4.5-1ubuntu0.3 -pkg zlib1g 1:1.3.dfsg-3.1ubuntu2.2 -pkg zlib1g-dev 1:1.3.dfsg-3.1ubuntu2.2 +pkg bash 5.2 +pkg coreutils 9.4 +pkg curl 8.5 +pkg dash 0.5 +pkg diffutils 3.10 +pkg findutils 4.9 +pkg git 2 +pkg grep 3.11 +pkg jq 1.7 +pkg libc6 2.39 +pkg libgcc-s1 14.2 +pkg libicu74 74.2 +pkg liblzma5 5.4 +pkg libstdc++6 14.2 +pkg libxml2 2.9 +pkg lsof 4.95 +pkg make 4.3 +pkg openssl 3.0 +pkg procps 4.0 +pkg python3 3.12 +pkg sed 4.9 +pkg sqlite3 3.45 +pkg tar 1.35 +pkg tmux 3.4 +pkg unzip 6.0 +pkg util-linux 2.39 +pkg xz-utils 5.4 +pkg zlib1g 1.3 +tool yq 4