Compare commits
1 commit
main
...
build/rbe-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73d4572bd2 |
5 changed files with 300 additions and 47 deletions
17
.github/workflows/nightly.yml
vendored
17
.github/workflows/nightly.yml
vendored
|
|
@ -120,6 +120,23 @@ jobs:
|
|||
# so this keeps sampling flaky and -race failures every night.
|
||||
fresh-test-results: true
|
||||
|
||||
# rbe-west's pool workers are gascity's and advertise the sha256 of
|
||||
# gascity's worker-env manifest; beads' tools/rbe/worker-env.txt is a
|
||||
# byte-for-byte copy and platforms/BUILD.bazel carries the same pin. A copy
|
||||
# that falls behind queues every remote action, so say so every night.
|
||||
rbe-worker-env-sync:
|
||||
name: rbe worker-env in step with gascity
|
||||
if: ${{ github.event_name == 'schedule' || inputs.suite == 'bazel' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Compare the worker-env manifest and pin with gascity's main
|
||||
run: tools/rbe/worker-env-sync main
|
||||
|
||||
full-test:
|
||||
name: Full Test Suite
|
||||
if: ${{ github.event_name == 'schedule' || inputs.suite == 'full-test' }}
|
||||
|
|
|
|||
|
|
@ -2,25 +2,28 @@
|
|||
# unconditionally with --extra_execution_platforms, so every action and test
|
||||
# executes on it and local, fork-cache and remote runs hash alike).
|
||||
#
|
||||
# Test actions exec host tools (bash, git, coreutils, ...) and load the
|
||||
# shared libraries of every cgo binary they link (glibc, libstdc++), so the
|
||||
# worker host is an input to every cached result. Its exec_properties enter
|
||||
# each action's key: worker-env is the sha256 of tools/rbe/worker-env.txt,
|
||||
# the manifest of the pinned worker host.
|
||||
# Test actions exec host tools (bash, git, python3, coreutils, ...) and load
|
||||
# the shared libraries of every cgo binary they link (glibc, libstdc++), so
|
||||
# the worker host is an input to every cached result. Its exec_properties
|
||||
# enter each action's key: worker-env is the sha256 of
|
||||
# tools/rbe/worker-env.txt, the toolchain manifest of the pinned worker host
|
||||
# (arch, OS release, Go, dolt, and the upstream releases of the libraries and
|
||||
# tools actions reach; not the image's security patch revisions).
|
||||
#
|
||||
# rbe-west's OSS pool is shared with gastownhall/gascity, which owns the
|
||||
# worker image: its tools/rbe/blacksmith-worker.sh advertises the sha256 of
|
||||
# the manifest it measures on its own host (gascity tools/rbe/worker-env),
|
||||
# and the schedulers match worker-env exactly, so these actions run only on
|
||||
# the pinned host. tools/rbe/worker-env.txt is a byte-for-byte copy of
|
||||
# gascity's: when gascity moves its pin (a new runner image, a package or Go
|
||||
# upgrade), copy its manifest here and update this sha256 in the same
|
||||
# commit, which is a new key for every action.
|
||||
# scripts/bazel_hermetic_cc_test.go keeps the pin and the manifest in step.
|
||||
# the pinned toolchain. tools/rbe/worker-env.txt is a byte-for-byte copy of
|
||||
# gascity's: when gascity moves its pin (a toolchain change: a distribution,
|
||||
# library or tool release, Go or dolt), copy its manifest here and update
|
||||
# this sha256 in the same commit, which is a new key for every action.
|
||||
# scripts/rbe_worker_env_test.go keeps the pin and the manifest in step, and
|
||||
# tools/rbe/worker-env-sync (nightly) compares both with gascity's main.
|
||||
platform(
|
||||
name = "rbe_worker",
|
||||
exec_properties = {
|
||||
"worker-env": "sha256:b2c8be1f53ba4c2f1603b1d65f31e0d4c7ddd498110b9928eda37f193c7eb674",
|
||||
"worker-env": "sha256:6be1381cf08e7fd66684cfce6f26aca57de2561e722c505d1f2f578e9fdeafa7",
|
||||
},
|
||||
parents = ["@bazel_tools//tools:host_platform"],
|
||||
visibility = ["//visibility:public"],
|
||||
|
|
|
|||
|
|
@ -4,7 +4,11 @@ import (
|
|||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
|
@ -15,13 +19,20 @@ import (
|
|||
// worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and
|
||||
// rbe-west's schedulers run an action only on a worker that advertises that
|
||||
// exact value. The OSS pool and its manifest are gastownhall/gascity's
|
||||
// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a copy of the
|
||||
// manifest so the pin is reviewable and moves only with it.
|
||||
// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a byte-for-byte
|
||||
// copy of the manifest so the pin is reviewable and moves only with it, and
|
||||
// tools/rbe/worker-env-sync (nightly) checks the copy against gascity's main.
|
||||
//
|
||||
// The manifest is the worker's toolchain, not its image: arch, OS release,
|
||||
// Go, dolt, and the upstream releases (at most major.minor) of the libraries
|
||||
// and tools actions reach. The Blacksmith image's Ubuntu security revisions
|
||||
// are not in it, so they neither re-key every action nor strand the pool.
|
||||
|
||||
const (
|
||||
rbeWorkerPlatformBuild = "platforms/BUILD.bazel"
|
||||
rbeWorkerEnvManifest = "tools/rbe/worker-env.txt"
|
||||
rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker"
|
||||
rbeWorkerEnvSync = "tools/rbe/worker-env-sync"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -117,3 +128,163 @@ func TestRBEWorkerPlatformGuards(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rbeWorkerEnvLineRE is every line gascity's tools/rbe/worker-env prints on
|
||||
// a worker that has what it measures: one arch, dolt, go, os and yq line,
|
||||
// and each package at its upstream release, at most major.minor.
|
||||
var rbeWorkerEnvLineRE = map[string]*regexp.Regexp{
|
||||
"arch": regexp.MustCompile(`^arch [a-z0-9_]+$`),
|
||||
"dolt": regexp.MustCompile(`^dolt dolt version \S+$`),
|
||||
"go": regexp.MustCompile(`^go go version go\S+ linux/\S+$`),
|
||||
"os": regexp.MustCompile(`^os ubuntu \d+\.\d+$`),
|
||||
"tool": regexp.MustCompile(`^tool yq \d+$`),
|
||||
"pkg": regexp.MustCompile(`^pkg [a-z0-9][a-z0-9+.-]* \d+(\.\d+)?$`),
|
||||
}
|
||||
|
||||
// checkRBEWorkerEnvManifest: manifest is a rendering of gascity's
|
||||
// tools/rbe/worker-env in its toolchain-only form. A copy of an older
|
||||
// gascity manifest (dpkg revisions such as 9.4-3ubuntu6.2), a hand edit, or
|
||||
// a truncated copy fails here rather than queueing every remote action.
|
||||
func checkRBEWorkerEnvManifest(manifest string) []error {
|
||||
if !strings.HasSuffix(manifest, "\n") {
|
||||
return []error{errors.New(rbeWorkerEnvManifest + " must end with a newline, as gascity's tools/rbe/worker-env prints it")}
|
||||
}
|
||||
lines := strings.Split(strings.TrimSuffix(manifest, "\n"), "\n")
|
||||
var errs []error
|
||||
if !slices.IsSorted(lines) {
|
||||
errs = append(errs, errors.New(rbeWorkerEnvManifest+" is not sorted (LC_ALL=C), as gascity's tools/rbe/worker-env prints it"))
|
||||
}
|
||||
seen := map[string]int{}
|
||||
for _, line := range lines {
|
||||
kind, _, _ := strings.Cut(line, " ")
|
||||
seen[kind]++
|
||||
re := rbeWorkerEnvLineRE[kind]
|
||||
if re == nil || !re.MatchString(line) {
|
||||
errs = append(errs, errors.New(rbeWorkerEnvManifest+": "+strings.TrimSpace(line)+
|
||||
" is not a gascity tools/rbe/worker-env line (packages are measured at their upstream release, at most major.minor, and installed)"))
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"arch", "dolt", "go", "os", "tool"} {
|
||||
if seen[kind] != 1 {
|
||||
errs = append(errs, errors.New(rbeWorkerEnvManifest+" needs exactly one "+kind+" line"))
|
||||
}
|
||||
}
|
||||
if seen["pkg"] == 0 {
|
||||
errs = append(errs, errors.New(rbeWorkerEnvManifest+" has no packages"))
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func TestRBEWorkerEnvManifestIsGascitysRendering(t *testing.T) {
|
||||
root := bazelPolicyRoot(t)
|
||||
for _, err := range checkRBEWorkerEnvManifest(readPolicyFile(t, root, rbeWorkerEnvManifest)) {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRBEWorkerEnvManifestGuards(t *testing.T) {
|
||||
good := "arch x86_64\ndolt dolt version 2.1.8\ngo go version go1.26.6 linux/amd64\nos ubuntu 24.04\n" +
|
||||
"pkg coreutils 9.4\npkg git 2\npkg libc6 2.39\ntool yq 4\n"
|
||||
if errs := checkRBEWorkerEnvManifest(good); len(errs) != 0 {
|
||||
t.Fatalf("good manifest fixture: %v", errs)
|
||||
}
|
||||
for name, bad := range map[string]string{
|
||||
"dpkg revision": strings.Replace(good, "pkg coreutils 9.4\n", "pkg coreutils 9.4-3ubuntu6.2\n", 1),
|
||||
"epoch": strings.Replace(good, "pkg git 2\n", "pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n", 1),
|
||||
"patch level": strings.Replace(good, "pkg libc6 2.39\n", "pkg libc6 2.39.0\n", 1),
|
||||
"missing package": strings.Replace(good, "pkg git 2\n", "pkg git missing\n", 1),
|
||||
"no yq line": strings.Replace(good, "tool yq 4\n", "", 1),
|
||||
"unsorted": strings.Replace(good, "arch x86_64\n", "", 1) + "arch x86_64\n",
|
||||
"no newline": strings.TrimSuffix(good, "\n"),
|
||||
"unknown line": good + "zzz extra\n",
|
||||
"two os lines": strings.Replace(good, "os ubuntu 24.04\n", "os ubuntu 24.04\nos ubuntu 26.04\n", 1),
|
||||
} {
|
||||
if len(checkRBEWorkerEnvManifest(bad)) == 0 {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local
|
||||
// gascity (file:// URLs): quiet success when the manifest and pin match
|
||||
// gascity's at REF, and otherwise exit 1 with the diff and the pin to commit
|
||||
// in the output and step summary, for a manifest or a pin that differs.
|
||||
func TestRBEWorkerEnvSync(t *testing.T) {
|
||||
requireHostTool(t, "curl")
|
||||
requireHostTool(t, "bash")
|
||||
script := filepath.Join(bazelPolicyRoot(t), rbeWorkerEnvSync)
|
||||
manifest := "arch x86_64\npkg git 2\n"
|
||||
sum := sha256.Sum256([]byte(manifest))
|
||||
pin := "sha256:" + hex.EncodeToString(sum[:])
|
||||
build := func(pin string) string {
|
||||
return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n"
|
||||
}
|
||||
write := func(path, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gascity := t.TempDir()
|
||||
write(filepath.Join(gascity, "main", "tools/rbe/worker-env.txt"), manifest)
|
||||
write(filepath.Join(gascity, "main", "platforms/BUILD.bazel"), build(pin))
|
||||
|
||||
run := func(ourManifest, ourPin string) (string, string, error) {
|
||||
t.Helper()
|
||||
beads := t.TempDir()
|
||||
write(filepath.Join(beads, rbeWorkerEnvManifest), ourManifest)
|
||||
write(filepath.Join(beads, rbeWorkerPlatformBuild), build(ourPin))
|
||||
summary := filepath.Join(beads, "summary.md")
|
||||
cmd := exec.Command("bash", script)
|
||||
cmd.Dir = beads
|
||||
cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity, "GITHUB_STEP_SUMMARY="+summary)
|
||||
out, err := cmd.CombinedOutput()
|
||||
b, _ := os.ReadFile(summary)
|
||||
return string(out), string(b), err
|
||||
}
|
||||
|
||||
out, summary, err := run(manifest, pin)
|
||||
if err != nil || !strings.Contains(out, "worker-env: in step with gascity main ("+pin+")") || summary != "" {
|
||||
t.Fatalf("in step: %v\n%s\nsummary:\n%s", err, out, summary)
|
||||
}
|
||||
|
||||
stale := "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n"
|
||||
staleSum := sha256.Sum256([]byte(stale))
|
||||
stalePin := "sha256:" + hex.EncodeToString(staleSum[:])
|
||||
for name, c := range map[string][2]string{
|
||||
"manifest and pin": {stale, stalePin},
|
||||
"pin only": {manifest, stalePin},
|
||||
"manifest only": {stale, pin},
|
||||
} {
|
||||
out, summary, err := run(c[0], c[1])
|
||||
if err == nil {
|
||||
t.Errorf("%s out of step: succeeded\n%s", name, out)
|
||||
continue
|
||||
}
|
||||
for _, want := range []string{
|
||||
"::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + pin,
|
||||
" \"worker-env\": \"" + pin + "\",\n",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("%s: output lacks %q:\n%s", name, want, out)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(summary, "### rbe worker-env out of step with gascity") {
|
||||
t.Errorf("%s: no step summary:\n%s", name, summary)
|
||||
}
|
||||
if c[0] != manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") {
|
||||
t.Errorf("%s: no manifest diff:\n%s", name, out)
|
||||
}
|
||||
}
|
||||
|
||||
// gascity unreachable (a REF that does not exist): an error, never "in step".
|
||||
cmd := exec.Command("bash", script, "no-such-ref")
|
||||
cmd.Dir = t.TempDir()
|
||||
cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity)
|
||||
if out, err := cmd.CombinedOutput(); err == nil {
|
||||
t.Errorf("worker-env-sync of a missing ref succeeded:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
67
tools/rbe/worker-env-sync
Executable file
67
tools/rbe/worker-env-sync
Executable file
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env bash
|
||||
# worker-env-sync: check that beads' worker-env pin is gascity's.
|
||||
#
|
||||
# tools/rbe/worker-env-sync [REF]
|
||||
#
|
||||
# rbe-west's OSS pool workers are gastownhall/gascity's: each advertises the
|
||||
# sha256 of the toolchain manifest it measures (gascity tools/rbe/worker-env),
|
||||
# and the schedulers match worker-env exactly. beads' actions request
|
||||
# //platforms:rbe_worker's pin, the sha256 of tools/rbe/worker-env.txt, a
|
||||
# byte-for-byte copy of gascity's. If the copies part, beads' remote actions
|
||||
# queue on a pin no worker serves. This compares the manifest and the pin
|
||||
# with gascity's at REF (default main) and, if either differs, prints the
|
||||
# diff and what to commit (stdout and $GITHUB_STEP_SUMMARY) and exits 1.
|
||||
# Run from the repository root.
|
||||
#
|
||||
# Env (tests only): WORKER_ENV_SYNC_URL, gascity's raw-content base (default
|
||||
# https://raw.githubusercontent.com/gastownhall/gascity).
|
||||
set -euo pipefail
|
||||
|
||||
ref=${1:-main}
|
||||
base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
summary() {
|
||||
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat >>"$GITHUB_STEP_SUMMARY"; else cat >/dev/null; fi
|
||||
}
|
||||
|
||||
# pin_of FILE: the worker-env pin in a platforms/BUILD.bazel ("none" if none).
|
||||
pin_of() {
|
||||
local pin
|
||||
pin=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' "$1")
|
||||
echo "${pin:-none}"
|
||||
}
|
||||
|
||||
curl -fsSL --retry 3 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt"
|
||||
curl -fsSL --retry 3 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel"
|
||||
ours=$(pin_of platforms/BUILD.bazel)
|
||||
theirs=$(pin_of "$tmp/BUILD.bazel")
|
||||
if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs" ]; then
|
||||
echo "worker-env: in step with gascity $ref ($ours)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/worker-env.txt" >"$tmp/diff" || true
|
||||
{
|
||||
cat <<-EOF
|
||||
### rbe worker-env out of step with gascity
|
||||
|
||||
beads pins \`worker-env=$ours\`; gascity \`$ref\` pins \`$theirs\`. rbe-west's pool workers advertise gascity's, so beads' remote actions on another pin only queue. Copy gascity's \`tools/rbe/worker-env.txt\` here byte for byte, and in \`platforms/BUILD.bazel\` set:
|
||||
|
||||
\`\`\`
|
||||
"worker-env": "$theirs",
|
||||
\`\`\`
|
||||
|
||||
<details><summary>diff (beads to gascity $ref)</summary>
|
||||
|
||||
\`\`\`diff
|
||||
$(cat "$tmp/diff")
|
||||
\`\`\`
|
||||
</details>
|
||||
EOF
|
||||
} >"$tmp/report.md"
|
||||
cat "$tmp/report.md"
|
||||
summary <"$tmp/report.md"
|
||||
echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: copy gascity's tools/rbe/worker-env.txt and its pin (the step summary has the diff)."
|
||||
exit 1
|
||||
|
|
@ -2,37 +2,32 @@ arch x86_64
|
|||
dolt dolt version 2.1.8
|
||||
go go version go1.26.6 linux/amd64
|
||||
os ubuntu 24.04
|
||||
pkg bash 5.2.21-2ubuntu4
|
||||
pkg cmake 3.28.3-1build7
|
||||
pkg coreutils 9.4-3ubuntu6.2
|
||||
pkg diffutils 1:3.10-1build1
|
||||
pkg findutils 4.9.0-5build1
|
||||
pkg git 1:2.55.0-0ppa1~ubuntu24.04.2
|
||||
pkg grep 3.11-4build1
|
||||
pkg jq 1.7.1-3ubuntu0.24.04.2
|
||||
pkg libbz2-dev 1.0.8-5.1ubuntu0.1
|
||||
pkg libc6 2.39-0ubuntu8.8
|
||||
pkg libexpat1-dev 2.6.1-2ubuntu0.6
|
||||
pkg libffi-dev 3.4.6-1build1
|
||||
pkg libgcc-s1 14.2.0-4ubuntu2~24.04.1
|
||||
pkg libicu74 74.2-1ubuntu3.1
|
||||
pkg liblzma-dev 5.6.1+really5.4.5-1ubuntu0.3
|
||||
pkg liblzma5 5.6.1+really5.4.5-1ubuntu0.3
|
||||
pkg libncurses-dev 6.4+20240113-1ubuntu2.2
|
||||
pkg libreadline-dev 8.2-4build1
|
||||
pkg libsqlite3-dev 3.45.1-1ubuntu2.8
|
||||
pkg libstdc++6 14.2.0-4ubuntu2~24.04.1
|
||||
pkg libxml2 2.9.14+dfsg-1.3ubuntu3.9
|
||||
pkg libxml2-dev 2.9.14+dfsg-1.3ubuntu3.9
|
||||
pkg lsof 4.95.0-1build3
|
||||
pkg make 4.3-4.1build2
|
||||
pkg procps 2:4.0.4-4ubuntu3.2
|
||||
pkg python3-dev 3.12.3-0ubuntu2.1
|
||||
pkg sed 4.9-2ubuntu0.24.04.1
|
||||
pkg sqlite3 3.45.1-1ubuntu2.8
|
||||
pkg tar 1.35+dfsg-3ubuntu0.4
|
||||
pkg tmux 3.4-1ubuntu0.1
|
||||
pkg util-linux 2.39.3-9ubuntu6.5
|
||||
pkg xz-utils 5.6.1+really5.4.5-1ubuntu0.3
|
||||
pkg zlib1g 1:1.3.dfsg-3.1ubuntu2.2
|
||||
pkg zlib1g-dev 1:1.3.dfsg-3.1ubuntu2.2
|
||||
pkg bash 5.2
|
||||
pkg coreutils 9.4
|
||||
pkg curl 8.5
|
||||
pkg dash 0.5
|
||||
pkg diffutils 3.10
|
||||
pkg findutils 4.9
|
||||
pkg git 2
|
||||
pkg grep 3.11
|
||||
pkg jq 1.7
|
||||
pkg libc6 2.39
|
||||
pkg libgcc-s1 14.2
|
||||
pkg libicu74 74.2
|
||||
pkg liblzma5 5.4
|
||||
pkg libstdc++6 14.2
|
||||
pkg libxml2 2.9
|
||||
pkg lsof 4.95
|
||||
pkg make 4.3
|
||||
pkg openssl 3.0
|
||||
pkg procps 4.0
|
||||
pkg python3 3.12
|
||||
pkg sed 4.9
|
||||
pkg sqlite3 3.45
|
||||
pkg tar 1.35
|
||||
pkg tmux 3.4
|
||||
pkg unzip 6.0
|
||||
pkg util-linux 2.39
|
||||
pkg xz-utils 5.4
|
||||
pkg zlib1g 1.3
|
||||
tool yq 4
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue