From b03f8f527ee27c07c40f75ad03f6b166354343c3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 03:49:06 +0000 Subject: [PATCH 1/2] ci: bigger runners for the slowest CI legs (MCP gate, Bazel test, Windows preflight) Runner-size A/B against merge group 37721729916 (3m49s): - package-mcp: 4 -> 8 vCPU with pytest -n 16 (BEADS_MCP_PYTEST_WORKERS, mode remote only; forks/ubuntu-latest keep package-mcp.sh's default 8). pytest was 70s, dominated by CPU-bound per-test `bd init` setup. Local, pinned with taskset: -n 8 on 4 CPUs 112s; -n 16 on 8 CPUs 60/57/58s, 228 passed / 5 skipped in all three runs. - bazel-test: 2 -> 4 vCPU. `bazel test //... --config=ci` spent ~63 of 81s in client-side loading/analysis (critical path 7.65s, every action a remote cache hit). - pr-preflight-platforms Windows leg: 4 -> 8 vCPU. main.yml's test-windows saver stays on 4 vCPU: the cache keys name runner.os/runner.arch, never the label, and actions/cache's version covers only paths + compression, so the entries are not size-specific; the other Windows consumers keep their exact label. TestWindowsPreflightLegReachesSaverAcrossSizes pins it. Runner labels live in the shared constants in scripts/ci_blacksmith_runner_test.go; TestBlacksmithBazelRunnerSizes checks every bazel.yml lane's runs-on and the MCP worker count per rbe mode with evalGHExpr. Revert any bump that does not save at least 15s. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 25 +++- .github/workflows/main.yml | 6 +- .github/workflows/pr.yml | 15 ++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 16 ++- scripts/ci/package-mcp.sh | 12 +- scripts/ci_blacksmith_runner_test.go | 151 +++++++++++++++++++++++- scripts/ci_workflow_test.go | 44 ++++--- scripts/pr_lanes_bazel_coverage_test.go | 4 +- 8 files changed, 232 insertions(+), 41 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6564862f0..66322c9d5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -728,8 +728,15 @@ jobs: # they miss on the runner, so they get the larger GitHub-hosted one. # Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted, # like every fork job: the runner is only a client. + # + # 4 vCPU, not the other lanes' 2: with every action a remote cache hit, + # this lane's `bazel test //... --config=ci` spent ~63 of 81s in + # client-side loading and analysis (1798 packages, 66k configured + # targets; critical path 7.65s, merge group 37721729916), which Bazel + # parallelizes across the client's cores. Revert to 2 vCPU if the A/B + # does not save at least 15s on that step. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed # worker pool shared with gascity). Skipped only in mode skip (same-repo # runs while RBE_WEST_WORKERS is unset). @@ -2210,9 +2217,12 @@ jobs: package-mcp: name: Package Gate (MCP) needs: rbe - # 4 vCPU on purpose: pytest-xdist -n 8 measured ~2.7-3x on 4 CPUs - # (tools/f3 timing, 2026-10), well past 2 vCPU's ceiling. - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # 8 vCPU with 16 xdist workers (BEADS_MCP_PYTEST_WORKERS below): pytest + # was 70s of this job at -n 8 on 4 vCPU (merge group 37721729916), + # dominated by per-test `bd init` fixture setup, which is CPU-bound. + # Was 4 vCPU / -n 8 (tools/f3 timing, 2026-10: ~2.7-3x on 4 CPUs); + # revert to that if the A/B does not save at least 15s. + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ inputs.package-gates == 'on' }} timeout-minutes: 30 outputs: @@ -2266,8 +2276,8 @@ jobs: echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" # Defense in depth: nothing after this step needs the Bazel server # or its RBE client key (uv sync and pytest run PR-controlled - # code). Shutting down also frees the JVM's memory for the 8 - # xdist workers that follow on this 4 vCPU runner. + # code). Shutting down also frees the JVM's memory for the 16 + # xdist workers that follow on this 8 vCPU runner. bazel shutdown rm -rf "$RUNNER_TEMP/bazel-ci-secret" @@ -2313,6 +2323,9 @@ jobs: if: ${{ steps.detect.outputs.mcp_package == 'true' }} env: GOCACHE: ${{ runner.temp }}/go-cache/non-race + # 16 workers on the 8 vCPU Blacksmith runner; package-mcp.sh's + # default 8 on the GitHub-hosted fallback. + BEADS_MCP_PYTEST_WORKERS: ${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }} run: make ci-package-mcp # Last step: the job's result as a workflow_call output for pr.yml's diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c64890e66..19ca95730 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -184,8 +184,10 @@ jobs: # F7b review fix (S2): warms the SAME non-race GOCACHE namespace that # pr-preflight-platforms'/worktree-remove-windows'/test-windows-liveness' - # Windows legs restore from, on the exact same runner selection - # (blacksmith-4vcpu-windows-2025), so their cold `cmd/bd` test compiles + # Windows legs restore from, on the same Windows image + # (blacksmith-4vcpu-windows-2025; pr-preflight-platforms' leg runs on + # 8 vCPU, which shares these entries because no key names the runner + # size), so their cold `cmd/bd` test compiles # (the real long pole; see pr-preflight-platforms' S2 comment) get a # true cache hit. Confirmed to run under Git Bash on Windows (reviewer # verification). diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 628ed5dcc..7f2895e43 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -506,6 +506,15 @@ jobs: # process creation, now parallel subtests), 27-58s to compile and link # ./cmd/bd's test binary, and about 25s for the rest. A second Windows leg # would pay the setup again to save about 20s, so it stays one leg. + # + # Runner-size A/B (2026-10-08): the Windows leg was the slowest Windows + # check at 129s on blacksmith-4vcpu-windows-2025 (merge group 37721729916), + # so it runs on 8 vCPU; the cache extraction and the ./cmd/bd test compile + # and link above are CPU-bound. It still restores main.yml's test-windows + # entries saved on 4 vCPU: the cache keys name runner.os/runner.arch, never + # the label, so nothing in an entry is size-specific + # (TestWindowsPreflightLegReachesSaverAcrossSizes). Revert to 4 vCPU if it + # does not save at least 15s. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) runs-on: ${{ matrix.runner }} @@ -519,7 +528,7 @@ jobs: runner: blacksmith-6vcpu-macos-26 - os: windows-latest expected_goos: windows - runner: blacksmith-4vcpu-windows-2025 + runner: blacksmith-8vcpu-windows-2025 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -538,8 +547,8 @@ jobs: restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- # F7b review (S2): restore-only non-race Go build cache, seeded by the - # push-to-main saver on the exact same Blacksmith label (Windows: - # main.yml's test-windows; macOS: main.yml's + # push-to-main saver on the same Blacksmith OS image (Windows: + # main.yml's test-windows, on 4 vCPU; macOS: main.yml's # blacksmith-macos-go-build-cache) so the cold cmd/bd compile below # (the real long pole, 250-300s uncached) gets a true cache hit instead # of starting cold. GOCACHE must match the seeder's exactly, and diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index fb22a7410..ed76084ce 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -883,9 +883,11 @@ Required` requires them to have run remotely and passed. `1.3.0 (dev)`, no commit); no consumer reads it. Verified 2026-10-02: both package gates pass with the Bazel-built bd (MCP: 228 passed, 5 skipped; npm: all tests and the pack dry run), the same as with a - `go build` bd. Both run on `blacksmith-4vcpu-ubuntu-2404` when - `rbe.outputs.enabled == 'true'` (4 vCPU: `pytest -n 8` is pinned to - timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own + `go build` bd. In mode remote, package-npm runs on + `blacksmith-4vcpu-ubuntu-2404` and package-mcp on + `blacksmith-8vcpu-ubuntu-2404` with `pytest -n 16` + (`BEADS_MCP_PYTEST_WORKERS`; the script's default is `-n 8`); + `ubuntu-latest` (and `-n 8`) otherwise. `bazel-test`'s own `bazel-ci-build-artifacts` upload is no longer consumed by anything; it is kept for the F3.5.3 SHA256SUMS comparison and for debugging. - The Dolt-backed domain, uow, tracker, doctor/fix and protocol suites @@ -1203,8 +1205,12 @@ Bazel on rbe-west. pr.yml's Windows jobs (`test-windows-liveness`, their `blacksmith-*vcpu-windows-2025` label literally; the two mixed-OS matrix jobs, `pr-preflight-platforms` and `check-doc-freshness-platforms`, run a macOS leg on `blacksmith-6vcpu-macos-26` (Apple Silicon) and a Windows -leg on `blacksmith-4vcpu-windows-2025`, each leg naming its label in the -matrix (`runs-on: ${{ matrix.runner }}`). There is no GitHub-hosted fallback: +leg (`blacksmith-8vcpu-windows-2025` for `pr-preflight-platforms`, +`blacksmith-4vcpu-windows-2025` for `check-doc-freshness-platforms`), each +leg naming its label in the matrix (`runs-on: ${{ matrix.runner }}`). The +8 vCPU leg restores main.yml's `test-windows` entries saved on 4 vCPU: the +cache keys name `runner.os` and `runner.arch`, never the label, and +`actions/cache`'s version hash covers only the paths and compression method. There is no GitHub-hosted fallback: Blacksmith serves this org's fork PRs (gascity's fork PRs run their CI on `blacksmith-*` labels). Fork runs get no secrets and a read-only token (`TestBlacksmithJobsReadNoSecrets` keeps every Blacksmith job free of secret diff --git a/scripts/ci/package-mcp.sh b/scripts/ci/package-mcp.sh index 5b14a7041..56490599d 100755 --- a/scripts/ci/package-mcp.sh +++ b/scripts/ci/package-mcp.sh @@ -55,8 +55,16 @@ mcp_pytest() { # Explicit worker count, not "auto": CI runners' reported CPU count # doesn't always match the cgroup quota, and -n 8 is pinned to measured # timing (tools/f3, 2026-10) on both 2 vCPU (2.7x) and 4 vCPU (~3x) - # runners. pytest-xdist is a pinned dev dependency (pyproject.toml). - uv run pytest -n 8 --durations=50 + # runners. bazel.yml's package-mcp sets BEADS_MCP_PYTEST_WORKERS=16 on + # its 8 vCPU Blacksmith runner (the suite is dominated by per-test + # `bd init` fixture setup, so two workers per CPU still pay off). + # pytest-xdist is a pinned dev dependency (pyproject.toml). + local workers="${BEADS_MCP_PYTEST_WORKERS:-8}" + if [[ ! "$workers" =~ ^[1-9][0-9]*$ ]]; then + echo "BEADS_MCP_PYTEST_WORKERS must be a positive integer, got '$workers'" >&2 + return 2 + fi + uv run pytest -n "$workers" --durations=50 } mcp_build() { diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index a6a1b3466..9aa28918d 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -55,6 +55,39 @@ const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (git const blacksmithWindows2vcpuRunsOn = "${{ 'blacksmith-2vcpu-windows-2025' }}" const blacksmithWindows4vcpuRunsOn = "${{ 'blacksmith-4vcpu-windows-2025' }}" +// bazel.yml's lanes pick their runner from the rbe job's decision alone +// (TestBazelRBEJobDecidesOnce): Blacksmith in mode remote, GitHub-hosted +// otherwise. bazelRemoteRunsOn builds that ternary for one Blacksmith size so +// every lane's pinned runs-on is the same literal apart from the label. +func bazelRemoteRunsOn(label string) string { + return "${{ needs.rbe.outputs.mode == 'remote' && '" + label + "' || 'ubuntu-latest' }}" +} + +// The default lane size: the runner is only an RBE client (every action +// executes on rbe-west), so 2 vCPU is enough. +var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") + +// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): bazel-test's +// `bazel test //... --config=ci` spent ~63 of its 81 s in client-side loading +// and analysis (1798 packages, 66k configured targets; critical path 7.65 s, +// every action a remote cache hit), which Skyframe parallelizes across the +// client's cores. 4 vCPU; revert to bazelLaneRunsOn if the PR run does not +// save at least 15 s on that step. +var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") + +// The package gates: package-npm at 4 vCPU (F3); package-mcp at 8 vCPU with +// pytest -n 16 (bazelMCPPytestWorkers), since its pytest run is dominated by +// per-test `bd init` fixture setup, which is CPU-bound and parallel. +var bazelPackageRunsOn = map[string]string{ + "package-mcp": bazelRemoteRunsOn("blacksmith-8vcpu-ubuntu-2404"), + "package-npm": bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404"), +} + +// package-mcp's BEADS_MCP_PYTEST_WORKERS: 16 xdist workers on the 8 vCPU +// Blacksmith runner (mode remote), package-mcp.sh's default 8 on the +// GitHub-hosted fallback (forks, Dependabot, rbe off/skip/cache). +const bazelMCPPytestWorkers = "${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }}" + // Blacksmith macOS (Apple Silicon M4, ARM64), pinned to macos-26 (the image // GitHub's macos-latest resolves to) so the PR legs and main.yml's // blacksmith-macos-go-build-cache saver share one image regardless of when @@ -62,18 +95,39 @@ const blacksmithWindows4vcpuRunsOn = "${{ 'blacksmith-4vcpu-windows-2025' }}" const blacksmithMacOSLabel = "blacksmith-6vcpu-macos-26" const blacksmithWindowsLabel = "blacksmith-4vcpu-windows-2025" +// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): pr-preflight-platforms' +// Windows leg, the slowest Windows check (129 s in merge group 37721729916), +// runs on 8 vCPU; its setup-go/cache extraction and the ./cmd/bd test +// compile and link are CPU-bound. It keeps restoring the non-race GOCACHE +// main.yml's test-windows saves on blacksmith-4vcpu-windows-2025: the cache +// key names runner.os and runner.arch, never the label, and actions/cache's +// version hash covers only the paths and compression method, so nothing in +// the entry is size-specific (Blacksmith documents branch scoping only). +// The saver stays on 4 vCPU so the other Windows consumers keep restoring +// from their exact label. Revert to blacksmithWindowsLabel if the PR run +// does not save at least 15 s or the leg's "Restore non-race Go build +// cache" step misses. +const blacksmithWindowsPreflightLabel = "blacksmith-8vcpu-windows-2025" + // platformsMatrixRunsOn: the mixed-OS matrix jobs (pr-preflight-platforms, // check-doc-freshness-platforms) name each leg's Blacksmith label in its // `include` entry's `runner` field. const platformsMatrixRunsOn = "${{ matrix.runner }}" -// platformsMatrixRunners: each mixed-OS matrix leg's `os` (which keeps the -// check names stable) -> its Blacksmith label. +// platformsMatrixRunners: check-doc-freshness-platforms' legs, `os` (which +// keeps the check names stable) -> its Blacksmith label. var platformsMatrixRunners = map[string]string{ "macos-latest": blacksmithMacOSLabel, "windows-latest": blacksmithWindowsLabel, } +// preflightMatrixRunners: pr-preflight-platforms' legs, the same shape with +// the larger Windows label. +var preflightMatrixRunners = map[string]string{ + "macos-latest": blacksmithMacOSLabel, + "windows-latest": blacksmithWindowsPreflightLabel, +} + // --- a minimal GitHub Actions expression evaluator ------------------------- // // Review SF-3 (2026-10-03) on the F7a review: the original @@ -512,9 +566,100 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { } } } - for _, leg := range []string{blacksmithMacOSLabel, blacksmithWindowsLabel} { + for _, leg := range []string{blacksmithMacOSLabel, blacksmithWindowsLabel, blacksmithWindowsPreflightLabel} { if got := mustEvalGHRunsOn(t, platformsMatrixRunsOn, map[string]string{"matrix.runner": leg}); got != leg { t.Errorf("%s with matrix.runner %q = %q", platformsMatrixRunsOn, leg, got) } } } + +// TestBlacksmithBazelRunnerSizes evaluates every bazel.yml lane's real +// runs-on (and package-mcp's xdist worker count) under each rbe mode: mode +// remote gets the lane's Blacksmith size, every other mode the GitHub-hosted +// runner, and package-mcp runs 16 xdist workers exactly when it is on its +// 8 vCPU runner (8 everywhere else, package-mcp.sh's default). +func TestBlacksmithBazelRunnerSizes(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + wantRemote := map[string]string{ + bazelJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelPackageMCPJobName: "blacksmith-8vcpu-ubuntu-2404", + bazelPackageNPMJobName: "blacksmith-4vcpu-ubuntu-2404", + } + mcpGate := workflow.job(t, bazelPackageMCPJobName).step(t, "Run MCP package gate") + if got := mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"]; got != bazelMCPPytestWorkers { + t.Errorf("package-mcp Run MCP package gate BEADS_MCP_PYTEST_WORKERS = %q, want %q", got, bazelMCPPytestWorkers) + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} { + ctx := map[string]string{"needs.rbe.outputs.mode": mode} + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + want := "ubuntu-latest" + if mode == "remote" { + want = "blacksmith-2vcpu-ubuntu-2404" + if label, ok := wantRemote[name]; ok { + want = label + } + } + if got := mustEvalGHRunsOn(t, job.RunsOn, ctx); got != want { + t.Errorf("%s runs-on under mode %s = %q, want %q", name, mode, got, want) + } + } + wantWorkers := "8" + if mustEvalGHRunsOn(t, workflow.job(t, bazelPackageMCPJobName).RunsOn, ctx) == "blacksmith-8vcpu-ubuntu-2404" { + wantWorkers = "16" + } + if got := mustEvalGHRunsOn(t, mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"], ctx); got != wantWorkers { + t.Errorf("package-mcp BEADS_MCP_PYTEST_WORKERS under mode %s = %q, want %q", mode, got, wantWorkers) + } + } +} + +// TestWindowsPreflightLegReachesSaverAcrossSizes: pr-preflight-platforms' +// Windows leg runs on a larger Blacksmith Windows size than main.yml's +// test-windows saver. That only works because nothing in the cache entry +// names the runner size: same OS image, the same cache path, and key +// templates built from runner.os/runner.arch (never the label, the matrix +// runner or a vCPU count), with the saver's key reachable from the leg's +// restore-keys prefix. +func TestWindowsPreflightLegReachesSaverAcrossSizes(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml") + saver := readCIWorkflow(t, "main.yml").job(t, "test-windows") + leg := pr.job(t, "pr-preflight-platforms") + + saverLabel := mustEvalGHRunsOn(t, saver.RunsOn, nil) + var legLabel string + for _, tuple := range leg.Strategy.Matrix.Include { + if tuple.OS == "windows-latest" { + legLabel = mustEvalGHRunsOn(t, leg.RunsOn, map[string]string{"matrix.runner": tuple.Runner, "matrix.os": tuple.OS}) + } + } + if legLabel != blacksmithWindowsPreflightLabel || saverLabel != blacksmithWindowsLabel { + t.Fatalf("Windows preflight leg = %q, saver = %q; want %q and %q", legLabel, saverLabel, blacksmithWindowsPreflightLabel, blacksmithWindowsLabel) + } + image := func(label string) string { return label[strings.LastIndex(label, "-windows-"):] } + if image(legLabel) != image(saverLabel) { + t.Errorf("Windows preflight leg image %q differs from the saver's %q; a different image may not share cache versions", image(legLabel), image(saverLabel)) + } + sizeSpecific := []string{"matrix.runner", "runner.name", "vcpu", "blacksmith"} + for _, pair := range [][2]string{ + {"Restore Go module cache", "Save Go module cache"}, + {"Restore non-race Go build cache", "Save non-race Go build cache"}, + } { + restore, save := leg.step(t, pair[0]), saver.step(t, pair[1]) + if restore.With["path"] != save.With["path"] { + t.Errorf("%q path %q != saver %q path %q", pair[0], restore.With["path"], pair[1], save.With["path"]) + } + if prefix := restore.With["restore-keys"]; prefix == "" || !strings.HasPrefix(save.With["key"], prefix) { + t.Errorf("saver %q key %q is not reachable from %q restore-keys %q", pair[1], save.With["key"], pair[0], prefix) + } + for _, s := range []string{restore.With["key"], restore.With["restore-keys"], save.With["key"]} { + for _, bad := range sizeSpecific { + if strings.Contains(strings.ToLower(s), bad) { + t.Errorf("cache key %q names %q; keys must not depend on the runner size", s, bad) + } + } + } + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index f5479a463..3d253f292 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -483,8 +483,8 @@ func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { t.Errorf("pr-preflight-platforms matrix os = %v, want the macOS and Windows hosts (Linux runs under Bazel)", got) } for _, tuple := range job.Strategy.Matrix.Include { - if tuple.Runner != platformsMatrixRunners[tuple.OS] { - t.Errorf("pr-preflight-platforms %s leg runner = %q, want %q", tuple.OS, tuple.Runner, platformsMatrixRunners[tuple.OS]) + if tuple.Runner != preflightMatrixRunners[tuple.OS] { + t.Errorf("pr-preflight-platforms %s leg runner = %q, want %q", tuple.OS, tuple.Runner, preflightMatrixRunners[tuple.OS]) } } if job.If != "" { @@ -1940,11 +1940,11 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp // caller left package-gates at its default "off". const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" -// F3: the package gates' runner. 4 vCPU, not bazel.yml's usual 2: pytest-xdist -// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). Mode -// remote only: the gates take no rbe-fork certificate, so fork modes +// F3: the package gates' runners (bazelPackageRunsOn, in +// ci_blacksmith_runner_test.go with the other runner sizes) are larger than +// bazel.yml's usual 2 vCPU because they run pytest/npm on the runner itself. +// Mode remote only: the gates take no rbe-fork certificate, so fork modes // (enabled too) build bd with go build on a GitHub-hosted runner, as before. -const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): mode // remote only (B1, security review of bdef342d5 - was remote or fork-rw). @@ -2110,7 +2110,7 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // runner. Only mode remote gets the secrets; fork modes get // BAZEL_FORK_REMOTE (setup-bazel mints the certificate), cache runs // BAZEL_FORK_CACHE. - const wantRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + wantRunsOn := bazelLaneRunsOn // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS // unset); remote-only lanes (27 race processes and more, an hour or more // on a GitHub-hosted runner, for tests the Go jobs already run there) @@ -2148,12 +2148,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } - // F3: the package gates use a 4 vCPU runner (pytest-xdist -n 8) and + // F3: the package gates use larger runners (bazelPackageRunsOn) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). wantJobRunsOn, wantJobIf, wantJobSetupEnv := wantRunsOn, wantIf, wantSetupEnv + if name == bazelJobName { + wantJobRunsOn = bazelTestLaneRunsOn + } if bazelPackageJobs[name] { - wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn, bazelPackageGatesIf, wantPackageSetupEnv + wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn[name], bazelPackageGatesIf, wantPackageSetupEnv } else if bazelRemoteOnlyJobs[name] { wantJobIf = wantRemoteOnlyIf } else if name == bazelIntegJobName || name == bazelCmdDoltJobName { @@ -2181,21 +2184,26 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { } } -// F3: bazelPackageRunsOn's comment and the package-mcp job's 4 vCPU runner +// F3: bazelPackageRunsOn's comment and the package-mcp job's runner size // are both premised on mcp_pytest() in package-mcp.sh passing an explicit -// "-n 8" (not "-n auto", which would reintroduce the CPU-count/cgroup-quota -// mismatch -n 8 was pinned to avoid), and on pytest-xdist itself being a -// locked dev dependency so that worker count is reproducible across CI runs. -// Neither half of that premise is checked by any other policy test, so pin -// both here directly against the script and the MCP package's manifest/lock. +// worker count (not "-n auto", which would reintroduce the CPU-count/ +// cgroup-quota mismatch the explicit count was pinned to avoid): the +// default 8, or BEADS_MCP_PYTEST_WORKERS, which bazel.yml's package-mcp sets +// to 16 exactly when it runs on its 8 vCPU Blacksmith runner +// (TestBlacksmithBazelRunnerSizes). It is also premised on pytest-xdist +// itself being a locked dev dependency so that worker count is reproducible +// across CI runs. Pin both here directly against the script and the MCP +// package's manifest/lock. func TestMCPPytestWorkerCountPinned(t *testing.T) { root := sourceRepoRoot(t) script := readPolicyFile(t, root, "scripts/ci/package-mcp.sh") - if !strings.Contains(script, "pytest -n 8") { - t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() does not pass an explicit -n 8") + for _, want := range []string{`local workers="${BEADS_MCP_PYTEST_WORKERS:-8}"`, `uv run pytest -n "$workers"`} { + if !strings.Contains(script, want) { + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() is missing %q (an explicit worker count, default 8)", want) + } } if strings.Contains(script, "-n auto") { - t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want the pinned -n 8") + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want an explicit worker count") } pyproject := readPolicyFile(t, root, "integrations/beads-mcp/pyproject.toml") if !regexp.MustCompile(`(?m)^\s*"pytest-xdist[><=]`).MatchString(pyproject) { diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 3983462d0..b2058509f 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -255,9 +255,9 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { if !ok { t.Fatalf("%s has no %s job", bazelWorkflowName, lane.job) } - if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn { + if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn[lane.job] { t.Errorf("%s: needs %v, if %q, runs-on %q; want needs [%s], if %q, runs-on %q", - lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn) + lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn[lane.job]) } detectCond := "steps.detect.outputs." + lane.detectOutput + " == 'true'" From 591c625738d9f78f1f665bdd61a9ea8a41b48764 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 04:02:50 +0000 Subject: [PATCH 2/2] ci(pr): keep the Windows preflight leg on 4 vCPU (A/B: 8 vCPU queued 63s) #7381's PR run put pr-preflight-platforms' Windows leg on blacksmith-8vcpu-windows-2025. The cache restore from main.yml's 4 vCPU test-windows saver hit, but the job waited 63s for an 8 vCPU Windows runner and finished at 144s against the 129s baseline on 4 vCPU, so the leg goes back to blacksmith-4vcpu-windows-2025. With saver and consumer on one label again, TestWindowsPreflightLegReachesSaverAcrossSizes and the separate preflight runner map have nothing left to pin and are dropped; the result is recorded next to platformsMatrixRunners, in pr.yml and in CI_REQUIRED_CHECK_TOPOLOGY.md. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/main.yml | 6 +- .github/workflows/pr.yml | 19 ++----- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 10 ++-- scripts/ci_blacksmith_runner_test.go | 80 +++------------------------ scripts/ci_workflow_test.go | 4 +- 5 files changed, 22 insertions(+), 97 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 19ca95730..c64890e66 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -184,10 +184,8 @@ jobs: # F7b review fix (S2): warms the SAME non-race GOCACHE namespace that # pr-preflight-platforms'/worktree-remove-windows'/test-windows-liveness' - # Windows legs restore from, on the same Windows image - # (blacksmith-4vcpu-windows-2025; pr-preflight-platforms' leg runs on - # 8 vCPU, which shares these entries because no key names the runner - # size), so their cold `cmd/bd` test compiles + # Windows legs restore from, on the exact same runner selection + # (blacksmith-4vcpu-windows-2025), so their cold `cmd/bd` test compiles # (the real long pole; see pr-preflight-platforms' S2 comment) get a # true cache hit. Confirmed to run under Git Bash on Windows (reviewer # verification). diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7f2895e43..9483e2b36 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -505,16 +505,9 @@ jobs: # restores, mostly tar extraction), 33-71s for TestPRPreflight (Git Bash # process creation, now parallel subtests), 27-58s to compile and link # ./cmd/bd's test binary, and about 25s for the rest. A second Windows leg - # would pay the setup again to save about 20s, so it stays one leg. - # - # Runner-size A/B (2026-10-08): the Windows leg was the slowest Windows - # check at 129s on blacksmith-4vcpu-windows-2025 (merge group 37721729916), - # so it runs on 8 vCPU; the cache extraction and the ./cmd/bd test compile - # and link above are CPU-bound. It still restores main.yml's test-windows - # entries saved on 4 vCPU: the cache keys name runner.os/runner.arch, never - # the label, so nothing in an entry is size-specific - # (TestWindowsPreflightLegReachesSaverAcrossSizes). Revert to 4 vCPU if it - # does not save at least 15s. + # would pay the setup again to save about 20s, so it stays one leg. An + # 8 vCPU Windows leg was tried (#7381): cache restores still hit, but it + # queued 63s for the larger runner and finished at 144s vs 129s here. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) runs-on: ${{ matrix.runner }} @@ -528,7 +521,7 @@ jobs: runner: blacksmith-6vcpu-macos-26 - os: windows-latest expected_goos: windows - runner: blacksmith-8vcpu-windows-2025 + runner: blacksmith-4vcpu-windows-2025 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -547,8 +540,8 @@ jobs: restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- # F7b review (S2): restore-only non-race Go build cache, seeded by the - # push-to-main saver on the same Blacksmith OS image (Windows: - # main.yml's test-windows, on 4 vCPU; macOS: main.yml's + # push-to-main saver on the exact same Blacksmith label (Windows: + # main.yml's test-windows; macOS: main.yml's # blacksmith-macos-go-build-cache) so the cold cmd/bd compile below # (the real long pole, 250-300s uncached) gets a true cache hit instead # of starting cold. GOCACHE must match the seeder's exactly, and diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index ed76084ce..e271b795b 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1205,12 +1205,10 @@ Bazel on rbe-west. pr.yml's Windows jobs (`test-windows-liveness`, their `blacksmith-*vcpu-windows-2025` label literally; the two mixed-OS matrix jobs, `pr-preflight-platforms` and `check-doc-freshness-platforms`, run a macOS leg on `blacksmith-6vcpu-macos-26` (Apple Silicon) and a Windows -leg (`blacksmith-8vcpu-windows-2025` for `pr-preflight-platforms`, -`blacksmith-4vcpu-windows-2025` for `check-doc-freshness-platforms`), each -leg naming its label in the matrix (`runs-on: ${{ matrix.runner }}`). The -8 vCPU leg restores main.yml's `test-windows` entries saved on 4 vCPU: the -cache keys name `runner.os` and `runner.arch`, never the label, and -`actions/cache`'s version hash covers only the paths and compression method. There is no GitHub-hosted fallback: +leg on `blacksmith-4vcpu-windows-2025`, each leg naming its label in the +matrix (`runs-on: ${{ matrix.runner }}`). An 8 vCPU Windows preflight leg +was tried (#7381): it restored the 4 vCPU saver's cache, but queued 63s for +the larger runner and finished slower (144s vs 129s), so it stays on 4 vCPU. There is no GitHub-hosted fallback: Blacksmith serves this org's fork PRs (gascity's fork PRs run their CI on `blacksmith-*` labels). Fork runs get no secrets and a read-only token (`TestBlacksmithJobsReadNoSecrets` keeps every Blacksmith job free of secret diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 9aa28918d..35685349a 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -95,39 +95,23 @@ const bazelMCPPytestWorkers = "${{ needs.rbe.outputs.mode == 'remote' && '16' || const blacksmithMacOSLabel = "blacksmith-6vcpu-macos-26" const blacksmithWindowsLabel = "blacksmith-4vcpu-windows-2025" -// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): pr-preflight-platforms' -// Windows leg, the slowest Windows check (129 s in merge group 37721729916), -// runs on 8 vCPU; its setup-go/cache extraction and the ./cmd/bd test -// compile and link are CPU-bound. It keeps restoring the non-race GOCACHE -// main.yml's test-windows saves on blacksmith-4vcpu-windows-2025: the cache -// key names runner.os and runner.arch, never the label, and actions/cache's -// version hash covers only the paths and compression method, so nothing in -// the entry is size-specific (Blacksmith documents branch scoping only). -// The saver stays on 4 vCPU so the other Windows consumers keep restoring -// from their exact label. Revert to blacksmithWindowsLabel if the PR run -// does not save at least 15 s or the leg's "Restore non-race Go build -// cache" step misses. -const blacksmithWindowsPreflightLabel = "blacksmith-8vcpu-windows-2025" - // platformsMatrixRunsOn: the mixed-OS matrix jobs (pr-preflight-platforms, // check-doc-freshness-platforms) name each leg's Blacksmith label in its // `include` entry's `runner` field. const platformsMatrixRunsOn = "${{ matrix.runner }}" -// platformsMatrixRunners: check-doc-freshness-platforms' legs, `os` (which -// keeps the check names stable) -> its Blacksmith label. +// platformsMatrixRunners: each mixed-OS matrix leg's `os` (which keeps the +// check names stable) -> its Blacksmith label. +// +// Runner-size A/B (#7381, 2026-10-08): pr-preflight-platforms' Windows leg on +// blacksmith-8vcpu-windows-2025 restored the 4 vCPU saver's cache fine, but +// waited 63s for an 8 vCPU Windows runner and finished at 144s against 129s +// on 4 vCPU, so it stays on blacksmithWindowsLabel. var platformsMatrixRunners = map[string]string{ "macos-latest": blacksmithMacOSLabel, "windows-latest": blacksmithWindowsLabel, } -// preflightMatrixRunners: pr-preflight-platforms' legs, the same shape with -// the larger Windows label. -var preflightMatrixRunners = map[string]string{ - "macos-latest": blacksmithMacOSLabel, - "windows-latest": blacksmithWindowsPreflightLabel, -} - // --- a minimal GitHub Actions expression evaluator ------------------------- // // Review SF-3 (2026-10-03) on the F7a review: the original @@ -566,7 +550,7 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { } } } - for _, leg := range []string{blacksmithMacOSLabel, blacksmithWindowsLabel, blacksmithWindowsPreflightLabel} { + for _, leg := range []string{blacksmithMacOSLabel, blacksmithWindowsLabel} { if got := mustEvalGHRunsOn(t, platformsMatrixRunsOn, map[string]string{"matrix.runner": leg}); got != leg { t.Errorf("%s with matrix.runner %q = %q", platformsMatrixRunsOn, leg, got) } @@ -615,51 +599,3 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) { } } } - -// TestWindowsPreflightLegReachesSaverAcrossSizes: pr-preflight-platforms' -// Windows leg runs on a larger Blacksmith Windows size than main.yml's -// test-windows saver. That only works because nothing in the cache entry -// names the runner size: same OS image, the same cache path, and key -// templates built from runner.os/runner.arch (never the label, the matrix -// runner or a vCPU count), with the saver's key reachable from the leg's -// restore-keys prefix. -func TestWindowsPreflightLegReachesSaverAcrossSizes(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml") - saver := readCIWorkflow(t, "main.yml").job(t, "test-windows") - leg := pr.job(t, "pr-preflight-platforms") - - saverLabel := mustEvalGHRunsOn(t, saver.RunsOn, nil) - var legLabel string - for _, tuple := range leg.Strategy.Matrix.Include { - if tuple.OS == "windows-latest" { - legLabel = mustEvalGHRunsOn(t, leg.RunsOn, map[string]string{"matrix.runner": tuple.Runner, "matrix.os": tuple.OS}) - } - } - if legLabel != blacksmithWindowsPreflightLabel || saverLabel != blacksmithWindowsLabel { - t.Fatalf("Windows preflight leg = %q, saver = %q; want %q and %q", legLabel, saverLabel, blacksmithWindowsPreflightLabel, blacksmithWindowsLabel) - } - image := func(label string) string { return label[strings.LastIndex(label, "-windows-"):] } - if image(legLabel) != image(saverLabel) { - t.Errorf("Windows preflight leg image %q differs from the saver's %q; a different image may not share cache versions", image(legLabel), image(saverLabel)) - } - sizeSpecific := []string{"matrix.runner", "runner.name", "vcpu", "blacksmith"} - for _, pair := range [][2]string{ - {"Restore Go module cache", "Save Go module cache"}, - {"Restore non-race Go build cache", "Save non-race Go build cache"}, - } { - restore, save := leg.step(t, pair[0]), saver.step(t, pair[1]) - if restore.With["path"] != save.With["path"] { - t.Errorf("%q path %q != saver %q path %q", pair[0], restore.With["path"], pair[1], save.With["path"]) - } - if prefix := restore.With["restore-keys"]; prefix == "" || !strings.HasPrefix(save.With["key"], prefix) { - t.Errorf("saver %q key %q is not reachable from %q restore-keys %q", pair[1], save.With["key"], pair[0], prefix) - } - for _, s := range []string{restore.With["key"], restore.With["restore-keys"], save.With["key"]} { - for _, bad := range sizeSpecific { - if strings.Contains(strings.ToLower(s), bad) { - t.Errorf("cache key %q names %q; keys must not depend on the runner size", s, bad) - } - } - } - } -} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 3d253f292..6905ece3c 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -483,8 +483,8 @@ func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { t.Errorf("pr-preflight-platforms matrix os = %v, want the macOS and Windows hosts (Linux runs under Bazel)", got) } for _, tuple := range job.Strategy.Matrix.Include { - if tuple.Runner != preflightMatrixRunners[tuple.OS] { - t.Errorf("pr-preflight-platforms %s leg runner = %q, want %q", tuple.OS, tuple.Runner, preflightMatrixRunners[tuple.OS]) + if tuple.Runner != platformsMatrixRunners[tuple.OS] { + t.Errorf("pr-preflight-platforms %s leg runner = %q, want %q", tuple.OS, tuple.Runner, platformsMatrixRunners[tuple.OS]) } } if job.If != "" {