diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6564862f0..66322c9d5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -728,8 +728,15 @@ jobs: # they miss on the runner, so they get the larger GitHub-hosted one. # Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted, # like every fork job: the runner is only a client. + # + # 4 vCPU, not the other lanes' 2: with every action a remote cache hit, + # this lane's `bazel test //... --config=ci` spent ~63 of 81s in + # client-side loading and analysis (1798 packages, 66k configured + # targets; critical path 7.65s, merge group 37721729916), which Bazel + # parallelizes across the client's cores. Revert to 2 vCPU if the A/B + # does not save at least 15s on that step. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed # worker pool shared with gascity). Skipped only in mode skip (same-repo # runs while RBE_WEST_WORKERS is unset). @@ -2210,9 +2217,12 @@ jobs: package-mcp: name: Package Gate (MCP) needs: rbe - # 4 vCPU on purpose: pytest-xdist -n 8 measured ~2.7-3x on 4 CPUs - # (tools/f3 timing, 2026-10), well past 2 vCPU's ceiling. - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # 8 vCPU with 16 xdist workers (BEADS_MCP_PYTEST_WORKERS below): pytest + # was 70s of this job at -n 8 on 4 vCPU (merge group 37721729916), + # dominated by per-test `bd init` fixture setup, which is CPU-bound. + # Was 4 vCPU / -n 8 (tools/f3 timing, 2026-10: ~2.7-3x on 4 CPUs); + # revert to that if the A/B does not save at least 15s. + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ inputs.package-gates == 'on' }} timeout-minutes: 30 outputs: @@ -2266,8 +2276,8 @@ jobs: echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" # Defense in depth: nothing after this step needs the Bazel server # or its RBE client key (uv sync and pytest run PR-controlled - # code). Shutting down also frees the JVM's memory for the 8 - # xdist workers that follow on this 4 vCPU runner. + # code). Shutting down also frees the JVM's memory for the 16 + # xdist workers that follow on this 8 vCPU runner. bazel shutdown rm -rf "$RUNNER_TEMP/bazel-ci-secret" @@ -2313,6 +2323,9 @@ jobs: if: ${{ steps.detect.outputs.mcp_package == 'true' }} env: GOCACHE: ${{ runner.temp }}/go-cache/non-race + # 16 workers on the 8 vCPU Blacksmith runner; package-mcp.sh's + # default 8 on the GitHub-hosted fallback. + BEADS_MCP_PYTEST_WORKERS: ${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }} run: make ci-package-mcp # Last step: the job's result as a workflow_call output for pr.yml's diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 628ed5dcc..9483e2b36 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -505,7 +505,9 @@ jobs: # restores, mostly tar extraction), 33-71s for TestPRPreflight (Git Bash # process creation, now parallel subtests), 27-58s to compile and link # ./cmd/bd's test binary, and about 25s for the rest. A second Windows leg - # would pay the setup again to save about 20s, so it stays one leg. + # would pay the setup again to save about 20s, so it stays one leg. An + # 8 vCPU Windows leg was tried (#7381): cache restores still hit, but it + # queued 63s for the larger runner and finished at 144s vs 129s here. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) runs-on: ${{ matrix.runner }} diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index fb22a7410..e271b795b 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -883,9 +883,11 @@ Required` requires them to have run remotely and passed. `1.3.0 (dev)`, no commit); no consumer reads it. Verified 2026-10-02: both package gates pass with the Bazel-built bd (MCP: 228 passed, 5 skipped; npm: all tests and the pack dry run), the same as with a - `go build` bd. Both run on `blacksmith-4vcpu-ubuntu-2404` when - `rbe.outputs.enabled == 'true'` (4 vCPU: `pytest -n 8` is pinned to - timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own + `go build` bd. In mode remote, package-npm runs on + `blacksmith-4vcpu-ubuntu-2404` and package-mcp on + `blacksmith-8vcpu-ubuntu-2404` with `pytest -n 16` + (`BEADS_MCP_PYTEST_WORKERS`; the script's default is `-n 8`); + `ubuntu-latest` (and `-n 8`) otherwise. `bazel-test`'s own `bazel-ci-build-artifacts` upload is no longer consumed by anything; it is kept for the F3.5.3 SHA256SUMS comparison and for debugging. - The Dolt-backed domain, uow, tracker, doctor/fix and protocol suites @@ -1204,7 +1206,9 @@ their `blacksmith-*vcpu-windows-2025` label literally; the two mixed-OS matrix jobs, `pr-preflight-platforms` and `check-doc-freshness-platforms`, run a macOS leg on `blacksmith-6vcpu-macos-26` (Apple Silicon) and a Windows leg on `blacksmith-4vcpu-windows-2025`, each leg naming its label in the -matrix (`runs-on: ${{ matrix.runner }}`). There is no GitHub-hosted fallback: +matrix (`runs-on: ${{ matrix.runner }}`). An 8 vCPU Windows preflight leg +was tried (#7381): it restored the 4 vCPU saver's cache, but queued 63s for +the larger runner and finished slower (144s vs 129s), so it stays on 4 vCPU. There is no GitHub-hosted fallback: Blacksmith serves this org's fork PRs (gascity's fork PRs run their CI on `blacksmith-*` labels). Fork runs get no secrets and a read-only token (`TestBlacksmithJobsReadNoSecrets` keeps every Blacksmith job free of secret diff --git a/scripts/ci/package-mcp.sh b/scripts/ci/package-mcp.sh index 5b14a7041..56490599d 100755 --- a/scripts/ci/package-mcp.sh +++ b/scripts/ci/package-mcp.sh @@ -55,8 +55,16 @@ mcp_pytest() { # Explicit worker count, not "auto": CI runners' reported CPU count # doesn't always match the cgroup quota, and -n 8 is pinned to measured # timing (tools/f3, 2026-10) on both 2 vCPU (2.7x) and 4 vCPU (~3x) - # runners. pytest-xdist is a pinned dev dependency (pyproject.toml). - uv run pytest -n 8 --durations=50 + # runners. bazel.yml's package-mcp sets BEADS_MCP_PYTEST_WORKERS=16 on + # its 8 vCPU Blacksmith runner (the suite is dominated by per-test + # `bd init` fixture setup, so two workers per CPU still pay off). + # pytest-xdist is a pinned dev dependency (pyproject.toml). + local workers="${BEADS_MCP_PYTEST_WORKERS:-8}" + if [[ ! "$workers" =~ ^[1-9][0-9]*$ ]]; then + echo "BEADS_MCP_PYTEST_WORKERS must be a positive integer, got '$workers'" >&2 + return 2 + fi + uv run pytest -n "$workers" --durations=50 } mcp_build() { diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index a6a1b3466..35685349a 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -55,6 +55,39 @@ const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (git const blacksmithWindows2vcpuRunsOn = "${{ 'blacksmith-2vcpu-windows-2025' }}" const blacksmithWindows4vcpuRunsOn = "${{ 'blacksmith-4vcpu-windows-2025' }}" +// bazel.yml's lanes pick their runner from the rbe job's decision alone +// (TestBazelRBEJobDecidesOnce): Blacksmith in mode remote, GitHub-hosted +// otherwise. bazelRemoteRunsOn builds that ternary for one Blacksmith size so +// every lane's pinned runs-on is the same literal apart from the label. +func bazelRemoteRunsOn(label string) string { + return "${{ needs.rbe.outputs.mode == 'remote' && '" + label + "' || 'ubuntu-latest' }}" +} + +// The default lane size: the runner is only an RBE client (every action +// executes on rbe-west), so 2 vCPU is enough. +var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") + +// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): bazel-test's +// `bazel test //... --config=ci` spent ~63 of its 81 s in client-side loading +// and analysis (1798 packages, 66k configured targets; critical path 7.65 s, +// every action a remote cache hit), which Skyframe parallelizes across the +// client's cores. 4 vCPU; revert to bazelLaneRunsOn if the PR run does not +// save at least 15 s on that step. +var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") + +// The package gates: package-npm at 4 vCPU (F3); package-mcp at 8 vCPU with +// pytest -n 16 (bazelMCPPytestWorkers), since its pytest run is dominated by +// per-test `bd init` fixture setup, which is CPU-bound and parallel. +var bazelPackageRunsOn = map[string]string{ + "package-mcp": bazelRemoteRunsOn("blacksmith-8vcpu-ubuntu-2404"), + "package-npm": bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404"), +} + +// package-mcp's BEADS_MCP_PYTEST_WORKERS: 16 xdist workers on the 8 vCPU +// Blacksmith runner (mode remote), package-mcp.sh's default 8 on the +// GitHub-hosted fallback (forks, Dependabot, rbe off/skip/cache). +const bazelMCPPytestWorkers = "${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }}" + // Blacksmith macOS (Apple Silicon M4, ARM64), pinned to macos-26 (the image // GitHub's macos-latest resolves to) so the PR legs and main.yml's // blacksmith-macos-go-build-cache saver share one image regardless of when @@ -69,6 +102,11 @@ const platformsMatrixRunsOn = "${{ matrix.runner }}" // platformsMatrixRunners: each mixed-OS matrix leg's `os` (which keeps the // check names stable) -> its Blacksmith label. +// +// Runner-size A/B (#7381, 2026-10-08): pr-preflight-platforms' Windows leg on +// blacksmith-8vcpu-windows-2025 restored the 4 vCPU saver's cache fine, but +// waited 63s for an 8 vCPU Windows runner and finished at 144s against 129s +// on 4 vCPU, so it stays on blacksmithWindowsLabel. var platformsMatrixRunners = map[string]string{ "macos-latest": blacksmithMacOSLabel, "windows-latest": blacksmithWindowsLabel, @@ -518,3 +556,46 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { } } } + +// TestBlacksmithBazelRunnerSizes evaluates every bazel.yml lane's real +// runs-on (and package-mcp's xdist worker count) under each rbe mode: mode +// remote gets the lane's Blacksmith size, every other mode the GitHub-hosted +// runner, and package-mcp runs 16 xdist workers exactly when it is on its +// 8 vCPU runner (8 everywhere else, package-mcp.sh's default). +func TestBlacksmithBazelRunnerSizes(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + wantRemote := map[string]string{ + bazelJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelPackageMCPJobName: "blacksmith-8vcpu-ubuntu-2404", + bazelPackageNPMJobName: "blacksmith-4vcpu-ubuntu-2404", + } + mcpGate := workflow.job(t, bazelPackageMCPJobName).step(t, "Run MCP package gate") + if got := mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"]; got != bazelMCPPytestWorkers { + t.Errorf("package-mcp Run MCP package gate BEADS_MCP_PYTEST_WORKERS = %q, want %q", got, bazelMCPPytestWorkers) + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} { + ctx := map[string]string{"needs.rbe.outputs.mode": mode} + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + want := "ubuntu-latest" + if mode == "remote" { + want = "blacksmith-2vcpu-ubuntu-2404" + if label, ok := wantRemote[name]; ok { + want = label + } + } + if got := mustEvalGHRunsOn(t, job.RunsOn, ctx); got != want { + t.Errorf("%s runs-on under mode %s = %q, want %q", name, mode, got, want) + } + } + wantWorkers := "8" + if mustEvalGHRunsOn(t, workflow.job(t, bazelPackageMCPJobName).RunsOn, ctx) == "blacksmith-8vcpu-ubuntu-2404" { + wantWorkers = "16" + } + if got := mustEvalGHRunsOn(t, mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"], ctx); got != wantWorkers { + t.Errorf("package-mcp BEADS_MCP_PYTEST_WORKERS under mode %s = %q, want %q", mode, got, wantWorkers) + } + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index f5479a463..6905ece3c 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1940,11 +1940,11 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp // caller left package-gates at its default "off". const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" -// F3: the package gates' runner. 4 vCPU, not bazel.yml's usual 2: pytest-xdist -// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). Mode -// remote only: the gates take no rbe-fork certificate, so fork modes +// F3: the package gates' runners (bazelPackageRunsOn, in +// ci_blacksmith_runner_test.go with the other runner sizes) are larger than +// bazel.yml's usual 2 vCPU because they run pytest/npm on the runner itself. +// Mode remote only: the gates take no rbe-fork certificate, so fork modes // (enabled too) build bd with go build on a GitHub-hosted runner, as before. -const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): mode // remote only (B1, security review of bdef342d5 - was remote or fork-rw). @@ -2110,7 +2110,7 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // runner. Only mode remote gets the secrets; fork modes get // BAZEL_FORK_REMOTE (setup-bazel mints the certificate), cache runs // BAZEL_FORK_CACHE. - const wantRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + wantRunsOn := bazelLaneRunsOn // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS // unset); remote-only lanes (27 race processes and more, an hour or more // on a GitHub-hosted runner, for tests the Go jobs already run there) @@ -2148,12 +2148,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } - // F3: the package gates use a 4 vCPU runner (pytest-xdist -n 8) and + // F3: the package gates use larger runners (bazelPackageRunsOn) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). wantJobRunsOn, wantJobIf, wantJobSetupEnv := wantRunsOn, wantIf, wantSetupEnv + if name == bazelJobName { + wantJobRunsOn = bazelTestLaneRunsOn + } if bazelPackageJobs[name] { - wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn, bazelPackageGatesIf, wantPackageSetupEnv + wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn[name], bazelPackageGatesIf, wantPackageSetupEnv } else if bazelRemoteOnlyJobs[name] { wantJobIf = wantRemoteOnlyIf } else if name == bazelIntegJobName || name == bazelCmdDoltJobName { @@ -2181,21 +2184,26 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { } } -// F3: bazelPackageRunsOn's comment and the package-mcp job's 4 vCPU runner +// F3: bazelPackageRunsOn's comment and the package-mcp job's runner size // are both premised on mcp_pytest() in package-mcp.sh passing an explicit -// "-n 8" (not "-n auto", which would reintroduce the CPU-count/cgroup-quota -// mismatch -n 8 was pinned to avoid), and on pytest-xdist itself being a -// locked dev dependency so that worker count is reproducible across CI runs. -// Neither half of that premise is checked by any other policy test, so pin -// both here directly against the script and the MCP package's manifest/lock. +// worker count (not "-n auto", which would reintroduce the CPU-count/ +// cgroup-quota mismatch the explicit count was pinned to avoid): the +// default 8, or BEADS_MCP_PYTEST_WORKERS, which bazel.yml's package-mcp sets +// to 16 exactly when it runs on its 8 vCPU Blacksmith runner +// (TestBlacksmithBazelRunnerSizes). It is also premised on pytest-xdist +// itself being a locked dev dependency so that worker count is reproducible +// across CI runs. Pin both here directly against the script and the MCP +// package's manifest/lock. func TestMCPPytestWorkerCountPinned(t *testing.T) { root := sourceRepoRoot(t) script := readPolicyFile(t, root, "scripts/ci/package-mcp.sh") - if !strings.Contains(script, "pytest -n 8") { - t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() does not pass an explicit -n 8") + for _, want := range []string{`local workers="${BEADS_MCP_PYTEST_WORKERS:-8}"`, `uv run pytest -n "$workers"`} { + if !strings.Contains(script, want) { + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() is missing %q (an explicit worker count, default 8)", want) + } } if strings.Contains(script, "-n auto") { - t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want the pinned -n 8") + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want an explicit worker count") } pyproject := readPolicyFile(t, root, "integrations/beads-mcp/pyproject.toml") if !regexp.MustCompile(`(?m)^\s*"pytest-xdist[><=]`).MatchString(pyproject) { diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 3983462d0..b2058509f 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -255,9 +255,9 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { if !ok { t.Fatalf("%s has no %s job", bazelWorkflowName, lane.job) } - if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn { + if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn[lane.job] { t.Errorf("%s: needs %v, if %q, runs-on %q; want needs [%s], if %q, runs-on %q", - lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn) + lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn[lane.job]) } detectCond := "steps.detect.outputs." + lane.detectOutput + " == 'true'"