Compare commits
2 commits
main
...
ci/bigger-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
591c625738 | ||
|
|
b03f8f527e |
7 changed files with 147 additions and 31 deletions
25
.github/workflows/bazel.yml
vendored
25
.github/workflows/bazel.yml
vendored
|
|
@ -728,8 +728,15 @@ jobs:
|
|||
# they miss on the runner, so they get the larger GitHub-hosted one.
|
||||
# Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted,
|
||||
# like every fork job: the runner is only a client.
|
||||
#
|
||||
# 4 vCPU, not the other lanes' 2: with every action a remote cache hit,
|
||||
# this lane's `bazel test //... --config=ci` spent ~63 of 81s in
|
||||
# client-side loading and analysis (1798 packages, 66k configured
|
||||
# targets; critical path 7.65s, merge group 37721729916), which Bazel
|
||||
# parallelizes across the client's cores. Revert to 2 vCPU if the A/B
|
||||
# does not save at least 15s on that step.
|
||||
needs: rbe
|
||||
runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
|
||||
# Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed
|
||||
# worker pool shared with gascity). Skipped only in mode skip (same-repo
|
||||
# runs while RBE_WEST_WORKERS is unset).
|
||||
|
|
@ -2210,9 +2217,12 @@ jobs:
|
|||
package-mcp:
|
||||
name: Package Gate (MCP)
|
||||
needs: rbe
|
||||
# 4 vCPU on purpose: pytest-xdist -n 8 measured ~2.7-3x on 4 CPUs
|
||||
# (tools/f3 timing, 2026-10), well past 2 vCPU's ceiling.
|
||||
runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
|
||||
# 8 vCPU with 16 xdist workers (BEADS_MCP_PYTEST_WORKERS below): pytest
|
||||
# was 70s of this job at -n 8 on 4 vCPU (merge group 37721729916),
|
||||
# dominated by per-test `bd init` fixture setup, which is CPU-bound.
|
||||
# Was 4 vCPU / -n 8 (tools/f3 timing, 2026-10: ~2.7-3x on 4 CPUs);
|
||||
# revert to that if the A/B does not save at least 15s.
|
||||
runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
|
||||
if: ${{ inputs.package-gates == 'on' }}
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
|
|
@ -2266,8 +2276,8 @@ jobs:
|
|||
echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV"
|
||||
# Defense in depth: nothing after this step needs the Bazel server
|
||||
# or its RBE client key (uv sync and pytest run PR-controlled
|
||||
# code). Shutting down also frees the JVM's memory for the 8
|
||||
# xdist workers that follow on this 4 vCPU runner.
|
||||
# code). Shutting down also frees the JVM's memory for the 16
|
||||
# xdist workers that follow on this 8 vCPU runner.
|
||||
bazel shutdown
|
||||
rm -rf "$RUNNER_TEMP/bazel-ci-secret"
|
||||
|
||||
|
|
@ -2313,6 +2323,9 @@ jobs:
|
|||
if: ${{ steps.detect.outputs.mcp_package == 'true' }}
|
||||
env:
|
||||
GOCACHE: ${{ runner.temp }}/go-cache/non-race
|
||||
# 16 workers on the 8 vCPU Blacksmith runner; package-mcp.sh's
|
||||
# default 8 on the GitHub-hosted fallback.
|
||||
BEADS_MCP_PYTEST_WORKERS: ${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }}
|
||||
run: make ci-package-mcp
|
||||
|
||||
# Last step: the job's result as a workflow_call output for pr.yml's
|
||||
|
|
|
|||
4
.github/workflows/pr.yml
vendored
4
.github/workflows/pr.yml
vendored
|
|
@ -505,7 +505,9 @@ jobs:
|
|||
# restores, mostly tar extraction), 33-71s for TestPRPreflight (Git Bash
|
||||
# process creation, now parallel subtests), 27-58s to compile and link
|
||||
# ./cmd/bd's test binary, and about 25s for the rest. A second Windows leg
|
||||
# would pay the setup again to save about 20s, so it stays one leg.
|
||||
# would pay the setup again to save about 20s, so it stays one leg. An
|
||||
# 8 vCPU Windows leg was tried (#7381): cache restores still hit, but it
|
||||
# queued 63s for the larger runner and finished at 144s vs 129s here.
|
||||
pr-preflight-platforms:
|
||||
name: PR preflight process (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
|
|
|
|||
|
|
@ -883,9 +883,11 @@ Required` requires them to have run remotely and passed.
|
|||
`1.3.0 (dev)`, no commit); no consumer reads it. Verified 2026-10-02:
|
||||
both package gates pass with the Bazel-built bd (MCP: 228 passed, 5
|
||||
skipped; npm: all tests and the pack dry run), the same as with a
|
||||
`go build` bd. Both run on `blacksmith-4vcpu-ubuntu-2404` when
|
||||
`rbe.outputs.enabled == 'true'` (4 vCPU: `pytest -n 8` is pinned to
|
||||
timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own
|
||||
`go build` bd. In mode remote, package-npm runs on
|
||||
`blacksmith-4vcpu-ubuntu-2404` and package-mcp on
|
||||
`blacksmith-8vcpu-ubuntu-2404` with `pytest -n 16`
|
||||
(`BEADS_MCP_PYTEST_WORKERS`; the script's default is `-n 8`);
|
||||
`ubuntu-latest` (and `-n 8`) otherwise. `bazel-test`'s own
|
||||
`bazel-ci-build-artifacts` upload is no longer consumed by anything; it
|
||||
is kept for the F3.5.3 SHA256SUMS comparison and for debugging.
|
||||
- The Dolt-backed domain, uow, tracker, doctor/fix and protocol suites
|
||||
|
|
@ -1204,7 +1206,9 @@ their `blacksmith-*vcpu-windows-2025` label literally; the two mixed-OS
|
|||
matrix jobs, `pr-preflight-platforms` and `check-doc-freshness-platforms`,
|
||||
run a macOS leg on `blacksmith-6vcpu-macos-26` (Apple Silicon) and a Windows
|
||||
leg on `blacksmith-4vcpu-windows-2025`, each leg naming its label in the
|
||||
matrix (`runs-on: ${{ matrix.runner }}`). There is no GitHub-hosted fallback:
|
||||
matrix (`runs-on: ${{ matrix.runner }}`). An 8 vCPU Windows preflight leg
|
||||
was tried (#7381): it restored the 4 vCPU saver's cache, but queued 63s for
|
||||
the larger runner and finished slower (144s vs 129s), so it stays on 4 vCPU. There is no GitHub-hosted fallback:
|
||||
Blacksmith serves this org's fork PRs (gascity's fork PRs run their CI on
|
||||
`blacksmith-*` labels). Fork runs get no secrets and a read-only token
|
||||
(`TestBlacksmithJobsReadNoSecrets` keeps every Blacksmith job free of secret
|
||||
|
|
|
|||
|
|
@ -55,8 +55,16 @@ mcp_pytest() {
|
|||
# Explicit worker count, not "auto": CI runners' reported CPU count
|
||||
# doesn't always match the cgroup quota, and -n 8 is pinned to measured
|
||||
# timing (tools/f3, 2026-10) on both 2 vCPU (2.7x) and 4 vCPU (~3x)
|
||||
# runners. pytest-xdist is a pinned dev dependency (pyproject.toml).
|
||||
uv run pytest -n 8 --durations=50
|
||||
# runners. bazel.yml's package-mcp sets BEADS_MCP_PYTEST_WORKERS=16 on
|
||||
# its 8 vCPU Blacksmith runner (the suite is dominated by per-test
|
||||
# `bd init` fixture setup, so two workers per CPU still pay off).
|
||||
# pytest-xdist is a pinned dev dependency (pyproject.toml).
|
||||
local workers="${BEADS_MCP_PYTEST_WORKERS:-8}"
|
||||
if [[ ! "$workers" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "BEADS_MCP_PYTEST_WORKERS must be a positive integer, got '$workers'" >&2
|
||||
return 2
|
||||
fi
|
||||
uv run pytest -n "$workers" --durations=50
|
||||
}
|
||||
|
||||
mcp_build() {
|
||||
|
|
|
|||
|
|
@ -55,6 +55,39 @@ const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (git
|
|||
const blacksmithWindows2vcpuRunsOn = "${{ 'blacksmith-2vcpu-windows-2025' }}"
|
||||
const blacksmithWindows4vcpuRunsOn = "${{ 'blacksmith-4vcpu-windows-2025' }}"
|
||||
|
||||
// bazel.yml's lanes pick their runner from the rbe job's decision alone
|
||||
// (TestBazelRBEJobDecidesOnce): Blacksmith in mode remote, GitHub-hosted
|
||||
// otherwise. bazelRemoteRunsOn builds that ternary for one Blacksmith size so
|
||||
// every lane's pinned runs-on is the same literal apart from the label.
|
||||
func bazelRemoteRunsOn(label string) string {
|
||||
return "${{ needs.rbe.outputs.mode == 'remote' && '" + label + "' || 'ubuntu-latest' }}"
|
||||
}
|
||||
|
||||
// The default lane size: the runner is only an RBE client (every action
|
||||
// executes on rbe-west), so 2 vCPU is enough.
|
||||
var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404")
|
||||
|
||||
// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): bazel-test's
|
||||
// `bazel test //... --config=ci` spent ~63 of its 81 s in client-side loading
|
||||
// and analysis (1798 packages, 66k configured targets; critical path 7.65 s,
|
||||
// every action a remote cache hit), which Skyframe parallelizes across the
|
||||
// client's cores. 4 vCPU; revert to bazelLaneRunsOn if the PR run does not
|
||||
// save at least 15 s on that step.
|
||||
var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404")
|
||||
|
||||
// The package gates: package-npm at 4 vCPU (F3); package-mcp at 8 vCPU with
|
||||
// pytest -n 16 (bazelMCPPytestWorkers), since its pytest run is dominated by
|
||||
// per-test `bd init` fixture setup, which is CPU-bound and parallel.
|
||||
var bazelPackageRunsOn = map[string]string{
|
||||
"package-mcp": bazelRemoteRunsOn("blacksmith-8vcpu-ubuntu-2404"),
|
||||
"package-npm": bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404"),
|
||||
}
|
||||
|
||||
// package-mcp's BEADS_MCP_PYTEST_WORKERS: 16 xdist workers on the 8 vCPU
|
||||
// Blacksmith runner (mode remote), package-mcp.sh's default 8 on the
|
||||
// GitHub-hosted fallback (forks, Dependabot, rbe off/skip/cache).
|
||||
const bazelMCPPytestWorkers = "${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }}"
|
||||
|
||||
// Blacksmith macOS (Apple Silicon M4, ARM64), pinned to macos-26 (the image
|
||||
// GitHub's macos-latest resolves to) so the PR legs and main.yml's
|
||||
// blacksmith-macos-go-build-cache saver share one image regardless of when
|
||||
|
|
@ -69,6 +102,11 @@ const platformsMatrixRunsOn = "${{ matrix.runner }}"
|
|||
|
||||
// platformsMatrixRunners: each mixed-OS matrix leg's `os` (which keeps the
|
||||
// check names stable) -> its Blacksmith label.
|
||||
//
|
||||
// Runner-size A/B (#7381, 2026-10-08): pr-preflight-platforms' Windows leg on
|
||||
// blacksmith-8vcpu-windows-2025 restored the 4 vCPU saver's cache fine, but
|
||||
// waited 63s for an 8 vCPU Windows runner and finished at 144s against 129s
|
||||
// on 4 vCPU, so it stays on blacksmithWindowsLabel.
|
||||
var platformsMatrixRunners = map[string]string{
|
||||
"macos-latest": blacksmithMacOSLabel,
|
||||
"windows-latest": blacksmithWindowsLabel,
|
||||
|
|
@ -518,3 +556,46 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlacksmithBazelRunnerSizes evaluates every bazel.yml lane's real
|
||||
// runs-on (and package-mcp's xdist worker count) under each rbe mode: mode
|
||||
// remote gets the lane's Blacksmith size, every other mode the GitHub-hosted
|
||||
// runner, and package-mcp runs 16 xdist workers exactly when it is on its
|
||||
// 8 vCPU runner (8 everywhere else, package-mcp.sh's default).
|
||||
func TestBlacksmithBazelRunnerSizes(t *testing.T) {
|
||||
workflow := readCIWorkflow(t, bazelWorkflowName)
|
||||
wantRemote := map[string]string{
|
||||
bazelJobName: "blacksmith-4vcpu-ubuntu-2404",
|
||||
bazelPackageMCPJobName: "blacksmith-8vcpu-ubuntu-2404",
|
||||
bazelPackageNPMJobName: "blacksmith-4vcpu-ubuntu-2404",
|
||||
}
|
||||
mcpGate := workflow.job(t, bazelPackageMCPJobName).step(t, "Run MCP package gate")
|
||||
if got := mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"]; got != bazelMCPPytestWorkers {
|
||||
t.Errorf("package-mcp Run MCP package gate BEADS_MCP_PYTEST_WORKERS = %q, want %q", got, bazelMCPPytestWorkers)
|
||||
}
|
||||
for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} {
|
||||
ctx := map[string]string{"needs.rbe.outputs.mode": mode}
|
||||
for name, job := range workflow.Jobs {
|
||||
if name == bazelRBEJobName {
|
||||
continue
|
||||
}
|
||||
want := "ubuntu-latest"
|
||||
if mode == "remote" {
|
||||
want = "blacksmith-2vcpu-ubuntu-2404"
|
||||
if label, ok := wantRemote[name]; ok {
|
||||
want = label
|
||||
}
|
||||
}
|
||||
if got := mustEvalGHRunsOn(t, job.RunsOn, ctx); got != want {
|
||||
t.Errorf("%s runs-on under mode %s = %q, want %q", name, mode, got, want)
|
||||
}
|
||||
}
|
||||
wantWorkers := "8"
|
||||
if mustEvalGHRunsOn(t, workflow.job(t, bazelPackageMCPJobName).RunsOn, ctx) == "blacksmith-8vcpu-ubuntu-2404" {
|
||||
wantWorkers = "16"
|
||||
}
|
||||
if got := mustEvalGHRunsOn(t, mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"], ctx); got != wantWorkers {
|
||||
t.Errorf("package-mcp BEADS_MCP_PYTEST_WORKERS under mode %s = %q, want %q", mode, got, wantWorkers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1940,11 +1940,11 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp
|
|||
// caller left package-gates at its default "off".
|
||||
const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}"
|
||||
|
||||
// F3: the package gates' runner. 4 vCPU, not bazel.yml's usual 2: pytest-xdist
|
||||
// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). Mode
|
||||
// remote only: the gates take no rbe-fork certificate, so fork modes
|
||||
// F3: the package gates' runners (bazelPackageRunsOn, in
|
||||
// ci_blacksmith_runner_test.go with the other runner sizes) are larger than
|
||||
// bazel.yml's usual 2 vCPU because they run pytest/npm on the runner itself.
|
||||
// Mode remote only: the gates take no rbe-fork certificate, so fork modes
|
||||
// (enabled too) build bd with go build on a GitHub-hosted runner, as before.
|
||||
const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}"
|
||||
|
||||
// rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): mode
|
||||
// remote only (B1, security review of bdef342d5 - was remote or fork-rw).
|
||||
|
|
@ -2110,7 +2110,7 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
|
|||
// runner. Only mode remote gets the secrets; fork modes get
|
||||
// BAZEL_FORK_REMOTE (setup-bazel mints the certificate), cache runs
|
||||
// BAZEL_FORK_CACHE.
|
||||
const wantRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}"
|
||||
wantRunsOn := bazelLaneRunsOn
|
||||
// Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS
|
||||
// unset); remote-only lanes (27 race processes and more, an hour or more
|
||||
// on a GitHub-hosted runner, for tests the Go jobs already run there)
|
||||
|
|
@ -2148,12 +2148,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
|
|||
if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) {
|
||||
t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName)
|
||||
}
|
||||
// F3: the package gates use a 4 vCPU runner (pytest-xdist -n 8) and
|
||||
// F3: the package gates use larger runners (bazelPackageRunsOn) and
|
||||
// their own if (the caller's package-gates input, not the rbe job's
|
||||
// mode - they never skip for execution-mode reasons).
|
||||
wantJobRunsOn, wantJobIf, wantJobSetupEnv := wantRunsOn, wantIf, wantSetupEnv
|
||||
if name == bazelJobName {
|
||||
wantJobRunsOn = bazelTestLaneRunsOn
|
||||
}
|
||||
if bazelPackageJobs[name] {
|
||||
wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn, bazelPackageGatesIf, wantPackageSetupEnv
|
||||
wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn[name], bazelPackageGatesIf, wantPackageSetupEnv
|
||||
} else if bazelRemoteOnlyJobs[name] {
|
||||
wantJobIf = wantRemoteOnlyIf
|
||||
} else if name == bazelIntegJobName || name == bazelCmdDoltJobName {
|
||||
|
|
@ -2181,21 +2184,26 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// F3: bazelPackageRunsOn's comment and the package-mcp job's 4 vCPU runner
|
||||
// F3: bazelPackageRunsOn's comment and the package-mcp job's runner size
|
||||
// are both premised on mcp_pytest() in package-mcp.sh passing an explicit
|
||||
// "-n 8" (not "-n auto", which would reintroduce the CPU-count/cgroup-quota
|
||||
// mismatch -n 8 was pinned to avoid), and on pytest-xdist itself being a
|
||||
// locked dev dependency so that worker count is reproducible across CI runs.
|
||||
// Neither half of that premise is checked by any other policy test, so pin
|
||||
// both here directly against the script and the MCP package's manifest/lock.
|
||||
// worker count (not "-n auto", which would reintroduce the CPU-count/
|
||||
// cgroup-quota mismatch the explicit count was pinned to avoid): the
|
||||
// default 8, or BEADS_MCP_PYTEST_WORKERS, which bazel.yml's package-mcp sets
|
||||
// to 16 exactly when it runs on its 8 vCPU Blacksmith runner
|
||||
// (TestBlacksmithBazelRunnerSizes). It is also premised on pytest-xdist
|
||||
// itself being a locked dev dependency so that worker count is reproducible
|
||||
// across CI runs. Pin both here directly against the script and the MCP
|
||||
// package's manifest/lock.
|
||||
func TestMCPPytestWorkerCountPinned(t *testing.T) {
|
||||
root := sourceRepoRoot(t)
|
||||
script := readPolicyFile(t, root, "scripts/ci/package-mcp.sh")
|
||||
if !strings.Contains(script, "pytest -n 8") {
|
||||
t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() does not pass an explicit -n 8")
|
||||
for _, want := range []string{`local workers="${BEADS_MCP_PYTEST_WORKERS:-8}"`, `uv run pytest -n "$workers"`} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() is missing %q (an explicit worker count, default 8)", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(script, "-n auto") {
|
||||
t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want the pinned -n 8")
|
||||
t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want an explicit worker count")
|
||||
}
|
||||
pyproject := readPolicyFile(t, root, "integrations/beads-mcp/pyproject.toml")
|
||||
if !regexp.MustCompile(`(?m)^\s*"pytest-xdist[><=]`).MatchString(pyproject) {
|
||||
|
|
|
|||
|
|
@ -255,9 +255,9 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) {
|
|||
if !ok {
|
||||
t.Fatalf("%s has no %s job", bazelWorkflowName, lane.job)
|
||||
}
|
||||
if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn {
|
||||
if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn[lane.job] {
|
||||
t.Errorf("%s: needs %v, if %q, runs-on %q; want needs [%s], if %q, runs-on %q",
|
||||
lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn)
|
||||
lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn[lane.job])
|
||||
}
|
||||
|
||||
detectCond := "steps.detect.outputs." + lane.detectOutput + " == 'true'"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue