perf(startup): initialize packages on four Ps, not every CPU (#7701)

* perf(startup): initialize packages on four Ps, not every CPU

bd links several thousand packages whose init functions allocate about
20 MB before main runs, which triggers four or five garbage collections.
Each one starts mark workers and stops the world across every P, and on a
large host that costs more than the collection itself: on a 192-CPU machine
`bd version` spent about 20 of its 110 ms there, paid again by every bd
process a script or an orchestrator starts.

internal/initprocs lowers GOMAXPROCS to 4 for the duration of package
initialization. It imports only runtime and syscall, so it initializes
ahead of the heavy packages, and main hands GOMAXPROCS back first thing:
to the GOMAXPROCS environment value when one is set, else to the runtime
default, which keeps tracking the CPU limit.

cmd/bd's TestMain does the same hand-back, since tests never run main, and
resets the -test.parallel default testing derived from the capped count.

Measured on that host (40 runs each, `bd version`): p50 103 -> 86 ms.

Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of gascity-agent
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* test(initprocs): tell an explicit GOMAXPROCS hand-back from the runtime default

Review follow-up. TestDoneHonorsAnExplicitGOMAXPROCS swaps the two runtime
calls for recorders and pins that Done restores the exact value when
GOMAXPROCS is set and calls SetDefaultGOMAXPROCS when it is not; the earlier
test only saw the resulting count, which is the same either way on a host
whose default equals the saved value.

Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of gascity-agent
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Julian Knutsen 2026-10-19 17:15:54 +00:00 • committed by GitHub
commit d28f56cd46
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 205 additions and 0 deletions

View file

@ -364,6 +364,7 @@ filegroup(
"//internal/httpclient/encode:repo_doc_files",
"//internal/httpclient/wire:repo_doc_files",
"//internal/idgen:repo_doc_files",
"//internal/initprocs:repo_doc_files",
"//internal/jira:repo_doc_files",
"//internal/linear:repo_doc_files",
"//internal/lockfile:repo_doc_files",
@ -600,6 +601,7 @@ filegroup(
"//internal/httpclient/encode:repo_go_srcs",
"//internal/httpclient/wire:repo_go_srcs",
"//internal/idgen:repo_go_srcs",
"//internal/initprocs:repo_go_srcs",
"//internal/jira:repo_go_srcs",
"//internal/linear:repo_go_srcs",
"//internal/lockfile:repo_go_srcs",
@ -821,6 +823,7 @@ filegroup(
"//internal/httpclient/encode:repo_go_test_srcs",
"//internal/httpclient/wire:repo_go_test_srcs",
"//internal/idgen:repo_go_test_srcs",
"//internal/initprocs:repo_go_test_srcs",
"//internal/jira:repo_go_test_srcs",
"//internal/linear:repo_go_test_srcs",
"//internal/lockfile:repo_go_test_srcs",
@ -1051,6 +1054,7 @@ filegroup(
"//internal/httpclient/encode:repo_other_files",
"//internal/httpclient/wire:repo_other_files",
"//internal/idgen:repo_other_files",
"//internal/initprocs:repo_other_files",
"//internal/jira:repo_other_files",
"//internal/linear:repo_other_files",
"//internal/lockfile:repo_other_files",

View file

@ -499,6 +499,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- `bd` initializes its packages with `GOMAXPROCS` capped at 4 and hands it
back before `main` runs anything, so the garbage collections package
initialization triggers do not stop the world across every CPU of a large
host. On a 192-CPU machine this takes about 17 ms off every bd process. An
explicit `GOMAXPROCS` is restored as set.
- `bd serve` normalizes every RFC 3339 time bound on the issue listing and
count (`created_*`, `updated_*`, `closed_*`, and the new `defer_*` and
`due_*`) to UTC before it reaches storage. A bound carrying an offset

View file

@ -421,6 +421,7 @@ go_library(
"//internal/httpapi",
"//internal/httpclient",
"//internal/httpclient/wire",
"//internal/initprocs",
"//internal/jira",
"//internal/linear",
"//internal/lockfile",
@ -1379,6 +1380,7 @@ go_test(
"//internal/httpclient",
"//internal/httpclient/encode",
"//internal/httpclient/wire",
"//internal/initprocs",
"//internal/linear",
"//internal/metrics",
"//internal/migration",

View file

@ -31,6 +31,7 @@ import (
"github.com/steveyegge/beads/internal/doltserver"
"github.com/steveyegge/beads/internal/hooks"
"github.com/steveyegge/beads/internal/httpclient"
"github.com/steveyegge/beads/internal/initprocs"
"github.com/steveyegge/beads/internal/metrics"
"github.com/steveyegge/beads/internal/migration"
"github.com/steveyegge/beads/internal/molecules"
@ -2878,6 +2879,7 @@ func validateWorkspaceIdentity(ctx context.Context, s storage.DoltStorage, beads
}
func main() {
initprocs.Done()
// BD_NAME overrides the binary name in help text (e.g. BD_NAME=ops makes
// "ops --help" show "ops" instead of "bd"). Useful for multi-instance
// setups where wrapper scripts set BEADS_DIR for routing.

View file

@ -2,11 +2,14 @@ package main
import (
"errors"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
"testing"
@ -15,6 +18,7 @@ import (
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/doltserver"
"github.com/steveyegge/beads/internal/git"
"github.com/steveyegge/beads/internal/initprocs"
"github.com/steveyegge/beads/internal/metrics"
"github.com/steveyegge/beads/internal/migration"
"github.com/steveyegge/beads/internal/testutil"
@ -78,6 +82,13 @@ const suiteRootPrefix = "beads-bd-tests-"
// Guardrail: ensure the cmd/bd test suite does not touch the real repo .beads state.
// Disable with BEADS_TEST_GUARD_DISABLE=1 (useful when running tests while actively using beads).
func TestMain(m *testing.M) {
// main() is not run here, so hand back the Ps initprocs held during init,
// and the -test.parallel default testing derived from the capped count.
// An explicit -test.parallel is parsed later, by m.Run, and still wins.
initprocs.Done()
if f := flag.Lookup("test.parallel"); f != nil {
_ = f.Value.Set(strconv.Itoa(runtime.GOMAXPROCS(0)))
}
if code, ok := credentialcmd.Dispatch(); ok {
os.Exit(code)
}

View file

@ -0,0 +1,75 @@
load("@rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "initprocs",
srcs = ["initprocs.go"],
importpath = "github.com/steveyegge/beads/internal/initprocs",
visibility = ["//:__subpackages__"],
)
go_test(
name = "initprocs_test",
srcs = ["initprocs_test.go"],
embed = [":initprocs"],
)
# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---
filegroup(
name = "repo_doc_files",
srcs = glob(
[
"**/*.md",
"**/*.mdx",
],
allow_empty = True,
exclude = ["**/node_modules/**"],
),
visibility = ["//:__subpackages__"],
)
filegroup(
name = "repo_go_srcs",
srcs = glob(
["**/*.go"],
allow_empty = True,
exclude = [
"**/*_test.go",
"**/node_modules/**",
],
),
visibility = ["//:__subpackages__"],
)
filegroup(
name = "repo_go_test_srcs",
srcs = glob(
["**/*_test.go"],
allow_empty = True,
exclude = ["**/node_modules/**"],
),
visibility = ["//:__subpackages__"],
)
filegroup(
name = "repo_other_files",
srcs = glob(
["**"],
exclude = [
"**/*.db",
"**/*.exe",
"**/*.go",
"**/*.md",
"**/*.mdx",
"**/*.out",
"**/*.prof",
"**/*.pyc",
"**/*.test",
"**/__pycache__/**",
"**/node_modules/**",
],
),
visibility = ["//:__subpackages__"],
)
# --- end repo_files ---

View file

@ -0,0 +1,51 @@
// Package initprocs runs package initialization on a few Ps rather than every
// CPU the host has.
//
// bd links several thousand packages whose init functions allocate about
// 20 MB before main runs, enough for a handful of garbage collections. Each
// collection starts a mark worker and stops the world across every P, which
// on a large host (192 CPUs on the machine this was measured on) costs more
// than the collection itself: about 20 ms of a 110 ms `bd version`.
//
// The package imports only runtime and syscall so that it initializes ahead of
// the heavy packages, and main calls Done before doing anything else.
package initprocs
import (
"runtime"
"syscall"
)
const initProcs = 4
var restore int
// setProcs and setDefault are runtime.GOMAXPROCS and
// runtime.SetDefaultGOMAXPROCS, swappable so a test can tell which one Done
// used.
var (
setProcs = runtime.GOMAXPROCS
setDefault = runtime.SetDefaultGOMAXPROCS
)
func init() {
if runtime.GOMAXPROCS(0) > initProcs {
restore = runtime.GOMAXPROCS(initProcs)
}
}
// Done returns GOMAXPROCS to what the process would have had without this
// package: the value GOMAXPROCS asked for when it is set, else the runtime's
// own default, which keeps tracking the CPU limit.
func Done() {
if restore == 0 {
return
}
n := restore
restore = 0
if _, set := syscall.Getenv("GOMAXPROCS"); set {
setProcs(n)
return
}
setDefault()
}

View file

@ -0,0 +1,54 @@
package initprocs
import (
"os"
"runtime"
"testing"
)
func TestDoneRestoresGOMAXPROCS(t *testing.T) {
want := restore
if want == 0 {
t.Skipf("GOMAXPROCS was already at most %d; init left it alone", initProcs)
}
if got := runtime.GOMAXPROCS(0); got != initProcs {
t.Fatalf("GOMAXPROCS before Done = %d, want the init cap %d", got, initProcs)
}
Done()
if got := runtime.GOMAXPROCS(0); got != want {
t.Fatalf("GOMAXPROCS after Done = %d, want %d", got, want)
}
Done()
if got := runtime.GOMAXPROCS(0); got != want {
t.Fatalf("a second Done changed GOMAXPROCS to %d", got)
}
}
// TestDoneHonorsAnExplicitGOMAXPROCS tells the two hand-backs apart: with
// GOMAXPROCS set, Done restores that exact value; without it, Done hands
// GOMAXPROCS back to the runtime's default, which keeps tracking the CPU limit.
func TestDoneHonorsAnExplicitGOMAXPROCS(t *testing.T) {
saved, savedProcs, savedDefault := restore, setProcs, setDefault
t.Cleanup(func() { restore, setProcs, setDefault = saved, savedProcs, savedDefault })
var procs []int
defaults := 0
setProcs = func(n int) int { procs = append(procs, n); return n }
setDefault = func() { defaults++ }
t.Setenv("GOMAXPROCS", "7")
restore = 7
Done()
if len(procs) != 1 || procs[0] != 7 || defaults != 0 {
t.Fatalf("GOMAXPROCS=7: GOMAXPROCS calls %v, default calls %d; want GOMAXPROCS(7) alone", procs, defaults)
}
procs = nil
if err := os.Unsetenv("GOMAXPROCS"); err != nil {
t.Fatal(err)
}
restore = 192
Done()
if len(procs) != 0 || defaults != 1 {
t.Fatalf("GOMAXPROCS unset: GOMAXPROCS calls %v, default calls %d; want SetDefaultGOMAXPROCS alone", procs, defaults)
}
}

View file

@ -222,6 +222,7 @@ release_cross_build(
"//internal/httpclient/wire:wire",
"//internal/httpclient:httpclient",
"//internal/idgen:idgen",
"//internal/initprocs:initprocs",
"//internal/jira:jira",
"//internal/linear:linear",
"//internal/lockfile:lockfile",