diff --git a/BUILD.bazel b/BUILD.bazel index b57ce7316..5a64b2365 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -364,6 +364,7 @@ filegroup( "//internal/httpclient/encode:repo_doc_files", "//internal/httpclient/wire:repo_doc_files", "//internal/idgen:repo_doc_files", + "//internal/initprocs:repo_doc_files", "//internal/jira:repo_doc_files", "//internal/linear:repo_doc_files", "//internal/lockfile:repo_doc_files", @@ -600,6 +601,7 @@ filegroup( "//internal/httpclient/encode:repo_go_srcs", "//internal/httpclient/wire:repo_go_srcs", "//internal/idgen:repo_go_srcs", + "//internal/initprocs:repo_go_srcs", "//internal/jira:repo_go_srcs", "//internal/linear:repo_go_srcs", "//internal/lockfile:repo_go_srcs", @@ -821,6 +823,7 @@ filegroup( "//internal/httpclient/encode:repo_go_test_srcs", "//internal/httpclient/wire:repo_go_test_srcs", "//internal/idgen:repo_go_test_srcs", + "//internal/initprocs:repo_go_test_srcs", "//internal/jira:repo_go_test_srcs", "//internal/linear:repo_go_test_srcs", "//internal/lockfile:repo_go_test_srcs", @@ -1051,6 +1054,7 @@ filegroup( "//internal/httpclient/encode:repo_other_files", "//internal/httpclient/wire:repo_other_files", "//internal/idgen:repo_other_files", + "//internal/initprocs:repo_other_files", "//internal/jira:repo_other_files", "//internal/linear:repo_other_files", "//internal/lockfile:repo_other_files", diff --git a/CHANGELOG.md b/CHANGELOG.md index f49bbd6c3..88539be6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -499,6 +499,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `bd` initializes its packages with `GOMAXPROCS` capped at 4 and hands it + back before `main` runs anything, so the garbage collections package + initialization triggers do not stop the world across every CPU of a large + host. On a 192-CPU machine this takes about 17 ms off every bd process. An + explicit `GOMAXPROCS` is restored as set. - `bd serve` normalizes every RFC 3339 time bound on the issue listing and count (`created_*`, `updated_*`, `closed_*`, and the new `defer_*` and `due_*`) to UTC before it reaches storage. A bound carrying an offset diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index e3ad53c74..bc8e9b54e 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -421,6 +421,7 @@ go_library( "//internal/httpapi", "//internal/httpclient", "//internal/httpclient/wire", + "//internal/initprocs", "//internal/jira", "//internal/linear", "//internal/lockfile", @@ -1379,6 +1380,7 @@ go_test( "//internal/httpclient", "//internal/httpclient/encode", "//internal/httpclient/wire", + "//internal/initprocs", "//internal/linear", "//internal/metrics", "//internal/migration", diff --git a/cmd/bd/main.go b/cmd/bd/main.go index c083fed90..6130b235b 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -31,6 +31,7 @@ import ( "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/hooks" "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/initprocs" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/molecules" @@ -2878,6 +2879,7 @@ func validateWorkspaceIdentity(ctx context.Context, s storage.DoltStorage, beads } func main() { + initprocs.Done() // BD_NAME overrides the binary name in help text (e.g. BD_NAME=ops makes // "ops --help" show "ops" instead of "bd"). Useful for multi-instance // setups where wrapper scripts set BEADS_DIR for routing. diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index e5876e5b5..e3c38c88a 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -2,11 +2,14 @@ package main import ( "errors" + "flag" "fmt" "os" "os/exec" "path/filepath" + "runtime" "slices" + "strconv" "strings" "testing" @@ -15,6 +18,7 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/initprocs" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/testutil" @@ -78,6 +82,13 @@ const suiteRootPrefix = "beads-bd-tests-" // Guardrail: ensure the cmd/bd test suite does not touch the real repo .beads state. // Disable with BEADS_TEST_GUARD_DISABLE=1 (useful when running tests while actively using beads). func TestMain(m *testing.M) { + // main() is not run here, so hand back the Ps initprocs held during init, + // and the -test.parallel default testing derived from the capped count. + // An explicit -test.parallel is parsed later, by m.Run, and still wins. + initprocs.Done() + if f := flag.Lookup("test.parallel"); f != nil { + _ = f.Value.Set(strconv.Itoa(runtime.GOMAXPROCS(0))) + } if code, ok := credentialcmd.Dispatch(); ok { os.Exit(code) } diff --git a/internal/initprocs/BUILD.bazel b/internal/initprocs/BUILD.bazel new file mode 100644 index 000000000..debe595db --- /dev/null +++ b/internal/initprocs/BUILD.bazel @@ -0,0 +1,75 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "initprocs", + srcs = ["initprocs.go"], + importpath = "github.com/steveyegge/beads/internal/initprocs", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "initprocs_test", + srcs = ["initprocs_test.go"], + embed = [":initprocs"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/internal/initprocs/initprocs.go b/internal/initprocs/initprocs.go new file mode 100644 index 000000000..4fcce751b --- /dev/null +++ b/internal/initprocs/initprocs.go @@ -0,0 +1,51 @@ +// Package initprocs runs package initialization on a few Ps rather than every +// CPU the host has. +// +// bd links several thousand packages whose init functions allocate about +// 20 MB before main runs, enough for a handful of garbage collections. Each +// collection starts a mark worker and stops the world across every P, which +// on a large host (192 CPUs on the machine this was measured on) costs more +// than the collection itself: about 20 ms of a 110 ms `bd version`. +// +// The package imports only runtime and syscall so that it initializes ahead of +// the heavy packages, and main calls Done before doing anything else. +package initprocs + +import ( + "runtime" + "syscall" +) + +const initProcs = 4 + +var restore int + +// setProcs and setDefault are runtime.GOMAXPROCS and +// runtime.SetDefaultGOMAXPROCS, swappable so a test can tell which one Done +// used. +var ( + setProcs = runtime.GOMAXPROCS + setDefault = runtime.SetDefaultGOMAXPROCS +) + +func init() { + if runtime.GOMAXPROCS(0) > initProcs { + restore = runtime.GOMAXPROCS(initProcs) + } +} + +// Done returns GOMAXPROCS to what the process would have had without this +// package: the value GOMAXPROCS asked for when it is set, else the runtime's +// own default, which keeps tracking the CPU limit. +func Done() { + if restore == 0 { + return + } + n := restore + restore = 0 + if _, set := syscall.Getenv("GOMAXPROCS"); set { + setProcs(n) + return + } + setDefault() +} diff --git a/internal/initprocs/initprocs_test.go b/internal/initprocs/initprocs_test.go new file mode 100644 index 000000000..c56b21350 --- /dev/null +++ b/internal/initprocs/initprocs_test.go @@ -0,0 +1,54 @@ +package initprocs + +import ( + "os" + "runtime" + "testing" +) + +func TestDoneRestoresGOMAXPROCS(t *testing.T) { + want := restore + if want == 0 { + t.Skipf("GOMAXPROCS was already at most %d; init left it alone", initProcs) + } + if got := runtime.GOMAXPROCS(0); got != initProcs { + t.Fatalf("GOMAXPROCS before Done = %d, want the init cap %d", got, initProcs) + } + Done() + if got := runtime.GOMAXPROCS(0); got != want { + t.Fatalf("GOMAXPROCS after Done = %d, want %d", got, want) + } + Done() + if got := runtime.GOMAXPROCS(0); got != want { + t.Fatalf("a second Done changed GOMAXPROCS to %d", got) + } +} + +// TestDoneHonorsAnExplicitGOMAXPROCS tells the two hand-backs apart: with +// GOMAXPROCS set, Done restores that exact value; without it, Done hands +// GOMAXPROCS back to the runtime's default, which keeps tracking the CPU limit. +func TestDoneHonorsAnExplicitGOMAXPROCS(t *testing.T) { + saved, savedProcs, savedDefault := restore, setProcs, setDefault + t.Cleanup(func() { restore, setProcs, setDefault = saved, savedProcs, savedDefault }) + var procs []int + defaults := 0 + setProcs = func(n int) int { procs = append(procs, n); return n } + setDefault = func() { defaults++ } + + t.Setenv("GOMAXPROCS", "7") + restore = 7 + Done() + if len(procs) != 1 || procs[0] != 7 || defaults != 0 { + t.Fatalf("GOMAXPROCS=7: GOMAXPROCS calls %v, default calls %d; want GOMAXPROCS(7) alone", procs, defaults) + } + + procs = nil + if err := os.Unsetenv("GOMAXPROCS"); err != nil { + t.Fatal(err) + } + restore = 192 + Done() + if len(procs) != 0 || defaults != 1 { + t.Fatalf("GOMAXPROCS unset: GOMAXPROCS calls %v, default calls %d; want SetDefaultGOMAXPROCS alone", procs, defaults) + } +} diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 73e966ddc..cf089b556 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -222,6 +222,7 @@ release_cross_build( "//internal/httpclient/wire:wire", "//internal/httpclient:httpclient", "//internal/idgen:idgen", + "//internal/initprocs:initprocs", "//internal/jira:jira", "//internal/linear:linear", "//internal/lockfile:lockfile",