diff --git a/CHANGELOG.md b/CHANGELOG.md index 380d4c8a9..6f9d51dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,15 +96,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `release/**` in all five (#7148). +- **Ordinary bd commands now wait out a maintenance operation instead of + failing at once.** While `bd init`, `bd backup restore`, `bd migrate` or + `bd bootstrap` holds the workspace gate exclusively, every other bd command + on that workspace — and, in shared-server mode, on every project sharing + the server (another project's ~8s `bd init`) — failed immediately with "a + maintenance operation is running". They now wait up to 30s (one "waiting + for another bd process" notice after 2s; Ctrl-C aborts), then fail with the + same error naming the bound (and, in shared-server mode, saying the holder + may be another project). Override with `BEADS_GATE_WAIT_TIMEOUT` (`1m`, + `90`); scripts that relied on the old fail-fast behavior should set + `BEADS_GATE_WAIT_TIMEOUT=0`. Git hooks stay fail-fast so a commit or + checkout is never stalled. Waiting is writer-fair: once a maintenance + operation is waiting for the gate, newly started commands queue behind it, + so a steady stream of short commands can no longer starve `bd init`. The + queue only ever delays a command, never fails it, and a maintenance + operation that cannot get in (for example behind a `bd list --watch`) + stops queueing others after 10s. Queued maintenance operations run one + after another, and a waiting command waits for all of them. The queue + marker is an OS lock beside the gate (`*.gate.lock.intent`, covered by the + existing `*.gate.lock*` gitignore pattern), so a crashed waiter never + leaves a stale queue behind. `bd init`'s own wait (`BEADS_INIT_GATE_TIMEOUT`) + rises from 30s to 60s, since back-to-back inits under load take 10-15s + each. + - **Concurrent `bd init --shared-server` runs in different projects no longer refuse each other.** Every shared-server project gates the one shared dolt data dir, and `bd init` holds that gate exclusively for its ~8s run but waited only 5s for it, so a second init (or an init during another project's long command) failed with "bd init refuses to run over live bd - activity". `bd init` now waits up to 30s, printing one "waiting for another - bd process on the shared server" notice after 2s, then fails with the same - refusal naming the bound. Override it with `BEADS_INIT_GATE_TIMEOUT` - (`2m`, `90`). Other exclusive operations keep their 5s wait. + activity". `bd init` now waits (60s by default, see the gate-wait entry + above), printing one "waiting for another bd process on the shared server" + notice after 2s, then fails with the same refusal naming the bound. + Override it with `BEADS_INIT_GATE_TIMEOUT` (`2m`, `90`). Other exclusive + operations keep their 5s wait. - **A proxied-server command against an unreachable external Dolt upstream now fails within about a second with a clear error instead of stalling diff --git a/cmd/bd/edit.go b/cmd/bd/edit.go index 786927550..5fe2bed53 100644 --- a/cmd/bd/edit.go +++ b/cmd/bd/edit.go @@ -119,6 +119,7 @@ Examples: editorParts := strings.Fields(editor) editorArgs := append(editorParts[1:], tmpPath) editorCmd := exec.Command(editorParts[0], editorArgs...) //nolint:gosec // G204: editor from trusted $EDITOR/$VISUAL env or known defaults + editorCmd.Env = envWithoutSharedHoldMarker() editorCmd.Stdin = os.Stdin editorCmd.Stdout = os.Stdout editorCmd.Stderr = os.Stderr diff --git a/cmd/bd/edit_proxied_server.go b/cmd/bd/edit_proxied_server.go index baee72575..e0c3af82b 100644 --- a/cmd/bd/edit_proxied_server.go +++ b/cmd/bd/edit_proxied_server.go @@ -96,6 +96,7 @@ func runEditProxiedServer(cmd *cobra.Command, ctx context.Context, args []string editorParts := strings.Fields(editor) editorArgs := append(editorParts[1:], tmpPath) editorCmd := exec.Command(editorParts[0], editorArgs...) //nolint:gosec // G204: editor from trusted $EDITOR/$VISUAL env or known defaults + editorCmd.Env = envWithoutSharedHoldMarker() editorCmd.Stdin = os.Stdin editorCmd.Stdout = os.Stdout editorCmd.Stderr = os.Stderr diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 033f4311a..f59f9ab99 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -2072,11 +2072,16 @@ func hookLinkedWorktreePrimaryRoot(hookRoot string) string { // exportJSONLForCommit. // // Nothing on either the export or the import path reads BD_GIT_HOOK to decide -// whether to do its work: its only consumers are maybeAutoExport -// (export_auto.go), maybeAutoBackup (backup_auto.go), the first-run metrics -// notice (metrics.go) and terminal color (internal/ui). Setting it therefore -// suppresses only the post-run writes the hook is doing itself, never the -// explicit command. +// whether to do its work: its consumers are maybeAutoExport (export_auto.go), +// maybeAutoBackup (backup_auto.go), the first-run metrics notice +// (metrics.go), terminal color (internal/ui), and the workspace gate +// (sharedGateWait in workspace_gate.go), which makes a hook subprocess fail +// fast instead of waiting up to BEADS_GATE_WAIT_TIMEOUT behind a maintenance +// operation. Setting it therefore suppresses only the post-run writes the +// hook is doing itself and keeps the hook from stalling the user's git +// command, never the explicit command. Dropping BD_GIT_HOOK here would make +// every commit/checkout during a bd init or restore hang for the gate wait; +// TestHookSubprocessEnvKeepsGateFailFast pins it. func hookSubprocessEnv(env []string) []string { return append(filterEnv(env, "BD_GIT_HOOK"), "BD_GIT_HOOK=1") } @@ -2226,8 +2231,10 @@ func importJSONLForSync(reason string) { warnJSONLWithoutDoltRemote(reason + " JSONL import") // Shell out to `bd import` — same pattern as exportJSONLForCommit, - // including BD_GIT_HOOK=1. No import path consults that variable, so - // clearing it suppressed nothing the import needs; all it did was let the + // including BD_GIT_HOOK=1 (which also keeps the subprocess's workspace + // gate acquisition fail-fast; see hookSubprocessEnv). No import path + // consults that variable to decide whether to import, so clearing it + // suppressed nothing the import needs; all it did was let the // subprocess's PersistentPostRun auto-export fire, and that export // re-resolves its destination from beads.FindBeadsDir() — the *primary* // checkout's JSONL. Harmless while this hook read the primary's copy too, diff --git a/cmd/bd/shared_server_integration_test.go b/cmd/bd/shared_server_integration_test.go index 2ded2763e..ac032988f 100644 --- a/cmd/bd/shared_server_integration_test.go +++ b/cmd/bd/shared_server_integration_test.go @@ -134,7 +134,7 @@ func TestSharedServerConcurrent(t *testing.T) { // the one shared dolt dir, and bd init holds that root's gate // EXCLUSIVELY (acquireInitMutationGate), so concurrent inits of // different projects serialize on it. Each waits up to - // initGateWaitDefault (30s) for the others, which covers the lane's + // initGateWaitDefault (60s) for the others, which covers the lane's // small BEADS_TEST_SS_DIRS at ~8s per init. Larger manual runs queue // numDirs inits behind one gate, so raise the bound to match rather // than reintroduce client-side serialization. @@ -178,9 +178,46 @@ func TestSharedServerConcurrent(t *testing.T) { t.Logf("init %d dirs: %s", numDirs, time.Since(phase)) // ── Fan out client workloads ──────────────────────────────────────── + // Plus one late `bd init` of a fresh project that starts while the + // workloads are running. It holds the shared physical root's gate + // EXCLUSIVELY for its whole run, so this exercises both directions of + // the gate wait: every client command that lands during the init waits + // for it (BEADS_GATE_WAIT_TIMEOUT, default 30s) instead of failing, and + // the init gets the gate despite the steady stream of short client + // commands (workspacegate writer fairness: once the init is queued, new + // shared acquirers wait behind it). Its bound is pinned to 30s, half the + // 60s default, rather than initEnv's raised one: a generous bound would + // mask a fairness regression. phase = time.Now() eg, egCtx = errgroup.WithContext(ctx) - eg.SetLimit(maxProcs) + eg.SetLimit(maxProcs + 1) // +1: the late init below must not take a client's slot + eg.Go(func() error { + select { + case <-time.After(time.Second): // let the workloads get going + case <-egCtx.Done(): + return egCtx.Err() + } + dir := filepath.Join(t.TempDir(), "projlate") + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("late project mkdir: %w", err) + } + if err := gitInit(egCtx, dir); err != nil { + return fmt.Errorf("late project git init: %w", err) + } + start := time.Now() + lateEnv := append(append([]string{}, baseEnv...), initGateTimeoutEnv+"=30s") + out, err := ssExec(egCtx, bdBinary, dir, lateEnv, + "init", "--shared-server", "--external", + "--prefix", "projlate", "--quiet", "--non-interactive") + if err != nil { + return fmt.Errorf("late init during client workloads: %s: %w", out, err) + } + t.Logf("late init during client workloads: %s", time.Since(start)) + if out, err := ssExec(egCtx, bdBinary, dir, baseEnv, "list", "--json", "--flat"); err != nil { + return fmt.Errorf("list in late project: %s: %w", out, err) + } + return nil + }) for c := range numClients { c := c eg.Go(func() error { diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 8a12e3ee9..ce3081d01 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/testutil" "github.com/steveyegge/beads/internal/testutil/credentialcmd" + "github.com/steveyegge/beads/internal/workspacegate" ) // beforeTestsHook is set by CGO-tagged test files to perform setup before tests run @@ -81,6 +82,9 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { + // A bd parent (say, a git hook or `bd` driving `go test`) must not make + // this binary's bd subprocesses skip the workspace gate's writer queue. + _ = os.Unsetenv(workspacegate.InheritedHoldEnv) origWD, _ := os.Getwd() // Computed once and reused below for the pin, the ceiling-var boundary, // and the guard's own watch setup, so all three agree on exactly which diff --git a/cmd/bd/workspace_gate.go b/cmd/bd/workspace_gate.go index 60b06eadb..e4ed66b22 100644 --- a/cmd/bd/workspace_gate.go +++ b/cmd/bd/workspace_gate.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" + "testing" "time" "github.com/spf13/cobra" @@ -36,10 +38,14 @@ import ( // beside a cross-user shared root lands here and proceeds ungated even // though that user may be mid-maintenance (workspacegate documents // cross-user shared roots as unsupported). -// - ErrBusy on SHARED means an exclusive maintenance operation is live on -// this workspace: abort with an actionable error naming the holder -// (the gate's busy detail carries pid/reason/host from the advisory -// sidecar). Proceeding would race a migration/restore mid-replace. +// - SHARED contention means an exclusive maintenance operation is live +// (or queued, see workspacegate's writer fairness) on this workspace: +// wait for it up to sharedGateWait() (BEADS_GATE_WAIT_TIMEOUT, default +// 30s; git hooks stay fail-fast), honoring Ctrl-C. ErrBusy past the +// bound aborts with an actionable error naming the holder (the gate's +// busy detail carries pid/reason/host from the advisory sidecar) and +// the knob. Proceeding would race a migration/restore mid-replace, and +// so would treating an interrupted wait as fail-open. // - EXCLUSIVE failures of any kind are hard errors: maintenance refuses // rather than pretends. // @@ -57,12 +63,74 @@ import ( // path, fail-open posture, or no workspace on disk). var workspaceGateHandle *workspacegate.MultiHandle +// inheritedHoldEnvSaved remembers workspacegate.InheritedHoldEnv as it was +// before this process advertised its own shared hold, so release restores +// it (an in-process test harness, or a child of a still-holding bd parent, +// must not lose or keep a stale marker). Valid when inheritedHoldEnvSet. +var ( + inheritedHoldEnvSaved string + inheritedHoldEnvHad bool + inheritedHoldEnvSet bool +) + +// advertiseSharedHold exports this process's PID in +// workspacegate.InheritedHoldEnv while it holds its command gates SHARED, so +// bd processes it spawns (`bd orphans` -> `bd close`, git hooks fired by a +// git command bd runs) skip the writer-fairness queue: a queued exclusive +// acquirer cannot get in until this process releases, and this process may +// be waiting on the child, so queueing the child would deadlock until its +// wait bound expired. +func advertiseSharedHold() { + if !advertiseSharedHoldEnabled { + return + } + own := strconv.Itoa(os.Getpid()) + cur, had := os.LookupEnv(workspacegate.InheritedHoldEnv) + // Save the pre-advertise value unless we are already advertising (a + // re-acquire without release). Keying on the live value rather than on + // inheritedHoldEnvSet alone keeps a missed withdraw from pinning a stale + // saved value forever. + if !inheritedHoldEnvSet || !had || cur != own { + inheritedHoldEnvSaved, inheritedHoldEnvHad = cur, had + } + inheritedHoldEnvSet = true + _ = os.Setenv(workspacegate.InheritedHoldEnv, own) +} + +// envWithoutSharedHoldMarker is os.Environ() minus +// workspacegate.InheritedHoldEnv, for user-facing children that are not bd +// (the $EDITOR `bd edit` opens can outlive this process by hours, e.g. a new +// IDE window). A bd run from such a descendant then queues normally instead +// of trusting a PID that may since have been recycled. bd's own children +// keep the marker. +func envWithoutSharedHoldMarker() []string { + return filterEnv(os.Environ(), workspacegate.InheritedHoldEnv) +} + +// advertiseSharedHoldEnabled is off inside test binaries: there in-process +// command tests run beside tests that spawn bd subprocesses, and a marker +// naming the (live) test process would let those children skip the queue +// and weaken their fairness assertions. Tests of the marker itself turn it +// on. +var advertiseSharedHoldEnabled = !testing.Testing() + +// withdrawSharedHold undoes advertiseSharedHold (no-op if never advertised). +func withdrawSharedHold() { + if !inheritedHoldEnvSet { + return + } + if inheritedHoldEnvHad { + _ = os.Setenv(workspacegate.InheritedHoldEnv, inheritedHoldEnvSaved) + } else { + _ = os.Unsetenv(workspacegate.InheritedHoldEnv) + } + inheritedHoldEnvSet = false +} + // exclusiveGateWait is how long EXCLUSIVE acquisitions poll before giving // up: long enough to ride out a short-lived `bd list` finishing, short // enough that a genuinely busy workspace fails with a clear message rather -// than hanging. SHARED acquisitions stay non-blocking (a live exclusive -// holder is a migration/restore — waiting seconds will not outlast it, and -// normal commands should fail fast with the holder's name). A var, not a +// than hanging. SHARED acquisitions use sharedGateWait instead. A var, not a // const, so tests can shorten it. var exclusiveGateWait = 5 * time.Second @@ -99,21 +167,26 @@ const initGateTimeoutEnv = "BEADS_INIT_GATE_TIMEOUT" // dolt data dir, so init in project A contends with init (or any gated // command) in project B. A single init holds the gate for ~8s, so the // generic 5s budget made two concurrent `bd init --shared-server` runs in -// different projects refuse each other. 30s rides out a few back-to-back -// inits while still failing with a clear error on a genuinely stuck holder. +// different projects refuse each other. Under load a single init holds the +// gate for 10-15s and each queued init waits for every one ahead of it, so +// 60s rides out a few back-to-back inits while still failing with a clear +// error on a genuinely stuck holder. (Waiting this long is only acceptable +// because a waiting init no longer holds ordinary commands back for its +// whole budget: workspacegate caps how long it queues them, maxIntentHold.) // The other exclusive callers (backup restore, bootstrap, migrate) keep // exclusiveGateWait: they are rare, deliberate maintenance operations rather // than routine setup that tooling fans out across many projects at once. -const initGateWaitDefault = 30 * time.Second +const initGateWaitDefault = 60 * time.Second -// initGateNoticeDelay is how long init waits silently before telling the -// user it is blocked: a sub-2s wait is not worth a line of output. A var so -// tests can observe contention without sleeping. -var initGateNoticeDelay = 2 * time.Second +// gateWaitNoticeDelay is how long a gate wait (bd init, or an ordinary +// command waiting out a maintenance operation) stays silent before telling +// the user it is blocked: a sub-2s wait is not worth a line of output. A var +// so tests can observe contention without sleeping. +var gateWaitNoticeDelay = 2 * time.Second -// initGateOnWait prints init's single "still waiting" notice. A var so -// tests can observe it. -var initGateOnWait = func(holder string) { +// gateWaitOnWait prints the single "still waiting" notice. A var so tests +// can observe it. +var gateWaitOnWait = func(holder string) { if quietFlag { return } @@ -126,21 +199,101 @@ var initGateOnWait = func(holder string) { fmt.Fprintf(os.Stderr, "waiting for another bd process on %s (%q)...\n", where, holder) //nolint:gosec // G705: stderr, not a browser context; %q additionally neutralizes terminal escapes } -// initGateWait resolves init's gate budget from initGateTimeoutEnv, falling -// back to initGateWaitDefault (with a warning) on unset, malformed, or -// non-positive values. -func initGateWait() time.Duration { - raw := strings.TrimSpace(os.Getenv(initGateTimeoutEnv)) +// delayedGateNotice returns an Options.OnWait that arms gateWaitOnWait to +// fire once after gateWaitNoticeDelay, plus a stop func the caller must run +// once acquisition returns. AcquireAll calls OnWait at most once, +// synchronously, before it returns, so the timer is set (if at all) by the +// time stop runs. +func delayedGateNotice() (onWait func(string), stop func()) { + var notice *time.Timer + onWait = func(holder string) { + notice = time.AfterFunc(gateWaitNoticeDelay, func() { gateWaitOnWait(holder) }) + } + stop = func() { + if notice != nil { + notice.Stop() + } + } + return onWait, stop +} + +// parseGateTimeoutEnv reads a BEADS_*_TIMEOUT-style knob: a Go duration +// ("90s", "2m") or bare whole seconds ("90"). ok is false when unset or +// malformed (malformed values warn once). +func parseGateTimeoutEnv(name string, fallback time.Duration) (d time.Duration, ok bool) { + raw := strings.TrimSpace(os.Getenv(name)) if raw == "" { - return initGateWaitDefault + return 0, false } d, err := time.ParseDuration(raw) if err != nil { d, err = time.ParseDuration(raw + "s") } - if err != nil || d <= 0 { + if err != nil { if !quietFlag { - fmt.Fprintf(os.Stderr, "warning: %s=%q is not a positive duration; using default %s\n", initGateTimeoutEnv, raw, initGateWaitDefault) + fmt.Fprintf(os.Stderr, "warning: %s=%q is not a duration; using default %s\n", name, raw, fallback) + } + return 0, false + } + return d, true +} + +// sharedGateWaitEnv overrides sharedGateWaitDefault for ordinary (SHARED) +// commands, same syntax as BEADS_INIT_GATE_TIMEOUT; 0 restores the old +// fail-fast behavior. +const sharedGateWaitEnv = "BEADS_GATE_WAIT_TIMEOUT" + +// sharedGateWaitDefault is how long an ordinary command waits for an +// exclusive maintenance holder (bd init ~8s, backup restore, migrate, +// bootstrap) — or for a queued one, see workspacegate's writer fairness — +// before failing. In shared-server mode every project shares one physical +// root gate, so another project's `bd init` briefly excludes every bd +// command on the machine; failing those instantly made routine tooling +// flaky. Queued maintenance operations run one after another (two inits +// fanned out across projects hold the gate back to back, 10-15s each under +// load), and a waiting command waits for all of them, so the default matches +// init's own patience rather than a single init: 30s. Humans see the notice +// after 2s and can Ctrl-C; agents and CI, the callers that suffered most +// from fail-fast, just get their command run. A genuinely long restore or +// migration still fails with a clear error naming the holder and the knob. +const sharedGateWaitDefault = 30 * time.Second + +// sharedGateWait resolves an ordinary command's gate budget. +// +// Git hooks stay fail-fast (0): the hook paths (`bd hooks run`, and the +// `bd export` / `bd import` children the pre-commit and post-merge/checkout +// hooks spawn) all run with BD_GIT_HOOK=1, treat a failure as a warning, +// and must never stall the user's `git commit` / `git checkout` for 30s +// behind a maintenance operation. A fail-fast acquisition also skips the +// writer-fairness queue, so the hook behavior is exactly what it was. +func sharedGateWait() time.Duration { + if os.Getenv("BD_GIT_HOOK") == "1" { + return 0 + } + d, ok := parseGateTimeoutEnv(sharedGateWaitEnv, sharedGateWaitDefault) + if !ok { + return sharedGateWaitDefault + } + if d < 0 { + if !quietFlag { + fmt.Fprintf(os.Stderr, "warning: %s=%s is negative; using default %s\n", sharedGateWaitEnv, d, sharedGateWaitDefault) + } + return sharedGateWaitDefault + } + return d +} + +// initGateWait resolves init's gate budget from initGateTimeoutEnv, falling +// back to initGateWaitDefault (with a warning) on unset, malformed, or +// non-positive values. +func initGateWait() time.Duration { + d, ok := parseGateTimeoutEnv(initGateTimeoutEnv, initGateWaitDefault) + if !ok { + return initGateWaitDefault + } + if d <= 0 { + if !quietFlag { + fmt.Fprintf(os.Stderr, "warning: %s=%s is not a positive duration; using default %s\n", initGateTimeoutEnv, d, initGateWaitDefault) } return initGateWaitDefault } @@ -183,6 +336,7 @@ func releaseWorkspaceGates() { _ = workspaceGateHandle.Release() // Best effort: the flock dies with the process anyway workspaceGateHandle = nil } + withdrawSharedHold() } // commandNeedsExclusiveGate classifies the store-opening commands that @@ -256,24 +410,58 @@ func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beads } mode := workspacegate.Shared - opts := workspacegate.Options{} + var opts workspacegate.Options + var sharedWait time.Duration + stopNotice := func() {} if exclusive { mode = workspacegate.Exclusive opts = exclusiveGateOptions("bd backup restore") + } else { + // Ordinary commands wait (bounded, Ctrl-C aware) for a live or + // queued exclusive maintenance holder instead of failing at once. + sharedWait = sharedGateWait() + opts = workspacegate.Options{ + Wait: sharedWait, + IgnoreQueuedExclusive: workspacegate.InheritedSharedHold(), + } + if sharedWait > 0 { + opts.OnWait, stopNotice = delayedGateNotice() + } } h, err := workspacegate.AcquireAll(ctx, mode, opts, gates...) + stopNotice() if err != nil { + // Interrupted while waiting (Ctrl-C / SIGTERM cancels rootCtx): + // abort. This must precede the SHARED fail-open branch below, or a + // canceled wait would "continue ungated" over the maintenance + // operation it was waiting out. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return HandleErrorRespectJSON("interrupted while waiting for the workspace gate: %v", err) + } if errors.Is(err, workspacegate.ErrBusy) { // Contention is never fail-open, in either mode — but the two // modes are blocked by OPPOSITE kinds of holders, so the // message must not claim "a maintenance operation" for both: // a SHARED attempt is blocked only by an exclusive maintenance - // holder, while an EXCLUSIVE attempt (backup restore) is - // usually blocked by ordinary shared commands. + // holder (or one queued for exclusive access), while an + // EXCLUSIVE attempt (backup restore) is usually blocked by + // ordinary shared commands. if exclusive { return HandleErrorRespectJSON("other bd commands are using this workspace; wait for them to finish and retry: %v", err) } - return HandleErrorRespectJSON("a maintenance operation is running on this workspace; retry when it completes: %v", err) + // A shared acquisition fails only when the gate itself is held + // exclusively (queued maintenance can delay it, never fail it — + // workspacegate's final attempt ignores the queue), so "running" + // is accurate. In shared-server mode the holder may be another + // project's operation; the error detail names it when known. + where := "this workspace" + if doltserver.IsSharedServerMode() { + where = "the shared server (possibly from another project)" + } + if sharedWait > 0 { + return HandleErrorRespectJSON("a maintenance operation is running on %s; retry when it completes (waited %s; set %s to wait longer): %v", where, sharedWait, sharedGateWaitEnv, err) + } + return HandleErrorRespectJSON("a maintenance operation is running on %s; retry when it completes: %v", where, err) } if exclusive { return HandleErrorRespectJSON("workspace gate: %v", err) @@ -284,6 +472,9 @@ func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beads return nil } workspaceGateHandle = h + if !exclusive { + advertiseSharedHold() + } return nil } @@ -366,24 +557,18 @@ func acquireExclusiveWorkspaceGatesWithOptions(ctx context.Context, beadsDir str // // Contention waits up to initGateWait() (not the generic exclusiveGateWait), // honoring ctx cancellation, and prints one notice if the wait outlasts -// initGateNoticeDelay. Exclusivity is unchanged; past the bound init fails +// gateWaitNoticeDelay. Exclusivity is unchanged; past the bound init fails // with the same refusal, naming the budget and the knob that raises it. func acquireInitMutationGate(ctx context.Context, beadsDir, physicalRoot string, preflight func() error) (*workspacegate.MultiHandle, error) { wait := initGateWait() - var notice *time.Timer + onWait, stopNotice := delayedGateNotice() opts := workspacegate.Options{ Wait: wait, Reason: "bd init", - // AcquireAll calls OnWait at most once, synchronously, before it - // returns, so notice is set (if at all) by the time Stop runs. - OnWait: func(holder string) { - notice = time.AfterFunc(initGateNoticeDelay, func() { initGateOnWait(holder) }) - }, + OnWait: onWait, } h, err := acquireExclusiveWorkspaceGatesWithOptions(ctx, beadsDir, opts, physicalRoot) - if notice != nil { - notice.Stop() - } + stopNotice() if err != nil { if errors.Is(err, workspacegate.ErrBusy) { return nil, fmt.Errorf("bd init refuses to run over live bd activity on this workspace (waited %s; set %s to wait longer): %w", wait, initGateTimeoutEnv, err) diff --git a/cmd/bd/workspace_gate_test.go b/cmd/bd/workspace_gate_test.go index 1dcb46dcd..ab85176fe 100644 --- a/cmd/bd/workspace_gate_test.go +++ b/cmd/bd/workspace_gate_test.go @@ -5,7 +5,9 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -19,6 +21,10 @@ import ( // central config) cannot change which gates these tests acquire. func resetGateTestEnv(t *testing.T) { t.Helper() + // Start from no held gates: an earlier in-process command test in the + // same binary can leave workspaceGateHandle set (and its shared hold + // advertised), which these tests assert on. + releaseWorkspaceGates() for _, k := range []string{ "BEADS_DOLT_SERVER_MODE", "BEADS_DOLT_SHARED_SERVER", @@ -27,6 +33,9 @@ func resetGateTestEnv(t *testing.T) { "BEADS_PROXIED_SERVER_ROOT_PATH", "BEADS_SHARED_SERVER_DIR", initGateTimeoutEnv, + sharedGateWaitEnv, + "BD_GIT_HOOK", + workspacegate.InheritedHoldEnv, } { t.Setenv(k, "") } @@ -93,11 +102,10 @@ func TestAcquireCommandWorkspaceGatesAbsentWorkspace(t *testing.T) { } } -func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { - resetGateTestEnv(t) - t.Cleanup(releaseWorkspaceGates) - beadsDir := newGateTestWorkspace(t) - +// holdWorkspaceGateExclusive takes beadsDir's workspace gate exclusively, +// as a maintenance operation (init/restore/migrate) would. +func holdWorkspaceGateExclusive(t *testing.T, beadsDir string) *workspacegate.Handle { + t.Helper() gate, err := workspacegate.ForWorkspace(beadsDir) if err != nil { t.Fatal(err) @@ -107,32 +115,327 @@ func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { if err != nil { t.Fatal(err) } - defer func() { _ = holder.Release() }() + t.Cleanup(func() { _ = holder.Release() }) + return holder +} + +// countGateNotices replaces the gate-wait notice with a counter. +func countGateNotices(t *testing.T) *atomic.Int32 { + t.Helper() + var n atomic.Int32 + old := gateWaitOnWait + gateWaitOnWait = func(string) { n.Add(1) } + t.Cleanup(func() { gateWaitOnWait = old }) + return &n +} + +// Past the bound, an ordinary command aborts with an error naming the +// holder, the budget, and the knob — and never proceeds ungated. +func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv(sharedGateWaitEnv, "300ms") + holdWorkspaceGateExclusive(t, beadsDir) list := &cobra.Command{Use: "list"} - if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err == nil { + var err error + start := time.Now() + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), list, beadsDir) + }) + if err == nil { t.Fatal("SHARED acquisition under a foreign exclusive holder must abort, got nil error") } + if waited := time.Since(start); waited < 250*time.Millisecond || waited > 3*time.Second { + t.Errorf("gave up after %s, want about the 300ms bound", waited) + } + for _, want := range []string{"a maintenance operation is running", "waited 300ms", sharedGateWaitEnv, "test maintenance"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr %q missing %q", stderr, want) + } + } if workspaceGateHandle != nil { t.Error("failed acquisition must leave no gate handle") } } +// The core behavior: an ordinary command that lands while a maintenance +// operation holds the gate waits for it (with one notice) and then runs. +func TestAcquireCommandWorkspaceGatesWaitsForExclusiveHolder(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + notices := countGateNotices(t) + if got := sharedGateWait(); got != sharedGateWaitDefault { + t.Fatalf("sharedGateWait() = %s, want default %s", got, sharedGateWaitDefault) + } + + oldDelay := gateWaitNoticeDelay + gateWaitNoticeDelay = 200 * time.Millisecond + t.Cleanup(func() { gateWaitNoticeDelay = oldDelay }) + + holder := holdWorkspaceGateExclusive(t, beadsDir) + holdFor := time.Second + released := time.AfterFunc(holdFor, func() { _ = holder.Release() }) + t.Cleanup(func() { released.Stop() }) + + list := &cobra.Command{Use: "list"} + start := time.Now() + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("ordinary command with a maintenance holder released after %s: %v", holdFor, err) + } + if waited := time.Since(start); waited < holdFor-500*time.Millisecond { + t.Fatalf("acquired after %s while the holder held the gate for %s", waited, holdFor) + } + if workspaceGateHandle == nil { + t.Fatal("expected a held gate handle") + } + if got := notices.Load(); got != 1 { + t.Fatalf("wait notices = %d, want exactly 1", got) + } +} + +// BEADS_GATE_WAIT_TIMEOUT=0 and git-hook context both keep the old +// fail-fast behavior. +func TestAcquireCommandWorkspaceGatesFailFastModes(t *testing.T) { + for _, tc := range []struct{ name, env, val string }{ + {"timeout zero", sharedGateWaitEnv, "0"}, + {"git hook", "BD_GIT_HOOK", "1"}, + } { + t.Run(tc.name, func(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv(tc.env, tc.val) + holdWorkspaceGateExclusive(t, beadsDir) + + list := &cobra.Command{Use: "list"} + var err error + start := time.Now() + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), list, beadsDir) + }) + if err == nil { + t.Fatal("fail-fast SHARED acquisition under an exclusive holder succeeded") + } + if waited := time.Since(start); waited > time.Second { + t.Errorf("fail-fast acquisition took %s", waited) + } + if strings.Contains(stderr, "waited") { + t.Errorf("fail-fast error claims a wait: %q", stderr) + } + }) + } +} + +// Ctrl-C (rootCtx cancellation) aborts the wait with an error. It must not +// fall through to the fail-open "continue ungated" path. +func TestAcquireCommandWorkspaceGatesHonorsCancellation(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + holdWorkspaceGateExclusive(t, beadsDir) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + list := &cobra.Command{Use: "list"} + var err error + start := time.Now() + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(ctx, list, beadsDir) + }) + if err == nil { + t.Fatal("canceled wait returned nil (proceeded ungated over a maintenance holder)") + } + if waited := time.Since(start); waited > 5*time.Second { + t.Fatalf("canceled wait kept going for %s", waited) + } + if !strings.Contains(stderr, "interrupted while waiting") || strings.Contains(stderr, "continuing ungated") { + t.Errorf("stderr = %q, want an interrupted error and no fail-open warning", stderr) + } + if workspaceGateHandle != nil { + t.Error("canceled acquisition must leave no gate handle") + } +} + +// Writer fairness end to end at the chokepoint: while bd init is queued for +// the gate, a new ordinary command waits behind it rather than slipping in — +// and an ordinary command spawned by a bd process that already holds the +// gate shared (InheritedHoldEnv) does slip in, since it cannot deadlock the +// queue it would otherwise join. +func TestAcquireCommandWorkspaceGatesQueuesBehindWaitingInit(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") + // Record every notice immediately. Installed before the init goroutine + // starts: its notice timer reads these globals. + var ( + noticeMu sync.Mutex + notices []string + ) + oldDelay, oldOnWait := gateWaitNoticeDelay, gateWaitOnWait + gateWaitNoticeDelay = 0 + gateWaitOnWait = func(h string) { noticeMu.Lock(); notices = append(notices, h); noticeMu.Unlock() } + t.Cleanup(func() { gateWaitNoticeDelay, gateWaitOnWait = oldDelay, oldOnWait }) + queuedNotice := func() bool { + noticeMu.Lock() + defer noticeMu.Unlock() + for _, h := range notices { + if strings.Contains(h, "queued for exclusive access") && strings.Contains(h, "bd init") { + return true + } + } + return false + } + + // An in-flight ordinary command holds the gate shared. + inflight, err := workspacegate.ForWorkspace(beadsDir) + if err != nil { + t.Fatal(err) + } + sh, err := inflight.Acquire(context.Background(), workspacegate.Shared, workspacegate.Options{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sh.Release() }) + + // bd init queues behind it. + initDone := make(chan error, 1) + var initHandle atomic.Pointer[workspacegate.MultiHandle] + go func() { + h, err := acquireInitMutationGate(context.Background(), beadsDir, physicalRoot, nil) + initHandle.Store(h) + initDone <- err + }() + t.Cleanup(func() { + if h := initHandle.Load(); h != nil { + _ = h.Release() + } + }) + queued := inflight.ExclusiveQueued + deadline := time.Now().Add(5 * time.Second) + for !queued() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !queued() { + t.Fatal("bd init never queued for the gate") + } + + // A new ordinary command must not jump the queue: it defers for its + // budget (the notice names the queued init), then its final attempt + // gets in anyway because nothing holds the gate exclusively — queued + // maintenance delays ordinary commands, it never fails them. + t.Setenv(sharedGateWaitEnv, "300ms") + list := &cobra.Command{Use: "list"} + start := time.Now() + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("ordinary command behind a queued (not running) init failed: %v", err) + } + if waited := time.Since(start); waited < 250*time.Millisecond { + t.Errorf("ordinary command jumped ahead of a queued bd init (acquired after %s)", waited) + } + deadlineN := time.Now().Add(time.Second) + for !queuedNotice() && time.Now().Before(deadlineN) { + time.Sleep(5 * time.Millisecond) + } + if !queuedNotice() { + noticeMu.Lock() + t.Errorf("no wait notice named the queued bd init; notices: %q", notices) + noticeMu.Unlock() + } + releaseWorkspaceGates() + + // ...unless its parent bd already holds the gate shared. + t.Setenv(workspacegate.InheritedHoldEnv, strconv.Itoa(os.Getppid())) + start = time.Now() + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("child of a shared holder queued behind init: %v", err) + } + if waited := time.Since(start); waited > 200*time.Millisecond { + t.Errorf("child of a shared holder still deferred to the queue for %s", waited) + } + releaseWorkspaceGates() + + // The in-flight command finishes; init gets the gate. + _ = sh.Release() + select { + case err := <-initDone: + if err != nil { + t.Fatalf("bd init after the in-flight command drained: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("bd init did not get the gate after the in-flight command released") + } +} + +// A successful shared hold advertises this PID to child processes, and +// release restores whatever value was inherited. +func TestSharedHoldAdvertisedToChildren(t *testing.T) { + resetGateTestEnv(t) + advertiseSharedHoldEnabled = true + t.Cleanup(func() { advertiseSharedHoldEnabled = false }) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv(workspacegate.InheritedHoldEnv, "inherited") + + list := &cobra.Command{Use: "list"} + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatal(err) + } + if got, want := os.Getenv(workspacegate.InheritedHoldEnv), strconv.Itoa(os.Getpid()); got != want { + t.Fatalf("%s while holding = %q, want %q", workspacegate.InheritedHoldEnv, got, want) + } + releaseWorkspaceGates() + if got := os.Getenv(workspacegate.InheritedHoldEnv); got != "inherited" { + t.Fatalf("%s after release = %q, want the inherited value restored", workspacegate.InheritedHoldEnv, got) + } +} + +func TestSharedGateWaitEnv(t *testing.T) { + for _, tc := range []struct { + raw, hook string + want time.Duration + }{ + {"", "", sharedGateWaitDefault}, + {"45", "", 45 * time.Second}, + {"2m", "", 2 * time.Minute}, + {"0", "", 0}, + {"0s", "", 0}, + {"-5s", "", sharedGateWaitDefault}, + {"soon", "", sharedGateWaitDefault}, + {"2m", "1", 0}, + {"", "1", 0}, + } { + t.Run(tc.raw+"/hook="+tc.hook, func(t *testing.T) { + t.Setenv(sharedGateWaitEnv, tc.raw) + t.Setenv("BD_GIT_HOOK", tc.hook) + oldQuiet := quietFlag + quietFlag = true + t.Cleanup(func() { quietFlag = oldQuiet }) + if got := sharedGateWait(); got != tc.want { + t.Fatalf("sharedGateWait() = %s, want %s", got, tc.want) + } + }) + } +} + func TestAcquireInitMutationGateKeepsReplacementExclusiveDuringPreflight(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - oldOnWait, oldDelay := initGateOnWait, initGateNoticeDelay + oldOnWait, oldDelay := gateWaitOnWait, gateWaitNoticeDelay secondWaited := make(chan struct{}, 1) - initGateNoticeDelay = 0 - initGateOnWait = func(string) { + gateWaitNoticeDelay = 0 + gateWaitOnWait = func(string) { select { case secondWaited <- struct{}{}: default: } } - t.Cleanup(func() { initGateOnWait, initGateNoticeDelay = oldOnWait, oldDelay }) + t.Cleanup(func() { gateWaitOnWait, gateWaitNoticeDelay = oldOnWait, oldDelay }) firstPreflightEntered := make(chan struct{}) allowFirstPreflight := make(chan struct{}) @@ -198,16 +501,6 @@ func holdInitGatesFor(t *testing.T, beadsDir, physicalRoot string) *workspacegat return h } -// countInitGateNotices replaces the init wait notice with a counter. -func countInitGateNotices(t *testing.T) *atomic.Int32 { - t.Helper() - var n atomic.Int32 - old := initGateOnWait - initGateOnWait = func(string) { n.Add(1) } - t.Cleanup(func() { initGateOnWait = old }) - return &n -} - // A holder that outlasts the generic 5s exclusive budget but releases within // init's bound (a concurrent init takes ~8s on a shared server): init waits, // prints exactly one notice, then proceeds. @@ -215,7 +508,7 @@ func TestAcquireInitMutationGateWaitsPastGenericBudget(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - notices := countInitGateNotices(t) + notices := countGateNotices(t) if got := initGateWait(); got != initGateWaitDefault || got <= exclusiveGateWait+time.Second { t.Fatalf("initGateWait() = %s, want default %s comfortably above exclusiveGateWait %s", got, initGateWaitDefault, exclusiveGateWait) @@ -246,10 +539,10 @@ func TestAcquireInitMutationGateFailsClearlyPastBound(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - notices := countInitGateNotices(t) - oldDelay := initGateNoticeDelay - initGateNoticeDelay = 5 * time.Second // longer than the bound below - t.Cleanup(func() { initGateNoticeDelay = oldDelay }) + notices := countGateNotices(t) + oldDelay := gateWaitNoticeDelay + gateWaitNoticeDelay = 5 * time.Second // longer than the bound below + t.Cleanup(func() { gateWaitNoticeDelay = oldDelay }) t.Setenv(initGateTimeoutEnv, "300ms") holder := holdInitGatesFor(t, beadsDir, physicalRoot) @@ -288,7 +581,7 @@ func TestAcquireInitMutationGateHonorsCancellation(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - countInitGateNotices(t) + countGateNotices(t) holder := holdInitGatesFor(t, beadsDir, physicalRoot) t.Cleanup(func() { _ = holder.Release() }) @@ -433,3 +726,83 @@ func TestChokepointSharedExcludesMigrateExclusive(t *testing.T) { } release() } + +// Hooks must never stall the user's git command behind a maintenance +// operation: the bd subprocesses a hook spawns get BD_GIT_HOOK=1 from +// hookSubprocessEnv, and that alone makes their gate acquisition fail fast, +// whatever BEADS_GATE_WAIT_TIMEOUT says. +func TestHookSubprocessEnvKeepsGateFailFast(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + t.Setenv(sharedGateWaitEnv, "2m") + var hookVal string + for _, kv := range hookSubprocessEnv([]string{"PATH=/bin"}) { + if v, ok := strings.CutPrefix(kv, "BD_GIT_HOOK="); ok { + hookVal = v + } + } + if hookVal == "" { + t.Fatal("hookSubprocessEnv does not set BD_GIT_HOOK; hook subprocesses would wait on the workspace gate") + } + t.Setenv("BD_GIT_HOOK", hookVal) + if got := sharedGateWait(); got != 0 { + t.Fatalf("sharedGateWait() in a hook subprocess env = %s, want 0 (fail fast)", got) + } + + beadsDir := newGateTestWorkspace(t) + holdWorkspaceGateExclusive(t, beadsDir) + start := time.Now() + var err error + captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), &cobra.Command{Use: "export"}, beadsDir) + }) + if err == nil { + t.Fatal("hook subprocess acquired the gate under an exclusive holder") + } + if waited := time.Since(start); waited > time.Second { + t.Fatalf("hook subprocess waited %s on the gate", waited) + } +} + +// In shared-server mode the exclusive holder may be another project's +// operation, so the error must not claim it is on "this workspace". +func TestAcquireCommandWorkspaceGatesSharedServerErrorText(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", t.TempDir()) + t.Setenv(sharedGateWaitEnv, "200ms") + holdWorkspaceGateExclusive(t, beadsDir) + + var err error + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), &cobra.Command{Use: "list"}, beadsDir) + }) + if err == nil { + t.Fatal("acquired under an exclusive holder") + } + for _, want := range []string{"running on the shared server (possibly from another project)", "test maintenance"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr %q missing %q", stderr, want) + } + } +} + +// bd edit's $EDITOR (which can outlive bd by hours) must not inherit the +// shared-hold marker. +func TestEnvWithoutSharedHoldMarker(t *testing.T) { + t.Setenv(workspacegate.InheritedHoldEnv, "12345") + t.Setenv("BEADS_GATE_TEST_KEEP", "1") + env := envWithoutSharedHoldMarker() + var kept bool + for _, kv := range env { + if strings.HasPrefix(kv, workspacegate.InheritedHoldEnv+"=") { + t.Fatalf("marker leaked into editor env: %q", kv) + } + kept = kept || kv == "BEADS_GATE_TEST_KEEP=1" + } + if !kept { + t.Fatal("envWithoutSharedHoldMarker dropped unrelated variables") + } +} diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0977b7b3f..0bb16251e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -509,7 +509,8 @@ Selected commonly-used variables: | `BEADS_ACTOR` | Actor identity (preferred over `BD_ACTOR`, which is a deprecated alias) | | `BEADS_IDENTITY` | Sender identity for `bd mail` | | `BEADS_FSCK_TIMEOUT` | Runtime-only timeout for the pre-push `dolt fsck --quiet` integrity check (default `30s`) | -| `BEADS_INIT_GATE_TIMEOUT` | How long `bd init` waits for another bd process (another init, or live commands on the same workspace or shared server) to release the workspace gate before refusing (default `30s`; accepts durations like `2m` or bare seconds like `90`) | +| `BEADS_INIT_GATE_TIMEOUT` | How long `bd init` waits for another bd process (another init, or live commands on the same workspace or shared server) to release the workspace gate before refusing (default `60s`; accepts durations like `2m` or bare seconds like `90`) | +| `BEADS_GATE_WAIT_TIMEOUT` | How long an ordinary bd command waits while a maintenance operation (`bd init`, backup restore, migrate, bootstrap — on the same workspace, or on any project sharing the server in shared-server mode) holds or is queued for the workspace gate, before failing (default `30s`; accepts durations like `1m` or bare seconds like `90`; `0` fails immediately, for scripts that want the old fail-fast behavior). Queued maintenance operations run one after another and a waiting command waits for all of them; a queued one only delays a command, never fails it, and stops queueing others after 10s if it cannot get in. Git hooks always fail fast | | `BEADS_MIGRATION_WATCHDOG_INTERVAL` | Interval between soft WARN lines while a schema migration's SQL keeps running (default `5m`; accepts durations like `10m` or bare seconds like `90`). Observability only — it never aborts a migration | | `BEADS_DOLT_SERVER_MODE`, `BEADS_DOLT_SHARED_SERVER`, `BEADS_DOLT_DATA_DIR`, `BEADS_DOLT_PORT`, ... | Embedded/server Dolt overrides | | `BEADS_DOLT_BIN` | Pin the exact external `dolt` CLI binary managed proxied-server mode spawns, overriding PATH lookup (highest precedence; an explicit path that fails validation is an error, not a silent fallback to PATH). On Windows the executable extension may be omitted — `C:\tools\dolt` finds `C:\tools\dolt.exe` via PATHEXT, though a file at the exact spelled path wins if both exist | diff --git a/internal/workspacegate/gate.go b/internal/workspacegate/gate.go index 8fc62d18f..f1f8d821c 100644 --- a/internal/workspacegate/gate.go +++ b/internal/workspacegate/gate.go @@ -45,6 +45,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "sync" "syscall" @@ -64,20 +65,15 @@ const ( // Shared is held by normal commands for the lifetime of their open // store or UOW provider. Any number of shared holders coexist. // - // Starvation note: a rolling sequence of shared holders (e.g. one bd - // command after another against a busy workspace) can starve an - // Exclusive acquirer indefinitely — acquisition is LOCK_NB polling - // with no intent signal, so a new shared holder has no way to know an - // exclusive attempt is queued and back off. A future extension could - // have Exclusive publish an advisory "waiting" marker (parallel to - // the holder-info sidecar) that new Shared acquisitions check and - // voluntarily defer to, but that does not exist yet: Options.Wait on - // an Exclusive acquisition is not a fairness guarantee. + // Fairness: a waiting Shared acquisition (Options.Wait > 0) defers to a + // queued Exclusive acquirer — see "Writer fairness" on Acquire — so a + // rolling sequence of short shared holders cannot starve maintenance. + // A fail-fast Shared attempt (Wait <= 0) does not consult the queue. Shared Mode = iota // Exclusive is held by maintenance operations. It conflicts with - // every other holder, shared or exclusive. See the starvation note - // on Shared: nothing here prevents shared holders from starving an - // Exclusive acquirer. + // every other holder, shared or exclusive. A waiting Exclusive + // acquisition publishes its intent so new waiting Shared acquisitions + // queue behind it (see Acquire). Exclusive ) @@ -98,14 +94,12 @@ var ErrBusy = errors.New("workspace gate busy") // attempt with no diagnostics callback. type Options struct { // Wait bounds how long Acquire keeps retrying after the first busy - // attempt. Zero or negative means exactly one non-blocking try. - // The underlying blocking lock primitives have no deadline support, - // so waiting is implemented as timed polling of the non-blocking - // primitive. This is not a fairness guarantee: see the starvation - // note on Exclusive — a Wait on an Exclusive acquisition can still - // exhaust its budget against a rolling sequence of Shared holders - // that each individually released before it, none of which had any - // signal that an exclusive acquirer was waiting. + // attempt. Zero or negative means exactly one non-blocking try (which + // also opts out of the writer-fairness queue in both directions: a + // fail-fast Exclusive publishes no intent, and a fail-fast Shared + // ignores published intent). The underlying blocking lock primitives + // have no deadline support, so waiting is implemented as timed polling + // of the non-blocking primitive. Wait time.Duration // PollInterval is the retry cadence while waiting (default 100ms). PollInterval time.Duration @@ -118,6 +112,14 @@ type Options struct { // effort, from the advisory sidecar). AcquireAll fires it at most // once across the whole gate set. OnWait func(holder string) + // IgnoreQueuedExclusive makes a waiting Shared acquisition skip the + // writer-fairness check. Set it only when an ANCESTOR process already + // holds this gate shared (see InheritedSharedHold): the queued + // exclusive acquirer cannot proceed until that ancestor releases, and + // the ancestor may be waiting on this process, so queueing would turn + // a nested bd invocation into a deadlock that only the wait bound + // breaks. Ignored for Exclusive. + IgnoreQueuedExclusive bool } // Gate identifies one gate file. Construct via ForWorkspace or @@ -231,8 +233,19 @@ type Info struct { func (g Gate) infoPath() string { return g.path + ".info" } -func (g Gate) readInfo() *Info { - data, err := os.ReadFile(g.infoPath()) +// intentPath is the writer-fairness lock: a waiting Exclusive acquirer +// holds it exclusively while it polls the gate, and waiting Shared +// acquirers defer while it is held. intentInfoPath is its advisory +// holder-info sidecar (a separate file because Windows LockFileEx locks +// are mandatory byte-range locks: other processes cannot read a locked +// file's contents). Both match the "*.gate.lock*" gitignore pattern. +func (g Gate) intentPath() string { return g.path + ".intent" } +func (g Gate) intentInfoPath() string { return g.path + ".intent.info" } + +func (g Gate) readInfo() *Info { return readInfoAt(g.infoPath()) } + +func readInfoAt(path string) *Info { + data, err := os.ReadFile(path) //nolint:gosec // G304: path derives from the gate location this package computed if err != nil { return nil } @@ -262,16 +275,7 @@ func (g Gate) busyDetail(mode Mode) string { } return "another bd process (no holder info recorded)" } - desc := fmt.Sprintf("pid %d", info.PID) - if info.Hostname != "" { - desc += " on " + info.Hostname - } - if info.Reason != "" { - desc += " (" + info.Reason + ")" - } - if !info.StartedAt.IsZero() { - desc += " since " + info.StartedAt.UTC().Format(time.RFC3339) - } + desc := info.describe() stale := func() bool { host, _ := os.Hostname() return host == info.Hostname && !pidAlive(info.PID) @@ -293,6 +297,42 @@ func (g Gate) busyDetail(mode Mode) string { return "possibly " + desc + ", or shared holders" } +// describe renders a holder record as "pid N on host (reason) since T". +func (info *Info) describe() string { + desc := fmt.Sprintf("pid %d", info.PID) + if info.Hostname != "" { + desc += " on " + info.Hostname + } + if info.Reason != "" { + desc += " (" + info.Reason + ")" + } + if !info.StartedAt.IsZero() { + desc += " since " + info.StartedAt.UTC().Format(time.RFC3339) + } + return desc +} + +// queuedDetail describes the queued exclusive acquirer a waiting Shared +// attempt is deferring to. The intent lock is held whenever this is +// called, so the sidecar (written right after the lock is taken) normally +// names a live process; it is only absent in the brief window before the +// write or when the write failed. +func (g Gate) queuedDetail() string { + info := readInfoAt(g.intentInfoPath()) + if info != nil { + // A sidecar whose removal failed (Windows refuses to delete a file + // another process has open) can outlive its writer; never present + // a dead process as the queued operation. + if host, _ := os.Hostname(); host == info.Hostname && !pidAlive(info.PID) { + info = nil + } + } + if info == nil { + return "a bd maintenance operation queued for exclusive access" + } + return "a bd maintenance operation queued for exclusive access: " + info.describe() +} + // pidAlive reports best-effort process liveness via the zero-signal // probe. Where the probe is unsupported it errs toward alive, which only // makes the diagnostic more cautious (the stale record is then presented @@ -367,6 +407,54 @@ func (h *Handle) Release() error { // inherited by spawned children (Go opens files close-on-exec on Unix and // non-inheritable on Windows), so a dolt child outliving its bd parent // does not keep the gate held. +// +// Writer fairness. flock and LockFileEx grant no queueing guarantees, and +// acquisition is non-blocking polling anyway, so without help a steady +// stream of short shared holders — each overlapping the next — would keep +// an Exclusive waiter out until its budget ran dry. A second lock file, +// the intent lock (.intent), fixes that: +// +// - a waiting Exclusive acquisition (Wait > 0) whose first attempt is +// busy takes the intent lock EXCLUSIVELY (non-blocking, retried each +// poll), holds it while it polls the gate, and drops it as soon as it +// owns the gate, gives up, or has held it for maxIntentHold without +// getting in (after which it keeps polling unqueued); +// - a waiting Shared acquisition (Wait > 0) checks the intent lock before +// every attempt but the last (a momentary non-blocking SHARED probe) +// and, while it is held, does not touch the gate — it waits, within its +// own budget, exactly as if the gate were held exclusively. Its final +// attempt tries the gate regardless. +// +// The two bounds keep the queue from ever costing availability. A queue can +// DELAY a shared acquirer but never FAIL it: the final attempt ignores +// intent, so a shared acquisition fails only when the gate itself is held +// exclusively. And a doomed exclusive waiter — one blocked behind a shared +// holder that will not leave soon (a --watch, a tail --follow, an open +// editor, a long-lived embedder) — stops queueing everyone else after +// maxIntentHold instead of for its whole budget. In-flight ordinary commands +// drain far faster than that, so the fairness win is kept. +// +// Queued exclusive operations still run one after another: while they take +// turns on the gate, a waiting shared acquirer waits for all of them (each +// new exclusive waiter re-publishes intent), bounded by its own budget. +// +// Shared holders that are already in keep running; the exclusive waiter +// waits only for them to drain, so its wait is bounded by the longest +// in-flight command rather than by the arrival rate. A shared acquirer that +// probed just before the intent was published can still slip in once, which +// costs at most one more command's duration. +// +// Stale intent cannot outlive its owner: it is an OS advisory lock, released +// by the kernel when the process exits or crashes (on every platform), never +// a marker file whose existence means anything. The intent file and its +// holder-info sidecar may linger; both are ignored whenever the lock itself +// is free. Every intent-lock failure other than "busy" (unsupported +// filesystem, permissions) degrades to the pre-fairness behavior instead of +// failing the acquisition: fairness is advisory, the gate is the authority. +// +// Lock order is unchanged in substance: the intent lock of a gate is taken +// only while acquiring that gate, so it slots directly before the gate in +// AcquireAll's sorted total order and cannot form a cycle. func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, error) { if g.path == "" { return nil, errors.New("workspacegate: zero Gate; use ForWorkspace/ForPhysicalRoot") @@ -396,32 +484,62 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er if mode == Exclusive { try = lockfile.FlockExclusiveNonBlock } + waiting := opts.Wait > 0 + defersToIntent := waiting && mode == Shared && !opts.IgnoreQueuedExclusive + publishesIntent := waiting && mode == Exclusive + var intentSince time.Time + + // intent is the writer-fairness lock while this Exclusive acquisition + // holds it. Released on every return path — after writeInfo on success, + // so a shared waiter never sees the gate unqueued-but-unclaimed. + var intent *os.File + defer func() { g.releaseIntent(intent) }() notified := false for { - err := try(f) - if err == nil { - h := &Handle{gate: g, mode: mode, f: f} - if mode == Exclusive { - g.writeInfo(opts.Reason) + var detail string + // The final attempt (budget spent) ignores intent: queued + // maintenance may delay a shared acquirer, never fail it. + final := time.Until(deadline) <= 0 + if defersToIntent && !final && g.ExclusiveQueued() { + detail = g.queuedDetail() + } else { + err := try(f) + if err == nil { + h := &Handle{gate: g, mode: mode, f: f} + if mode == Exclusive { + g.writeInfo(opts.Reason) + } + return h, nil } - return h, nil - } - if !errors.Is(err, lockfile.ErrLockBusy) && !lockfile.IsLocked(err) { - _ = f.Close() - return nil, fmt.Errorf("workspacegate: lock %s: %w", g.path, err) + if !errors.Is(err, lockfile.ErrLockBusy) && !lockfile.IsLocked(err) { + _ = f.Close() + return nil, fmt.Errorf("workspacegate: lock %s: %w", g.path, err) + } + detail = g.busyDetail(mode) } if !notified { notified = true if opts.OnWait != nil { - opts.OnWait(g.busyDetail(mode)) + opts.OnWait(detail) } } remaining := time.Until(deadline) - if opts.Wait <= 0 || remaining <= 0 { + if !waiting || remaining <= 0 { _ = f.Close() return nil, fmt.Errorf("workspacegate: %s (%s mode) held by %s: %w", - g.path, mode, g.busyDetail(mode), ErrBusy) + g.path, mode, detail, ErrBusy) + } + if intent != nil && time.Since(intentSince) >= maxIntentHold { + // Doomed or very slow: stop holding everyone else back. + g.releaseIntent(intent) + intent = nil + publishesIntent = false + } + if publishesIntent && intent == nil { + if intent = g.tryTakeIntent(opts.Reason); intent != nil { + intentSince = time.Now() + } } // Never sleep past the wait budget: a Wait shorter than the poll // interval must still come back within (about) Wait, and the @@ -439,13 +557,71 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er } } -// writeInfo records the advisory exclusive-holder sidecar. Failures are +// maxIntentHold caps how long one Exclusive acquisition keeps the intent +// lock without getting the gate. It must stay well below the shared wait +// budget callers use (bd: BEADS_GATE_WAIT_TIMEOUT, default 30s) and well +// above how long ordinary in-flight commands take to drain. A var so tests +// can shorten it. +var maxIntentHold = 10 * time.Second + +// ExclusiveQueued reports whether a waiting Exclusive acquirer currently +// holds this gate's intent lock. The probe opens read-only and treats a +// missing file as "no intent", so ordinary shared commands never create +// intent files; any error other than "busy" reads as "not queued" (see the +// degradation rule on Acquire). +func (g Gate) ExclusiveQueued() bool { + f, err := os.OpenFile(g.intentPath(), os.O_RDONLY, 0o600) + if err != nil { + return false + } + defer f.Close() + lerr := lockfile.FlockSharedNonBlock(f) + if lerr == nil { + _ = lockfile.FlockUnlock(f) + return false + } + return errors.Is(lerr, lockfile.ErrLockBusy) || lockfile.IsLocked(lerr) +} + +// tryTakeIntent makes one non-blocking attempt at the intent lock and, on +// success, records the advisory intent sidecar. It returns nil when the +// lock is busy (another exclusive waiter is queued first; this acquirer +// keeps polling the gate and retries the intent next round) or unusable. +func (g Gate) tryTakeIntent(reason string) *os.File { + f, err := os.OpenFile(g.intentPath(), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil + } + if err := lockfile.FlockExclusiveNonBlock(f); err != nil { + _ = f.Close() + return nil + } + writeInfoAt(g.intentInfoPath(), reason) + return f +} + +// releaseIntent drops a held intent lock (nil-safe). As with the gate's own +// sidecar, the intent sidecar is removed BEFORE unlocking so a late removal +// can never delete the next waiter's record. +func (g Gate) releaseIntent(f *os.File) { + if f == nil { + return + } + _ = os.Remove(g.intentInfoPath()) + _ = lockfile.FlockUnlock(f) + _ = f.Close() +} + +// writeInfo records the advisory exclusive-holder sidecar. +func (g Gate) writeInfo(reason string) { writeInfoAt(g.infoPath(), reason) } + +// writeInfoAt writes an advisory holder-info sidecar. Failures are // deliberately swallowed: diagnostics must never block the operation that // already holds the authoritative lock. The write goes to an O_EXCL temp // file renamed into place, which (a) never follows a pre-planted symlink // at either path — plain WriteFile would truncate the symlink's target — // and (b) is atomic, so concurrent readers cannot see torn JSON. -func (g Gate) writeInfo(reason string) { +func writeInfoAt(path, reason string) { host, _ := os.Hostname() data, err := json.Marshal(Info{ PID: os.Getpid(), @@ -456,7 +632,7 @@ func (g Gate) writeInfo(reason string) { if err != nil { return } - tmp := fmt.Sprintf("%s.%d.tmp", g.infoPath(), os.Getpid()) + tmp := fmt.Sprintf("%s.%d.tmp", path, os.Getpid()) _ = os.Remove(tmp) f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) //nolint:gosec // G304: path derives from the gate location this package computed, not request input; the preceding os.Remove clears a pre-planted file, and O_EXCL closes the remove-then-open window so a symlink replanted in between is refused rather than followed if err != nil { @@ -469,16 +645,40 @@ func (g Gate) writeInfo(reason string) { return } // Rename replaces a symlink itself rather than following it. - if err := os.Rename(tmp, g.infoPath()); err != nil { + if err := os.Rename(tmp, path); err != nil { // Windows can refuse to replace an existing file; retry once // after removing the destination. Still best effort. - _ = os.Remove(g.infoPath()) - if err := os.Rename(tmp, g.infoPath()); err != nil { + _ = os.Remove(path) + if err := os.Rename(tmp, path); err != nil { _ = os.Remove(tmp) } } } +// InheritedHoldEnv names the environment variable a bd process sets (to its +// PID) while it holds its command's gates SHARED, so that bd processes it +// spawns — `bd orphans` running `bd close`, git hooks under a bd-driven +// commit — can tell that an ancestor already holds the gate. See +// Options.IgnoreQueuedExclusive and InheritedSharedHold. +const InheritedHoldEnv = "BEADS_GATE_SHARED_HOLDER_PID" + +// InheritedSharedHold reports whether InheritedHoldEnv names a live process +// other than this one, i.e. whether this process was (probably) spawned by a +// bd command that still holds its gates shared. A stale or recycled PID only +// costs fairness, never safety: the flag merely skips the writer queue, and +// the gate itself still excludes every exclusive holder. +func InheritedSharedHold() bool { + raw := strings.TrimSpace(os.Getenv(InheritedHoldEnv)) + if raw == "" { + return false + } + pid, err := strconv.Atoi(raw) + if err != nil || pid == os.Getpid() { + return false + } + return pidAlive(pid) +} + // MultiHandle holds several gates acquired by AcquireAll and releases // them in reverse acquisition order. type MultiHandle struct { @@ -540,6 +740,9 @@ func AcquireAll(ctx context.Context, mode Mode, opts Options, gates ...Gate) (*M m := &MultiHandle{handles: make([]*Handle, 0, len(ordered))} for _, g := range ordered { if opts.Wait > 0 { + // An exhausted budget leaves later gates one non-blocking + // attempt (Wait 0), which ignores the writer-fairness queue — + // the same as the final attempt of a waiting acquisition. perGate.Wait = time.Until(deadline) if perGate.Wait < 0 { perGate.Wait = 0 diff --git a/internal/workspacegate/gate_test.go b/internal/workspacegate/gate_test.go index 4bcdd3ff1..b838e0f6d 100644 --- a/internal/workspacegate/gate_test.go +++ b/internal/workspacegate/gate_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "runtime" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -34,6 +36,10 @@ func TestMain(m *testing.M) { // // hold — acquire, print ACQUIRED, // hold until stdin closes, release, print RELEASED. +// intent — hold the gate's writer-fairness intent lock +// (as a queued exclusive acquirer does) until stdin closes. +// inherited-acquire — waiting shared acquisition with +// IgnoreQueuedExclusive = InheritedSharedHold(); prints the outcome. // sleep — sleep long without touching any gate (handle-inheritance probe). // exit — exit immediately, so the caller has a real, now-dead PID (used // to fabricate a stale holder-info sidecar in tests). @@ -64,6 +70,43 @@ func helperMain() { os.Exit(1) } fmt.Println("RELEASED") + case "intent": + // Hold the writer-fairness intent lock for /.beads's gate, as + // a queued exclusive acquirer would, until stdin closes. + g, err := ForWorkspace(filepath.Join(os.Args[1], ".beads")) + if err != nil { + fmt.Println("ERR", err) + os.Exit(1) + } + f := g.tryTakeIntent("test-intent-helper") + if f == nil { + fmt.Println("ERR intent lock unavailable") + os.Exit(1) + } + fmt.Println("ACQUIRED") + _, _ = io.Copy(io.Discard, os.Stdin) + g.releaseIntent(f) + fmt.Println("RELEASED") + case "inherited-acquire": + // Waiting shared acquisition the way bd's chokepoint does it, + // honoring an inherited shared hold; reports how long it took. + g, err := ForWorkspace(filepath.Join(os.Args[1], ".beads")) + if err != nil { + fmt.Println("ERR", err) + os.Exit(1) + } + inherited := InheritedSharedHold() + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + Wait: 3 * time.Second, PollInterval: 20 * time.Millisecond, + IgnoreQueuedExclusive: inherited, + }) + if err != nil { + fmt.Println("ERR", err) + os.Exit(1) + } + _ = h.Release() + fmt.Printf("inherited=%v waited_ms=%d\n", inherited, time.Since(start).Milliseconds()) case "sleep": time.Sleep(30 * time.Second) } @@ -595,13 +638,20 @@ func (h *holderProc) stop() { // spawnHolder starts this test binary in helper mode and waits until it // reports the lock acquired. func spawnHolder(t *testing.T, dir, mode string) *holderProc { + t.Helper() + return spawnHelperProc(t, "hold", dir, mode) +} + +// spawnHelperProc starts this test binary in the given helper mode and +// waits for it to print ACQUIRED. +func spawnHelperProc(t *testing.T, helperMode string, args ...string) *holderProc { t.Helper() exe, err := os.Executable() if err != nil { t.Fatalf("os.Executable: %v", err) } - cmd := exec.Command(exe, dir, mode) - cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=hold") + cmd := exec.Command(exe, args...) + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER="+helperMode) stdin, err := cmd.StdinPipe() if err != nil { t.Fatal(err) @@ -727,3 +777,543 @@ func TestSpawnedChildDoesNotInheritGate(t *testing.T) { } _ = h2.Release() } + +// --- writer fairness (exclusive intent) --- + +// A continuous stream of overlapping shared holders keeps the gate busy +// forever, so without writer fairness a waiting exclusive acquirer would +// exhaust its budget. With it, the exclusive acquirer gets the gate as soon +// as the shared holders that were already in drain, and the stream resumes +// after it releases. +func TestExclusiveWaiterNotStarvedBySharedStream(t *testing.T) { + g, _ := testGate(t) + first := mustAcquire(t, g, Shared, Options{}) + + const ( + workers = 4 + hold = 150 * time.Millisecond + ) + stop := make(chan struct{}) + done := make(chan struct{}) + var acquired, failed atomic.Int64 + go func() { + defer close(done) + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + // Stagger so the gate is never free between holders. + time.Sleep(time.Duration(w) * hold / workers) + for { + select { + case <-stop: + return + default: + } + h, err := g.Acquire(context.Background(), Shared, + Options{Wait: 10 * time.Second, PollInterval: 10 * time.Millisecond}) + if err != nil { + failed.Add(1) + continue + } + acquired.Add(1) + time.Sleep(hold) + _ = h.Release() + } + }(w) + } + wg.Wait() + }() + t.Cleanup(func() { close(stop); <-done }) + + // Let the stream get going, then release the original holder: from + // here on the gate is only ever held by overlapping stream members. + time.Sleep(2 * hold) + _ = first.Release() + + start := time.Now() + h, err := g.Acquire(context.Background(), Exclusive, + Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond, Reason: "test exclusive waiter"}) + if err != nil { + t.Fatalf("exclusive acquirer starved by a shared stream: %v", err) + } + waited := time.Since(start) + // Bounded by the in-flight holders draining (one hold), plus slack. + if waited > 2*time.Second { + t.Errorf("exclusive waited %s behind a stream of %s shared holds", waited, hold) + } + if g.ExclusiveQueued() { + t.Error("intent lock still held after the exclusive acquirer got the gate") + } + before := acquired.Load() + time.Sleep(3 * hold) + if got := acquired.Load(); got != before { + t.Errorf("shared stream acquired %d times while the exclusive holder held the gate", got-before) + } + _ = h.Release() + + // The stream resumes once the exclusive holder is gone. + deadline := time.Now().Add(3 * time.Second) + for acquired.Load() == before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if acquired.Load() == before { + t.Error("shared stream did not resume after the exclusive holder released") + } + if n := failed.Load(); n != 0 { + t.Errorf("%d shared acquisitions failed; with a 10s budget all should wait", n) + } +} + +// A waiting shared acquirer defers to queued intent (and says so), but a +// fail-fast one and one whose ancestor holds the gate do not. +func TestSharedWaitDefersToQueuedExclusive(t *testing.T) { + g, _ := testGate(t) + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent on a fresh gate returned nil") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + // Deferred for its whole budget, then the final attempt ignores the + // queue: delayed, never failed, because nothing holds the gate. + var holder string + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + Wait: 300 * time.Millisecond, PollInterval: 20 * time.Millisecond, + OnWait: func(h string) { holder = h }, + }) + if err != nil { + t.Fatalf("waiting shared under queued intent with a free gate: %v, want success on the final attempt", err) + } + _ = h.Release() + if waited := time.Since(start); waited < 250*time.Millisecond { + t.Errorf("shared acquirer did not defer to queued intent (acquired after %s)", waited) + } + if !strings.Contains(holder, "queued for exclusive access") || !strings.Contains(holder, "test queued maintenance") { + t.Errorf("wait notice %q does not name the queued exclusive acquirer", holder) + } + + h = mustAcquire(t, g, Shared, Options{}) // fail-fast: legacy behavior + _ = h.Release() + h = mustAcquire(t, g, Shared, Options{Wait: time.Second, IgnoreQueuedExclusive: true}) + _ = h.Release() + + // Once the intent is withdrawn, a waiting shared acquirer proceeds. + g.releaseIntent(intent) + intent = nil + h = mustAcquire(t, g, Shared, Options{Wait: time.Second}) + _ = h.Release() +} + +// Ordinary shared traffic never creates intent files, and an uncontended +// exclusive acquisition does not either; only a waiting exclusive does, and +// it withdraws its intent when it gives up. +func TestIntentLifecycle(t *testing.T) { + g, _ := testGate(t) + h := mustAcquire(t, g, Shared, Options{Wait: time.Second}) + _ = h.Release() + h = mustAcquire(t, g, Exclusive, Options{Wait: time.Second}) + _ = h.Release() + if _, err := os.Stat(g.intentPath()); !os.IsNotExist(err) { + t.Fatalf("intent file exists without any contended exclusive wait (stat err %v)", err) + } + + sh := mustAcquire(t, g, Shared, Options{}) + _, err := g.Acquire(context.Background(), Exclusive, + Options{Wait: 300 * time.Millisecond, PollInterval: 20 * time.Millisecond, Reason: "test giving up"}) + if !errors.Is(err, ErrBusy) { + t.Fatalf("exclusive under shared holder: %v, want ErrBusy", err) + } + if _, err := os.Stat(g.intentPath()); err != nil { + t.Fatalf("contended exclusive wait did not publish intent: %v", err) + } + if g.ExclusiveQueued() { + t.Error("intent still held after the exclusive acquirer gave up") + } + if _, err := os.Stat(g.intentInfoPath()); !os.IsNotExist(err) { + t.Errorf("intent sidecar left behind by an orderly give-up (stat err %v)", err) + } + _ = sh.Release() +} + +// Intent from a crashed exclusive waiter is released by the OS with the +// process; the leftover intent sidecar must not hold anyone back. +func TestStaleIntentFromCrashedProcess(t *testing.T) { + dir := t.TempDir() + g, err := ForWorkspace(filepath.Join(dir, ".beads")) + if err != nil { + t.Fatal(err) + } + hp := spawnHelperProc(t, "intent", dir) + + var holder string + h0, err := g.Acquire(context.Background(), Shared, Options{ + Wait: 200 * time.Millisecond, PollInterval: 20 * time.Millisecond, + OnWait: func(h string) { holder = h }, + }) + if err != nil { + t.Fatalf("shared wait under another process's intent: %v", err) + } + _ = h0.Release() + if !strings.Contains(holder, fmt.Sprintf("pid %d", hp.cmd.Process.Pid)) { + t.Fatalf("wait notice %q does not name the intent holder pid %d", holder, hp.cmd.Process.Pid) + } + + if err := hp.cmd.Process.Kill(); err != nil { + t.Fatalf("kill helper: %v", err) + } + <-hp.drained + _ = hp.cmd.Wait() + if _, err := os.Stat(g.intentInfoPath()); err != nil { + t.Fatalf("expected the crashed helper's intent sidecar to linger: %v", err) + } + + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, + Options{Wait: 10 * time.Second, PollInterval: 25 * time.Millisecond}) + if err != nil { + t.Fatalf("shared acquisition after intent holder crashed: %v", err) + } + _ = h.Release() + if waited := time.Since(start); waited > 3*time.Second { + t.Errorf("shared acquisition took %s after the intent holder died", waited) + } + // A new exclusive waiter can take the intent over the stale sidecar. + if f := g.tryTakeIntent("test successor"); f == nil { + t.Error("intent lock not reacquirable after its holder crashed") + } else { + g.releaseIntent(f) + } +} + +func TestContextCancelWhileDeferringToIntent(t *testing.T) { + g, _ := testGate(t) + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent returned nil") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + start := time.Now() + _, err := g.Acquire(ctx, Shared, Options{Wait: 30 * time.Second}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want context.DeadlineExceeded", err) + } + if waited := time.Since(start); waited > 3*time.Second { + t.Fatalf("cancellation took %s to stop the wait", waited) + } +} + +func TestInheritedSharedHold(t *testing.T) { + for _, tc := range []struct { + name, val string + want bool + }{ + {"unset", "", false}, + {"garbage", "not-a-pid", false}, + {"self", fmt.Sprint(os.Getpid()), false}, + {"dead", fmt.Sprint(deadPID(t)), false}, + {"live parent", fmt.Sprint(os.Getppid()), true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(InheritedHoldEnv, tc.val) + if got := InheritedSharedHold(); got != tc.want { + t.Fatalf("InheritedSharedHold() with %s=%q = %v, want %v", InheritedHoldEnv, tc.val, got, tc.want) + } + }) + } +} + +// setMaxIntentHold shortens maxIntentHold for one test. +func setMaxIntentHold(t *testing.T, d time.Duration) { + t.Helper() + old := maxIntentHold + maxIntentHold = d + t.Cleanup(func() { maxIntentHold = old }) +} + +// acquireAsync runs AcquireAll in a goroutine; the result arrives on the +// returned channel and any acquired handle is released at test end. +type acqResult struct { + h *MultiHandle + err error + at time.Time +} + +func acquireAsync(t *testing.T, mode Mode, opts Options, gates ...Gate) <-chan acqResult { + t.Helper() + ch := make(chan acqResult, 1) + go func() { + h, err := AcquireAll(context.Background(), mode, opts, gates...) + ch <- acqResult{h: h, err: err, at: time.Now()} + }() + return ch +} + +func waitQueued(t *testing.T, g Gate) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !g.ExclusiveQueued() { + if time.Now().After(deadline) { + t.Fatal("exclusive waiter never published intent") + } + time.Sleep(5 * time.Millisecond) + } +} + +func recv(t *testing.T, ch <-chan acqResult, what string) acqResult { + t.Helper() + select { + case r := <-ch: + if r.err != nil { + t.Fatalf("%s: %v", what, r.err) + } + t.Cleanup(func() { _ = r.h.Release() }) + return r + case <-time.After(10 * time.Second): + t.Fatalf("%s: no result within 10s (deadlock?)", what) + } + return acqResult{} +} + +// pending fails the test if the acquisition has already completed. +func pending(t *testing.T, ch <-chan acqResult, what string) { + t.Helper() + select { + case r := <-ch: + if r.h != nil { + _ = r.h.Release() + } + t.Fatalf("%s finished early (err %v)", what, r.err) + default: + } +} + +// The availability regression this design must not have: a long-lived +// shared holder (a --watch, an open editor, an embedder) plus an exclusive +// waiter that therefore cannot get in. NEW shared commands must still run — +// before the intent cap they failed for the exclusive waiter's whole budget. +func TestDoomedExclusiveWaiterDoesNotFailNewSharedCommands(t *testing.T) { + t.Run("final attempt ignores queue", func(t *testing.T) { + g, _ := testGate(t) + long := mustAcquire(t, g, Shared, Options{}) + t.Cleanup(func() { _ = long.Release() }) + x := acquireAsync(t, Exclusive, Options{Wait: 3 * time.Second, PollInterval: 20 * time.Millisecond, Reason: "test doomed init"}, g) + waitQueued(t, g) + + h, err := g.Acquire(context.Background(), Shared, Options{Wait: time.Second, PollInterval: 20 * time.Millisecond}) + if err != nil { + t.Fatalf("new shared command failed behind a doomed exclusive waiter: %v", err) + } + _ = h.Release() + if r := <-x; !errors.Is(r.err, ErrBusy) { + t.Fatalf("doomed exclusive waiter: %v, want ErrBusy", r.err) + } + }) + t.Run("intent dropped after maxIntentHold", func(t *testing.T) { + setMaxIntentHold(t, 300*time.Millisecond) + g, _ := testGate(t) + long := mustAcquire(t, g, Shared, Options{}) + t.Cleanup(func() { _ = long.Release() }) + x := acquireAsync(t, Exclusive, Options{Wait: 3 * time.Second, PollInterval: 20 * time.Millisecond}, g) + waitQueued(t, g) + time.Sleep(500 * time.Millisecond) + if g.ExclusiveQueued() { + t.Fatal("exclusive waiter still holds intent past maxIntentHold") + } + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{Wait: time.Second, PollInterval: 20 * time.Millisecond}) + if err != nil { + t.Fatalf("shared after the intent cap: %v", err) + } + _ = h.Release() + if waited := time.Since(start); waited > 200*time.Millisecond { + t.Errorf("shared acquirer still deferred %s after the intent was dropped", waited) + } + pending(t, x, "exclusive waiter") // still waiting, just no longer queueing others + _ = long.Release() + recv(t, x, "exclusive waiter after the long holder left") + }) +} + +// Lock order with intent across a gate set (a < b): a shared AcquireAll that +// holds a and defers on b's intent, while an exclusive waiter on b alone +// drains b. Nobody deadlocks, and the exclusive waiter goes first. +func TestIntentLockOrderSharedHoldsEarlierGate(t *testing.T) { + dir := t.TempDir() + a, _ := ForWorkspace(filepath.Join(dir, "a")) + b, _ := ForPhysicalRoot(filepath.Join(dir, "b")) + if gateKey(a.Path()) >= gateKey(b.Path()) { + t.Fatalf("test assumes %s sorts before %s", a.Path(), b.Path()) + } + hb := mustAcquire(t, b, Shared, Options{}) + x := acquireAsync(t, Exclusive, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, b) + waitQueued(t, b) + s := acquireAsync(t, Shared, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, a, b) + time.Sleep(150 * time.Millisecond) + if held, _, _ := a.ExclusiveHolder(); held { + t.Fatal("unexpected exclusive holder on a") + } + pending(t, s, "shared {a,b}") + pending(t, x, "exclusive {b}") + + _ = hb.Release() + xr := recv(t, x, "exclusive waiter on b") + time.Sleep(100 * time.Millisecond) + pending(t, s, "shared {a,b} while the exclusive holder has b") + xRelease := time.Now() + _ = xr.h.Release() + sr := recv(t, s, "shared {a,b} behind the queued exclusive") + if sr.at.Before(xRelease) { + t.Error("shared acquirer got b while the exclusive holder had it") + } +} + +// ...and the other direction: an exclusive AcquireAll holding a and the +// intent on b, a shared waiter on b alone, and a shared waiter on {a,b} +// blocked on a. All three complete once b's shared holder leaves. +func TestIntentLockOrderExclusiveHoldsEarlierGate(t *testing.T) { + dir := t.TempDir() + a, _ := ForWorkspace(filepath.Join(dir, "a")) + b, _ := ForPhysicalRoot(filepath.Join(dir, "b")) + hb := mustAcquire(t, b, Shared, Options{}) + x := acquireAsync(t, Exclusive, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, a, b) + waitQueued(t, b) + if held, _, _ := a.ExclusiveHolder(); !held { + t.Fatal("exclusive waiter should hold a while queued on b") + } + s1 := acquireAsync(t, Shared, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, b) + s2 := acquireAsync(t, Shared, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, a, b) + time.Sleep(150 * time.Millisecond) + pending(t, s1, "shared {b}") + pending(t, s2, "shared {a,b}") + + _ = hb.Release() + xr := recv(t, x, "exclusive {a,b}") + _ = xr.h.Release() + recv(t, s1, "shared {b}") + recv(t, s2, "shared {a,b}") +} + +// An exhausted AcquireAll budget leaves later gates exactly one attempt, +// and queued intent on such a gate cannot fail the set (the gate is free). +func TestAcquireAllExhaustedBudgetFinalAttemptIgnoresIntent(t *testing.T) { + dir := t.TempDir() + a, _ := ForWorkspace(filepath.Join(dir, "a")) + b, _ := ForPhysicalRoot(filepath.Join(dir, "b")) + intent := b.tryTakeIntent("test queued on b") + if intent == nil { + t.Fatal("tryTakeIntent on b failed") + } + t.Cleanup(func() { b.releaseIntent(intent) }) + // a is held exclusively past the whole budget's midpoint and released + // only after the budget is spent, so b gets no budget at all. + ha := mustAcquire(t, a, Exclusive, Options{}) + time.AfterFunc(400*time.Millisecond, func() { _ = ha.Release() }) + _, err := AcquireAll(context.Background(), Shared, + Options{Wait: 300 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) + if !errors.Is(err, ErrBusy) { + t.Fatalf("a held past the budget: %v, want ErrBusy", err) + } + time.Sleep(200 * time.Millisecond) // a is free now + start := time.Now() + m, err := AcquireAll(context.Background(), Shared, + Options{Wait: 200 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) + if err != nil { + t.Fatalf("shared set with free gates and queued intent on b: %v", err) + } + _ = m.Release() + if waited := time.Since(start); waited < 150*time.Millisecond { + t.Errorf("b's queued intent was not honored before the final attempt (took %s)", waited) + } +} + +// Two exclusive waiters behind one shared holder: no deadlock, both get the +// gate in turn, and the intent is handed off and finally released. +func TestTwoExclusiveWaiters(t *testing.T) { + g, _ := testGate(t) + sh := mustAcquire(t, g, Shared, Options{}) + opts := Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond} + x1 := acquireAsync(t, Exclusive, opts, g) + x2 := acquireAsync(t, Exclusive, opts, g) + waitQueued(t, g) + time.AfterFunc(200*time.Millisecond, func() { _ = sh.Release() }) + + var second <-chan acqResult + var r acqResult + select { + case r = <-x1: + second = x2 + case r = <-x2: + second = x1 + case <-time.After(5 * time.Second): + t.Fatal("neither exclusive waiter got the gate") + } + if r.err != nil { + t.Fatalf("first exclusive waiter: %v", r.err) + } + time.Sleep(100 * time.Millisecond) + _ = r.h.Release() + r = recv(t, second, "second exclusive waiter") + _ = r.h.Release() + if g.ExclusiveQueued() { + t.Error("intent still held after both exclusive waiters finished") + } +} + +// The nested-bd escape hatch works through a real exec: a child whose +// environment names a live ancestor skips the queue; without it, it defers. +func TestInheritedHoldThroughSubprocess(t *testing.T) { + dir := t.TempDir() + g, err := ForWorkspace(filepath.Join(dir, ".beads")) + if err != nil { + t.Fatal(err) + } + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent failed") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + run := func(extra ...string) string { + t.Helper() + cmd := exec.Command(exe, dir) + env := []string{"WORKSPACEGATE_HELPER=inherited-acquire"} + for _, kv := range bazeltest.ShardFreeEnv(os.Environ()) { + if !strings.HasPrefix(kv, InheritedHoldEnv+"=") { + env = append(env, kv) + } + } + cmd.Env = append(env, extra...) + out, err := cmd.Output() + if err != nil { + t.Fatalf("helper: %v (%s)", err, out) + } + return strings.TrimSpace(string(out)) + } + parse := func(out string) (bool, time.Duration) { + var inherited bool + var ms int64 + if _, err := fmt.Sscanf(out, "inherited=%t waited_ms=%d", &inherited, &ms); err != nil { + t.Fatalf("helper output %q: %v", out, err) + } + return inherited, time.Duration(ms) * time.Millisecond + } + + inh, waited := parse(run(fmt.Sprintf("%s=%d", InheritedHoldEnv, os.Getpid()))) + if !inh || waited > time.Second { + t.Errorf("child of a live shared holder: inherited=%v waited=%s, want true and no queueing", inh, waited) + } + inh, waited = parse(run()) + if inh || waited < 2*time.Second { + t.Errorf("unrelated child: inherited=%v waited=%s, want false and a full deferral", inh, waited) + } +}