fix(circuit): stop the legacy sweep churning against live TMPDIR-less writers (#5640)

The legacy /tmp/beads-circuit closed-file sweep (GH#4636) assumed the
directory is abandoned — 'no live process left to reuse or clean' its
files. That premise is false wherever os.TempDir() still resolves to /tmp:
any TMPDIR-less process (launchd, cron, a bare ssh command) uses the
legacy path as its LIVE breaker directory and rewrites its closed file on
every successful connection (RecordSuccess). Every TMPDIR-set invocation
then deleted that live file and logged
'[circuit-breaker] removed legacy closed breaker file: ...' — recreate,
delete, log, forever, one stderr line per bd invocation on any machine
mixing TMPDIR-set and TMPDIR-less contexts (macOS most of all, where
interactive shells get a per-user TMPDIR and background contexts do not).

Deterministic repro at tip: 'env -u TMPDIR bd list' creates the file; the
next normal 'bd list' logs and deletes it; repeat.

Fix: gate the closed-file removal on an mtime TTL (legacyClosedSweepTTL,
24h) and drop the per-file log line. A live writer keeps its file's mtime
fresh, so it is never fought; a genuinely abandoned file — the GH#4636
ephemeral-port accumulation this sweep exists for — only ages, and still
gets cleaned. Open/half-open handling is unchanged.

Test updated to pin the split: aged closed file removed, fresh closed
file (a live writer's state) survives, fresh open file survives.
Red-before-green: the old code removes the fresh closed file
unconditionally.

(bd-uann8, from the post-wave startup audit bd-7x3ov)
This commit is contained in:
Steve Yegge 2026-08-11 00:14:51 -07:00 • committed by GitHub
commit ffeffdda45
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 67 additions and 24 deletions

View file

@ -44,6 +44,21 @@ const (
// ago or the machine was rebooted. The TTL is based on the TrippedAt timestamp
// (or LastFailure if TrippedAt is zero).
circuitStaleTTL = 5 * time.Minute
// legacyClosedSweepTTL is how long a CLOSED breaker file in the legacy
// "/tmp/beads-circuit" directory must sit unmodified before the legacy
// sweep removes it. The sweep's original premise — that the directory is
// abandoned, with "no live process left to reuse or clean" its files
// (GH#4636) — is false wherever os.TempDir() still resolves to /tmp: any
// TMPDIR-less process (launchd, cron, a bare ssh command) uses the legacy
// path as its LIVE directory and rewrites its closed file on every
// successful connection. An unconditional remove therefore fought those
// writers forever — recreate, delete, log, on every alternation of
// TMPDIR-set and TMPDIR-less invocations. A live writer keeps its file's
// mtime fresh, so an mtime TTL preserves the GH#4636 cleanup (a genuinely
// abandoned file only ages) without the churn. Generous on purpose: a
// writer that runs even once a day keeps its state.
legacyClosedSweepTTL = 24 * time.Hour
)
// circuitState is the shared file-based circuit breaker state.
@ -372,10 +387,13 @@ func CleanStaleCircuitBreakerFiles() {
// Also sweep the legacy hardcoded "/tmp/beads-circuit" location
// (C:\tmp\beads-circuit on Windows) where older builds accumulated files
// before the os.TempDir() fix (GH#4636). Unlike the live directory above,
// remove *closed* files here too: every successful circuit reset leaves a
// closed sidecar (RecordSuccess), and ephemeral ports mint a distinct
// filename each time, so closed files accumulate indefinitely in this
// abandoned directory with no live process left to reuse or clean them.
// remove *closed* files here too — but only past an mtime TTL: this
// directory is NOT fully abandoned. Any TMPDIR-less process (launchd,
// cron, a bare ssh command) resolves os.TempDir() to /tmp and uses it as
// its live directory, rewriting closed state on every success; only
// files no writer has touched in legacyClosedSweepTTL are the GH#4636
// accumulation this sweep exists for (ephemeral ports minting a distinct
// filename each time, never reused).
if os.Getenv(testCircuitBreakerDirEnv) == "" {
if legacy := filepath.Join("/tmp", "beads-circuit"); legacy != dir {
cleanStaleCircuitBreakerFilesIn(legacy, true)
@ -386,14 +404,15 @@ func CleanStaleCircuitBreakerFiles() {
// cleanStaleCircuitBreakerFilesIn is the testable implementation of
// CleanStaleCircuitBreakerFiles that accepts a directory parameter.
//
// removeClosed additionally removes closed-state files, regardless of age.
// This is only safe for directories that no longer receive live writes (e.g.
// the abandoned legacy "/tmp/beads-circuit" directory, GH#4636): a closed
// state file is written on every successful RecordSuccess, and since
// ephemeral ports mint a distinct filename per connection, closed sidecars
// accumulate indefinitely in a directory with no other cleanup path. The
// live directory must not pass true here — a closed file there reflects a
// healthy, currently-in-use breaker.
// removeClosed additionally removes closed-state files whose mtime is older
// than legacyClosedSweepTTL. It exists for the legacy "/tmp/beads-circuit"
// directory (GH#4636): a closed state file is written on every successful
// RecordSuccess, and since ephemeral ports mint a distinct filename per
// connection, closed sidecars accumulate there with no other cleanup path.
// The TTL is what makes it safe alongside the directory's remaining LIVE
// writers (TMPDIR-less processes, whose os.TempDir() is /tmp — they keep
// their file's mtime fresh). The live directory must not pass true here —
// a closed file there reflects a healthy, currently-in-use breaker.
func cleanStaleCircuitBreakerFilesIn(dir string, removeClosed bool) {
pattern := filepath.Join(dir, "beads-dolt-circuit-*.json")
matches, err := filepath.Glob(pattern)
@ -422,9 +441,16 @@ func cleanStaleCircuitBreakerFilesIn(dir string, removeClosed bool) {
continue
}
if state.State == circuitClosed {
// Legacy-directory mode only — and only past an mtime TTL: the
// legacy dir has LIVE writers wherever os.TempDir() is /tmp
// (TMPDIR-less launchd/cron/ssh processes), so an unconditional
// remove churned against them forever (see legacyClosedSweepTTL).
// Silent on purpose: removing routine closed-state hygiene is not
// worth a stderr line per file per invocation.
if removeClosed {
_ = os.Remove(path)
log.Printf("[circuit-breaker] removed legacy closed breaker file: %s", path)
if info, statErr := os.Stat(path); statErr == nil && time.Since(info.ModTime()) > legacyClosedSweepTTL {
_ = os.Remove(path)
}
}
continue
}

View file

@ -420,18 +420,32 @@ func TestCleanStaleCircuitBreakerFiles(t *testing.T) {
}
// TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed verifies that, in
// legacy-directory mode (removeClosed=true), closed breaker files are swept
// too — unlike the live directory. A closed sidecar in the abandoned legacy
// "/tmp/beads-circuit" location (GH#4636) has no process left to reuse or
// clean it: every successful circuit reset writes a closed file, and
// ephemeral ports mint a distinct filename each time, so they would
// otherwise accumulate there forever.
// legacy-directory mode (removeClosed=true), closed breaker files ARE swept —
// unlike the live directory — but only past the legacyClosedSweepTTL mtime
// threshold. The legacy "/tmp/beads-circuit" location (GH#4636) is not fully
// abandoned: TMPDIR-less processes (launchd, cron, bare ssh) resolve
// os.TempDir() to /tmp and rewrite live closed state there on every success,
// and the old unconditional remove churned against them forever — recreate,
// delete, log, on every TMPDIR-set invocation (bd-uann8). A fresh closed file
// is a live writer's state and must survive; an aged one is the GH#4636
// ephemeral-port accumulation and must go.
func TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed(t *testing.T) {
dir := t.TempDir()
closedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-9999.json")
// A closed file no writer has touched past the TTL: the true GH#4636 case.
agedClosedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-9999.json")
closedData, _ := json.Marshal(circuitState{State: circuitClosed})
if err := os.WriteFile(closedFile, closedData, 0600); err != nil {
if err := os.WriteFile(agedClosedFile, closedData, 0600); err != nil {
t.Fatal(err)
}
aged := time.Now().Add(-legacyClosedSweepTTL - time.Hour)
if err := os.Chtimes(agedClosedFile, aged, aged); err != nil {
t.Fatal(err)
}
// A freshly-written closed file: a live TMPDIR-less writer's state.
freshClosedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-8888.json")
if err := os.WriteFile(freshClosedFile, closedData, 0600); err != nil {
t.Fatal(err)
}
@ -445,8 +459,11 @@ func TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed(t *testing.T) {
cleanStaleCircuitBreakerFilesIn(dir, true)
if _, err := os.Stat(closedFile); !os.IsNotExist(err) {
t.Errorf("legacy closed breaker file should have been removed: %s", closedFile)
if _, err := os.Stat(agedClosedFile); !os.IsNotExist(err) {
t.Errorf("aged legacy closed breaker file should have been removed: %s", agedClosedFile)
}
if _, err := os.Stat(freshClosedFile); err != nil {
t.Errorf("fresh closed breaker file (live TMPDIR-less writer) should NOT have been removed: %s", freshClosedFile)
}
if _, err := os.Stat(freshFile); err != nil {
t.Errorf("fresh open breaker file should NOT have been removed: %s", freshFile)