From ffeffdda45256fe3fe070a30c8bab05a4fd8a6bd Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Tue, 11 Aug 2026 00:14:51 -0700 Subject: [PATCH] fix(circuit): stop the legacy sweep churning against live TMPDIR-less writers (#5640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy /tmp/beads-circuit closed-file sweep (GH#4636) assumed the directory is abandoned — 'no live process left to reuse or clean' its files. That premise is false wherever os.TempDir() still resolves to /tmp: any TMPDIR-less process (launchd, cron, a bare ssh command) uses the legacy path as its LIVE breaker directory and rewrites its closed file on every successful connection (RecordSuccess). Every TMPDIR-set invocation then deleted that live file and logged '[circuit-breaker] removed legacy closed breaker file: ...' — recreate, delete, log, forever, one stderr line per bd invocation on any machine mixing TMPDIR-set and TMPDIR-less contexts (macOS most of all, where interactive shells get a per-user TMPDIR and background contexts do not). Deterministic repro at tip: 'env -u TMPDIR bd list' creates the file; the next normal 'bd list' logs and deletes it; repeat. Fix: gate the closed-file removal on an mtime TTL (legacyClosedSweepTTL, 24h) and drop the per-file log line. A live writer keeps its file's mtime fresh, so it is never fought; a genuinely abandoned file — the GH#4636 ephemeral-port accumulation this sweep exists for — only ages, and still gets cleaned. Open/half-open handling is unchanged. Test updated to pin the split: aged closed file removed, fresh closed file (a live writer's state) survives, fresh open file survives. Red-before-green: the old code removes the fresh closed file unconditionally. (bd-uann8, from the post-wave startup audit bd-7x3ov) --- internal/storage/dolt/circuit.go | 54 ++++++++++++++++++++------- internal/storage/dolt/circuit_test.go | 37 +++++++++++++----- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/internal/storage/dolt/circuit.go b/internal/storage/dolt/circuit.go index 82a1ec746..e330a298d 100644 --- a/internal/storage/dolt/circuit.go +++ b/internal/storage/dolt/circuit.go @@ -44,6 +44,21 @@ const ( // ago or the machine was rebooted. The TTL is based on the TrippedAt timestamp // (or LastFailure if TrippedAt is zero). circuitStaleTTL = 5 * time.Minute + + // legacyClosedSweepTTL is how long a CLOSED breaker file in the legacy + // "/tmp/beads-circuit" directory must sit unmodified before the legacy + // sweep removes it. The sweep's original premise — that the directory is + // abandoned, with "no live process left to reuse or clean" its files + // (GH#4636) — is false wherever os.TempDir() still resolves to /tmp: any + // TMPDIR-less process (launchd, cron, a bare ssh command) uses the legacy + // path as its LIVE directory and rewrites its closed file on every + // successful connection. An unconditional remove therefore fought those + // writers forever — recreate, delete, log, on every alternation of + // TMPDIR-set and TMPDIR-less invocations. A live writer keeps its file's + // mtime fresh, so an mtime TTL preserves the GH#4636 cleanup (a genuinely + // abandoned file only ages) without the churn. Generous on purpose: a + // writer that runs even once a day keeps its state. + legacyClosedSweepTTL = 24 * time.Hour ) // circuitState is the shared file-based circuit breaker state. @@ -372,10 +387,13 @@ func CleanStaleCircuitBreakerFiles() { // Also sweep the legacy hardcoded "/tmp/beads-circuit" location // (C:\tmp\beads-circuit on Windows) where older builds accumulated files // before the os.TempDir() fix (GH#4636). Unlike the live directory above, - // remove *closed* files here too: every successful circuit reset leaves a - // closed sidecar (RecordSuccess), and ephemeral ports mint a distinct - // filename each time, so closed files accumulate indefinitely in this - // abandoned directory with no live process left to reuse or clean them. + // remove *closed* files here too — but only past an mtime TTL: this + // directory is NOT fully abandoned. Any TMPDIR-less process (launchd, + // cron, a bare ssh command) resolves os.TempDir() to /tmp and uses it as + // its live directory, rewriting closed state on every success; only + // files no writer has touched in legacyClosedSweepTTL are the GH#4636 + // accumulation this sweep exists for (ephemeral ports minting a distinct + // filename each time, never reused). if os.Getenv(testCircuitBreakerDirEnv) == "" { if legacy := filepath.Join("/tmp", "beads-circuit"); legacy != dir { cleanStaleCircuitBreakerFilesIn(legacy, true) @@ -386,14 +404,15 @@ func CleanStaleCircuitBreakerFiles() { // cleanStaleCircuitBreakerFilesIn is the testable implementation of // CleanStaleCircuitBreakerFiles that accepts a directory parameter. // -// removeClosed additionally removes closed-state files, regardless of age. -// This is only safe for directories that no longer receive live writes (e.g. -// the abandoned legacy "/tmp/beads-circuit" directory, GH#4636): a closed -// state file is written on every successful RecordSuccess, and since -// ephemeral ports mint a distinct filename per connection, closed sidecars -// accumulate indefinitely in a directory with no other cleanup path. The -// live directory must not pass true here — a closed file there reflects a -// healthy, currently-in-use breaker. +// removeClosed additionally removes closed-state files whose mtime is older +// than legacyClosedSweepTTL. It exists for the legacy "/tmp/beads-circuit" +// directory (GH#4636): a closed state file is written on every successful +// RecordSuccess, and since ephemeral ports mint a distinct filename per +// connection, closed sidecars accumulate there with no other cleanup path. +// The TTL is what makes it safe alongside the directory's remaining LIVE +// writers (TMPDIR-less processes, whose os.TempDir() is /tmp — they keep +// their file's mtime fresh). The live directory must not pass true here — +// a closed file there reflects a healthy, currently-in-use breaker. func cleanStaleCircuitBreakerFilesIn(dir string, removeClosed bool) { pattern := filepath.Join(dir, "beads-dolt-circuit-*.json") matches, err := filepath.Glob(pattern) @@ -422,9 +441,16 @@ func cleanStaleCircuitBreakerFilesIn(dir string, removeClosed bool) { continue } if state.State == circuitClosed { + // Legacy-directory mode only — and only past an mtime TTL: the + // legacy dir has LIVE writers wherever os.TempDir() is /tmp + // (TMPDIR-less launchd/cron/ssh processes), so an unconditional + // remove churned against them forever (see legacyClosedSweepTTL). + // Silent on purpose: removing routine closed-state hygiene is not + // worth a stderr line per file per invocation. if removeClosed { - _ = os.Remove(path) - log.Printf("[circuit-breaker] removed legacy closed breaker file: %s", path) + if info, statErr := os.Stat(path); statErr == nil && time.Since(info.ModTime()) > legacyClosedSweepTTL { + _ = os.Remove(path) + } } continue } diff --git a/internal/storage/dolt/circuit_test.go b/internal/storage/dolt/circuit_test.go index 0b0aa8940..613bf5e33 100644 --- a/internal/storage/dolt/circuit_test.go +++ b/internal/storage/dolt/circuit_test.go @@ -420,18 +420,32 @@ func TestCleanStaleCircuitBreakerFiles(t *testing.T) { } // TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed verifies that, in -// legacy-directory mode (removeClosed=true), closed breaker files are swept -// too — unlike the live directory. A closed sidecar in the abandoned legacy -// "/tmp/beads-circuit" location (GH#4636) has no process left to reuse or -// clean it: every successful circuit reset writes a closed file, and -// ephemeral ports mint a distinct filename each time, so they would -// otherwise accumulate there forever. +// legacy-directory mode (removeClosed=true), closed breaker files ARE swept — +// unlike the live directory — but only past the legacyClosedSweepTTL mtime +// threshold. The legacy "/tmp/beads-circuit" location (GH#4636) is not fully +// abandoned: TMPDIR-less processes (launchd, cron, bare ssh) resolve +// os.TempDir() to /tmp and rewrite live closed state there on every success, +// and the old unconditional remove churned against them forever — recreate, +// delete, log, on every TMPDIR-set invocation (bd-uann8). A fresh closed file +// is a live writer's state and must survive; an aged one is the GH#4636 +// ephemeral-port accumulation and must go. func TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed(t *testing.T) { dir := t.TempDir() - closedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-9999.json") + // A closed file no writer has touched past the TTL: the true GH#4636 case. + agedClosedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-9999.json") closedData, _ := json.Marshal(circuitState{State: circuitClosed}) - if err := os.WriteFile(closedFile, closedData, 0600); err != nil { + if err := os.WriteFile(agedClosedFile, closedData, 0600); err != nil { + t.Fatal(err) + } + aged := time.Now().Add(-legacyClosedSweepTTL - time.Hour) + if err := os.Chtimes(agedClosedFile, aged, aged); err != nil { + t.Fatal(err) + } + + // A freshly-written closed file: a live TMPDIR-less writer's state. + freshClosedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-8888.json") + if err := os.WriteFile(freshClosedFile, closedData, 0600); err != nil { t.Fatal(err) } @@ -445,8 +459,11 @@ func TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed(t *testing.T) { cleanStaleCircuitBreakerFilesIn(dir, true) - if _, err := os.Stat(closedFile); !os.IsNotExist(err) { - t.Errorf("legacy closed breaker file should have been removed: %s", closedFile) + if _, err := os.Stat(agedClosedFile); !os.IsNotExist(err) { + t.Errorf("aged legacy closed breaker file should have been removed: %s", agedClosedFile) + } + if _, err := os.Stat(freshClosedFile); err != nil { + t.Errorf("fresh closed breaker file (live TMPDIR-less writer) should NOT have been removed: %s", freshClosedFile) } if _, err := os.Stat(freshFile); err != nil { t.Errorf("fresh open breaker file should NOT have been removed: %s", freshFile)