fix(circuit): stop the legacy sweep churning against live TMPDIR-less writers (#5640)
The legacy /tmp/beads-circuit closed-file sweep (GH#4636) assumed the directory is abandoned — 'no live process left to reuse or clean' its files. That premise is false wherever os.TempDir() still resolves to /tmp: any TMPDIR-less process (launchd, cron, a bare ssh command) uses the legacy path as its LIVE breaker directory and rewrites its closed file on every successful connection (RecordSuccess). Every TMPDIR-set invocation then deleted that live file and logged '[circuit-breaker] removed legacy closed breaker file: ...' — recreate, delete, log, forever, one stderr line per bd invocation on any machine mixing TMPDIR-set and TMPDIR-less contexts (macOS most of all, where interactive shells get a per-user TMPDIR and background contexts do not). Deterministic repro at tip: 'env -u TMPDIR bd list' creates the file; the next normal 'bd list' logs and deletes it; repeat. Fix: gate the closed-file removal on an mtime TTL (legacyClosedSweepTTL, 24h) and drop the per-file log line. A live writer keeps its file's mtime fresh, so it is never fought; a genuinely abandoned file — the GH#4636 ephemeral-port accumulation this sweep exists for — only ages, and still gets cleaned. Open/half-open handling is unchanged. Test updated to pin the split: aged closed file removed, fresh closed file (a live writer's state) survives, fresh open file survives. Red-before-green: the old code removes the fresh closed file unconditionally. (bd-uann8, from the post-wave startup audit bd-7x3ov)
This commit is contained in:
parent
0498b22a75
commit
ffeffdda45
2 changed files with 67 additions and 24 deletions
|
|
@ -44,6 +44,21 @@ const (
|
|||
// ago or the machine was rebooted. The TTL is based on the TrippedAt timestamp
|
||||
// (or LastFailure if TrippedAt is zero).
|
||||
circuitStaleTTL = 5 * time.Minute
|
||||
|
||||
// legacyClosedSweepTTL is how long a CLOSED breaker file in the legacy
|
||||
// "/tmp/beads-circuit" directory must sit unmodified before the legacy
|
||||
// sweep removes it. The sweep's original premise — that the directory is
|
||||
// abandoned, with "no live process left to reuse or clean" its files
|
||||
// (GH#4636) — is false wherever os.TempDir() still resolves to /tmp: any
|
||||
// TMPDIR-less process (launchd, cron, a bare ssh command) uses the legacy
|
||||
// path as its LIVE directory and rewrites its closed file on every
|
||||
// successful connection. An unconditional remove therefore fought those
|
||||
// writers forever — recreate, delete, log, on every alternation of
|
||||
// TMPDIR-set and TMPDIR-less invocations. A live writer keeps its file's
|
||||
// mtime fresh, so an mtime TTL preserves the GH#4636 cleanup (a genuinely
|
||||
// abandoned file only ages) without the churn. Generous on purpose: a
|
||||
// writer that runs even once a day keeps its state.
|
||||
legacyClosedSweepTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// circuitState is the shared file-based circuit breaker state.
|
||||
|
|
@ -372,10 +387,13 @@ func CleanStaleCircuitBreakerFiles() {
|
|||
// Also sweep the legacy hardcoded "/tmp/beads-circuit" location
|
||||
// (C:\tmp\beads-circuit on Windows) where older builds accumulated files
|
||||
// before the os.TempDir() fix (GH#4636). Unlike the live directory above,
|
||||
// remove *closed* files here too: every successful circuit reset leaves a
|
||||
// closed sidecar (RecordSuccess), and ephemeral ports mint a distinct
|
||||
// filename each time, so closed files accumulate indefinitely in this
|
||||
// abandoned directory with no live process left to reuse or clean them.
|
||||
// remove *closed* files here too — but only past an mtime TTL: this
|
||||
// directory is NOT fully abandoned. Any TMPDIR-less process (launchd,
|
||||
// cron, a bare ssh command) resolves os.TempDir() to /tmp and uses it as
|
||||
// its live directory, rewriting closed state on every success; only
|
||||
// files no writer has touched in legacyClosedSweepTTL are the GH#4636
|
||||
// accumulation this sweep exists for (ephemeral ports minting a distinct
|
||||
// filename each time, never reused).
|
||||
if os.Getenv(testCircuitBreakerDirEnv) == "" {
|
||||
if legacy := filepath.Join("/tmp", "beads-circuit"); legacy != dir {
|
||||
cleanStaleCircuitBreakerFilesIn(legacy, true)
|
||||
|
|
@ -386,14 +404,15 @@ func CleanStaleCircuitBreakerFiles() {
|
|||
// cleanStaleCircuitBreakerFilesIn is the testable implementation of
|
||||
// CleanStaleCircuitBreakerFiles that accepts a directory parameter.
|
||||
//
|
||||
// removeClosed additionally removes closed-state files, regardless of age.
|
||||
// This is only safe for directories that no longer receive live writes (e.g.
|
||||
// the abandoned legacy "/tmp/beads-circuit" directory, GH#4636): a closed
|
||||
// state file is written on every successful RecordSuccess, and since
|
||||
// ephemeral ports mint a distinct filename per connection, closed sidecars
|
||||
// accumulate indefinitely in a directory with no other cleanup path. The
|
||||
// live directory must not pass true here — a closed file there reflects a
|
||||
// healthy, currently-in-use breaker.
|
||||
// removeClosed additionally removes closed-state files whose mtime is older
|
||||
// than legacyClosedSweepTTL. It exists for the legacy "/tmp/beads-circuit"
|
||||
// directory (GH#4636): a closed state file is written on every successful
|
||||
// RecordSuccess, and since ephemeral ports mint a distinct filename per
|
||||
// connection, closed sidecars accumulate there with no other cleanup path.
|
||||
// The TTL is what makes it safe alongside the directory's remaining LIVE
|
||||
// writers (TMPDIR-less processes, whose os.TempDir() is /tmp — they keep
|
||||
// their file's mtime fresh). The live directory must not pass true here —
|
||||
// a closed file there reflects a healthy, currently-in-use breaker.
|
||||
func cleanStaleCircuitBreakerFilesIn(dir string, removeClosed bool) {
|
||||
pattern := filepath.Join(dir, "beads-dolt-circuit-*.json")
|
||||
matches, err := filepath.Glob(pattern)
|
||||
|
|
@ -422,9 +441,16 @@ func cleanStaleCircuitBreakerFilesIn(dir string, removeClosed bool) {
|
|||
continue
|
||||
}
|
||||
if state.State == circuitClosed {
|
||||
// Legacy-directory mode only — and only past an mtime TTL: the
|
||||
// legacy dir has LIVE writers wherever os.TempDir() is /tmp
|
||||
// (TMPDIR-less launchd/cron/ssh processes), so an unconditional
|
||||
// remove churned against them forever (see legacyClosedSweepTTL).
|
||||
// Silent on purpose: removing routine closed-state hygiene is not
|
||||
// worth a stderr line per file per invocation.
|
||||
if removeClosed {
|
||||
_ = os.Remove(path)
|
||||
log.Printf("[circuit-breaker] removed legacy closed breaker file: %s", path)
|
||||
if info, statErr := os.Stat(path); statErr == nil && time.Since(info.ModTime()) > legacyClosedSweepTTL {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -420,18 +420,32 @@ func TestCleanStaleCircuitBreakerFiles(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed verifies that, in
|
||||
// legacy-directory mode (removeClosed=true), closed breaker files are swept
|
||||
// too — unlike the live directory. A closed sidecar in the abandoned legacy
|
||||
// "/tmp/beads-circuit" location (GH#4636) has no process left to reuse or
|
||||
// clean it: every successful circuit reset writes a closed file, and
|
||||
// ephemeral ports mint a distinct filename each time, so they would
|
||||
// otherwise accumulate there forever.
|
||||
// legacy-directory mode (removeClosed=true), closed breaker files ARE swept —
|
||||
// unlike the live directory — but only past the legacyClosedSweepTTL mtime
|
||||
// threshold. The legacy "/tmp/beads-circuit" location (GH#4636) is not fully
|
||||
// abandoned: TMPDIR-less processes (launchd, cron, bare ssh) resolve
|
||||
// os.TempDir() to /tmp and rewrite live closed state there on every success,
|
||||
// and the old unconditional remove churned against them forever — recreate,
|
||||
// delete, log, on every TMPDIR-set invocation (bd-uann8). A fresh closed file
|
||||
// is a live writer's state and must survive; an aged one is the GH#4636
|
||||
// ephemeral-port accumulation and must go.
|
||||
func TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
closedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-9999.json")
|
||||
// A closed file no writer has touched past the TTL: the true GH#4636 case.
|
||||
agedClosedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-9999.json")
|
||||
closedData, _ := json.Marshal(circuitState{State: circuitClosed})
|
||||
if err := os.WriteFile(closedFile, closedData, 0600); err != nil {
|
||||
if err := os.WriteFile(agedClosedFile, closedData, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
aged := time.Now().Add(-legacyClosedSweepTTL - time.Hour)
|
||||
if err := os.Chtimes(agedClosedFile, aged, aged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A freshly-written closed file: a live TMPDIR-less writer's state.
|
||||
freshClosedFile := filepath.Join(dir, "beads-dolt-circuit-127-0-0-1-8888.json")
|
||||
if err := os.WriteFile(freshClosedFile, closedData, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
|
@ -445,8 +459,11 @@ func TestCleanStaleCircuitBreakerFilesIn_LegacyRemovesClosed(t *testing.T) {
|
|||
|
||||
cleanStaleCircuitBreakerFilesIn(dir, true)
|
||||
|
||||
if _, err := os.Stat(closedFile); !os.IsNotExist(err) {
|
||||
t.Errorf("legacy closed breaker file should have been removed: %s", closedFile)
|
||||
if _, err := os.Stat(agedClosedFile); !os.IsNotExist(err) {
|
||||
t.Errorf("aged legacy closed breaker file should have been removed: %s", agedClosedFile)
|
||||
}
|
||||
if _, err := os.Stat(freshClosedFile); err != nil {
|
||||
t.Errorf("fresh closed breaker file (live TMPDIR-less writer) should NOT have been removed: %s", freshClosedFile)
|
||||
}
|
||||
if _, err := os.Stat(freshFile); err != nil {
|
||||
t.Errorf("fresh open breaker file should NOT have been removed: %s", freshFile)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue