diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 35c5193eb..7d54b41ba 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -50,10 +50,14 @@ type testRunner interface { func runTestsAndSweep(m testRunner) int { code := m.Run() - doltserver.SweepOrphanedTestServers(testTempRoot) - return code + swept := doltserver.SweepOrphanedTestServers(testTempRoot) + return doltserver.ApplyLeakPolicy("cmd/bd", code, swept) } +// suiteRootPrefix is testMainInner's os.MkdirTemp pattern without its random +// tail. It is what SweepDeadSuiteRoots globs for, so the two must not drift. +const suiteRootPrefix = "beads-bd-tests-" + // Guardrail: ensure the cmd/bd test suite does not touch the real repo .beads state. // Disable with BEADS_TEST_GUARD_DISABLE=1 (useful when running tests while actively using beads). func TestMain(m *testing.M) { @@ -68,13 +72,27 @@ func testMainInner(m *testing.M) int { // Many tests expect default config values; running from within this repo would // cause config.Initialize() to walk up from CWD and load `.beads/config.yaml`, // which may set non-default config values and makes tests assert the wrong behavior. - tmp, err := os.MkdirTemp("", "beads-bd-tests-*") + // Before claiming a root of our own, clear out the roots of EARLIER runs + // of this suite whose process is gone — a `go test -timeout` panic skips + // both the defer below and the post-Run sweep, so the servers those runs + // started outlive every cleanup this process installs, and nothing else + // ever looks at a dead run's tree again (wy-j2zc8q). Roots with no owner + // marker, and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + tmp, err := os.MkdirTemp("", suiteRootPrefix+"*") if err != nil { fmt.Fprintf(os.Stderr, "failed to create temp dir: %v\n", err) return 1 } defer func() { _ = forceRemoveAll(tmp) }() + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(tmp); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", tmp, err) + } + // Anchor package-level sync.Once builders (test binaries, isolated // HOMEs) under this directory so the defer above sweeps them up too. // Without this, those helpers leaked ~179MB-1.4GB per test run into diff --git a/internal/doltserver/sweep_darwin.go b/internal/doltserver/sweep_darwin.go index 86a69af1b..aa044219e 100644 --- a/internal/doltserver/sweep_darwin.go +++ b/internal/doltserver/sweep_darwin.go @@ -97,6 +97,10 @@ func isDoltServerProcess(pid int) bool { return isDoltServerCmdline(strings.TrimSpace(string(out))) } +// statFunc matches os.Stat. parseDarwinCwd takes one so its +// deleted-directory probe can be exercised without a live process. +type statFunc func(string) (os.FileInfo, error) + // readDarwinCwd resolves pid's cwd from lsof's machine-readable field output. // lsof emits the name as an `n` field after selecting descriptor `cwd`. func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { @@ -106,7 +110,27 @@ func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { if err != nil { return "", false, false } - for _, line := range strings.Split(string(out), "\n") { + return parseDarwinCwd(out, os.Stat) +} + +// parseDarwinCwd extracts the working directory from `lsof -Fn` output and +// decides whether that directory still exists. +// +// Unlike Linux's /proc//cwd symlink, macOS lsof does NOT append a +// " (deleted)" marker when the directory a process is sitting in has been +// unlinked — it prints the bare, now-dangling path. The suffix is still +// honored (harmless, and it keeps this parser shaped like readProcCwd), but +// when it is absent the path is stat'ed and ENOENT is the deletion signal. +// Without that probe the deleted-cwd arm of selectOrphanTestServerPIDs could +// never fire on darwin, so a `go test -timeout` panic — which skips every +// t.Cleanup and TestMain defer — left its dolt sql-server running forever +// even though the temp tree it served was long gone (wy-j2zc8q). +// +// Any other stat error (a permission wall, an unreadable parent) is treated +// as "not deleted": this sweep only ever escalates on positive proof, never +// on a failed read. +func parseDarwinCwd(lsofOutput []byte, stat statFunc) (cwd string, deleted bool, ok bool) { + for _, line := range strings.Split(string(lsofOutput), "\n") { if !strings.HasPrefix(line, "n") { continue } @@ -115,9 +139,13 @@ func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { if strings.HasSuffix(cwd, deletedSuffix) { return strings.TrimSuffix(cwd, deletedSuffix), true, true } - if cwd != "" { - return cwd, false, true + if cwd == "" { + continue } + if _, err := stat(cwd); err != nil && os.IsNotExist(err) { + return cwd, true, true + } + return cwd, false, true } return "", false, false } diff --git a/internal/doltserver/sweep_darwin_test.go b/internal/doltserver/sweep_darwin_test.go new file mode 100644 index 000000000..0f5045337 --- /dev/null +++ b/internal/doltserver/sweep_darwin_test.go @@ -0,0 +1,120 @@ +//go:build darwin + +package doltserver + +import ( + "io/fs" + "os" + "testing" +) + +// TestParseDarwinCwdOrphanDetection pins the darwin-only half of the +// deleted-cwd signal. macOS lsof prints the bare path of a working directory +// that has since been removed — no " (deleted)" suffix, unlike Linux's +// /proc//cwd — so the stat probe is the only thing that lets +// selectOrphanTestServerPIDs recognize a leaked test server on this platform +// (wy-j2zc8q). +func TestParseDarwinCwdOrphanDetection(t *testing.T) { + const livePath = "/private/var/folders/xx/T/some-suite/002" + const gonePath = "/private/var/folders/xx/T/TestUOWDependencyEditorContract1/002" + + statLive := func(string) (os.FileInfo, error) { return nil, nil } + statGone := func(string) (os.FileInfo, error) { + return nil, &fs.PathError{Op: "stat", Path: gonePath, Err: fs.ErrNotExist} + } + statDenied := func(string) (os.FileInfo, error) { + return nil, &fs.PathError{Op: "stat", Path: livePath, Err: fs.ErrPermission} + } + statPanics := func(string) (os.FileInfo, error) { + t.Helper() + t.Fatal("stat must not be called when lsof already reported the directory deleted") + return nil, nil + } + + cases := []struct { + name string + lsofOutput string + stat statFunc + wantCwd string + wantDeleted bool + wantOK bool + }{ + { + name: "existing directory is live", + lsofOutput: "p123\nn" + livePath + "\n", + stat: statLive, + wantCwd: livePath, + wantOK: true, + }, + { + name: "missing directory is the leak signature even with no suffix", + lsofOutput: "p123\nn" + gonePath + "\n", + stat: statGone, + wantCwd: gonePath, + wantDeleted: true, + wantOK: true, + }, + { + name: "explicit (deleted) suffix still wins without touching the filesystem", + lsofOutput: "p123\nn" + gonePath + " (deleted)\n", + stat: statPanics, + wantCwd: gonePath, + wantDeleted: true, + wantOK: true, + }, + { + name: "a stat error that is not ENOENT is never read as deleted", + lsofOutput: "p123\nn" + livePath + "\n", + stat: statDenied, + wantCwd: livePath, + wantOK: true, + }, + { + name: "no n field at all is unknown, not deleted", + lsofOutput: "p123\nfcwd\n", + stat: statPanics, + wantOK: false, + }, + { + name: "an empty n field is skipped in favour of the next one", + lsofOutput: "p123\nn\nn" + livePath + "\n", + stat: statLive, + wantCwd: livePath, + wantOK: true, + }, + { + name: "empty output is unknown", + lsofOutput: "", + stat: statPanics, + wantOK: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cwd, deleted, ok := parseDarwinCwd([]byte(tc.lsofOutput), tc.stat) + if cwd != tc.wantCwd || deleted != tc.wantDeleted || ok != tc.wantOK { + t.Errorf("parseDarwinCwd() = (%q, %v, %v), want (%q, %v, %v)", + cwd, deleted, ok, tc.wantCwd, tc.wantDeleted, tc.wantOK) + } + }) + } +} + +// TestParseDarwinCwdAgainstRealFilesystem exercises the same path with the +// real os.Stat, so the ENOENT classification is checked against the kernel +// rather than a hand-written error value. +func TestParseDarwinCwdAgainstRealFilesystem(t *testing.T) { + live := t.TempDir() + gone := t.TempDir() + if err := os.RemoveAll(gone); err != nil { + t.Fatalf("RemoveAll(%s): %v", gone, err) + } + + if _, deleted, ok := parseDarwinCwd([]byte("n"+live+"\n"), os.Stat); !ok || deleted { + t.Errorf("live dir %s: deleted=%v ok=%v, want deleted=false ok=true", live, deleted, ok) + } + if cwd, deleted, ok := parseDarwinCwd([]byte("n"+gone+"\n"), os.Stat); !ok || !deleted || cwd != gone { + t.Errorf("removed dir %s: cwd=%q deleted=%v ok=%v, want cwd=%s deleted=true ok=true", gone, cwd, deleted, ok, gone) + } +} diff --git a/internal/doltserver/sweep_suite.go b/internal/doltserver/sweep_suite.go new file mode 100644 index 000000000..af2ecb3d7 --- /dev/null +++ b/internal/doltserver/sweep_suite.go @@ -0,0 +1,176 @@ +package doltserver + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// SuiteOwnerMarkerName is the file a test suite's TestMain writes at the root +// of the temp tree it owns, recording the PID of the process that created it. +// +// It exists so a LATER run can tell two indistinguishable-looking leftovers +// apart: the temp root of a sibling run that is alive right now (a parallel +// package under scripts/test.sh -p N, or a second `go test` in another +// terminal), and the temp root of a run whose process is gone — killed by +// `go test -timeout`, a CI cancel, or Ctrl-C — which skipped every defer and +// left its dolt sql-server behind (wy-j2zc8q). +const SuiteOwnerMarkerName = "testmain.pid" + +// FailOnLeakEnv turns a post-run sweep that actually reaped something from a +// warning into a suite failure. It is opt-in on purpose: a single leaky test +// would otherwise redden a whole package for everyone, so the default is a +// loud line on stderr and the code the tests themselves produced. +const FailOnLeakEnv = "BEADS_TEST_FAIL_ON_LEAK" + +// WriteSuiteOwnerMarker records this process as the owner of root by writing +// SuiteOwnerMarkerName inside it. Call it from TestMain immediately after +// creating the suite's temp root; SweepDeadSuiteRoots in a later run reads it. +// +// A root with no marker is never swept, so failing to write one only costs +// the next run its chance to clean up — it is never unsafe. +func WriteSuiteOwnerMarker(root string) error { + if root == "" { + return fmt.Errorf("doltserver: WriteSuiteOwnerMarker: empty root") + } + path := filepath.Join(root, SuiteOwnerMarkerName) + return os.WriteFile(path, []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600) +} + +// readSuiteOwnerMarker returns the PID recorded in root's owner marker. +// ok is false when the marker is absent, unreadable, or does not hold a +// plausible PID — all of which mean "owner unknown", never "owner dead". +func readSuiteOwnerMarker(root string) (pid int, ok bool) { + // #nosec G304 -- the path is this package's own constant joined to a + // temp root the caller vouches for; nothing here comes from user input. + data, err := os.ReadFile(filepath.Join(root, SuiteOwnerMarkerName)) + if err != nil { + return 0, false + } + pid, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +// suiteRootAction is what SweepDeadSuiteRoots does with one candidate root. +type suiteRootAction int + +const ( + // leaveSuiteRoot means the root is not provably debris: either nobody + // claimed it (a pre-marker leftover, or another tool's directory that + // merely shares the prefix) or its owning process is still running. + leaveSuiteRoot suiteRootAction = iota + // sweepSuiteRoot means a run claimed this root and that run is gone, so + // anything still alive under it is leaked debris. + sweepSuiteRoot +) + +// decideSuiteRoot is the whole safety judgment of SweepDeadSuiteRoots, +// isolated from the filesystem and from process signaling so it can be +// tabled in a unit test. +// +// Only one combination reaps: a marker exists AND the PID it names is gone. +// "No marker" deliberately does NOT reap — a directory nobody claimed cannot +// be proven to be ours, and deleting it (plus SIGTERMing servers under it) +// on a guess is exactly the cross-suite killer sweep.go's contract forbids. +func decideSuiteRoot(hasMarker, ownerAlive bool) suiteRootAction { + if !hasMarker || ownerAlive { + return leaveSuiteRoot + } + return sweepSuiteRoot +} + +// SweepDeadSuiteRoots reaps the temp roots left behind by earlier runs of the +// calling suite whose owning process is dead: it kills the dolt sql-servers +// still running under each such root and removes the tree. +// +// parentDir is where the suite creates its roots (os.TempDir() at call time) +// and prefix is the suite's own os.MkdirTemp pattern minus the random tail +// (e.g. "beads-bd-tests-"). Both must be non-empty; an empty prefix would +// glob the entire temp directory and is refused. +// +// Call it from TestMain BEFORE creating this run's own root. It is +// best-effort and returns the roots it removed. +// +// Safety rests entirely on decideSuiteRoot: a root is only ever touched when +// it carries this suite's own owner marker and that owner is gone. Live +// sibling runs (marker + live PID) and unclaimed leftovers (no marker) are +// both left exactly as they are. The per-root SweepOrphanedTestServers call +// therefore honors that function's contract — a caller-vouched suite root, +// never a shared temp dir. +func SweepDeadSuiteRoots(parentDir, prefix string) []string { + return sweepDeadSuiteRoots(parentDir, prefix, processAlive, SweepOrphanedTestServers, os.RemoveAll) +} + +// sweepDeadSuiteRoots is SweepDeadSuiteRoots with its three effects injected: +// the liveness probe, the orphan-server reaper, and the tree removal. +func sweepDeadSuiteRoots( + parentDir, prefix string, + alive func(int) bool, + sweepServers func(...string) []int, + removeAll func(string) error, +) []string { + if parentDir == "" || prefix == "" { + return nil + } + matches, err := filepath.Glob(filepath.Join(parentDir, prefix+"*")) + if err != nil { + return nil + } + + var swept []string + for _, root := range matches { + // Lstat, not Stat: a symlink that happens to match the prefix must + // not be followed out of parentDir and removed. + info, err := os.Lstat(root) + if err != nil || !info.IsDir() { + continue + } + pid, hasMarker := readSuiteOwnerMarker(root) + if decideSuiteRoot(hasMarker, hasMarker && alive(pid)) != sweepSuiteRoot { + continue + } + sweepServers(root) + if err := removeAll(root); err != nil { + continue + } + swept = append(swept, root) + } + + if len(swept) > 0 { + fmt.Fprintf(os.Stderr, "Info: swept %d dead test-suite temp root(s): %v\n", len(swept), swept) + } + return swept +} + +// ApplyLeakPolicy folds the result of a post-run SweepOrphanedTestServers +// call into the suite's exit code. suite names the package for the log line. +// +// A non-empty sweep means the suite leaked a dolt sql-server: the tests +// finished, but a server survived every t.Cleanup and TestMain defer. That is +// always reported loudly on stderr; it only fails the run when +// FailOnLeakEnv is set to "1", so one flaky test cannot redden a package for +// everyone who did not opt in. A code the tests already failed with is +// preserved, never downgraded. +func ApplyLeakPolicy(suite string, code int, swept []int) int { + if len(swept) == 0 { + return code + } + if os.Getenv(FailOnLeakEnv) == "1" { + fmt.Fprintf(os.Stderr, + "FAIL: %s leaked %d dolt sql-server process(es) %v; swept them, and %s=1 makes that a failure\n", + suite, len(swept), swept, FailOnLeakEnv) + if code == 0 { + return 1 + } + return code + } + fmt.Fprintf(os.Stderr, + "Warning: %d leaked dolt sql-server(s) swept after %s: %v (set %s=1 to fail the run on this)\n", + len(swept), suite, swept, FailOnLeakEnv) + return code +} diff --git a/internal/doltserver/sweep_suite_other.go b/internal/doltserver/sweep_suite_other.go new file mode 100644 index 000000000..4fa55ed47 --- /dev/null +++ b/internal/doltserver/sweep_suite_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package doltserver + +// processAlive reports every owner as alive on platforms where this package +// has no liveness probe, which makes SweepDeadSuiteRoots a no-op there: it +// can never conclude a root's owner is dead, so it never removes anything. +// Same posture as SweepOrphanedTestServers in sweep_other.go — the stub keeps +// callers (test TestMains) portable without giving them a destructive +// best-guess. +func processAlive(_ int) bool { return true } diff --git a/internal/doltserver/sweep_suite_test.go b/internal/doltserver/sweep_suite_test.go new file mode 100644 index 000000000..171d438da --- /dev/null +++ b/internal/doltserver/sweep_suite_test.go @@ -0,0 +1,219 @@ +package doltserver + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "strconv" + "testing" +) + +// TestDecideSuiteRoot pins the safety judgment behind SweepDeadSuiteRoots: +// exactly one of the four states — claimed by a process that is gone — is +// debris. In particular an UNCLAIMED root is left alone, because a directory +// that merely matches the suite's prefix cannot be proven to be this suite's, +// and reaping it would resurrect the cross-suite killer that +// selectOrphanTestServerPIDs' contract exists to prevent (wy-j2zc8q). +func TestDecideSuiteRoot(t *testing.T) { + cases := []struct { + name string + hasMarker bool + ownerAlive bool + want suiteRootAction + }{ + {"claimed by a dead owner is debris", true, false, sweepSuiteRoot}, + {"claimed by a live owner is a concurrent run", true, true, leaveSuiteRoot}, + {"unclaimed root is never provably ours", false, false, leaveSuiteRoot}, + {"unclaimed root stays unclaimed even if some pid is alive", false, true, leaveSuiteRoot}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := decideSuiteRoot(tc.hasMarker, tc.ownerAlive); got != tc.want { + t.Errorf("decideSuiteRoot(%v, %v) = %v, want %v", tc.hasMarker, tc.ownerAlive, got, tc.want) + } + }) + } +} + +// TestSuiteOwnerMarkerRoundTrip covers the marker read/write pair, including +// every way a read can be inconclusive — each of which must report "owner +// unknown" (ok=false), never "owner dead". +func TestSuiteOwnerMarkerRoundTrip(t *testing.T) { + root := t.TempDir() + if err := WriteSuiteOwnerMarker(root); err != nil { + t.Fatalf("WriteSuiteOwnerMarker: %v", err) + } + pid, ok := readSuiteOwnerMarker(root) + if !ok || pid != os.Getpid() { + t.Errorf("readSuiteOwnerMarker() = (%d, %v), want (%d, true)", pid, ok, os.Getpid()) + } + + if err := WriteSuiteOwnerMarker(""); err == nil { + t.Error("WriteSuiteOwnerMarker(\"\") = nil, want an error") + } + + if _, ok := readSuiteOwnerMarker(t.TempDir()); ok { + t.Error("readSuiteOwnerMarker() on a root with no marker reported ok=true") + } + + for _, body := range []string{"", "not-a-pid", "0", "-1"} { + bad := t.TempDir() + if err := os.WriteFile(filepath.Join(bad, SuiteOwnerMarkerName), []byte(body), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + if _, ok := readSuiteOwnerMarker(bad); ok { + t.Errorf("readSuiteOwnerMarker() on marker %q reported ok=true, want unknown", body) + } + } +} + +// TestSweepDeadSuiteRootsSelection walks a synthetic temp directory holding +// one of every case and asserts both halves of the outcome: which roots were +// removed, and which roots SweepOrphanedTestServers was vouched for. Nothing +// here signals a real process — the liveness probe and the reaper are +// injected. +func TestSweepDeadSuiteRootsSelection(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-bd-tests-" + + mkRoot := func(name string, markerPID int) string { + dir := filepath.Join(parent, name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if markerPID != 0 { + body := []byte(strconv.Itoa(markerPID) + "\n") + if err := os.WriteFile(filepath.Join(dir, SuiteOwnerMarkerName), body, 0o600); err != nil { + t.Fatalf("write marker in %s: %v", dir, err) + } + } + return dir + } + + const deadPID, livePID = 4242, 4243 + dead := mkRoot(prefix+"dead", deadPID) + live := mkRoot(prefix+"live", livePID) + unclaimed := mkRoot(prefix+"unclaimed", 0) + otherSuite := mkRoot("beads-storage-dolt-tests-dead", deadPID) + + // A plain file and a symlink that both match the prefix: neither is a + // directory this suite created, and neither may be removed. + stray := filepath.Join(parent, prefix+"stray-file") + if err := os.WriteFile(stray, []byte("x"), 0o600); err != nil { + t.Fatalf("write stray file: %v", err) + } + elsewhere := t.TempDir() + link := filepath.Join(parent, prefix+"symlink") + if err := os.Symlink(elsewhere, link); err != nil { + t.Fatalf("symlink: %v", err) + } + + var vouched []string + swept := sweepDeadSuiteRoots( + parent, prefix, + func(pid int) bool { return pid == livePID }, + func(roots ...string) []int { + vouched = append(vouched, roots...) + return nil + }, + os.RemoveAll, + ) + + if want := []string{dead}; !reflect.DeepEqual(swept, want) { + t.Errorf("sweepDeadSuiteRoots() = %v, want %v", swept, want) + } + if want := []string{dead}; !reflect.DeepEqual(vouched, want) { + t.Errorf("SweepOrphanedTestServers vouched for %v, want %v", vouched, want) + } + if _, err := os.Stat(dead); !errors.Is(err, os.ErrNotExist) { + t.Errorf("dead-owner root %s still present: %v", dead, err) + } + for _, keep := range []string{live, unclaimed, otherSuite, stray, link, elsewhere} { + if _, err := os.Lstat(keep); err != nil { + t.Errorf("%s must have been left alone: %v", keep, err) + } + } +} + +// TestSweepDeadSuiteRootsRefusesUnboundedGlob guards the one input that would +// turn this helper into a temp-directory shredder. +func TestSweepDeadSuiteRootsRefusesUnboundedGlob(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "anything") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, SuiteOwnerMarkerName), []byte("4242\n"), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + + dead := func(int) bool { return false } + fatalSweep := func(roots ...string) []int { + t.Errorf("SweepOrphanedTestServers must not be called: %v", roots) + return nil + } + fatalRemove := func(path string) error { + t.Errorf("removeAll must not be called: %s", path) + return nil + } + + if got := sweepDeadSuiteRoots(parent, "", dead, fatalSweep, fatalRemove); got != nil { + t.Errorf("empty prefix swept %v, want nothing", got) + } + if got := sweepDeadSuiteRoots("", "beads-bd-tests-", dead, fatalSweep, fatalRemove); got != nil { + t.Errorf("empty parent dir swept %v, want nothing", got) + } +} + +// TestSweepDeadSuiteRootsSkipsSelf is the belt-and-braces check that the +// running process's own root survives, since TestMain writes its marker into +// a directory the very same glob will match on the next call. +func TestSweepDeadSuiteRootsSkipsSelf(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-self-tests-" + mine := filepath.Join(parent, prefix+"mine") + if err := os.MkdirAll(mine, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := WriteSuiteOwnerMarker(mine); err != nil { + t.Fatalf("WriteSuiteOwnerMarker: %v", err) + } + + // Real liveness probe this time: our own PID must read as alive. + swept := sweepDeadSuiteRoots(parent, prefix, processAlive, SweepOrphanedTestServers, os.RemoveAll) + if len(swept) != 0 { + t.Fatalf("sweepDeadSuiteRoots removed %v, including this process's own root", swept) + } + if _, err := os.Stat(mine); err != nil { + t.Errorf("own root %s was removed: %v", mine, err) + } +} + +// TestApplyLeakPolicyForSuite covers the exit-code arithmetic of the +// env-gated leak-as-failure rule. +func TestApplyLeakPolicyForSuite(t *testing.T) { + cases := []struct { + name string + env string + code int + swept []int + want int + }{ + {name: "no leak, passing suite", code: 0, swept: nil, want: 0}, + {name: "no leak, failing suite", code: 2, swept: nil, want: 2}, + {name: "leak warns but does not fail by default", code: 0, swept: []int{101}, want: 0}, + {name: "leak fails a passing suite when opted in", env: "1", code: 0, swept: []int{101}, want: 1}, + {name: "leak never downgrades an existing failure", env: "1", code: 2, swept: []int{101}, want: 2}, + {name: "any value other than 1 stays advisory", env: "true", code: 0, swept: []int{101}, want: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(FailOnLeakEnv, tc.env) + if got := ApplyLeakPolicy("internal/doltserver", tc.code, tc.swept); got != tc.want { + t.Errorf("ApplyLeakPolicy(%d, %v) with %s=%q = %d, want %d", + tc.code, tc.swept, FailOnLeakEnv, tc.env, got, tc.want) + } + }) + } +} diff --git a/internal/doltserver/sweep_suite_unix.go b/internal/doltserver/sweep_suite_unix.go new file mode 100644 index 000000000..bc21b27bb --- /dev/null +++ b/internal/doltserver/sweep_suite_unix.go @@ -0,0 +1,27 @@ +//go:build darwin || linux + +package doltserver + +import ( + "errors" + "syscall" +) + +// processAlive reports whether pid still names a running process, using the +// signal-0 probe (kill(2) performs its permission and existence checks but +// delivers nothing). +// +// Only ESRCH — "no such process" — counts as dead. EPERM means the process +// exists and belongs to another user, and any other errno means the probe +// itself failed; both report alive, so SweepDeadSuiteRoots leaves the root +// untouched rather than deleting a tree on an inconclusive read. +func processAlive(pid int) bool { + // kill(0, sig) signals the caller's whole process group and kill(-1, …) + // every process it may signal, so a nonsensical PID must never reach the + // syscall. Report it alive: an unusable marker is not proof of death. + if pid <= 0 { + return true + } + err := syscall.Kill(pid, 0) + return !errors.Is(err, syscall.ESRCH) +} diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index ab4a28018..8f382454f 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -60,12 +60,23 @@ func testMainInner(m *testing.M) int { // Suite-owned root for the orphan-server sweep below. Must never be a // shared/global temp dir (see SweepOrphanedTestServers) — this one is // unique to this test run and removed when it exits. - suiteTempRoot, tempRootErr := os.MkdirTemp("", "beads-storage-dolt-tests-*") + // + // Before claiming one, clear out the roots of earlier runs of this suite + // whose process is gone: a `go test -timeout` panic skips both the defer + // below and the post-Run sweep, so those runs' servers outlive every + // cleanup installed here (wy-j2zc8q). Unclaimed roots, and roots whose + // owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + suiteTempRoot, tempRootErr := os.MkdirTemp("", suiteRootPrefix+"*") if tempRootErr != nil { fmt.Fprintf(os.Stderr, "FATAL: failed to create suite temp root: %v\n", tempRootErr) return 1 } else { defer os.RemoveAll(suiteTempRoot) + if err := doltserver.WriteSuiteOwnerMarker(suiteTempRoot); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", suiteTempRoot, err) + } if err := os.Setenv(testCircuitBreakerDirEnv, filepath.Join(suiteTempRoot, "circuit")); err != nil { fmt.Fprintf(os.Stderr, "FATAL: failed to isolate circuit state: %v\n", err) return 1 @@ -129,7 +140,8 @@ func testMainInner(m *testing.M) int { // suite's own temp root (e.g. a SIGKILLed run of the multiprocess // schema init tests, which call doltserver.Start directly) — see // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(suiteTempRoot) + swept := doltserver.SweepOrphanedTestServers(suiteTempRoot) + code = doltserver.ApplyLeakPolicy("internal/storage/dolt", code, swept) testServerPort = 0 os.Unsetenv("BEADS_DOLT_PORT") @@ -138,6 +150,10 @@ func testMainInner(m *testing.M) int { return code } +// suiteRootPrefix is testMainInner's os.MkdirTemp pattern without its random +// tail. It is what SweepDeadSuiteRoots globs for, so the two must not drift. +const suiteRootPrefix = "beads-storage-dolt-tests-" + // initSharedSchema creates a store on the shared DB, sets config, and commits // so that branches inherit the full schema. func initSharedSchema(port int) error { diff --git a/internal/storage/uow/testmain_test.go b/internal/storage/uow/testmain_test.go new file mode 100644 index 000000000..2f55e14fb --- /dev/null +++ b/internal/storage/uow/testmain_test.go @@ -0,0 +1,100 @@ +package uow + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// suiteRootPrefix is this suite's os.MkdirTemp pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-storage-uow-tests-" + +// suiteTempRoot is the TestMain-owned temp directory that every t.TempDir() +// in this package lands under, so SweepOrphanedTestServers has a root it can +// vouch for when it reaps leaked dolt sql-servers. +var suiteTempRoot string + +// TestMain gives this package the suite-lifecycle it was missing. +// +// The integration tests here (newTestUOWProvider) start a real proxied +// `dolt sql-server` whose data directory is a t.TempDir(). Cleanup was purely +// per-test — proxy.Shutdown in a t.Cleanup — and there was no TestMain at +// all, so a run killed by `go test -timeout` (which panics the binary and +// skips every Cleanup) left the server running with its data directory +// deleted out from under it, forever. Observed live on a dev box as a +// TestUOWDependencyEditorContract server still serving a temp dir that had +// been gone for hours (wy-j2zc8q). +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. Roots with no owner marker, and + // roots whose owner is still running (a parallel package under + // scripts/test.sh, a second `go test`), are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin TMPDIR under a suite-owned root so every t.TempDir() — including + // the storeRootDir a provider serves — is nested under something the + // post-run sweep may vouch for. Same pattern as cmd/bd/doctor's TestMain. + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + + code := m.Run() + + // Best-effort reap of any dolt sql-server still running under this run's + // own root — the backstop for a test whose Cleanup did not get to run. + swept := doltserver.SweepOrphanedTestServers(root) + return doltserver.ApplyLeakPolicy("internal/storage/uow", code, swept) +} + +// TestTempDirLandsUnderSuiteSweepRoot guards the pinning above: if t.TempDir() +// ever stops landing under suiteTempRoot, the post-run sweep silently loses +// its scope and leaked sql-servers become unreapable again. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + dir := t.TempDir() + if !pathUnderRoot(dir, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", dir, suiteTempRoot) + } +} + +// pathUnderRoot reports whether dir is root or nested under it, comparing +// symlink-resolved paths (macOS hands out /var paths that resolve to +// /private/var). +func pathUnderRoot(dir, root string) bool { + dir = evalOrSelf(dir) + root = evalOrSelf(root) + if dir == root { + return true + } + sep := string(os.PathSeparator) + return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) +} + +func evalOrSelf(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p +} diff --git a/internal/storage/uow/version_reconcile_integration_test.go b/internal/storage/uow/version_reconcile_integration_test.go index 7fb4ebda0..ccdc57dd9 100644 --- a/internal/storage/uow/version_reconcile_integration_test.go +++ b/internal/storage/uow/version_reconcile_integration_test.go @@ -2,13 +2,19 @@ package uow import ( "context" + "errors" + "os" "os/exec" "path/filepath" + "syscall" "testing" + "time" "github.com/stretchr/testify/require" + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" "github.com/steveyegge/beads/internal/testutil" "github.com/steveyegge/beads/internal/workapi" publicops "github.com/steveyegge/beads/issueops" @@ -32,9 +38,14 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) t.Cleanup(func() { + // Read the backend PID BEFORE shutting down: a successful + // proxy.Shutdown removes the pid file, so afterwards there is + // nothing left to verify against. + pid := backendServerPID(t, storeRootDir) if err := proxy.Shutdown(storeRootDir); err != nil { t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) } + requireBackendExited(t, pid) }) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") @@ -60,6 +71,78 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { return provider } +const ( + // backendExitTimeout is how long a test waits for the dolt sql-server to + // leave the process table after proxy.Shutdown returned. Shutdown itself + // already polls for confirmation, so anything approaching this budget is + // a real failure to stop, not slow teardown. + backendExitTimeout = 5 * time.Second + backendExitPoll = 50 * time.Millisecond +) + +// backendServerPID returns the PID of the dolt sql-server the proxy started +// under rootDir, read from the backend pid file the proxy's child manager +// writes (server.PIDFileName). It returns 0 when there is no usable record — +// no server was ever started, or it already shut down and removed the file. +func backendServerPID(t *testing.T, rootDir string) int { + t.Helper() + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil { + t.Logf("read %s in %s: %v", server.PIDFileName, rootDir, err) + return 0 + } + if pf == nil || pf.Pid <= 0 { + return 0 + } + return pf.Pid +} + +// requireBackendExited fails the test if the dolt sql-server is still running +// a short while after proxy.Shutdown claimed to have stopped it, then +// force-kills it so the leak does not outlive the run. +// +// This is deliberately a t.Errorf and not a t.Logf: a surviving server holds +// the temp tree the test is about to delete, which is precisely how this +// package produced sql-servers still serving directories that had been gone +// for hours (wy-j2zc8q). A leak must be visible where it is caused, not +// discovered later by a sweep. +func requireBackendExited(t *testing.T, pid int) { + t.Helper() + if pid <= 0 { + return + } + deadline := time.Now().Add(backendExitTimeout) + for { + if !processRunning(pid) { + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(backendExitPoll) + } + if proc, err := os.FindProcess(pid); err == nil { + _ = proc.Kill() + } + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s (force-killed)", pid, backendExitTimeout) +} + +// processRunning reports whether pid is still in the process table, using the +// signal-0 probe (the kernel runs its existence and permission checks but +// delivers nothing). EPERM means the process exists but belongs to someone +// else, which still counts as running. +func processRunning(pid int) bool { + proc, err := os.FindProcess(pid) + if err != nil { + return false + } + err = proc.Signal(syscall.Signal(0)) + if err == nil { + return true + } + return errors.Is(err, syscall.EPERM) +} + // TestReconcileVersionPersistsAcrossUOW is the one version assertion that // stays out of the conformance contract, because it is about this backend's // TRANSACTION rather than about the role: a marker written inside a unit of