diff --git a/internal/doltserver/sweep.go b/internal/doltserver/sweep.go index 88026a6dd..e717444e5 100644 --- a/internal/doltserver/sweep.go +++ b/internal/doltserver/sweep.go @@ -1,6 +1,8 @@ package doltserver import ( + "os" + "path/filepath" "strconv" "strings" ) @@ -24,10 +26,9 @@ type serverCandidate struct { // reap as leaked test debris. A candidate qualifies only when its cmdline // names a dolt sql-server AND either: // -// - its working directory has been deleted (the temp dir it was serving -// no longer exists — this is the signature of a SIGKILLed test run -// whose t.TempDir() cleanup ran on top of a still-live server), or -// - its working directory sits under one of suiteRoots. +// - its working directory sits under one of suiteRoots, or +// - its working directory has been deleted AND the path it used to name +// sits under one of tempRoots. // // suiteRoots MUST be directories owned by the calling test suite alone // (e.g. that suite's own testTempRoot) — never a shared/global temp dir @@ -38,30 +39,97 @@ type serverCandidate struct { // data dirs live somewhere under os.TempDir() too. Passing a global root // here would turn this safety net into a cross-suite server killer. // +// tempRoots (see tempDirRoots) bounds the deleted-cwd arm to throwaway +// directories. A deleted working directory is a strong leak signal — a +// t.TempDir() cleanup ran on top of a still-live detached server — but it is +// NOT by itself proof of a TEST server: a production server is spawned with +// cmd.Dir = /.beads/dolt, so a developer who moved or deleted that +// workspace, or whose external volume was unmounted, would have their live +// server reaped by any test run on the box. Requiring the deleted path to +// have been under a temp dir keeps the arm pointed at test debris only. +// // This is intentionally conservative in the "never kill production" sense: -// a real shared server's data directory is a persistent, non-temp path that -// still exists and is never one of a test suite's own scoped roots, so it -// matches neither condition and is left alone. -func selectOrphanTestServerPIDs(candidates []serverCandidate, suiteRoots []string) []int { +// a real shared server's data directory is a persistent, non-temp path, so it +// is neither under a suite's scoped roots nor — deleted or not — under a temp +// root, and matches neither condition. +func selectOrphanTestServerPIDs(candidates []serverCandidate, suiteRoots, tempRoots []string) []int { + return selectCandidatePIDs(candidates, func(c serverCandidate) bool { + if underAnyRoot(c.cwd, suiteRoots) { + return true + } + return c.cwdDeleted && underAnyRoot(c.cwd, tempRoots) + }) +} + +// selectServersUnderRoots returns the PIDs of candidates whose working +// directory sits under one of roots, and nothing else. It is the strictly +// root-scoped selection: no deleted-cwd arm, so it can never reach a process +// outside the caller's own trees. +// +// SweepDeadSuiteRoots uses it because that sweep runs at suite START, while +// sibling packages (go test -p N) are mid-run: whatever it reaps must be +// provably inside the one dead root it is cleaning up, never merely +// "somewhere temporary". +func selectServersUnderRoots(candidates []serverCandidate, roots []string) []int { + return selectCandidatePIDs(candidates, func(c serverCandidate) bool { + return underAnyRoot(c.cwd, roots) + }) +} + +// selectCandidatePIDs applies want to every candidate that is a dolt +// sql-server with a known working directory. A candidate whose cwd could not +// be resolved is always skipped: unknown is not provably debris. +func selectCandidatePIDs(candidates []serverCandidate, want func(serverCandidate) bool) []int { var pids []int for _, c := range candidates { - if !isDoltServerCmdline(c.cmdline) { + if !isDoltServerCmdline(c.cmdline) || c.cwd == "" { continue } - if c.cwdDeleted { - pids = append(pids, c.pid) - continue - } - if c.cwd == "" { - continue - } - if underAnyRoot(c.cwd, suiteRoots) { + if want(c) { pids = append(pids, c.pid) } } return pids } +// tempDirRoots is the set of directories under which a deleted working +// directory is credible evidence of leaked TEST debris rather than a moved +// production workspace: the process temp dir (honoring TMPDIR, which the +// suites pin to their own root) and /tmp, which os.MkdirTemp uses when TMPDIR +// is unset and which several suites hardcode. +func tempDirRoots() []string { + return canonicalRoots([]string{os.TempDir(), "/tmp"}) +} + +// canonicalRoots expands each non-empty root into every form a process's +// working directory may be reported in: the path as given and, when it +// differs, its symlink-resolved form. macOS is why — os.MkdirTemp hands back +// /var/folders/… while lsof reports the /private/var/folders/… that symlink +// points at — but /tmp is a symlink to /private/tmp there too, and on Linux +// /tmp can be a symlink as well. Roots that cannot be resolved are kept +// as-is; duplicates are dropped. +func canonicalRoots(roots []string) []string { + var out []string + seen := make(map[string]bool, len(roots)*2) + add := func(path string) { + if path == "" || seen[path] { + return + } + seen[path] = true + out = append(out, path) + } + for _, root := range roots { + if root == "" { + continue + } + add(root) + if resolved, err := filepath.EvalSymlinks(root); err == nil { + add(resolved) + } + } + return out +} + // isDoltServerCmdline reports whether cmdline looks like a dolt sql-server // invocation. Mirrors the substring check in listDoltProcessPIDs (both // "dolt" and "sql-server" must appear) rather than an exact match, since diff --git a/internal/doltserver/sweep_darwin.go b/internal/doltserver/sweep_darwin.go index aa044219e..5c0aa85b1 100644 --- a/internal/doltserver/sweep_darwin.go +++ b/internal/doltserver/sweep_darwin.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "strconv" "strings" "syscall" @@ -25,8 +24,28 @@ import ( // Returns the PIDs it sent a kill signal to. func SweepOrphanedTestServers(suiteTempRoots ...string) []int { candidates := gatherDoltServerCandidates() - pids := selectOrphanTestServerPIDs(candidates, canonicalDarwinRoots(suiteTempRoots)) + pids := selectOrphanTestServerPIDs(candidates, canonicalRoots(suiteTempRoots), tempDirRoots()) + return reapServerPIDs(pids) +} +// sweepServersUnderRoots reaps only the dolt sql-servers whose working +// directory sits under one of suiteTempRoots. It is SweepOrphanedTestServers +// without the deleted-cwd arm, for callers that must not reach outside the +// trees they name — see selectServersUnderRoots. +// +// Returns the PIDs it sent a kill signal to. +func sweepServersUnderRoots(suiteTempRoots ...string) []int { + candidates := gatherDoltServerCandidates() + pids := selectServersUnderRoots(candidates, canonicalRoots(suiteTempRoots)) + return reapServerPIDs(pids) +} + +// reapServerPIDs SIGTERMs each selected PID, then SIGKILLs whatever is still +// alive a moment later, revalidating the process identity immediately before +// each signal so a PID recycled since selection cannot be targeted. +// +// Returns the PIDs it sent a kill signal to. +func reapServerPIDs(pids []int) []int { self := os.Getpid() var killed []int for _, pid := range pids { @@ -62,25 +81,6 @@ func SweepOrphanedTestServers(suiteTempRoots ...string) []int { return killed } -// canonicalDarwinRoots makes caller roots comparable with lsof output. macOS -// commonly reports a cwd below /private/var while os.MkdirTemp returned the -// equivalent /var path. -func canonicalDarwinRoots(roots []string) []string { - canonical := make([]string, 0, len(roots)) - for _, root := range roots { - if root == "" { - continue - } - resolved, err := filepath.EvalSymlinks(root) - if err != nil { - canonical = append(canonical, root) - continue - } - canonical = append(canonical, resolved) - } - return canonical -} - func gatherDoltServerCandidates() []serverCandidate { out, err := exec.Command("ps", "-axo", "pid=,command=").Output() if err != nil { diff --git a/internal/doltserver/sweep_darwin_test.go b/internal/doltserver/sweep_darwin_test.go index 0f5045337..87e419985 100644 --- a/internal/doltserver/sweep_darwin_test.go +++ b/internal/doltserver/sweep_darwin_test.go @@ -76,7 +76,7 @@ func TestParseDarwinCwdOrphanDetection(t *testing.T) { wantOK: false, }, { - name: "an empty n field is skipped in favour of the next one", + name: "an empty n field is skipped in favor of the next one", lsofOutput: "p123\nn\nn" + livePath + "\n", stat: statLive, wantCwd: livePath, diff --git a/internal/doltserver/sweep_linux.go b/internal/doltserver/sweep_linux.go index 6764b8ccb..87276105a 100644 --- a/internal/doltserver/sweep_linux.go +++ b/internal/doltserver/sweep_linux.go @@ -30,10 +30,13 @@ import ( // caller vouches for as its own. // // A server whose working directory has been deleted (cwdDeleted, see -// readProcCwd) is reaped unconditionally regardless of suiteTempRoots — -// that is the unambiguous leak signature (a t.TempDir() cleanup ran out -// from under a still-live detached server) and cannot occur for any -// server, from any suite, that is still legitimately in use. +// readProcCwd) is reaped regardless of suiteTempRoots, but only when the +// path it used to name was under a temp dir (tempDirRoots). A deleted cwd +// is the leak signature — a t.TempDir() cleanup ran out from under a +// still-live detached server — yet on its own it does not distinguish a +// test server from a production one whose workspace was moved, deleted, or +// unmounted (a production server is spawned with cmd.Dir = the workspace's +// .beads/dolt). The temp-dir bound keeps that arm on test debris. // // Safety is the whole point: this must never touch a developer's real // shared server. It only reads /proc (no killing) to build the candidate @@ -46,8 +49,28 @@ import ( // Returns the PIDs it sent a kill signal to. func SweepOrphanedTestServers(suiteTempRoots ...string) []int { candidates := gatherDoltServerCandidates() - pids := selectOrphanTestServerPIDs(candidates, suiteTempRoots) + pids := selectOrphanTestServerPIDs(candidates, canonicalRoots(suiteTempRoots), tempDirRoots()) + return reapServerPIDs(pids) +} +// sweepServersUnderRoots reaps only the dolt sql-servers whose working +// directory sits under one of suiteTempRoots. It is SweepOrphanedTestServers +// without the deleted-cwd arm, for callers that must not reach outside the +// trees they name — see selectServersUnderRoots. +// +// Returns the PIDs it sent a kill signal to. +func sweepServersUnderRoots(suiteTempRoots ...string) []int { + candidates := gatherDoltServerCandidates() + pids := selectServersUnderRoots(candidates, canonicalRoots(suiteTempRoots)) + return reapServerPIDs(pids) +} + +// reapServerPIDs SIGTERMs each selected PID, then SIGKILLs whatever is still +// alive a moment later, revalidating the process identity immediately before +// each signal so a PID recycled since selection cannot be targeted. +// +// Returns the PIDs it sent a kill signal to. +func reapServerPIDs(pids []int) []int { self := os.Getpid() var killed []int for _, pid := range pids { diff --git a/internal/doltserver/sweep_other.go b/internal/doltserver/sweep_other.go index 70f2f90b6..6b82fe656 100644 --- a/internal/doltserver/sweep_other.go +++ b/internal/doltserver/sweep_other.go @@ -8,3 +8,9 @@ package doltserver func SweepOrphanedTestServers(_ ...string) []int { return nil } + +// sweepServersUnderRoots is the root-scoped sibling of +// SweepOrphanedTestServers and is a no-op here for the same reason. +func sweepServersUnderRoots(_ ...string) []int { + return nil +} diff --git a/internal/doltserver/sweep_suite.go b/internal/doltserver/sweep_suite.go index af2ecb3d7..a9124daf4 100644 --- a/internal/doltserver/sweep_suite.go +++ b/internal/doltserver/sweep_suite.go @@ -32,11 +32,18 @@ const FailOnLeakEnv = "BEADS_TEST_FAIL_ON_LEAK" // A root with no marker is never swept, so failing to write one only costs // the next run its chance to clean up — it is never unsafe. func WriteSuiteOwnerMarker(root string) error { + return writeSuiteOwnerMarkerPID(root, os.Getpid()) +} + +// writeSuiteOwnerMarkerPID writes an owner marker naming an arbitrary PID. +// SweepDeadSuiteRoots uses it to put a marker BACK, naming the same dead +// owner, when it could not finish removing a root. +func writeSuiteOwnerMarkerPID(root string, pid int) error { if root == "" { return fmt.Errorf("doltserver: WriteSuiteOwnerMarker: empty root") } path := filepath.Join(root, SuiteOwnerMarkerName) - return os.WriteFile(path, []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600) + return os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o600) } // readSuiteOwnerMarker returns the PID recorded in root's owner marker. @@ -97,13 +104,40 @@ func decideSuiteRoot(hasMarker, ownerAlive bool) suiteRootAction { // best-effort and returns the roots it removed. // // Safety rests entirely on decideSuiteRoot: a root is only ever touched when -// it carries this suite's own owner marker and that owner is gone. Live -// sibling runs (marker + live PID) and unclaimed leftovers (no marker) are -// both left exactly as they are. The per-root SweepOrphanedTestServers call -// therefore honors that function's contract — a caller-vouched suite root, -// never a shared temp dir. +// it is owned by the current user, carries this suite's own owner marker, and +// that owner is gone. Live sibling runs (marker + live PID) and unclaimed +// leftovers (no marker) are both left exactly as they are. +// +// The per-root reap is sweepServersUnderRoots, NOT SweepOrphanedTestServers: +// this runs at suite START, while sibling packages (go test -p N) are mid-run, +// so it must only ever reach processes provably inside the dead root it is +// cleaning up. SweepOrphanedTestServers' extra deleted-cwd arm spans every +// temp dir on the box and belongs only at end-of-run, where the suite is the +// last thing standing. func SweepDeadSuiteRoots(parentDir, prefix string) []string { - return sweepDeadSuiteRoots(parentDir, prefix, processAlive, SweepOrphanedTestServers, os.RemoveAll) + return sweepDeadSuiteRoots(parentDir, prefix, processAlive, sweepServersUnderRoots, removeSuiteRoot) +} + +// removeSuiteRoot removes a suite temp root, making unwritable directories +// writable first. cmd/bd's root doubles as an isolated $HOME, so it can hold +// read-only Go module cache entries that plain os.RemoveAll refuses; this +// mirrors forceRemoveAll in cmd/bd's TestMain. +func removeSuiteRoot(root string) error { + _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() && info.Mode()&0o200 == 0 { + // #nosec G122 -- filepath.Walk lstats, so this only ever fires on + // a real directory (never a symlink), inside a root the caller + // already proved is owned by this user and abandoned by a dead + // process. Widening its owner-write bit is also the least + // consequential thing that could be done to a swapped path. + _ = os.Chmod(path, info.Mode()|0o200) + } + return nil + }) + return os.RemoveAll(root) } // sweepDeadSuiteRoots is SweepDeadSuiteRoots with its three effects injected: @@ -130,12 +164,30 @@ func sweepDeadSuiteRoots( if err != nil || !info.IsDir() { continue } + // On a shared /tmp anyone can plant a directory with this prefix and + // a marker naming a dead PID. Only trust a root this user owns. + if !rootOwnedBySelf(info) { + continue + } pid, hasMarker := readSuiteOwnerMarker(root) if decideSuiteRoot(hasMarker, hasMarker && alive(pid)) != sweepSuiteRoot { continue } sweepServers(root) if err := removeAll(root); err != nil { + // Removal is not atomic: the marker is usually among the first + // entries deleted, so a partial failure would leave an unclaimed + // root that no later run may ever touch again. Put the marker + // back, naming the same dead owner, so the next run retries. + if writeErr := writeSuiteOwnerMarkerPID(root, pid); writeErr != nil { + fmt.Fprintf(os.Stderr, + "Warning: could not remove dead test-suite root %s (%v) and could not restore its owner marker (%v); it will not be retried\n", + root, err, writeErr) + } else { + fmt.Fprintf(os.Stderr, + "Warning: could not remove dead test-suite root %s: %v (marker restored; the next run retries)\n", + root, err) + } continue } swept = append(swept, root) diff --git a/internal/doltserver/sweep_suite_other.go b/internal/doltserver/sweep_suite_other.go index 4fa55ed47..af7e052ee 100644 --- a/internal/doltserver/sweep_suite_other.go +++ b/internal/doltserver/sweep_suite_other.go @@ -2,6 +2,13 @@ package doltserver +import "os" + +// rootOwnedBySelf accepts every root on platforms with no POSIX ownership in +// os.FileInfo.Sys(). It costs nothing: processAlive below reports every owner +// alive here, so SweepDeadSuiteRoots never removes anything anyway. +func rootOwnedBySelf(_ os.FileInfo) bool { return true } + // processAlive reports every owner as alive on platforms where this package // has no liveness probe, which makes SweepDeadSuiteRoots a no-op there: it // can never conclude a root's owner is dead, so it never removes anything. diff --git a/internal/doltserver/sweep_suite_test.go b/internal/doltserver/sweep_suite_test.go index 171d438da..3df9f661b 100644 --- a/internal/doltserver/sweep_suite_test.go +++ b/internal/doltserver/sweep_suite_test.go @@ -70,9 +70,8 @@ func TestSuiteOwnerMarkerRoundTrip(t *testing.T) { // TestSweepDeadSuiteRootsSelection walks a synthetic temp directory holding // one of every case and asserts both halves of the outcome: which roots were -// removed, and which roots SweepOrphanedTestServers was vouched for. Nothing -// here signals a real process — the liveness probe and the reaper are -// injected. +// removed, and which roots the server reaper was vouched for. Nothing here +// signals a real process — the liveness probe and the reaper are injected. func TestSweepDeadSuiteRootsSelection(t *testing.T) { parent := t.TempDir() const prefix = "beads-bd-tests-" @@ -106,7 +105,10 @@ func TestSweepDeadSuiteRootsSelection(t *testing.T) { elsewhere := t.TempDir() link := filepath.Join(parent, prefix+"symlink") if err := os.Symlink(elsewhere, link); err != nil { - t.Fatalf("symlink: %v", err) + // Windows without the create-symlink privilege; the rest of the + // case still carries its weight. + t.Logf("skipping the symlink leg: %v", err) + link = "" } var vouched []string @@ -124,18 +126,98 @@ func TestSweepDeadSuiteRootsSelection(t *testing.T) { t.Errorf("sweepDeadSuiteRoots() = %v, want %v", swept, want) } if want := []string{dead}; !reflect.DeepEqual(vouched, want) { - t.Errorf("SweepOrphanedTestServers vouched for %v, want %v", vouched, want) + t.Errorf("the server reaper was vouched for %v, want %v", vouched, want) } if _, err := os.Stat(dead); !errors.Is(err, os.ErrNotExist) { t.Errorf("dead-owner root %s still present: %v", dead, err) } for _, keep := range []string{live, unclaimed, otherSuite, stray, link, elsewhere} { + if keep == "" { + continue + } if _, err := os.Lstat(keep); err != nil { t.Errorf("%s must have been left alone: %v", keep, err) } } } +// TestSweepDeadSuiteRootsRestoresMarkerOnPartialRemoval covers the recovery +// path for a removal that fails halfway. os.RemoveAll is not atomic and the +// marker is an early casualty, so without the rewrite a root that could not +// be fully deleted (a read-only Go module cache entry under cmd/bd's isolated +// $HOME, say) would come back UNCLAIMED — and an unclaimed root is never +// swept again, by design. The marker must go back naming the same dead owner. +func TestSweepDeadSuiteRootsRestoresMarkerOnPartialRemoval(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-partial-tests-" + const deadPID = 4242 + + root := filepath.Join(parent, prefix+"stuck") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := writeSuiteOwnerMarkerPID(root, deadPID); err != nil { + t.Fatalf("write marker: %v", err) + } + + // Stand in for a partial os.RemoveAll: the marker is gone, the root is + // not, and an error comes back. + stubbornRemove := func(path string) error { + if err := os.Remove(filepath.Join(path, SuiteOwnerMarkerName)); err != nil { + t.Fatalf("simulate partial removal: %v", err) + } + return errors.New("permission denied") + } + + swept := sweepDeadSuiteRoots(parent, prefix, func(int) bool { return false }, func(...string) []int { return nil }, stubbornRemove) + if len(swept) != 0 { + t.Errorf("sweepDeadSuiteRoots() = %v, want nothing reported as removed", swept) + } + if _, err := os.Stat(root); err != nil { + t.Fatalf("root should still be there: %v", err) + } + pid, ok := readSuiteOwnerMarker(root) + if !ok || pid != deadPID { + t.Fatalf("marker after a failed removal = (%d, %v), want (%d, true) so the next run retries", pid, ok, deadPID) + } + + // And the next run, with removal working, finishes the job. + swept = sweepDeadSuiteRoots(parent, prefix, func(int) bool { return false }, func(...string) []int { return nil }, removeSuiteRoot) + if want := []string{root}; !reflect.DeepEqual(swept, want) { + t.Errorf("retry swept %v, want %v", swept, want) + } +} + +// TestSweepDeadSuiteRootsRemovesReadOnlyDirs pins removeSuiteRoot's chmod +// walk: cmd/bd's suite root doubles as an isolated $HOME and can hold +// read-only Go module cache directories, which plain os.RemoveAll refuses. +func TestSweepDeadSuiteRootsRemovesReadOnlyDirs(t *testing.T) { + root := filepath.Join(t.TempDir(), "root") + nested := filepath.Join(root, "pkg", "mod", "readonly") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(nested, "file"), []byte("x"), 0o400); err != nil { + t.Fatalf("write: %v", err) + } + if err := os.Chmod(nested, 0o500); err != nil { + t.Fatalf("chmod: %v", err) + } + // Restore write permission whatever happens, so the test's own temp + // cleanup cannot fail. + t.Cleanup(func() { _ = os.Chmod(nested, 0o700) }) + + if err := os.RemoveAll(root); err == nil { + t.Skip("this platform's os.RemoveAll already handles read-only directories") + } + if err := removeSuiteRoot(root); err != nil { + t.Fatalf("removeSuiteRoot: %v", err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Errorf("root still present after removeSuiteRoot: %v", err) + } +} + // TestSweepDeadSuiteRootsRefusesUnboundedGlob guards the one input that would // turn this helper into a temp-directory shredder. func TestSweepDeadSuiteRootsRefusesUnboundedGlob(t *testing.T) { @@ -181,7 +263,7 @@ func TestSweepDeadSuiteRootsSkipsSelf(t *testing.T) { } // Real liveness probe this time: our own PID must read as alive. - swept := sweepDeadSuiteRoots(parent, prefix, processAlive, SweepOrphanedTestServers, os.RemoveAll) + swept := sweepDeadSuiteRoots(parent, prefix, processAlive, sweepServersUnderRoots, removeSuiteRoot) if len(swept) != 0 { t.Fatalf("sweepDeadSuiteRoots removed %v, including this process's own root", swept) } diff --git a/internal/doltserver/sweep_suite_unix.go b/internal/doltserver/sweep_suite_unix.go index bc21b27bb..1bdbb8cb5 100644 --- a/internal/doltserver/sweep_suite_unix.go +++ b/internal/doltserver/sweep_suite_unix.go @@ -4,9 +4,27 @@ package doltserver import ( "errors" + "os" "syscall" ) +// rootOwnedBySelf reports whether the directory info describes is owned by +// the user running this process. +// +// SweepDeadSuiteRoots globs a world-writable temp directory, so on a shared +// box (/tmp, a CI runner with several users) anyone could plant a directory +// carrying this suite's prefix and an owner marker naming a PID that is not +// running, and have the sweep delete a tree of their choosing under the +// caller's identity. Ownership is the cheap check that closes it. An +// unreadable or unexpected stat shape reports false — not ours, leave it. +func rootOwnedBySelf(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return false + } + return int(stat.Uid) == os.Getuid() +} + // processAlive reports whether pid still names a running process, using the // signal-0 probe (kill(2) performs its permission and existence checks but // delivers nothing). diff --git a/internal/doltserver/sweep_test.go b/internal/doltserver/sweep_test.go index 291941864..1ad51e3b7 100644 --- a/internal/doltserver/sweep_test.go +++ b/internal/doltserver/sweep_test.go @@ -1,6 +1,8 @@ package doltserver import ( + "os" + "path/filepath" "reflect" "testing" ) @@ -15,11 +17,18 @@ import ( // servers all living under os.TempDir(), and only a suite's own scoped // root may reap its own debris, not everyone else's (gastownhall/beads // mybd-q6cz). +// +// The second bound is on the deleted-cwd arm: a deleted working directory +// only counts as debris when the path it named was under a TEMP root. A +// production server is spawned with cmd.Dir = /.beads/dolt, so +// without that bound a moved workspace or an unmounted volume would make a +// developer's live server look exactly like test debris (wy-j2zc8q). func TestSelectOrphanTestServerPIDs(t *testing.T) { cases := []struct { name string candidates []serverCandidate suiteRoots []string + tempRoots []string want []int }{ { @@ -28,6 +37,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 100, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/beads-bd-tests-xyz/.beads/dolt", cwdDeleted: true}, }, suiteRoots: nil, + tempRoots: []string{"/tmp"}, want: []int{100}, }, { @@ -36,6 +46,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 101, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{101}, }, { @@ -44,6 +55,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 102, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/other-suite-xyz/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -52,6 +64,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 103, cmdline: "dolt --prof cpu --prof-path /tmp/my-suite-root/dolt-pprof sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{103}, }, { @@ -60,6 +73,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 200, cmdline: "dolt sql-server -H 127.0.0.1 -P 3307", cwd: "/home/dev/project/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -68,6 +82,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 201, cmdline: "some-other-binary --flag", cwd: "/tmp/my-suite-root/whatever"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -76,6 +91,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 202, cmdline: "dolt status", cwd: "/tmp/my-suite-root/whatever"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -84,6 +100,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 203, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: ""}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -92,6 +109,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 204, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root2/evil"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -100,6 +118,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 205, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/some-suite/.beads/dolt"}, }, suiteRoots: nil, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -111,13 +130,68 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 303, cmdline: "dolt sql-server -P 4", cwdDeleted: true, cwd: "/tmp/whatever-else/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{302, 303}, }, + { + name: "a PRODUCTION server whose workspace was moved or deleted is left alone", + candidates: []serverCandidate{ + {pid: 400, cmdline: "dolt sql-server --config /Users/dev/project/.beads/dolt-server-config.yaml", cwd: "/Users/dev/project/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp", "/private/var/folders/c1/xx/T"}, + want: nil, + }, + { + name: "a production server on an unmounted volume is left alone", + candidates: []serverCandidate{ + {pid: 401, cmdline: "dolt sql-server -P 3307", cwd: "/Volumes/external/work/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/tmp"}, + want: nil, + }, + { + name: "the observed orphan: deleted cwd under the macOS private temp form is reaped with no suite roots", + candidates: []serverCandidate{ + {pid: 402, cmdline: "dolt sql-server --config /private/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/003/config.yaml", cwd: "/private/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/002", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/var/folders/c1/xx/T", "/private/var/folders/c1/xx/T"}, + want: []int{402}, + }, + { + name: "the same orphan reported in the unresolved /var form is reaped too", + candidates: []serverCandidate{ + {pid: 403, cmdline: "dolt sql-server -P 1", cwd: "/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/002", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/var/folders/c1/xx/T", "/private/var/folders/c1/xx/T"}, + want: []int{403}, + }, + { + name: "a deleted cwd inside the caller's own suite root needs no temp root at all", + candidates: []serverCandidate{ + {pid: 404, cmdline: "dolt sql-server -P 1", cwd: "/opt/scratch/my-suite-root/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/opt/scratch/my-suite-root"}, + tempRoots: nil, + want: []int{404}, + }, + { + name: "no temp roots at all disables the deleted-cwd arm entirely", + candidates: []serverCandidate{ + {pid: 405, cmdline: "dolt sql-server -P 1", cwd: "/tmp/some-suite/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: nil, + want: nil, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := selectOrphanTestServerPIDs(tc.candidates, tc.suiteRoots) + got := selectOrphanTestServerPIDs(tc.candidates, tc.suiteRoots, tc.tempRoots) if !reflect.DeepEqual(got, tc.want) { t.Errorf("selectOrphanTestServerPIDs() = %v, want %v", got, tc.want) } @@ -175,9 +249,91 @@ not-a-pid dolt sql-server t.Fatalf("gatherPSCandidates() = %#v, want %#v", candidates, wantCandidates) } - gotPIDs := selectOrphanTestServerPIDs(candidates, []string{"/tmp/my-suite"}) + gotPIDs := selectOrphanTestServerPIDs(candidates, []string{"/tmp/my-suite"}, []string{"/tmp"}) wantPIDs := []int{101, 103} if !reflect.DeepEqual(gotPIDs, wantPIDs) { t.Errorf("darwin ps selection path = %v, want %v", gotPIDs, wantPIDs) } } + +// TestSelectServersUnderSuiteRoots pins the strictly root-scoped selection +// SweepDeadSuiteRoots relies on. It runs at suite START, with sibling +// packages mid-run, so it must reap only what is provably inside the roots +// it was handed: no deleted-cwd arm, no temp-dir reasoning of any kind. +func TestSelectServersUnderSuiteRoots(t *testing.T) { + candidates := []serverCandidate{ + {pid: 500, cmdline: "dolt sql-server -P 1", cwd: "/tmp/dead-root/.beads/dolt"}, + {pid: 501, cmdline: "dolt sql-server -P 2", cwd: "/tmp/dead-root/nested/deeper/dolt", cwdDeleted: true}, + {pid: 502, cmdline: "dolt sql-server -P 3", cwd: "/tmp/live-sibling-root/.beads/dolt"}, + {pid: 503, cmdline: "dolt sql-server -P 4", cwd: "/tmp/some-other-suite/.beads/dolt", cwdDeleted: true}, + {pid: 504, cmdline: "dolt sql-server -P 5", cwd: "/Users/dev/project/.beads/dolt", cwdDeleted: true}, + {pid: 505, cmdline: "not-dolt --flag", cwd: "/tmp/dead-root/whatever"}, + {pid: 506, cmdline: "dolt sql-server -P 6", cwd: ""}, + } + + got := selectServersUnderRoots(candidates, []string{"/tmp/dead-root"}) + want := []int{500, 501} + if !reflect.DeepEqual(got, want) { + t.Errorf("selectServersUnderRoots() = %v, want %v", got, want) + } + + if got := selectServersUnderRoots(candidates, nil); got != nil { + t.Errorf("selectServersUnderRoots() with no roots = %v, want nothing", got) + } +} + +// TestCanonicalRootsForSweep covers the path-form expansion that lets a root +// match however the OS reports a process's cwd. +func TestCanonicalRootsForSweep(t *testing.T) { + if got := canonicalRoots(nil); got != nil { + t.Errorf("canonicalRoots(nil) = %v, want nothing", got) + } + if got := canonicalRoots([]string{"", ""}); got != nil { + t.Errorf("canonicalRoots(empties) = %v, want nothing", got) + } + + // A path that cannot be resolved is still kept as given. + missing := filepath.Join(t.TempDir(), "does-not-exist") + if got := canonicalRoots([]string{missing}); !reflect.DeepEqual(got, []string{missing}) { + t.Errorf("canonicalRoots(%q) = %v, want just the literal path", missing, got) + } + + // A real directory reached through a symlink yields both forms, in that + // order, with no duplicates — this is what makes macOS's + // /var/folders/… vs /private/var/folders/… pair match. + target := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + resolved, err := filepath.EvalSymlinks(link) + if err != nil { + t.Fatalf("EvalSymlinks(%s): %v", link, err) + } + got := canonicalRoots([]string{link, link, resolved}) + want := []string{link, resolved} + if !reflect.DeepEqual(got, want) { + t.Errorf("canonicalRoots() = %v, want %v", got, want) + } +} + +// TestTempDirRootsBoundTheOrphanArm checks that the deleted-cwd bound +// actually covers the directory this process's own t.TempDir() lands in — +// if it did not, the arm would silently never fire. +func TestTempDirRootsBoundTheOrphanArm(t *testing.T) { + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty; the deleted-cwd arm could never fire") + } + dir := t.TempDir() + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + resolved = dir + } + if !underAnyRoot(dir, roots) && !underAnyRoot(resolved, roots) { + t.Errorf("t.TempDir() %q (resolved %q) is under none of tempDirRoots() %v", dir, resolved, roots) + } + if underAnyRoot("/Users/dev/project/.beads/dolt", roots) { + t.Errorf("a production workspace path matched tempDirRoots() %v", roots) + } +} diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index 8f382454f..069aa9be2 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -57,6 +57,30 @@ func testMainInner(m *testing.M) int { // separate flag from BEADS_TEST_MODE. os.Setenv("BEADS_TEST_PDEATHSIG", "1") + // AD-01 (be-c5p): the test/bench harness opens a process-local dolt + // sql-server (testcontainer or external port). The new database-name + // firewall in dolt.New refuses test-named DBs unless this opt-in is set. + os.Setenv("BEADS_TEST_SERVER", "1") + if isHelperSubprocess() { + // A helper subprocess is handed its server's port by the parent test + // and never looks at this suite's own server. Starting one anyway + // costs a Dolt container plus a full migration chain per subprocess: + // TestMultiProcessSchemaInit spawns 8 at once, so the machine ran 9 + // Dolt servers to measure a lock race between 8 clients of the + // parent's single server. Worse, every helper exits through os.Exit, + // which skips the deferred TerminateDoltContainer below and leaks the + // container. That load is what pushed the contended GET_LOCK past its + // 5s wait in CI shard 3. + // + // It returns BEFORE the suite temp root below for the same reason: + // eight concurrent helpers would each run the dead-root sweep and each + // claim a root of its own, and os.Exit would abandon every one of + // them. A helper needs neither — it inherits the parent's live + // BEADS_TEST_CIRCUIT_DIR through the BEADS_ allowlist in + // integration.FilterEnv, and the parent's root outlives it. + return m.Run() + } + // Suite-owned root for the orphan-server sweep below. Must never be a // shared/global temp dir (see SweepOrphanedTestServers) — this one is // unique to this test run and removed when it exits. @@ -83,23 +107,6 @@ func testMainInner(m *testing.M) int { } defer os.Unsetenv(testCircuitBreakerDirEnv) } - - // AD-01 (be-c5p): the test/bench harness opens a process-local dolt - // sql-server (testcontainer or external port). The new database-name - // firewall in dolt.New refuses test-named DBs unless this opt-in is set. - os.Setenv("BEADS_TEST_SERVER", "1") - if isHelperSubprocess() { - // A helper subprocess is handed its server's port by the parent test - // and never looks at this suite's own server. Starting one anyway - // costs a Dolt container plus a full migration chain per subprocess: - // TestMultiProcessSchemaInit spawns 8 at once, so the machine ran 9 - // Dolt servers to measure a lock race between 8 clients of the - // parent's single server. Worse, every helper exits through os.Exit, - // which skips the deferred TerminateDoltContainer below and leaks the - // container. That load is what pushed the contended GET_LOCK past its - // 5s wait in CI shard 3. - return m.Run() - } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { diff --git a/internal/storage/uow/version_reconcile_integration_test.go b/internal/storage/uow/version_reconcile_integration_test.go index ccdc57dd9..5786ffc10 100644 --- a/internal/storage/uow/version_reconcile_integration_test.go +++ b/internal/storage/uow/version_reconcile_integration_test.go @@ -2,16 +2,14 @@ package uow import ( "context" - "errors" - "os" "os/exec" "path/filepath" - "syscall" "testing" "time" "github.com/stretchr/testify/require" + "github.com/steveyegge/beads/internal/procid" "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/dbproxy/server" @@ -38,14 +36,19 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) t.Cleanup(func() { - // Read the backend PID BEFORE shutting down: a successful + // Read the backend's record BEFORE shutting down: a successful // proxy.Shutdown removes the pid file, so afterwards there is // nothing left to verify against. - pid := backendServerPID(t, storeRootDir) - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) + record := backendServerRecord(t, storeRootDir) + err := proxy.Shutdown(storeRootDir) + if err == nil { + // Shutdown killed the backend AND confirmed its exit + // (waitForRecordedProcessExit) before removing the record. + // There is nothing left to check. + return } - requireBackendExited(t, pid) + t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) + requireBackendExited(t, record) }) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") @@ -73,47 +76,68 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { const ( // backendExitTimeout is how long a test waits for the dolt sql-server to - // leave the process table after proxy.Shutdown returned. Shutdown itself - // already polls for confirmation, so anything approaching this budget is - // a real failure to stop, not slow teardown. - backendExitTimeout = 5 * time.Second + // leave the process table after proxy.Shutdown failed. It is generous on + // purpose: Shutdown's own budget is already 5s of confirmation plus a 2s + // post-kill minimum, and this only runs on the error path, where the + // interesting outcome is "still there", not "took a while". + backendExitTimeout = 30 * time.Second backendExitPoll = 50 * time.Millisecond ) -// backendServerPID returns the PID of the dolt sql-server the proxy started -// under rootDir, read from the backend pid file the proxy's child manager -// writes (server.PIDFileName). It returns 0 when there is no usable record — -// no server was ever started, or it already shut down and removed the file. -func backendServerPID(t *testing.T, rootDir string) int { +// backendServerRecord returns the pid file the proxy's child manager wrote +// for the dolt sql-server under rootDir (server.PIDFileName). nil means there +// is nothing to verify: no server was started, or it shut down cleanly and +// removed its record. +// +// The whole record is returned, not just the PID: Birth is the process-birth +// token that makes the PID safe to act on at all. +func backendServerRecord(t *testing.T, rootDir string) *pidfile.PidFile { t.Helper() pf, err := pidfile.Read(rootDir, server.PIDFileName) if err != nil { t.Logf("read %s in %s: %v", server.PIDFileName, rootDir, err) - return 0 + return nil } if pf == nil || pf.Pid <= 0 { - return 0 + return nil } - return pf.Pid + return pf } -// requireBackendExited fails the test if the dolt sql-server is still running -// a short while after proxy.Shutdown claimed to have stopped it, then -// force-kills it so the leak does not outlive the run. +// requireBackendExited fails the test if the recorded dolt sql-server is +// still running after proxy.Shutdown returned an ERROR, then force-kills it +// so the leak does not outlive the run. // -// This is deliberately a t.Errorf and not a t.Logf: a surviving server holds +// It is deliberately a t.Errorf and not a t.Logf: a surviving server holds // the temp tree the test is about to delete, which is precisely how this // package produced sql-servers still serving directories that had been gone // for hours (wy-j2zc8q). A leak must be visible where it is caused, not // discovered later by a sweep. -func requireBackendExited(t *testing.T, pid int) { +// +// Every step is gated on procid birth identity rather than the bare PID. The +// server is not this process's child, so its PID is reusable the instant it +// exits, and proxy.Shutdown deliberately REFUSES to signal a record it cannot +// verify (ErrUnverifiableProcess) — killing that same PID here would defeat +// the exact protection the package went out of its way to provide. A record +// with no Birth token, or one whose token no longer matches, is therefore +// left alone: not the server we recorded, nothing to report. +func requireBackendExited(t *testing.T, pf *pidfile.PidFile) { t.Helper() - if pid <= 0 { + if pf == nil || pf.Pid <= 0 || pf.Birth == "" { return } + token := procid.Token(pf.Birth) + deadline := time.Now().Add(backendExitTimeout) for { - if !processRunning(pid) { + same, err := procid.Verify(pf.Pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pf.Pid, err) + return + } + if !same { + // Either the process is gone, or the PID now belongs to + // something else. Either way our server is not running. return } if time.Now().After(deadline) { @@ -121,26 +145,21 @@ func requireBackendExited(t *testing.T, pid int) { } time.Sleep(backendExitPoll) } - if proc, err := os.FindProcess(pid); err == nil { - _ = proc.Kill() - } - t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s (force-killed)", pid, backendExitTimeout) -} -// processRunning reports whether pid is still in the process table, using the -// signal-0 probe (the kernel runs its existence and permission checks but -// delivers nothing). EPERM means the process exists but belongs to someone -// else, which still counts as running. -func processRunning(pid int) bool { - proc, err := os.FindProcess(pid) + handle, err := procid.Open(pf.Pid, token) if err != nil { - return false + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s and could not be opened to kill: %v", + pf.Pid, backendExitTimeout, err) + return } - err = proc.Signal(syscall.Signal(0)) - if err == nil { - return true + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s and could not be killed: %v", + pf.Pid, backendExitTimeout, killErr) + return } - return errors.Is(err, syscall.EPERM) + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s (force-killed)", pf.Pid, backendExitTimeout) } // TestReconcileVersionPersistsAcrossUOW is the one version assertion that