diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 2944dd3e3..575514bf1 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -62,10 +62,18 @@ func testMainInner(m *testing.M) int { // doctor_pkg_shared and doctest_*-prefixed databases through. os.Setenv("BEADS_TEST_SERVER", "1") - // Pin t.TempDir() under a suite-owned root so SweepOrphanedTestServers + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin t.TempDir() under a suite-owned root so SweepSuiteTestServers // can reap AutoStart leftovers whose t.TempDir() cleanup failed because // the live child still holds the tree (gastownhall/beads#5631). - root, pinErr := testutil.PinSuiteTempRoot("beads-doctor-tests-*") + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if pinErr != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) return 1 @@ -73,6 +81,12 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + if err := testutil.EnsureDoltContainerForTestMain(); err != nil { fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { @@ -101,7 +115,8 @@ func testMainInner(m *testing.M) int { // Best-effort reap of any dolt sql-server left running under a temp dir // this suite created (e.g. a SIGKILLed run) — see // gastownhall/beads mybd-q6cz / #5631. - doltserver.SweepOrphanedTestServers(root, testBDDir) + swept := doltserver.SweepSuiteTestServers(root, testBDDir) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor", code, swept) os.Unsetenv("BEADS_DOLT_PORT") os.Unsetenv("BEADS_TEST_MODE") diff --git a/cmd/bd/doctor/fix/suite_temp_test.go b/cmd/bd/doctor/fix/suite_temp_test.go index 91f53ec63..7aba7e642 100644 --- a/cmd/bd/doctor/fix/suite_temp_test.go +++ b/cmd/bd/doctor/fix/suite_temp_test.go @@ -1,39 +1,25 @@ package fix import ( - "os" - "path/filepath" - "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil" ) // suiteTempRoot is the TestMain-owned temp directory that t.TempDir must -// land under so SweepOrphanedTestServers can reap leaked sql-servers. +// land under so SweepSuiteTestServers can reap leaked sql-servers. var suiteTempRoot string +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its +// random tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-fix-tests-" + func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { if suiteTempRoot == "" { t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") } tmp := t.TempDir() - if !pathUnderRoot(tmp, suiteTempRoot) { + if !testutil.PathUnderSuiteRoot(tmp, suiteTempRoot) { t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", tmp, suiteTempRoot) } } - -func pathUnderRoot(dir, root string) bool { - dir = evalOrSelf(dir) - root = evalOrSelf(root) - if dir == root { - return true - } - sep := string(os.PathSeparator) - return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) -} - -func evalOrSelf(p string) string { - if r, err := filepath.EvalSymlinks(p); err == nil { - return r - } - return p -} diff --git a/cmd/bd/doctor/fix/testmain_cgo_test.go b/cmd/bd/doctor/fix/testmain_cgo_test.go index 88fa31362..b93c2b2f1 100644 --- a/cmd/bd/doctor/fix/testmain_cgo_test.go +++ b/cmd/bd/doctor/fix/testmain_cgo_test.go @@ -31,11 +31,19 @@ func testMainInner(m *testing.M) int { defer testutil.TerminateDoltContainer() } + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + // Pin t.TempDir() under a suite-owned root so the sweep below can reap // AutoStart leftovers whose directory cleanup failed because the live // child still holds the tree (gastownhall/beads#5631). Must never be a - // shared/global temp dir (see SweepOrphanedTestServers). - root, pinErr := testutil.PinSuiteTempRoot("beads-fix-tests-*") + // shared/global temp dir (see SweepSuiteTestServers). + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if pinErr != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) return 1 @@ -43,12 +51,19 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + code := m.Run() // Best-effort reap of any dolt sql-server left running under this // suite's own temp root (e.g. a SIGKILLed run) — see // gastownhall/beads mybd-q6cz / #5631. - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor/fix", code, swept) os.Unsetenv("BEADS_DOLT_PORT") os.Unsetenv("BEADS_TEST_MODE") os.Unsetenv("BEADS_TEST_SERVER") diff --git a/cmd/bd/doctor/fix/testmain_nocgo_test.go b/cmd/bd/doctor/fix/testmain_nocgo_test.go index db4e45f99..2329d310e 100644 --- a/cmd/bd/doctor/fix/testmain_nocgo_test.go +++ b/cmd/bd/doctor/fix/testmain_nocgo_test.go @@ -19,7 +19,15 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { - root, err := testutil.PinSuiteTempRoot("beads-fix-tests-*") + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) return 1 @@ -27,7 +35,14 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + code := m.Run() - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor/fix", code, swept) return code } diff --git a/cmd/bd/doctor/suite_temp_test.go b/cmd/bd/doctor/suite_temp_test.go index 668a3a535..b22ccfb3d 100644 --- a/cmd/bd/doctor/suite_temp_test.go +++ b/cmd/bd/doctor/suite_temp_test.go @@ -1,39 +1,25 @@ package doctor import ( - "os" - "path/filepath" - "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil" ) // suiteTempRoot is the TestMain-owned temp directory that t.TempDir must -// land under so SweepOrphanedTestServers can reap leaked sql-servers. +// land under so SweepSuiteTestServers can reap leaked sql-servers. var suiteTempRoot string +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its +// random tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-doctor-tests-" + func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { if suiteTempRoot == "" { t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") } tmp := t.TempDir() - if !pathUnderRoot(tmp, suiteTempRoot) { + if !testutil.PathUnderSuiteRoot(tmp, suiteTempRoot) { t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", tmp, suiteTempRoot) } } - -func pathUnderRoot(dir, root string) bool { - dir = evalOrSelf(dir) - root = evalOrSelf(root) - if dir == root { - return true - } - sep := string(os.PathSeparator) - return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) -} - -func evalOrSelf(p string) string { - if r, err := filepath.EvalSymlinks(p); err == nil { - return r - } - return p -} diff --git a/cmd/bd/doctor/testmain_nocgo_test.go b/cmd/bd/doctor/testmain_nocgo_test.go index e626d7732..108dd6ed6 100644 --- a/cmd/bd/doctor/testmain_nocgo_test.go +++ b/cmd/bd/doctor/testmain_nocgo_test.go @@ -19,7 +19,15 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { - root, err := testutil.PinSuiteTempRoot("beads-doctor-tests-*") + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) return 1 @@ -27,7 +35,14 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + code := m.Run() - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor", code, swept) return code } diff --git a/cmd/bd/dolt_worktree_remote_test.go b/cmd/bd/dolt_worktree_remote_test.go index fd9de5b7c..bbfc4efe8 100644 --- a/cmd/bd/dolt_worktree_remote_test.go +++ b/cmd/bd/dolt_worktree_remote_test.go @@ -32,6 +32,12 @@ func TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig(t *testing.T) { "BEADS_DOLT_SERVER_PORT="+strconv.Itoa(port), ) + // `bd init` in shared-server mode DAEMONIZES a dolt sql-server under + // $HOME/.beads/shared-server and returns; nothing here used to stop it, + // so it outlived the suite that deleted its tree (wy-j2zc8q). Registered + // before the first subprocess so the t.Fatalf paths are covered too. + stopSharedServerCleanup(t) + initCmd := exec.Command(bd, "init", "--prefix", "remote-sync", "--skip-hooks", "--quiet") initCmd.Dir = worktreeDir initCmd.Env = sharedEnv diff --git a/cmd/bd/init_metrics_test.go b/cmd/bd/init_metrics_test.go index 2b55202be..b284940d6 100644 --- a/cmd/bd/init_metrics_test.go +++ b/cmd/bd/init_metrics_test.go @@ -8,10 +8,12 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/testutil" ) type metricsEvent struct { @@ -95,7 +97,7 @@ func metricsTestEnv(home string, extra ...string) []string { return append(out, extra...) } -func runBdInitForMetrics(t *testing.T, home string, args ...string) { +func runBdInitForMetrics(t *testing.T, home string, extraEnv []string, args ...string) { t.Helper() bd := buildEmbeddedBD(t) repo, err := testTempDir("bd-metrics-repo-*") @@ -104,10 +106,20 @@ func runBdInitForMetrics(t *testing.T, home string, args ...string) { } initGitRepoAt(t, repo) + // `bd init --shared-server` daemonizes a dolt sql-server under + // /.beads/shared-server — being the HOME this fixture gives + // ONLY to the subprocess, so the in-process SharedServerPath would look in + // the wrong tree — and `--proxied-server` leaves a proxy plus its backend + // server under /.beads/dolt. Neither was ever stopped; the post-run + // sweep named this fixture by both cwds (wy-j2zc8q). Registered before + // the subprocess so the t.Fatalf paths are covered too. + stopDoltServerCleanup(t, filepath.Join(home, ".beads", "shared-server")) + stopProxiedServerCleanup(t, filepath.Join(repo, ".beads", "dolt")) + full := append([]string{"init", "--non-interactive", "--quiet"}, args...) cmd := exec.Command(bd, full...) cmd.Dir = repo - cmd.Env = metricsTestEnv(home) + cmd.Env = metricsTestEnv(home, extraEnv...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -120,6 +132,7 @@ func TestInitMetricsEmittedPerDoltMode(t *testing.T) { extraArgs []string extraEnv []string expectedCommand string + freeServerPort bool }{ { name: "embedded_default", @@ -134,6 +147,11 @@ func TestInitMetricsEmittedPerDoltMode(t *testing.T) { name: "shared_server_via_flag", extraArgs: []string{"--shared-server"}, expectedCommand: "init-shared-server", + // The shared server binds the default port unless told + // otherwise; on a box where 3308 is already someone's dolt the + // init fails before it starts anything, and this case would + // silently stop exercising the shared path. + freeServerPort: true, }, { name: "proxied_server_via_flag", @@ -149,7 +167,15 @@ func TestInitMetricsEmittedPerDoltMode(t *testing.T) { if err != nil { t.Fatalf("temp home: %v", err) } - runBdInitForMetrics(t, home, tc.extraArgs...) + env := tc.extraEnv + if tc.freeServerPort { + port, err := testutil.FindFreePort() + if err != nil { + t.Fatalf("find free port: %v", err) + } + env = append(env, "BEADS_DOLT_SERVER_PORT="+strconv.Itoa(port)) + } + runBdInitForMetrics(t, home, env, tc.extraArgs...) evt := readInitEvent(t, home, tc.expectedCommand) if evt.AppName != "beads" { diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 1ec32dd6e..294dc65f3 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -2093,6 +2093,12 @@ func TestInitDatabaseFlag(t *testing.T) { bareDir, worktreeDir := setupBareParentInitWorktree(t) bareBeadsDir := filepath.Join(bareDir, ".beads") + // Shared-server init daemonizes a dolt sql-server that nothing here + // stopped; it outlived the suite whose temp HOME it served + // (wy-j2zc8q). Registered before the first subprocess so the + // t.Fatalf paths are covered too. + stopSharedServerCleanup(t) + cmd := exec.Command(bd, "init", "--prefix", "bare-fallback", "--skip-hooks", "--quiet") cmd.Dir = worktreeDir cmd.Env = append(os.Environ(), "BEADS_DOLT_SHARED_SERVER=1") @@ -2316,6 +2322,9 @@ func TestBareParentWorktreeCoreCommandsWithoutRedirect(t *testing.T) { bareBeadsDir := filepath.Join(bareDir, ".beads") sharedEnv := append(os.Environ(), "BEADS_DOLT_SHARED_SERVER=1") + // Same daemonized shared server, same missing stop (wy-j2zc8q). + stopSharedServerCleanup(t) + initCmd := exec.Command(bd, "init", "--prefix", "bare-core", "--skip-hooks", "--quiet") initCmd.Dir = worktreeDir initCmd.Env = sharedEnv diff --git a/cmd/bd/shared_server_cleanup_test.go b/cmd/bd/shared_server_cleanup_test.go new file mode 100644 index 000000000..72cc49bce --- /dev/null +++ b/cmd/bd/shared_server_cleanup_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// stopSharedServerCleanup is the cleanup every fixture in this package that +// runs a subprocess `bd init` with BEADS_DOLT_SHARED_SERVER=1 owes. +// +// That init takes the shared-global branch in init.go, which calls +// doltserver.Start(sharedDir) — and Start DAEMONIZES: cmd.Process.Release(), +// no Wait, cmd.Dir = /dolt (doltserver.go:1443-1446). The `bd` +// subprocess then exits and the sql-server keeps running, serving a directory +// under the suite's temp HOME that TestMain deletes on the way out. Nothing in +// those fixtures ever stopped it, so a full cmd/bd run finished with a live +// `dolt sql-server --config /.beads/shared-server/…` still in the +// process table — the leak this bead is about (wy-j2zc8q). +// +// Because every one of those fixtures resolves the SAME $HOME/.beads/ +// shared-server, only the last one to start a server leaves a survivor, which +// is why the process table names one fixture while three carry the defect. +// +// Register it after the fixture's directories are built and BEFORE the first +// subprocess `bd` runs, so it covers the t.Fatalf paths too. +func stopSharedServerCleanup(t *testing.T) { + t.Helper() + // SharedServerPath, not SharedServerDir: resolving must not create the + // tree the command under test is supposed to create. + sharedDir, err := doltserver.SharedServerPath() + if err != nil { + t.Fatalf("resolve shared server dir: %v", err) + } + stopDoltServerCleanup(t, sharedDir) +} + +// stopDoltServerCleanup is the same cleanup for a server whose pid file lives +// under an explicit beads directory: a per-project server under +// /.beads, or a shared server under a HOME the fixture hands ONLY to +// the subprocess (`cmd.Env = ...HOME=...`), where this process's own +// SharedServerPath would resolve the wrong tree. Register it BEFORE the first +// subprocess `bd` runs; the directory need not exist yet. +func stopDoltServerCleanup(t *testing.T, beadsDir string) { + t.Helper() + sharedDir := beadsDir + t.Cleanup(func() { + // Read the record BEFORE stopping: a clean Stop removes the pid + // file, so afterwards there is nothing left to verify against. + state, err := doltserver.IsRunning(sharedDir) + if err != nil { + t.Errorf("doltserver.IsRunning(%s): %v", sharedDir, err) + return + } + if state == nil || !state.Running { + return + } + pid := state.PID + // The birth token makes the PID safe to ask about after the stop: + // the server is not this process's child, so its PID is reusable the + // instant it exits. + token, tokenErr := procid.Capture(pid) + + if stopErr := doltserver.Stop(sharedDir); stopErr != nil && + !errors.Is(stopErr, doltserver.ErrServerNotRunning) { + t.Errorf("doltserver.Stop(%s) for pid %d: %v", sharedDir, pid, stopErr) + } + + if tokenErr != nil { + t.Logf("procid.Capture(%d): %v; skipping the exit check", pid, tokenErr) + return + } + requireSharedServerExited(t, pid, token) + }) +} + +// stopProxiedServerCleanup is the same duty for a subprocess `bd init +// --proxied-server`: that init leaves a proxy (proxy.pid) and its backend +// dolt sql-server (proxy-child.pid) running under /dolt, with no +// dolt-server.pid for stopDoltServerCleanup to find — the post-run sweep +// named the metrics fixture's repo by exactly that cwd. proxy.Shutdown is +// the verified stop `bd dolt stop` performs and covers both records. A root +// that never started (the embedded and per-project cases of the same table) +// has neither record and is not an error. +func stopProxiedServerCleanup(t *testing.T, proxyRoot string) { + t.Helper() + t.Cleanup(func() { + recorded := false + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(proxyRoot, name)); err == nil { + recorded = true + } + } + if !recorded { + return + } + if err := proxy.Shutdown(proxyRoot); err != nil { + t.Errorf("proxy.Shutdown(%s): %v", proxyRoot, err) + } + }) +} + +const ( + // sharedServerExitTimeout is how long the cleanup waits for the shared + // dolt sql-server to leave the process table. doltserver.Stop's own + // gracefulStop budget is 5s, so this only ever elapses when the stop did + // not take — the outcome worth reporting. + sharedServerExitTimeout = 30 * time.Second + sharedServerExitPoll = 50 * time.Millisecond +) + +// requireSharedServerExited fails the test if the recorded shared server is +// still running after doltserver.Stop, then force-kills it so the leak does +// not outlive the run. +// +// t.Errorf, not t.Logf: a surviving server holds the temp HOME the suite is +// about to delete, and reporting it where it is caused is the whole point — +// the previous machinery printed a stderr line and exited 0, which is how the +// same leak was fixed three times and rediscovered from a process table +// (wy-j2zc8q). +func requireSharedServerExited(t *testing.T, pid int, token procid.Token) { + t.Helper() + deadline := time.Now().Add(sharedServerExitTimeout) + for { + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pid, err) + return + } + if !same { + // Gone, or the PID now belongs to something else. Either way + // our server is not running. + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(sharedServerExitPoll) + } + + handle, err := procid.Open(pid, token) + if err != nil { + if sharedServerStillRunning(t, pid, token) { + t.Errorf("shared dolt sql-server pid %d survived Stop by more than %s and could not be opened to kill: %v", + pid, sharedServerExitTimeout, err) + } + return + } + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + if sharedServerStillRunning(t, pid, token) { + t.Errorf("shared dolt sql-server pid %d survived Stop by more than %s and could not be killed: %v", + pid, sharedServerExitTimeout, killErr) + } + return + } + t.Errorf("shared dolt sql-server pid %d survived Stop by more than %s (force-killed)", + pid, sharedServerExitTimeout) +} + +// sharedServerStillRunning re-checks birth identity after procid.Open or +// Handle.Kill failed. Both verify the token themselves and return a plain +// "does not match token" error when it no longer does — exactly what a +// process that exited between the last poll and the kill attempt produces. +// Without this second look, a server that shut down a few milliseconds late +// would be reported as one that survived Stop entirely. +func sharedServerStillRunning(t *testing.T, pid int, token procid.Token) bool { + t.Helper() + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d) after a failed kill attempt: %v", pid, err) + return false + } + return same +} diff --git a/cmd/bd/suite_temp_test.go b/cmd/bd/suite_temp_test.go new file mode 100644 index 000000000..9d5aa3836 --- /dev/null +++ b/cmd/bd/suite_temp_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Guard the actual TestMain wiring: both APIs must allocate fixtures under +// the owned root or a failed fixture cleanup can leave a green, leaking suite. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + root := testTempRoot + if root == "" || root == "." { + t.Fatal("TestMain did not pin its suite temp root") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, root) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, root) + } + if testutil.PathUnderSuiteRoot(dir, os.Getenv("HOME")) { + t.Fatal("fixture directories must not inherit the suite's ~/.beads state") + } +} diff --git a/cmd/bd/test_repo_beads_guard_darwin_test.go b/cmd/bd/test_repo_beads_guard_darwin_test.go index a1621dd70..2181fd405 100644 --- a/cmd/bd/test_repo_beads_guard_darwin_test.go +++ b/cmd/bd/test_repo_beads_guard_darwin_test.go @@ -9,6 +9,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/steveyegge/beads/internal/doltserver" ) type sweepTestRunner struct { @@ -19,47 +21,72 @@ func (r sweepTestRunner) Run() int { return r.run() } +// TestRunTestsAndSweepReapsOrphanedServer pins both halves of the leak +// contract on a suite whose tests all pass but which leaves a detached +// `dolt sql-server` behind: the sweep always reaps it, and the reaped orphan +// now FAILS the run (exit 1) unless doltserver.AllowLeakEnv is set to "1". +// +// The failure is the point (wy-j2zc8q). Until it landed, a swept leak was a +// stderr line on an exit-0 run, so all three previous fixes for this leak +// (wy-9byjk, wy-5ce39p, and this one) rode a green CI for weeks and the +// regression was rediscovered from a dev box's process table instead. func TestRunTestsAndSweepReapsOrphanedServer(t *testing.T) { - oldRoot := testTempRoot - testTempRoot = t.TempDir() - t.Cleanup(func() { testTempRoot = oldRoot }) - - serverDir := filepath.Join(testTempRoot, "orphan", ".beads", "dolt") - if err := os.MkdirAll(serverDir, 0o755); err != nil { - t.Fatalf("mkdir server dir: %v", err) - } - script := filepath.Join(testTempRoot, "dolt") - if err := os.WriteFile(script, []byte("#!/bin/sh\ntrap 'exit 0' TERM\nwhile :; do sleep 1; done\n"), 0o755); err != nil { - t.Fatalf("write fake dolt: %v", err) + cases := []struct { + name string + allowEnv string + want int + }{ + {name: "a swept leak fails an otherwise passing suite", allowEnv: "", want: 1}, + {name: "the opt-out downgrades the leak to a warning", allowEnv: "1", want: 0}, } - var cmd *exec.Cmd - runner := sweepTestRunner{run: func() int { - cmd = exec.Command(script, "sql-server") - cmd.Dir = serverDir - if err := cmd.Start(); err != nil { - t.Fatalf("start fake dolt sql-server: %v", err) - } - t.Cleanup(func() { _ = cmd.Process.Kill() }) - return 0 - }} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Both cases must reap; only the exit code differs. + t.Setenv(doltserver.AllowLeakEnv, tc.allowEnv) - if code := runTestsAndSweep(runner); code != 0 { - t.Fatalf("runTestsAndSweep() = %d, want 0", code) - } - if cmd == nil || cmd.Process == nil { - t.Fatal("runner did not start fake dolt sql-server") - } + oldRoot := testTempRoot + testTempRoot = t.TempDir() + t.Cleanup(func() { testTempRoot = oldRoot }) - waitDone := make(chan error, 1) - go func() { waitDone <- cmd.Wait() }() - select { - case err := <-waitDone: - var exitErr *exec.ExitError - if err != nil && !errors.As(err, &exitErr) { - t.Fatalf("fake dolt sql-server exited unexpectedly: %v", err) - } - case <-time.After(3 * time.Second): - t.Fatal("runTestsAndSweep left fake dolt sql-server running") + serverDir := filepath.Join(testTempRoot, "orphan", ".beads", "dolt") + if err := os.MkdirAll(serverDir, 0o755); err != nil { + t.Fatalf("mkdir server dir: %v", err) + } + script := filepath.Join(testTempRoot, "dolt") + if err := os.WriteFile(script, []byte("#!/bin/sh\ntrap 'exit 0' TERM\nwhile :; do sleep 1; done\n"), 0o755); err != nil { + t.Fatalf("write fake dolt: %v", err) + } + + var cmd *exec.Cmd + runner := sweepTestRunner{run: func() int { + cmd = exec.Command(script, "sql-server") + cmd.Dir = serverDir + if err := cmd.Start(); err != nil { + t.Fatalf("start fake dolt sql-server: %v", err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + return 0 + }} + + if code := runTestsAndSweep(runner); code != tc.want { + t.Fatalf("runTestsAndSweep() = %d, want %d", code, tc.want) + } + if cmd == nil || cmd.Process == nil { + t.Fatal("runner did not start fake dolt sql-server") + } + + waitDone := make(chan error, 1) + go func() { waitDone <- cmd.Wait() }() + select { + case err := <-waitDone: + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + t.Fatalf("fake dolt sql-server exited unexpectedly: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("runTestsAndSweep left fake dolt sql-server running") + } + }) } } diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 5149084b8..b43c58221 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -51,10 +51,14 @@ type testRunner interface { func runTestsAndSweep(m testRunner) int { code := m.Run() - doltserver.SweepOrphanedTestServers(testTempRoot) - return code + swept := doltserver.SweepSuiteTestServers(testTempRoot) + return doltserver.ApplyLeakPolicy("cmd/bd", code, swept) } +// suiteRootPrefix is testMainInner's PinSuiteTempRoot pattern without its random +// tail. It is what SweepDeadSuiteRoots globs for, so the two must not drift. +const suiteRootPrefix = "beads-bd-tests-" + // Guardrail: ensure the cmd/bd test suite does not touch the real repo .beads state. // Disable with BEADS_TEST_GUARD_DISABLE=1 (useful when running tests while actively using beads). func TestMain(m *testing.M) { @@ -69,13 +73,27 @@ func testMainInner(m *testing.M) int { // Many tests expect default config values; running from within this repo would // cause config.Initialize() to walk up from CWD and load `.beads/config.yaml`, // which may set non-default config values and makes tests assert the wrong behavior. - tmp, err := os.MkdirTemp("", "beads-bd-tests-*") + // Before claiming a root of our own, clear out the roots of EARLIER runs + // of this suite whose process is gone — a `go test -timeout` panic skips + // both the defer below and the post-Run sweep, so the servers those runs + // started outlive every cleanup this process installs, and nothing else + // ever looks at a dead run's tree again (wy-j2zc8q). Roots with no owner + // marker, and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + tmp, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "failed to create temp dir: %v\n", err) return 1 } defer func() { _ = forceRemoveAll(tmp) }() + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(tmp); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", tmp, err) + } + // Anchor package-level sync.Once builders (test binaries, isolated // HOMEs) under this directory so the defer above sweeps them up too. // Without this, those helpers leaked ~179MB-1.4GB per test run into @@ -108,10 +126,18 @@ func testMainInner(m *testing.M) int { // daemons like OrbStack (bd-84kos). testutil.PinDockerHostFromContext() - _ = os.Setenv("HOME", tmp) - _ = os.Setenv("USERPROFILE", tmp) // Windows compatibility - _ = os.Setenv("APPDATA", filepath.Join(tmp, "AppData", "Roaming")) - _ = os.Setenv("XDG_CONFIG_HOME", filepath.Join(tmp, "xdg-config")) + // Keep HOME beside the fixture directories, not above them. Tests may + // create ~/.beads; putting it on their ancestry would make repository + // discovery pick up unrelated suite state before trying worktree fallback. + home := filepath.Join(tmp, "home") + if err := os.Mkdir(home, 0o700); err != nil { + fmt.Fprintf(os.Stderr, "failed to create test home: %v\n", err) + return 1 + } + _ = os.Setenv("HOME", home) + _ = os.Setenv("USERPROFILE", home) // Windows compatibility + _ = os.Setenv("APPDATA", filepath.Join(home, "AppData", "Roaming")) + _ = os.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "xdg-config")) _ = os.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") // Keep telemetry out of the test suite entirely (wy-12x1p). diff --git a/internal/beads/suite_temp_test.go b/internal/beads/suite_temp_test.go new file mode 100644 index 000000000..bb1c110e6 --- /dev/null +++ b/internal/beads/suite_temp_test.go @@ -0,0 +1,28 @@ +package beads + +import ( + "os" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Guard the actual TestMain wiring: both APIs must allocate fixtures under +// the owned root or a failed fixture cleanup can leave a green, leaking suite. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + root := suiteTempRoot + if root == "" || root == "." { + t.Fatal("TestMain did not pin its suite temp root") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, root) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, root) + } +} diff --git a/internal/beads/testmain_test.go b/internal/beads/testmain_test.go index 1ee0ace29..538b2a3ee 100644 --- a/internal/beads/testmain_test.go +++ b/internal/beads/testmain_test.go @@ -7,15 +7,38 @@ import ( "testing" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" ) +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-internal-tests-" + +var suiteTempRoot string + func TestMain(m *testing.M) { - root, err := os.MkdirTemp("", "beads-internal-tests-*") + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every cleanup below AND the post-run sweep, so the servers such a run + // started outlive everything this process installs and nothing ever looks + // at that run's tree again (wy-j2zc8q). Roots with no owner marker, and + // roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "failed to create test temp dir: %v\n", err) os.Exit(1) } + suiteTempRoot = root + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + home := filepath.Join(root, "home") if err := os.MkdirAll(home, 0700); err != nil { fmt.Fprintf(os.Stderr, "failed to create test home dir: %v\n", err) @@ -47,7 +70,8 @@ func TestMain(m *testing.M) { // suite's temp root (e.g. auto-started by the BEADS_TEST_BD_BINARY // this TestMain builds, if a SIGKILLed run left one behind) — see // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("internal/beads", code, swept) integrationCleanup() _ = os.RemoveAll(root) diff --git a/internal/doltserver/socket_integration_test.go b/internal/doltserver/socket_integration_test.go index 56608bdae..25683cdd9 100644 --- a/internal/doltserver/socket_integration_test.go +++ b/internal/doltserver/socket_integration_test.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "syscall" "testing" "time" @@ -24,7 +25,9 @@ import ( func TestUnixSocket_ConnectAndQuery(t *testing.T) { doltBin := integration.RequireDolt(t) - tmpDir := t.TempDir() + // Exercise data paths longer than either Linux's or macOS's socket-path + // budget, as happens with a long TMPDIR plus a suite-owned root. + tmpDir := filepath.Join(t.TempDir(), strings.Repeat("long-", 24)) beadsDir := filepath.Join(tmpDir, ".beads") doltDir := filepath.Join(beadsDir, "dolt") if err := os.MkdirAll(doltDir, 0700); err != nil { @@ -40,7 +43,15 @@ func TestUnixSocket_ConnectAndQuery(t *testing.T) { t.Fatalf("dolt init: %v\n%s", err, out) } - socketPath := filepath.Join(tmpDir, "dolt.sock") + // Keep only the socket outside the owned data root: Unix socket paths + // have a small fixed budget (104 bytes on macOS), independent of TMPDIR. + // The private directory is removed after the server cleanup registered below. + socketDir, err := os.MkdirTemp("/tmp", "bd-sock-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(socketDir) }) + socketPath := filepath.Join(socketDir, "dolt.sock") portListener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("allocate TCP port: %v", err) diff --git a/internal/doltserver/sweep.go b/internal/doltserver/sweep.go index 88026a6dd..9c7d5663d 100644 --- a/internal/doltserver/sweep.go +++ b/internal/doltserver/sweep.go @@ -1,6 +1,9 @@ package doltserver import ( + "os" + "path/filepath" + "runtime" "strconv" "strings" ) @@ -20,14 +23,44 @@ type serverCandidate struct { cwdDeleted bool } -// selectOrphanTestServerPIDs returns the PIDs of candidates that are safe to -// reap as leaked test debris. A candidate qualifies only when its cmdline -// names a dolt sql-server AND either: +// SweptServer names one leaked `dolt sql-server` a sweep selected: the PID it +// signaled and the working directory that process was serving. // -// - its working directory has been deleted (the temp dir it was serving -// no longer exists — this is the signature of a SIGKILLed test run -// whose t.TempDir() cleanup ran on top of a still-live server), or -// - its working directory sits under one of suiteRoots. +// The cwd is the whole reason this type exists. A sweep report of bare PIDs +// ("swept 1 orphaned test dolt sql-server process(es): [21881]") says a +// fixture leaked but not WHICH one, and the PID is gone by the time anyone +// reads the log — so a CI failure on a package with hundreds of tests left +// the reader nothing to grep for. A leaked server's cwd is the .beads/dolt +// directory under the suite's own temp tree, and Go names those after the +// test that made them (a t.TempDir() is /TestSomeName1234/001), so the +// directory identifies the leaking test directly. +type SweptServer struct { + // PID is the process that was signaled. + PID int + // Cwd is the working directory the process was serving. Never empty for + // a selected server: selectServers skips candidates with no known cwd, + // because unknown is not provably debris. + Cwd string +} + +// String renders a swept server as " cwd=", which is what the +// sweep's stderr lines print for each entry (fmt applies this to every +// element of a []SweptServer formatted with %v). +func (s SweptServer) String() string { + if s.Cwd == "" { + return strconv.Itoa(s.PID) + } + return strconv.Itoa(s.PID) + " cwd=" + s.Cwd +} + +// selectOrphanTestServers returns the candidates that are safe to reap as +// leaked test debris, each carrying the working directory it was serving. A +// candidate qualifies only when its cmdline names a dolt sql-server AND +// either: +// +// - its working directory sits under one of suiteRoots, or +// - its working directory has been deleted AND the path it used to name +// sits under one of tempRoots. // // suiteRoots MUST be directories owned by the calling test suite alone // (e.g. that suite's own testTempRoot) — never a shared/global temp dir @@ -38,28 +71,204 @@ type serverCandidate struct { // data dirs live somewhere under os.TempDir() too. Passing a global root // here would turn this safety net into a cross-suite server killer. // +// tempRoots (see tempDirRoots) bounds the deleted-cwd arm to throwaway +// directories. A deleted working directory is a strong leak signal — a +// t.TempDir() cleanup ran on top of a still-live detached server — but it is +// NOT by itself proof of a TEST server: a production server is spawned with +// cmd.Dir = /.beads/dolt, so a developer who moved or deleted that +// workspace, or whose external volume was unmounted, would have their live +// server reaped by any test run on the box. Requiring the deleted path to +// have been under a temp dir keeps the arm pointed at test debris only. +// // This is intentionally conservative in the "never kill production" sense: -// a real shared server's data directory is a persistent, non-temp path that -// still exists and is never one of a test suite's own scoped roots, so it -// matches neither condition and is left alone. -func selectOrphanTestServerPIDs(candidates []serverCandidate, suiteRoots []string) []int { - var pids []int - for _, c := range candidates { - if !isDoltServerCmdline(c.cmdline) { - continue - } - if c.cwdDeleted { - pids = append(pids, c.pid) - continue - } - if c.cwd == "" { - continue - } +// a real shared server's data directory is a persistent, non-temp path, so it +// is neither under a suite's scoped roots nor — deleted or not — under a temp +// root, and matches neither condition. +func selectOrphanTestServers(candidates []serverCandidate, suiteRoots, tempRoots []string) []SweptServer { + return selectServers(candidates, func(c serverCandidate) bool { if underAnyRoot(c.cwd, suiteRoots) { - pids = append(pids, c.pid) + return true + } + return c.cwdDeleted && underAnyRoot(c.cwd, tempRoots) + }) +} + +// selectServersUnderRoots returns the candidates whose working directory sits +// under one of roots, and nothing else. It is the strictly +// root-scoped selection: no deleted-cwd arm, so it can never reach a process +// outside the caller's own trees. +// +// SweepDeadSuiteRoots uses it because that sweep runs at suite START, while +// sibling packages (go test -p N) are mid-run: whatever it reaps must be +// provably inside the one dead root it is cleaning up, never merely +// "somewhere temporary". +func selectServersUnderRoots(candidates []serverCandidate, roots []string) []SweptServer { + return selectServers(candidates, func(c serverCandidate) bool { + return underAnyRoot(c.cwd, roots) + }) +} + +// selectServers applies want to every candidate that is a dolt sql-server +// with a known working directory, returning the survivors as SweptServers. A +// candidate whose cwd could not be resolved is always skipped: unknown is not +// provably debris — which is also why every SweptServer it returns has a +// non-empty Cwd for the sweep's report to name. +func selectServers(candidates []serverCandidate, want func(serverCandidate) bool) []SweptServer { + var selected []SweptServer + for _, c := range candidates { + if !isDoltServerCmdline(c.cmdline) || c.cwd == "" { + continue + } + if want(c) { + selected = append(selected, SweptServer{PID: c.pid, Cwd: c.cwd}) } } - return pids + return selected +} + +// tempDirRoots is the set of directories under which a deleted working +// directory is credible evidence of leaked TEST debris rather than a moved +// production workspace: the process temp dir (honoring TMPDIR, which the +// suites pin to their own root) and /tmp, which os.MkdirTemp uses when TMPDIR +// is unset and which several suites hardcode. +// +// Roots too broad to be evidence of anything are dropped — see +// isCredibleTempRoot. TMPDIR is an environment variable, so "/" or a home +// directory can land here, and a root that broad would restore exactly the +// unbounded deleted-cwd arm this bound exists to remove. +func tempDirRoots() []string { + home, err := os.UserHomeDir() + if err != nil { + home = "" + } + var roots []string + for _, root := range canonicalRoots([]string{os.TempDir(), "/tmp"}) { + if !isCredibleTempRoot(root, home) { + continue + } + roots = append(roots, root) + } + return roots +} + +// isCredibleTempRoot reports whether root is narrow enough to bound the +// deleted-cwd arm. It rejects the filesystem root, a relative or empty path, +// and any directory that IS the user's home or contains it — "/", "/Users", +// "/home", $HOME itself. Everything a real temp dir looks like (/tmp, +// /private/tmp, /var/folders/xx/yy/T, a pinned suite root) passes. +// +// The home test doubles as the breadth test: a directory holding the user's +// home holds their workspaces too, so a deleted .beads/dolt under it would +// again look like test debris. +// +// The one home that anchors no workspaces is a SANDBOX home: CI harnesses +// (scripts/ci/lib/test-env.sh) export HOME under a mktemp -d root so a test +// can never read or write the runner's real dotfiles, and several suites pin +// HOME to a t.TempDir() for the same reason. Such a home lives under a fixed +// system temp location (isSandboxHome), and without this carve-out it would +// disqualify that location itself, leaving tempDirRoots empty and the +// deleted-cwd arm silently disabled on exactly the boxes whose killed runs it +// exists to clean up after. A root that merely contains a sandbox home stays +// credible; a root that IS the home does not, whatever the home looks like. +func isCredibleTempRoot(root, home string) bool { + cleaned := filepath.Clean(root) + if cleaned == "" || cleaned == "." || cleaned == string(filepath.Separator) { + return false + } + if !filepath.IsAbs(cleaned) { + return false + } + if home == "" { + return true + } + for _, h := range canonicalRoots([]string{home}) { + h = filepath.Clean(h) + if h == cleaned { + return false + } + if isUnderDir(h, cleaned) && !isSandboxHome(h) { + return false + } + } + return true +} + +// isSandboxHome reports whether home lives inside a FIXED system temp +// location — the shape a test harness's throwaway HOME takes. It is judged +// against hardcoded locations, never os.TempDir(), so TMPDIR cannot vote on +// its own credibility: TMPDIR=/home with HOME=/home/runner must still read as +// a real home under an overbroad root (TestTempDirRootsRejectsOverbroadTMPDIR). +func isSandboxHome(home string) bool { + return isUnderFixedTempRoots(home, runtime.GOOS) +} + +// isUnderFixedTempRoots reports whether path lies under one of the temp +// locations goos puts at a FIXED, non-configurable place. goos is a parameter +// rather than a read of runtime.GOOS so the platform table is testable from +// any platform (TestFixedTempRootsAreCredibleSandboxHomes). +// +// The list is deliberately hardcoded — see isSandboxHome — and canonicalRoots +// expands each entry so macOS's /private/var/folders form matches too. +func isUnderFixedTempRoots(path, goos string) bool { + return underAnyRoot(path, canonicalRoots(fixedTempRoots(goos))) +} + +// fixedTempRoots is the platform table behind isUnderFixedTempRoots. +// +// /tmp is everywhere: it is os.MkdirTemp's fallback when TMPDIR is unset, and +// CI harnesses (scripts/ci/lib/test-env.sh) mktemp -d their sandbox HOME under +// it. macOS additionally gives every user a per-user temp tree under +// /var/folders and points TMPDIR at it, so a suite that pins HOME to a +// t.TempDir() (cmd/bd/test_repo_beads_guard_test.go, internal/beads/testmain_test.go) +// puts HOME under /var/folders/xx/yy/T — which without this entry disqualified +// os.TempDir() itself and left the deleted-cwd arm inert on every Mac +// (wy-j2zc8q). +// +// The darwin row spells out the symlink-RESOLVED forms ("/private/tmp", +// "/private/var/folders") literally instead of leaving them to canonicalRoots. +// canonicalRoots resolves with filepath.EvalSymlinks, which reads the HOST's +// filesystem: on a Mac it turns /tmp and /var/folders into their /private/… +// targets, and on a Linux runner it adds nothing at all. But this table is a +// claim ABOUT darwin that any platform may be asked to judge — the platform +// row is pinned from Linux CI by TestSandboxHomeUnderPerUserTempRoot, and a +// Mac's lsof reports cwds in the /private/… form — so the answer must not +// depend on where the judging happens. canonicalRoots dedups, so on a real +// Mac these literals cost nothing: they are exactly what it would have added. +func fixedTempRoots(goos string) []string { + roots := []string{"/tmp"} + if goos == "darwin" { + roots = append(roots, "/private/tmp", "/var/folders", "/private/var/folders") + } + return roots +} + +// canonicalRoots expands each non-empty root into every form a process's +// working directory may be reported in: the path as given and, when it +// differs, its symlink-resolved form. macOS is why — os.MkdirTemp hands back +// /var/folders/… while lsof reports the /private/var/folders/… that symlink +// points at — but /tmp is a symlink to /private/tmp there too, and on Linux +// /tmp can be a symlink as well. Roots that cannot be resolved are kept +// as-is; duplicates are dropped. +func canonicalRoots(roots []string) []string { + var out []string + seen := make(map[string]bool, len(roots)*2) + add := func(path string) { + if path == "" || seen[path] { + return + } + seen[path] = true + out = append(out, path) + } + for _, root := range roots { + if root == "" { + continue + } + add(root) + if resolved, err := filepath.EvalSymlinks(root); err == nil { + add(resolved) + } + } + return out } // isDoltServerCmdline reports whether cmdline looks like a dolt sql-server diff --git a/internal/doltserver/sweep_darwin.go b/internal/doltserver/sweep_darwin.go index 86a69af1b..244a8e325 100644 --- a/internal/doltserver/sweep_darwin.go +++ b/internal/doltserver/sweep_darwin.go @@ -3,14 +3,10 @@ package doltserver import ( - "fmt" "os" "os/exec" - "path/filepath" "strconv" "strings" - "syscall" - "time" ) // SweepOrphanedTestServers reaps `dolt sql-server` processes that are @@ -22,63 +18,28 @@ import ( // a shared/global temp directory. This is best-effort: process-listing errors // and candidates whose cwd cannot be resolved are ignored. // -// Returns the PIDs it sent a kill signal to. -func SweepOrphanedTestServers(suiteTempRoots ...string) []int { +// Deprecated: use SweepSuiteTestServers for suite shutdown and +// SweepDeadSuiteRoots for abandoned runs. This global sweep can consume leak +// evidence belonging to a still-running foreign suite; do not call it from +// TestMain or pass its result to ApplyLeakPolicy. +// +// Returns the servers (pid + cwd) it sent a kill signal to. +func SweepOrphanedTestServers(suiteTempRoots ...string) []SweptServer { candidates := gatherDoltServerCandidates() - pids := selectOrphanTestServerPIDs(candidates, canonicalDarwinRoots(suiteTempRoots)) - - self := os.Getpid() - var killed []int - for _, pid := range pids { - if pid == self { - continue - } - // Re-read the process command immediately before signaling so a PID - // recycled since candidate collection cannot target an unrelated - // process. - if !isDoltServerProcess(pid) { - continue - } - if err := syscall.Kill(pid, syscall.SIGTERM); err == nil { - killed = append(killed, pid) - } - } - - if len(killed) == 0 { - return killed - } - - fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed) - - time.Sleep(300 * time.Millisecond) - for _, pid := range killed { - // Revalidate again before escalation for the same PID-reuse reason. - if !isDoltServerProcess(pid) { - continue - } - _ = syscall.Kill(pid, syscall.SIGKILL) - } - - return killed + selected := selectOrphanTestServers(candidates, canonicalRoots(suiteTempRoots), tempDirRoots()) + return reapServers(selected, isDoltServerProcess) } -// canonicalDarwinRoots makes caller roots comparable with lsof output. macOS -// commonly reports a cwd below /private/var while os.MkdirTemp returned the -// equivalent /var path. -func canonicalDarwinRoots(roots []string) []string { - canonical := make([]string, 0, len(roots)) - for _, root := range roots { - if root == "" { - continue - } - resolved, err := filepath.EvalSymlinks(root) - if err != nil { - canonical = append(canonical, root) - continue - } - canonical = append(canonical, resolved) - } - return canonical +// sweepServersUnderRoots reaps only the dolt sql-servers whose working +// directory sits under one of suiteTempRoots. It is SweepOrphanedTestServers +// without the deleted-cwd arm, for callers that must not reach outside the +// trees they name — see selectServersUnderRoots. +// +// Returns the servers (pid + cwd) it sent a kill signal to. +func sweepServersUnderRoots(suiteTempRoots ...string) []SweptServer { + candidates := gatherDoltServerCandidates() + selected := selectServersUnderRoots(candidates, canonicalRoots(suiteTempRoots)) + return reapServers(selected, isDoltServerProcess) } func gatherDoltServerCandidates() []serverCandidate { @@ -97,6 +58,10 @@ func isDoltServerProcess(pid int) bool { return isDoltServerCmdline(strings.TrimSpace(string(out))) } +// statFunc matches os.Stat. parseDarwinCwd takes one so its +// deleted-directory probe can be exercised without a live process. +type statFunc func(string) (os.FileInfo, error) + // readDarwinCwd resolves pid's cwd from lsof's machine-readable field output. // lsof emits the name as an `n` field after selecting descriptor `cwd`. func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { @@ -106,7 +71,27 @@ func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { if err != nil { return "", false, false } - for _, line := range strings.Split(string(out), "\n") { + return parseDarwinCwd(out, os.Stat) +} + +// parseDarwinCwd extracts the working directory from `lsof -Fn` output and +// decides whether that directory still exists. +// +// Unlike Linux's /proc//cwd symlink, macOS lsof does NOT append a +// " (deleted)" marker when the directory a process is sitting in has been +// unlinked — it prints the bare, now-dangling path. The suffix is still +// honored (harmless, and it keeps this parser shaped like readProcCwd), but +// when it is absent the path is stat'ed and ENOENT is the deletion signal. +// Without that probe the deleted-cwd arm of selectOrphanTestServers could +// never fire on darwin, so a `go test -timeout` panic — which skips every +// t.Cleanup and TestMain defer — left its dolt sql-server running forever +// even though the temp tree it served was long gone (wy-j2zc8q). +// +// Any other stat error (a permission wall, an unreadable parent) is treated +// as "not deleted": this sweep only ever escalates on positive proof, never +// on a failed read. +func parseDarwinCwd(lsofOutput []byte, stat statFunc) (cwd string, deleted bool, ok bool) { + for _, line := range strings.Split(string(lsofOutput), "\n") { if !strings.HasPrefix(line, "n") { continue } @@ -115,9 +100,13 @@ func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { if strings.HasSuffix(cwd, deletedSuffix) { return strings.TrimSuffix(cwd, deletedSuffix), true, true } - if cwd != "" { - return cwd, false, true + if cwd == "" { + continue } + if _, err := stat(cwd); err != nil && os.IsNotExist(err) { + return cwd, true, true + } + return cwd, false, true } return "", false, false } diff --git a/internal/doltserver/sweep_darwin_test.go b/internal/doltserver/sweep_darwin_test.go new file mode 100644 index 000000000..87e419985 --- /dev/null +++ b/internal/doltserver/sweep_darwin_test.go @@ -0,0 +1,120 @@ +//go:build darwin + +package doltserver + +import ( + "io/fs" + "os" + "testing" +) + +// TestParseDarwinCwdOrphanDetection pins the darwin-only half of the +// deleted-cwd signal. macOS lsof prints the bare path of a working directory +// that has since been removed — no " (deleted)" suffix, unlike Linux's +// /proc//cwd — so the stat probe is the only thing that lets +// selectOrphanTestServerPIDs recognize a leaked test server on this platform +// (wy-j2zc8q). +func TestParseDarwinCwdOrphanDetection(t *testing.T) { + const livePath = "/private/var/folders/xx/T/some-suite/002" + const gonePath = "/private/var/folders/xx/T/TestUOWDependencyEditorContract1/002" + + statLive := func(string) (os.FileInfo, error) { return nil, nil } + statGone := func(string) (os.FileInfo, error) { + return nil, &fs.PathError{Op: "stat", Path: gonePath, Err: fs.ErrNotExist} + } + statDenied := func(string) (os.FileInfo, error) { + return nil, &fs.PathError{Op: "stat", Path: livePath, Err: fs.ErrPermission} + } + statPanics := func(string) (os.FileInfo, error) { + t.Helper() + t.Fatal("stat must not be called when lsof already reported the directory deleted") + return nil, nil + } + + cases := []struct { + name string + lsofOutput string + stat statFunc + wantCwd string + wantDeleted bool + wantOK bool + }{ + { + name: "existing directory is live", + lsofOutput: "p123\nn" + livePath + "\n", + stat: statLive, + wantCwd: livePath, + wantOK: true, + }, + { + name: "missing directory is the leak signature even with no suffix", + lsofOutput: "p123\nn" + gonePath + "\n", + stat: statGone, + wantCwd: gonePath, + wantDeleted: true, + wantOK: true, + }, + { + name: "explicit (deleted) suffix still wins without touching the filesystem", + lsofOutput: "p123\nn" + gonePath + " (deleted)\n", + stat: statPanics, + wantCwd: gonePath, + wantDeleted: true, + wantOK: true, + }, + { + name: "a stat error that is not ENOENT is never read as deleted", + lsofOutput: "p123\nn" + livePath + "\n", + stat: statDenied, + wantCwd: livePath, + wantOK: true, + }, + { + name: "no n field at all is unknown, not deleted", + lsofOutput: "p123\nfcwd\n", + stat: statPanics, + wantOK: false, + }, + { + name: "an empty n field is skipped in favor of the next one", + lsofOutput: "p123\nn\nn" + livePath + "\n", + stat: statLive, + wantCwd: livePath, + wantOK: true, + }, + { + name: "empty output is unknown", + lsofOutput: "", + stat: statPanics, + wantOK: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cwd, deleted, ok := parseDarwinCwd([]byte(tc.lsofOutput), tc.stat) + if cwd != tc.wantCwd || deleted != tc.wantDeleted || ok != tc.wantOK { + t.Errorf("parseDarwinCwd() = (%q, %v, %v), want (%q, %v, %v)", + cwd, deleted, ok, tc.wantCwd, tc.wantDeleted, tc.wantOK) + } + }) + } +} + +// TestParseDarwinCwdAgainstRealFilesystem exercises the same path with the +// real os.Stat, so the ENOENT classification is checked against the kernel +// rather than a hand-written error value. +func TestParseDarwinCwdAgainstRealFilesystem(t *testing.T) { + live := t.TempDir() + gone := t.TempDir() + if err := os.RemoveAll(gone); err != nil { + t.Fatalf("RemoveAll(%s): %v", gone, err) + } + + if _, deleted, ok := parseDarwinCwd([]byte("n"+live+"\n"), os.Stat); !ok || deleted { + t.Errorf("live dir %s: deleted=%v ok=%v, want deleted=false ok=true", live, deleted, ok) + } + if cwd, deleted, ok := parseDarwinCwd([]byte("n"+gone+"\n"), os.Stat); !ok || !deleted || cwd != gone { + t.Errorf("removed dir %s: cwd=%q deleted=%v ok=%v, want cwd=%s deleted=true ok=true", gone, cwd, deleted, ok, gone) + } +} diff --git a/internal/doltserver/sweep_linux.go b/internal/doltserver/sweep_linux.go index 6764b8ccb..0056646a6 100644 --- a/internal/doltserver/sweep_linux.go +++ b/internal/doltserver/sweep_linux.go @@ -3,94 +3,39 @@ package doltserver import ( - "fmt" "os" "path/filepath" "strconv" "strings" - "syscall" - "time" ) -// SweepOrphanedTestServers reaps `dolt sql-server` processes that are -// provably leaked test debris: their working directory has been deleted, or -// sits under one of suiteTempRoots. It is meant to be called once, from a -// test suite's TestMain, after m.Run() returns — a backstop for servers -// that survived an interrupted (e.g. SIGKILLed) test run despite the -// test-mode Pdeathsig in procattr_linux.go. +// SweepOrphanedTestServers reaps dolt sql-servers under suiteTempRoots and +// deleted-cwd servers under credible global temp roots. A root must belong +// exclusively to the caller, never be a shared/global temp directory. // -// suiteTempRoots MUST be directories owned by (anchored under) the calling -// suite alone — e.g. a package's own testTempRoot — never a shared/global -// temp dir such as os.TempDir(). scripts/test.sh runs packages in parallel -// (-p N), so a global root would make every *other* suite's still-running -// server (whose data dir also happens to live under os.TempDir(), which is -// true of essentially all of them) look like debris and get SIGTERM'd -// mid-run. That is why this function never defaults to os.TempDir() itself: -// a live server is only ever reaped when its cwd is nested under a root the -// caller vouches for as its own. +// Deprecated: use SweepSuiteTestServers for suite shutdown and +// SweepDeadSuiteRoots for abandoned runs. This global sweep can consume leak +// evidence belonging to a still-running foreign suite; do not call it from +// TestMain or pass its result to ApplyLeakPolicy. // -// A server whose working directory has been deleted (cwdDeleted, see -// readProcCwd) is reaped unconditionally regardless of suiteTempRoots — -// that is the unambiguous leak signature (a t.TempDir() cleanup ran out -// from under a still-live detached server) and cannot occur for any -// server, from any suite, that is still legitimately in use. -// -// Safety is the whole point: this must never touch a developer's real -// shared server. It only reads /proc (no killing) to build the candidate -// list, and selectOrphanTestServerPIDs only matches processes whose data -// directory is gone or explicitly caller-scoped — a production server's -// data directory is neither. Errors reading /proc for any single PID just -// drop that PID from consideration; this function is best-effort and never -// returns an error itself. -// -// Returns the PIDs it sent a kill signal to. -func SweepOrphanedTestServers(suiteTempRoots ...string) []int { +// Returns the servers (pid + cwd) it sent a kill signal to. Process-listing +// errors and candidates whose cwd cannot be resolved are ignored. +func SweepOrphanedTestServers(suiteTempRoots ...string) []SweptServer { candidates := gatherDoltServerCandidates() - pids := selectOrphanTestServerPIDs(candidates, suiteTempRoots) + selected := selectOrphanTestServers(candidates, canonicalRoots(suiteTempRoots), tempDirRoots()) + return reapServers(selected, isDoltServerProcess) +} - self := os.Getpid() - var killed []int - for _, pid := range pids { - if pid == self { - continue - } - // Revalidate identity right before signaling: candidate selection - // above already did its own /proc read, and in a PID-reuse window - // the kernel could have recycled this PID to an unrelated process - // in between. isDoltServerProcess re-reads /proc//cmdline so - // we only ever signal something that still looks like the - // dolt sql-server we selected. - if !isDoltServerProcess(pid) { - continue - } - if err := syscall.Kill(pid, syscall.SIGTERM); err == nil { - killed = append(killed, pid) - } - } - - if len(killed) == 0 { - return killed - } - - fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed) - - // Give SIGTERM a moment, then force anything still alive. This runs at - // suite exit, so a short bounded wait here is acceptable. - time.Sleep(300 * time.Millisecond) - for _, pid := range killed { - // Revalidate again before escalating to SIGKILL: the original - // server may have exited cleanly during the grace period, and in - // a PID-reuse window (the kernel cycling the whole PID space - // within 300ms) this PID could now belong to an unrelated - // process. Recheck it still looks like a dolt sql-server before - // force-killing it. - if !isDoltServerProcess(pid) { - continue - } - _ = syscall.Kill(pid, syscall.SIGKILL) - } - - return killed +// sweepServersUnderRoots reaps only the dolt sql-servers whose working +// directory sits under one of suiteTempRoots. It is SweepOrphanedTestServers +// without the deleted-cwd arm, for callers that must not reach outside the +// trees they name — see selectServersUnderRoots. +// +// Returns the servers (pid + cwd) it sent a kill signal to. +func sweepServersUnderRoots(suiteTempRoots ...string) []SweptServer { + candidates := gatherDoltServerCandidates() + selected := selectServersUnderRoots(candidates, canonicalRoots(suiteTempRoots)) + return reapServers(selected, isDoltServerProcess) } // isDoltServerProcess re-reads /proc//cmdline and reports whether pid diff --git a/internal/doltserver/sweep_other.go b/internal/doltserver/sweep_other.go index 70f2f90b6..c4ff67e4a 100644 --- a/internal/doltserver/sweep_other.go +++ b/internal/doltserver/sweep_other.go @@ -5,6 +5,15 @@ package doltserver // SweepOrphanedTestServers is a no-op on platforms where process command // lines and working directories cannot be inspected by an implementation in // this package. The stub keeps callers (test TestMains) portable. -func SweepOrphanedTestServers(_ ...string) []int { +// +// Deprecated: use SweepSuiteTestServers for suite shutdown and +// SweepDeadSuiteRoots for abandoned runs. +func SweepOrphanedTestServers(_ ...string) []SweptServer { + return nil +} + +// sweepServersUnderRoots is the root-scoped sibling of +// SweepOrphanedTestServers and is a no-op here for the same reason. +func sweepServersUnderRoots(_ ...string) []SweptServer { return nil } diff --git a/internal/doltserver/sweep_reap_unix.go b/internal/doltserver/sweep_reap_unix.go new file mode 100644 index 000000000..cc7ab08f1 --- /dev/null +++ b/internal/doltserver/sweep_reap_unix.go @@ -0,0 +1,63 @@ +//go:build darwin || linux + +package doltserver + +import ( + "fmt" + "os" + "syscall" + "time" +) + +// reapServers SIGTERMs each selected server, then SIGKILLs whatever is still +// alive a moment later. isServer re-reads the process's identity and reports +// whether it still looks like a dolt sql-server; it is consulted immediately +// before EVERY signal, because the kernel could have recycled the PID onto an +// unrelated process since the candidate list was built (and again during the +// grace period below). The caller passes its platform's probe — /proc on +// linux, ps on darwin. +// +// This lives in a darwin||linux file rather than the portable sweep.go +// because the signaling itself is POSIX; the selection logic that decides +// WHICH servers get here is in sweep.go and is platform-independent. +// +// The Info line prints each entry as " cwd=" (SweptServer.String). +// The directory, not the PID, is what a reader can act on: it is the temp +// tree the leaked server was serving, which Go names after the test that +// created it, so the report identifies the leaking fixture instead of a +// number that no longer refers to anything (wy-j2zc8q). +// +// Returns the servers it sent a kill signal to. +func reapServers(servers []SweptServer, isServer func(int) bool) []SweptServer { + self := os.Getpid() + var killed []SweptServer + for _, server := range servers { + if server.PID == self { + continue + } + if !isServer(server.PID) { + continue + } + if err := syscall.Kill(server.PID, syscall.SIGTERM); err == nil { + killed = append(killed, server) + } + } + + if len(killed) == 0 { + return killed + } + + fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed) + + // Give SIGTERM a moment, then force anything still alive. This runs at a + // suite boundary, so a short bounded wait here is acceptable. + time.Sleep(300 * time.Millisecond) + for _, server := range killed { + if !isServer(server.PID) { + continue + } + _ = syscall.Kill(server.PID, syscall.SIGKILL) + } + + return killed +} diff --git a/internal/doltserver/sweep_reap_unix_test.go b/internal/doltserver/sweep_reap_unix_test.go new file mode 100644 index 000000000..4617e656a --- /dev/null +++ b/internal/doltserver/sweep_reap_unix_test.go @@ -0,0 +1,53 @@ +//go:build darwin || linux + +package doltserver + +import ( + "os/exec" + "reflect" + "strings" + "testing" + "time" +) + +// TestReapServersReportsTheLeakedDirectory pins the sweep's Info line against +// a real process. The line is the FIRST thing a reader sees when a suite +// leaks, and until it carried the reaped server's working directory it named +// only a PID that had already been killed — nothing to grep, nothing to fix +// (wy-j2zc8q). The cwd is the temp tree the server was serving, which Go +// names after the test that created it. +func TestReapServersReportsTheLeakedDirectory(t *testing.T) { + dir := t.TempDir() + cmd := exec.Command("/bin/sh", "-c", "trap 'exit 0' TERM; while :; do sleep 1; done") + cmd.Dir = dir + if err := cmd.Start(); err != nil { + t.Fatalf("start stand-in server: %v", err) + } + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + t.Cleanup(func() { _ = cmd.Process.Kill() }) + + server := SweptServer{PID: cmd.Process.Pid, Cwd: dir} + var killed []SweptServer + out := captureStderr(t, func() { + // The identity probe is stubbed: this stand-in is a shell, not a + // dolt binary, and the probe's own behavior is covered elsewhere. + killed = reapServers([]SweptServer{server}, func(int) bool { return true }) + }) + + if want := []SweptServer{server}; !reflect.DeepEqual(killed, want) { + t.Fatalf("reapServers() = %v, want %v", killed, want) + } + if !strings.Contains(out, "Info: swept 1 orphaned test dolt sql-server process(es)") { + t.Errorf("stderr = %q, want the Info line", out) + } + if want := "cwd=" + dir; !strings.Contains(out, want) { + t.Errorf("stderr = %q, want it to name the leaked directory %q", out, want) + } + + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("reapServers left the stand-in server running") + } +} diff --git a/internal/doltserver/sweep_suite.go b/internal/doltserver/sweep_suite.go new file mode 100644 index 000000000..abca5af75 --- /dev/null +++ b/internal/doltserver/sweep_suite.go @@ -0,0 +1,257 @@ +package doltserver + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// SuiteOwnerMarkerName is the file a test suite's TestMain writes at the root +// of the temp tree it owns, recording the PID of the process that created it. +// +// It exists so a LATER run can tell two indistinguishable-looking leftovers +// apart: the temp root of a sibling run that is alive right now (a parallel +// package under scripts/test.sh -p N, or a second `go test` in another +// terminal), and the temp root of a run whose process is gone — killed by +// `go test -timeout`, a CI cancel, or Ctrl-C — which skipped every defer and +// left its dolt sql-server behind (wy-j2zc8q). +const SuiteOwnerMarkerName = "testmain.pid" + +// AllowLeakEnv downgrades a post-run sweep that actually reaped something +// from a suite failure to a warning. Setting it to "1" is the ONLY way a +// leaked `dolt sql-server` leaves a green run. +// +// The default is the failure, not the warning. This leak has now been +// diagnosed and closed three times (wy-9byjk, wy-5ce39p, wy-j2zc8q) and every +// one of those fixes was landed alongside machinery that reported the leak on +// stderr and exited 0 — so each regression rode a green suite for weeks, and +// the leak was rediscovered from a dev box's process table rather than from +// CI. A reaped server means a test finished with a detached daemon still +// serving a directory the suite is about to delete; that is a defect in the +// suite, and the run should say so in the only channel anybody reads. +// +// The escape hatch exists for the bisect/triage case — someone who needs the +// tests' own exit code out of a package with a known leak — not as a lane +// default. +const AllowLeakEnv = "BEADS_TEST_ALLOW_LEAK" + +// WriteSuiteOwnerMarker records this process as the owner of root by writing +// SuiteOwnerMarkerName inside it. Call it from TestMain immediately after +// creating the suite's temp root; SweepDeadSuiteRoots in a later run reads it. +// +// A root with no marker is never swept, so failing to write one only costs +// the next run its chance to clean up — it is never unsafe. +func WriteSuiteOwnerMarker(root string) error { + return writeSuiteOwnerMarkerPID(root, os.Getpid()) +} + +// writeSuiteOwnerMarkerPID writes an owner marker naming an arbitrary PID. +// SweepDeadSuiteRoots uses it to put a marker BACK, naming the same dead +// owner, when it could not finish removing a root. +func writeSuiteOwnerMarkerPID(root string, pid int) error { + if root == "" { + return fmt.Errorf("doltserver: WriteSuiteOwnerMarker: empty root") + } + path := filepath.Join(root, SuiteOwnerMarkerName) + return os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o600) +} + +// readSuiteOwnerMarker returns the PID recorded in root's owner marker. +// ok is false when the marker is absent, unreadable, or does not hold a +// plausible PID — all of which mean "owner unknown", never "owner dead". +func readSuiteOwnerMarker(root string) (pid int, ok bool) { + // #nosec G304 -- the path is this package's own constant joined to a + // temp root the caller vouches for; nothing here comes from user input. + data, err := os.ReadFile(filepath.Join(root, SuiteOwnerMarkerName)) + if err != nil { + return 0, false + } + pid, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +// suiteRootAction is what SweepDeadSuiteRoots does with one candidate root. +type suiteRootAction int + +const ( + // leaveSuiteRoot means the root is not provably debris: either nobody + // claimed it (a pre-marker leftover, or another tool's directory that + // merely shares the prefix) or its owning process is still running. + leaveSuiteRoot suiteRootAction = iota + // sweepSuiteRoot means a run claimed this root and that run is gone, so + // anything still alive under it is leaked debris. + sweepSuiteRoot +) + +// decideSuiteRoot is the whole safety judgment of SweepDeadSuiteRoots, +// isolated from the filesystem and from process signaling so it can be +// tabled in a unit test. +// +// Only one combination reaps: a marker exists AND the PID it names is gone. +// "No marker" deliberately does NOT reap — a directory nobody claimed cannot +// be proven to be ours, and deleting it (plus SIGTERMing servers under it) +// on a guess is exactly the cross-suite killer sweep.go's contract forbids. +func decideSuiteRoot(hasMarker, ownerAlive bool) suiteRootAction { + if !hasMarker || ownerAlive { + return leaveSuiteRoot + } + return sweepSuiteRoot +} + +// SweepDeadSuiteRoots reaps the temp roots left behind by earlier runs of the +// calling suite whose owning process is dead: it kills the dolt sql-servers +// still running under each such root and removes the tree. +// +// parentDir is where the suite creates its roots (os.TempDir() at call time) +// and prefix is the suite's own os.MkdirTemp pattern minus the random tail +// (e.g. "beads-bd-tests-"). Both must be non-empty; an empty prefix would +// glob the entire temp directory and is refused. +// +// Call it from TestMain BEFORE creating this run's own root. It is +// best-effort and returns the roots it removed. +// +// Safety rests entirely on decideSuiteRoot: a root is only ever touched when +// it is owned by the current user, carries this suite's own owner marker, and +// that owner is gone. Live sibling runs (marker + live PID) and unclaimed +// leftovers (no marker) are both left exactly as they are. +// +// The per-root reap is strictly root-scoped. Sibling packages may still be +// running at either startup or shutdown, so neither sweep may consume their +// leak evidence. Abandoned-run cleanup is separate from this run's leak policy. +func SweepDeadSuiteRoots(parentDir, prefix string) []string { + return sweepDeadSuiteRoots(parentDir, prefix, processAlive, sweepServersUnderRoots, removeSuiteRoot) +} + +// removeSuiteRoot removes a suite temp root, making unwritable directories +// writable first. cmd/bd's root doubles as an isolated $HOME, so it can hold +// read-only Go module cache entries that plain os.RemoveAll refuses; this +// mirrors forceRemoveAll in cmd/bd's TestMain. +func removeSuiteRoot(root string) error { + _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() && info.Mode()&0o200 == 0 { + // #nosec G122 -- filepath.Walk lstats, so this only ever fires on + // a real directory (never a symlink), inside a root the caller + // already proved is owned by this user and abandoned by a dead + // process. Widening its owner-write bit is also the least + // consequential thing that could be done to a swapped path. + _ = os.Chmod(path, info.Mode()|0o200) + } + return nil + }) + return os.RemoveAll(root) +} + +// sweepDeadSuiteRoots is SweepDeadSuiteRoots with its three effects injected: +// the liveness probe, the orphan-server reaper, and the tree removal. +func sweepDeadSuiteRoots( + parentDir, prefix string, + alive func(int) bool, + sweepServers func(...string) []SweptServer, + removeAll func(string) error, +) []string { + if parentDir == "" || prefix == "" { + return nil + } + matches, err := filepath.Glob(filepath.Join(parentDir, prefix+"*")) + if err != nil { + return nil + } + + var swept []string + for _, root := range matches { + // Lstat, not Stat: a symlink that happens to match the prefix must + // not be followed out of parentDir and removed. + info, err := os.Lstat(root) + if err != nil || !info.IsDir() { + continue + } + // On a shared /tmp anyone can plant a directory with this prefix and + // a marker naming a dead PID. Only trust a root this user owns. + if !rootOwnedBySelf(info) { + continue + } + pid, hasMarker := readSuiteOwnerMarker(root) + if decideSuiteRoot(hasMarker, hasMarker && alive(pid)) != sweepSuiteRoot { + continue + } + sweepServers(root) + if err := removeAll(root); err != nil { + // Removal is not atomic: the marker is usually among the first + // entries deleted, so a partial failure would leave an unclaimed + // root that no later run may ever touch again. Put the marker + // back, naming the same dead owner, so the next run retries. + if writeErr := writeSuiteOwnerMarkerPID(root, pid); writeErr != nil { + fmt.Fprintf(os.Stderr, + "Warning: could not remove dead test-suite root %s (%v) and could not restore its owner marker (%v); it will not be retried\n", + root, err, writeErr) + } else { + fmt.Fprintf(os.Stderr, + "Warning: could not remove dead test-suite root %s: %v (marker restored; the next run retries)\n", + root, err) + } + continue + } + swept = append(swept, root) + } + + if len(swept) > 0 { + fmt.Fprintf(os.Stderr, "Info: swept %d dead test-suite temp root(s): %v\n", len(swept), swept) + } + return swept +} + +// SweepSuiteTestServers reaps only dolt sql-servers under suiteTempRoots, +// including servers whose working directory has been deleted. Call it after +// m.Run, before removing the roots, and pass its result to ApplyLeakPolicy. +// Every root must belong exclusively to this suite; never pass a shared temp +// directory. No roots means no servers are selected. +// +// Unlike SweepOrphanedTestServers, it never reaps deleted-cwd debris outside +// those roots: a foreign suite may still be running and must retain its own +// leak evidence. Use SweepDeadSuiteRoots separately at startup to clean up +// abandoned runs without charging their leaks to the current run. +func SweepSuiteTestServers(suiteTempRoots ...string) []SweptServer { + return sweepServersUnderRoots(suiteTempRoots...) +} + +// ApplyLeakPolicy folds the result of a post-run SweepSuiteTestServers +// call into the suite's exit code. suite names the package for the log line. +// +// A non-empty sweep means the suite leaked a dolt sql-server: the tests +// finished, but a server survived every t.Cleanup and TestMain defer. That +// fails the run, so the regression is caught by the suite that caused it +// instead of by someone reading `ps` days later — unless AllowLeakEnv is set +// to "1", which downgrades it to the loud stderr line it used to be. A code +// the tests already failed with is preserved, never overwritten. +// +// Both lines print each leak as " cwd=" (SweptServer.String): the +// PID is dead by the time anyone reads the log, but the directory names the +// test whose fixture leaked, which is the only part of the report a reader +// can act on (wy-j2zc8q). +func ApplyLeakPolicy(suite string, code int, swept []SweptServer) int { + if len(swept) == 0 { + return code + } + if os.Getenv(AllowLeakEnv) == "1" { + fmt.Fprintf(os.Stderr, + "Warning: %d leaked dolt sql-server(s) swept after %s: %v (%s=1 is downgrading this from a failure)\n", + len(swept), suite, swept, AllowLeakEnv) + return code + } + fmt.Fprintf(os.Stderr, + "FAIL: %s leaked %d dolt sql-server process(es) %v; swept them. A test finished with a detached server still\n"+ + " serving a temp dir this suite deletes. Fix the fixture's cleanup, or set %s=1 to downgrade this.\n", + suite, len(swept), swept, AllowLeakEnv) + if code == 0 { + return 1 + } + return code +} diff --git a/internal/doltserver/sweep_suite_linux_test.go b/internal/doltserver/sweep_suite_linux_test.go new file mode 100644 index 000000000..c96771962 --- /dev/null +++ b/internal/doltserver/sweep_suite_linux_test.go @@ -0,0 +1,82 @@ +//go:build linux + +package doltserver + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +// Exercise the public sweep and real signal path in the order parallel suites +// can finish: clean B first, leaking A second. Linux retains a deleted cwd's +// path in /proc, so this does not depend on a Dolt installation or lsof. +func TestSweepSuiteTestServersPreservesForeignLeak(t *testing.T) { + t.Setenv(AllowLeakEnv, "") + rootA, rootB := t.TempDir(), t.TempDir() + cwd := filepath.Join(rootA, "leaked-server") + if err := os.Mkdir(cwd, 0o700); err != nil { + t.Fatal(err) + } + // The selector recognizes dolt/sql-server in the command line. A shell + // blocked on stdin is a deterministic stand-in with no child processes. + cmd := exec.Command("/bin/sh", "-c", "read line", "dolt", "sql-server") + cmd.Dir = cwd + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + defer stdin.Close() + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + exited := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + if err := os.Remove(cwd); err != nil { + t.Fatal(err) + } + if got, deleted := readProcCwd(cmd.Process.Pid); !deleted || got == "" { + t.Fatalf("stand-in cwd = %q, deleted = %v; want a deleted cwd", got, deleted) + } + + for _, roots := range [][]string{nil, {rootB}} { + var swept []SweptServer + var code int + out := captureStderr(t, func() { + swept = SweepSuiteTestServers(roots...) + code = ApplyLeakPolicy("clean B", 0, swept) + }) + if len(swept) != 0 || code != 0 || out != "" { + t.Fatalf("clean sweep %v: swept=%v code=%d stderr=%q", roots, swept, code, out) + } + select { + case <-exited: + t.Fatal("clean suite consumed the leaking suite's evidence") + default: + } + } + + var swept []SweptServer + var code int + captureStderr(t, func() { + swept = SweepSuiteTestServers(rootA) + code = ApplyLeakPolicy("leaking A", 0, swept) + }) + if len(swept) != 1 || swept[0].PID != cmd.Process.Pid || code != 1 { + t.Fatalf("owning suite: swept=%v code=%d; want its server and failure", swept, code) + } + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("owning suite left its leaked server running") + } +} diff --git a/internal/doltserver/sweep_suite_other.go b/internal/doltserver/sweep_suite_other.go new file mode 100644 index 000000000..af7e052ee --- /dev/null +++ b/internal/doltserver/sweep_suite_other.go @@ -0,0 +1,18 @@ +//go:build !linux && !darwin + +package doltserver + +import "os" + +// rootOwnedBySelf accepts every root on platforms with no POSIX ownership in +// os.FileInfo.Sys(). It costs nothing: processAlive below reports every owner +// alive here, so SweepDeadSuiteRoots never removes anything anyway. +func rootOwnedBySelf(_ os.FileInfo) bool { return true } + +// processAlive reports every owner as alive on platforms where this package +// has no liveness probe, which makes SweepDeadSuiteRoots a no-op there: it +// can never conclude a root's owner is dead, so it never removes anything. +// Same posture as SweepOrphanedTestServers in sweep_other.go — the stub keeps +// callers (test TestMains) portable without giving them a destructive +// best-guess. +func processAlive(_ int) bool { return true } diff --git a/internal/doltserver/sweep_suite_test.go b/internal/doltserver/sweep_suite_test.go new file mode 100644 index 000000000..89b5a81b6 --- /dev/null +++ b/internal/doltserver/sweep_suite_test.go @@ -0,0 +1,384 @@ +package doltserver + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +// captureStderr runs fn with os.Stderr redirected to a pipe and returns what +// fn wrote there. The sweep reports its findings on stderr and nowhere else, +// so the content of those lines is only testable this way. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + orig := os.Stderr + os.Stderr = w + collected := make(chan string, 1) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + collected <- buf.String() + }() + fn() + os.Stderr = orig + _ = w.Close() + out := <-collected + _ = r.Close() + return out +} + +// TestDecideSuiteRoot pins the safety judgment behind SweepDeadSuiteRoots: +// exactly one of the four states — claimed by a process that is gone — is +// debris. In particular an UNCLAIMED root is left alone, because a directory +// that merely matches the suite's prefix cannot be proven to be this suite's, +// and reaping it would resurrect the cross-suite killer that +// selectOrphanTestServers' contract exists to prevent (wy-j2zc8q). +func TestDecideSuiteRoot(t *testing.T) { + cases := []struct { + name string + hasMarker bool + ownerAlive bool + want suiteRootAction + }{ + {"claimed by a dead owner is debris", true, false, sweepSuiteRoot}, + {"claimed by a live owner is a concurrent run", true, true, leaveSuiteRoot}, + {"unclaimed root is never provably ours", false, false, leaveSuiteRoot}, + {"unclaimed root stays unclaimed even if some pid is alive", false, true, leaveSuiteRoot}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := decideSuiteRoot(tc.hasMarker, tc.ownerAlive); got != tc.want { + t.Errorf("decideSuiteRoot(%v, %v) = %v, want %v", tc.hasMarker, tc.ownerAlive, got, tc.want) + } + }) + } +} + +// TestSuiteOwnerMarkerRoundTrip covers the marker read/write pair, including +// every way a read can be inconclusive — each of which must report "owner +// unknown" (ok=false), never "owner dead". +func TestSuiteOwnerMarkerRoundTrip(t *testing.T) { + root := t.TempDir() + if err := WriteSuiteOwnerMarker(root); err != nil { + t.Fatalf("WriteSuiteOwnerMarker: %v", err) + } + pid, ok := readSuiteOwnerMarker(root) + if !ok || pid != os.Getpid() { + t.Errorf("readSuiteOwnerMarker() = (%d, %v), want (%d, true)", pid, ok, os.Getpid()) + } + + if err := WriteSuiteOwnerMarker(""); err == nil { + t.Error("WriteSuiteOwnerMarker(\"\") = nil, want an error") + } + + if _, ok := readSuiteOwnerMarker(t.TempDir()); ok { + t.Error("readSuiteOwnerMarker() on a root with no marker reported ok=true") + } + + for _, body := range []string{"", "not-a-pid", "0", "-1"} { + bad := t.TempDir() + if err := os.WriteFile(filepath.Join(bad, SuiteOwnerMarkerName), []byte(body), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + if _, ok := readSuiteOwnerMarker(bad); ok { + t.Errorf("readSuiteOwnerMarker() on marker %q reported ok=true, want unknown", body) + } + } +} + +// TestSweepDeadSuiteRootsSelection walks a synthetic temp directory holding +// one of every case and asserts both halves of the outcome: which roots were +// removed, and which roots the server reaper was vouched for. Nothing here +// signals a real process — the liveness probe and the reaper are injected. +func TestSweepDeadSuiteRootsSelection(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-bd-tests-" + + mkRoot := func(name string, markerPID int) string { + dir := filepath.Join(parent, name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if markerPID != 0 { + body := []byte(strconv.Itoa(markerPID) + "\n") + if err := os.WriteFile(filepath.Join(dir, SuiteOwnerMarkerName), body, 0o600); err != nil { + t.Fatalf("write marker in %s: %v", dir, err) + } + } + return dir + } + + const deadPID, livePID = 4242, 4243 + dead := mkRoot(prefix+"dead", deadPID) + live := mkRoot(prefix+"live", livePID) + unclaimed := mkRoot(prefix+"unclaimed", 0) + otherSuite := mkRoot("beads-storage-dolt-tests-dead", deadPID) + + // A plain file and a symlink that both match the prefix: neither is a + // directory this suite created, and neither may be removed. + stray := filepath.Join(parent, prefix+"stray-file") + if err := os.WriteFile(stray, []byte("x"), 0o600); err != nil { + t.Fatalf("write stray file: %v", err) + } + elsewhere := t.TempDir() + link := filepath.Join(parent, prefix+"symlink") + if err := os.Symlink(elsewhere, link); err != nil { + // Windows without the create-symlink privilege; the rest of the + // case still carries its weight. + t.Logf("skipping the symlink leg: %v", err) + link = "" + } + + var vouched []string + swept := sweepDeadSuiteRoots( + parent, prefix, + func(pid int) bool { return pid == livePID }, + func(roots ...string) []SweptServer { + vouched = append(vouched, roots...) + return nil + }, + os.RemoveAll, + ) + + if want := []string{dead}; !reflect.DeepEqual(swept, want) { + t.Errorf("sweepDeadSuiteRoots() = %v, want %v", swept, want) + } + if want := []string{dead}; !reflect.DeepEqual(vouched, want) { + t.Errorf("the server reaper was vouched for %v, want %v", vouched, want) + } + if _, err := os.Stat(dead); !errors.Is(err, os.ErrNotExist) { + t.Errorf("dead-owner root %s still present: %v", dead, err) + } + for _, keep := range []string{live, unclaimed, otherSuite, stray, link, elsewhere} { + if keep == "" { + continue + } + if _, err := os.Lstat(keep); err != nil { + t.Errorf("%s must have been left alone: %v", keep, err) + } + } +} + +// TestSweepDeadSuiteRootsRestoresMarkerOnPartialRemoval covers the recovery +// path for a removal that fails halfway. os.RemoveAll is not atomic and the +// marker is an early casualty, so without the rewrite a root that could not +// be fully deleted (a read-only Go module cache entry under cmd/bd's isolated +// $HOME, say) would come back UNCLAIMED — and an unclaimed root is never +// swept again, by design. The marker must go back naming the same dead owner. +func TestSweepDeadSuiteRootsRestoresMarkerOnPartialRemoval(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-partial-tests-" + const deadPID = 4242 + + root := filepath.Join(parent, prefix+"stuck") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := writeSuiteOwnerMarkerPID(root, deadPID); err != nil { + t.Fatalf("write marker: %v", err) + } + + // Stand in for a partial os.RemoveAll: the marker is gone, the root is + // not, and an error comes back. + stubbornRemove := func(path string) error { + if err := os.Remove(filepath.Join(path, SuiteOwnerMarkerName)); err != nil { + t.Fatalf("simulate partial removal: %v", err) + } + return errors.New("permission denied") + } + + swept := sweepDeadSuiteRoots(parent, prefix, func(int) bool { return false }, func(...string) []SweptServer { return nil }, stubbornRemove) + if len(swept) != 0 { + t.Errorf("sweepDeadSuiteRoots() = %v, want nothing reported as removed", swept) + } + if _, err := os.Stat(root); err != nil { + t.Fatalf("root should still be there: %v", err) + } + pid, ok := readSuiteOwnerMarker(root) + if !ok || pid != deadPID { + t.Fatalf("marker after a failed removal = (%d, %v), want (%d, true) so the next run retries", pid, ok, deadPID) + } + + // And the next run, with removal working, finishes the job. + swept = sweepDeadSuiteRoots(parent, prefix, func(int) bool { return false }, func(...string) []SweptServer { return nil }, removeSuiteRoot) + if want := []string{root}; !reflect.DeepEqual(swept, want) { + t.Errorf("retry swept %v, want %v", swept, want) + } +} + +// TestSweepDeadSuiteRootsRemovesReadOnlyDirs pins removeSuiteRoot's chmod +// walk: cmd/bd's suite root doubles as an isolated $HOME and can hold +// read-only Go module cache directories, which plain os.RemoveAll refuses. +func TestSweepDeadSuiteRootsRemovesReadOnlyDirs(t *testing.T) { + root := filepath.Join(t.TempDir(), "root") + nested := filepath.Join(root, "pkg", "mod", "readonly") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(nested, "file"), []byte("x"), 0o400); err != nil { + t.Fatalf("write: %v", err) + } + if err := os.Chmod(nested, 0o500); err != nil { + t.Fatalf("chmod: %v", err) + } + // Restore write permission whatever happens, so the test's own temp + // cleanup cannot fail. + t.Cleanup(func() { _ = os.Chmod(nested, 0o700) }) + + if err := os.RemoveAll(root); err == nil { + t.Skip("this platform's os.RemoveAll already handles read-only directories") + } + if err := removeSuiteRoot(root); err != nil { + t.Fatalf("removeSuiteRoot: %v", err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Errorf("root still present after removeSuiteRoot: %v", err) + } +} + +// TestSweepDeadSuiteRootsRefusesUnboundedGlob guards the one input that would +// turn this helper into a temp-directory shredder. +func TestSweepDeadSuiteRootsRefusesUnboundedGlob(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "anything") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, SuiteOwnerMarkerName), []byte("4242\n"), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + + dead := func(int) bool { return false } + fatalSweep := func(roots ...string) []SweptServer { + t.Errorf("SweepOrphanedTestServers must not be called: %v", roots) + return nil + } + fatalRemove := func(path string) error { + t.Errorf("removeAll must not be called: %s", path) + return nil + } + + if got := sweepDeadSuiteRoots(parent, "", dead, fatalSweep, fatalRemove); got != nil { + t.Errorf("empty prefix swept %v, want nothing", got) + } + if got := sweepDeadSuiteRoots("", "beads-bd-tests-", dead, fatalSweep, fatalRemove); got != nil { + t.Errorf("empty parent dir swept %v, want nothing", got) + } +} + +// TestSweepDeadSuiteRootsSkipsSelf is the belt-and-braces check that the +// running process's own root survives, since TestMain writes its marker into +// a directory the very same glob will match on the next call. +func TestSweepDeadSuiteRootsSkipsSelf(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-self-tests-" + mine := filepath.Join(parent, prefix+"mine") + if err := os.MkdirAll(mine, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := WriteSuiteOwnerMarker(mine); err != nil { + t.Fatalf("WriteSuiteOwnerMarker: %v", err) + } + + // Real liveness probe this time: our own PID must read as alive. + swept := sweepDeadSuiteRoots(parent, prefix, processAlive, sweepServersUnderRoots, removeSuiteRoot) + if len(swept) != 0 { + t.Fatalf("sweepDeadSuiteRoots removed %v, including this process's own root", swept) + } + if _, err := os.Stat(mine); err != nil { + t.Errorf("own root %s was removed: %v", mine, err) + } +} + +// TestApplyLeakPolicyForSuite covers the exit-code arithmetic of the +// leak-as-failure rule and its env-gated downgrade. +func TestApplyLeakPolicyForSuite(t *testing.T) { + oneLeak := []SweptServer{{PID: 101, Cwd: "/tmp/TestLeaky123/001/.beads/dolt"}} + cases := []struct { + name string + env string + code int + swept []SweptServer + want int + }{ + {name: "no leak, passing suite", code: 0, swept: nil, want: 0}, + {name: "no leak, failing suite", code: 2, swept: nil, want: 2}, + {name: "leak fails a passing suite by default", code: 0, swept: oneLeak, want: 1}, + {name: "leak never overwrites an existing failure", code: 2, swept: oneLeak, want: 2}, + {name: "leak warns instead when opted out", env: "1", code: 0, swept: oneLeak, want: 0}, + {name: "opt-out never revives a failing suite", env: "1", code: 2, swept: oneLeak, want: 2}, + {name: "any value other than 1 still fails", env: "true", code: 0, swept: oneLeak, want: 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(AllowLeakEnv, tc.env) + // Capture stderr even though this test asserts only the exit + // code. ApplyLeakPolicy's sole side effect is printing, and the + // line it prints for the leak cases is byte-identical to a real + // leak failure — so an uncaptured table run emits + // "FAIL: internal/doltserver leaked 1 dolt sql-server ..." from a + // PASSING test on every green run of this package. That trains + // readers and CI log scrapers to ignore the exact signal this + // package exists to raise. The sibling + // TestApplyLeakPolicyNamesTheLeakingDirectory proves captureStderr + // composes with assertions on the same call. + var got int + captureStderr(t, func() { got = ApplyLeakPolicy("internal/doltserver", tc.code, tc.swept) }) + if got != tc.want { + t.Errorf("ApplyLeakPolicy(%d, %v) with %s=%q = %d, want %d", + tc.code, tc.swept, AllowLeakEnv, tc.env, got, tc.want) + } + }) + } +} + +// TestApplyLeakPolicyNamesTheLeakingDirectory pins the payload of both leak +// lines. A PID alone is useless to whoever reads the CI log — the process is +// long gone and the number identifies nothing — whereas the working directory +// is the temp tree the leaked server was serving, which Go names after the +// test that created it. Reporting it is what turns "cmd/bd leaked a server" +// into a specific fixture to fix (wy-j2zc8q). +func TestApplyLeakPolicyNamesTheLeakingDirectory(t *testing.T) { + const cwd = "/tmp/beads-bd-tests-xyz/TestLeakyFixture123/001/.beads/dolt" + swept := []SweptServer{{PID: 21881, Cwd: cwd}} + + t.Run("the failure line", func(t *testing.T) { + t.Setenv(AllowLeakEnv, "") + var code int + out := captureStderr(t, func() { code = ApplyLeakPolicy("cmd/bd", 0, swept) }) + if code != 1 { + t.Errorf("ApplyLeakPolicy() = %d, want 1", code) + } + if !strings.Contains(out, "FAIL: cmd/bd leaked") { + t.Errorf("stderr = %q, want the FAIL line", out) + } + if !strings.Contains(out, "21881 cwd="+cwd) { + t.Errorf("stderr = %q, want it to name %q", out, cwd) + } + }) + + t.Run("the downgraded warning line", func(t *testing.T) { + t.Setenv(AllowLeakEnv, "1") + var code int + out := captureStderr(t, func() { code = ApplyLeakPolicy("cmd/bd", 0, swept) }) + if code != 0 { + t.Errorf("ApplyLeakPolicy() = %d, want 0", code) + } + if !strings.Contains(out, "Warning: 1 leaked dolt sql-server(s)") { + t.Errorf("stderr = %q, want the Warning line", out) + } + if !strings.Contains(out, "21881 cwd="+cwd) { + t.Errorf("stderr = %q, want it to name %q", out, cwd) + } + }) +} diff --git a/internal/doltserver/sweep_suite_unix.go b/internal/doltserver/sweep_suite_unix.go new file mode 100644 index 000000000..1bdbb8cb5 --- /dev/null +++ b/internal/doltserver/sweep_suite_unix.go @@ -0,0 +1,45 @@ +//go:build darwin || linux + +package doltserver + +import ( + "errors" + "os" + "syscall" +) + +// rootOwnedBySelf reports whether the directory info describes is owned by +// the user running this process. +// +// SweepDeadSuiteRoots globs a world-writable temp directory, so on a shared +// box (/tmp, a CI runner with several users) anyone could plant a directory +// carrying this suite's prefix and an owner marker naming a PID that is not +// running, and have the sweep delete a tree of their choosing under the +// caller's identity. Ownership is the cheap check that closes it. An +// unreadable or unexpected stat shape reports false — not ours, leave it. +func rootOwnedBySelf(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return false + } + return int(stat.Uid) == os.Getuid() +} + +// processAlive reports whether pid still names a running process, using the +// signal-0 probe (kill(2) performs its permission and existence checks but +// delivers nothing). +// +// Only ESRCH — "no such process" — counts as dead. EPERM means the process +// exists and belongs to another user, and any other errno means the probe +// itself failed; both report alive, so SweepDeadSuiteRoots leaves the root +// untouched rather than deleting a tree on an inconclusive read. +func processAlive(pid int) bool { + // kill(0, sig) signals the caller's whole process group and kill(-1, …) + // every process it may signal, so a nonsensical PID must never reach the + // syscall. Report it alive: an unusable marker is not proof of death. + if pid <= 0 { + return true + } + err := syscall.Kill(pid, 0) + return !errors.Is(err, syscall.ESRCH) +} diff --git a/internal/doltserver/sweep_test.go b/internal/doltserver/sweep_test.go index 291941864..4865ceee1 100644 --- a/internal/doltserver/sweep_test.go +++ b/internal/doltserver/sweep_test.go @@ -1,7 +1,11 @@ package doltserver import ( + "fmt" + "os" + "path/filepath" "reflect" + "slices" "testing" ) @@ -15,11 +19,18 @@ import ( // servers all living under os.TempDir(), and only a suite's own scoped // root may reap its own debris, not everyone else's (gastownhall/beads // mybd-q6cz). +// +// The second bound is on the deleted-cwd arm: a deleted working directory +// only counts as debris when the path it named was under a TEMP root. A +// production server is spawned with cmd.Dir = /.beads/dolt, so +// without that bound a moved workspace or an unmounted volume would make a +// developer's live server look exactly like test debris (wy-j2zc8q). func TestSelectOrphanTestServerPIDs(t *testing.T) { cases := []struct { name string candidates []serverCandidate suiteRoots []string + tempRoots []string want []int }{ { @@ -28,6 +39,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 100, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/beads-bd-tests-xyz/.beads/dolt", cwdDeleted: true}, }, suiteRoots: nil, + tempRoots: []string{"/tmp"}, want: []int{100}, }, { @@ -36,6 +48,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 101, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{101}, }, { @@ -44,6 +57,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 102, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/other-suite-xyz/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -52,6 +66,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 103, cmdline: "dolt --prof cpu --prof-path /tmp/my-suite-root/dolt-pprof sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{103}, }, { @@ -60,6 +75,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 200, cmdline: "dolt sql-server -H 127.0.0.1 -P 3307", cwd: "/home/dev/project/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -68,6 +84,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 201, cmdline: "some-other-binary --flag", cwd: "/tmp/my-suite-root/whatever"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -76,6 +93,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 202, cmdline: "dolt status", cwd: "/tmp/my-suite-root/whatever"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -84,6 +102,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 203, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: ""}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -92,6 +111,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 204, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root2/evil"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -100,6 +120,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 205, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/some-suite/.beads/dolt"}, }, suiteRoots: nil, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -111,15 +132,70 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 303, cmdline: "dolt sql-server -P 4", cwdDeleted: true, cwd: "/tmp/whatever-else/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{302, 303}, }, + { + name: "a PRODUCTION server whose workspace was moved or deleted is left alone", + candidates: []serverCandidate{ + {pid: 400, cmdline: "dolt sql-server --config /Users/dev/project/.beads/dolt-server-config.yaml", cwd: "/Users/dev/project/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp", "/private/var/folders/c1/xx/T"}, + want: nil, + }, + { + name: "a production server on an unmounted volume is left alone", + candidates: []serverCandidate{ + {pid: 401, cmdline: "dolt sql-server -P 3307", cwd: "/Volumes/external/work/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/tmp"}, + want: nil, + }, + { + name: "the observed orphan: deleted cwd under the macOS private temp form is reaped with no suite roots", + candidates: []serverCandidate{ + {pid: 402, cmdline: "dolt sql-server --config /private/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/003/config.yaml", cwd: "/private/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/002", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/var/folders/c1/xx/T", "/private/var/folders/c1/xx/T"}, + want: []int{402}, + }, + { + name: "the same orphan reported in the unresolved /var form is reaped too", + candidates: []serverCandidate{ + {pid: 403, cmdline: "dolt sql-server -P 1", cwd: "/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/002", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/var/folders/c1/xx/T", "/private/var/folders/c1/xx/T"}, + want: []int{403}, + }, + { + name: "a deleted cwd inside the caller's own suite root needs no temp root at all", + candidates: []serverCandidate{ + {pid: 404, cmdline: "dolt sql-server -P 1", cwd: "/opt/scratch/my-suite-root/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/opt/scratch/my-suite-root"}, + tempRoots: nil, + want: []int{404}, + }, + { + name: "no temp roots at all disables the deleted-cwd arm entirely", + candidates: []serverCandidate{ + {pid: 405, cmdline: "dolt sql-server -P 1", cwd: "/tmp/some-suite/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: nil, + want: nil, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := selectOrphanTestServerPIDs(tc.candidates, tc.suiteRoots) + got := sweptPIDs(selectOrphanTestServers(tc.candidates, tc.suiteRoots, tc.tempRoots)) if !reflect.DeepEqual(got, tc.want) { - t.Errorf("selectOrphanTestServerPIDs() = %v, want %v", got, tc.want) + t.Errorf("selectOrphanTestServers() = %v, want %v", got, tc.want) } }) } @@ -175,9 +251,344 @@ not-a-pid dolt sql-server t.Fatalf("gatherPSCandidates() = %#v, want %#v", candidates, wantCandidates) } - gotPIDs := selectOrphanTestServerPIDs(candidates, []string{"/tmp/my-suite"}) - wantPIDs := []int{101, 103} - if !reflect.DeepEqual(gotPIDs, wantPIDs) { - t.Errorf("darwin ps selection path = %v, want %v", gotPIDs, wantPIDs) + // The darwin ps/lsof path must hand the selection a cwd for every + // survivor, because that cwd is what the sweep's report prints + // (wy-j2zc8q). Assert the whole SweptServer, not just the PIDs. + gotServers := selectOrphanTestServers(candidates, []string{"/tmp/my-suite"}, []string{"/tmp"}) + wantServers := []SweptServer{ + {PID: 101, Cwd: "/tmp/my-suite/.beads/dolt"}, + {PID: 103, Cwd: "/tmp/deleted-suite/.beads/dolt"}, + } + if !reflect.DeepEqual(gotServers, wantServers) { + t.Errorf("darwin ps selection path = %v, want %v", gotServers, wantServers) } } + +// sweptPIDs projects a selection result down to its PIDs so the big tables +// above can stay readable. The cwd every entry carries is pinned directly by +// TestGatherPSCandidates and TestSweptServerNamesItsDirectory. +func sweptPIDs(servers []SweptServer) []int { + if len(servers) == 0 { + return nil + } + pids := make([]int, 0, len(servers)) + for _, server := range servers { + pids = append(pids, server.PID) + } + return pids +} + +// TestSweptServerNamesItsDirectory pins the rendering both sweep stderr lines +// rely on: a reader gets the leaked server's working directory, which names +// the test that leaked it, not just a PID that is already dead (wy-j2zc8q). +func TestSweptServerNamesItsDirectory(t *testing.T) { + server := SweptServer{PID: 21881, Cwd: "/tmp/TestFoo123/001/.beads/dolt"} + const want = "21881 cwd=/tmp/TestFoo123/001/.beads/dolt" + if got := server.String(); got != want { + t.Errorf("SweptServer.String() = %q, want %q", got, want) + } + if got := fmt.Sprintf("%v", []SweptServer{server}); got != "["+want+"]" { + t.Errorf("formatted slice = %q, want %q", got, "["+want+"]") + } + // A cwd-less server can never be selected (selectServers skips them), + // but the renderer must not print a dangling "cwd=" if one appears. + if got := (SweptServer{PID: 7}).String(); got != "7" { + t.Errorf("SweptServer with no cwd = %q, want %q", got, "7") + } +} + +// TestSelectServersUnderSuiteRoots pins the strictly root-scoped selection +// SweepDeadSuiteRoots relies on. It runs at suite START, with sibling +// packages mid-run, so it must reap only what is provably inside the roots +// it was handed: no deleted-cwd arm, no temp-dir reasoning of any kind. +func TestSelectServersUnderSuiteRoots(t *testing.T) { + candidates := []serverCandidate{ + {pid: 500, cmdline: "dolt sql-server -P 1", cwd: "/tmp/dead-root/.beads/dolt"}, + {pid: 501, cmdline: "dolt sql-server -P 2", cwd: "/tmp/dead-root/nested/deeper/dolt", cwdDeleted: true}, + {pid: 502, cmdline: "dolt sql-server -P 3", cwd: "/tmp/live-sibling-root/.beads/dolt"}, + {pid: 503, cmdline: "dolt sql-server -P 4", cwd: "/tmp/some-other-suite/.beads/dolt", cwdDeleted: true}, + {pid: 504, cmdline: "dolt sql-server -P 5", cwd: "/Users/dev/project/.beads/dolt", cwdDeleted: true}, + {pid: 505, cmdline: "not-dolt --flag", cwd: "/tmp/dead-root/whatever"}, + {pid: 506, cmdline: "dolt sql-server -P 6", cwd: ""}, + } + + got := selectServersUnderRoots(candidates, []string{"/tmp/dead-root"}) + want := []SweptServer{ + {PID: 500, Cwd: "/tmp/dead-root/.beads/dolt"}, + {PID: 501, Cwd: "/tmp/dead-root/nested/deeper/dolt"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("selectServersUnderRoots() = %v, want %v", got, want) + } + + if got := selectServersUnderRoots(candidates, nil); got != nil { + t.Errorf("selectServersUnderRoots() with no roots = %v, want nothing", got) + } +} + +// TestCanonicalRootsForSweep covers the path-form expansion that lets a root +// match however the OS reports a process's cwd. +func TestCanonicalRootsForSweep(t *testing.T) { + if got := canonicalRoots(nil); got != nil { + t.Errorf("canonicalRoots(nil) = %v, want nothing", got) + } + if got := canonicalRoots([]string{"", ""}); got != nil { + t.Errorf("canonicalRoots(empties) = %v, want nothing", got) + } + + // A path that cannot be resolved is still kept as given. + missing := filepath.Join(t.TempDir(), "does-not-exist") + if got := canonicalRoots([]string{missing}); !reflect.DeepEqual(got, []string{missing}) { + t.Errorf("canonicalRoots(%q) = %v, want just the literal path", missing, got) + } + + // A real directory reached through a symlink yields both forms, in that + // order, with no duplicates — this is what makes macOS's + // /var/folders/… vs /private/var/folders/… pair match. + target := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + resolved, err := filepath.EvalSymlinks(link) + if err != nil { + t.Fatalf("EvalSymlinks(%s): %v", link, err) + } + got := canonicalRoots([]string{link, link, resolved}) + want := []string{link, resolved} + if !reflect.DeepEqual(got, want) { + t.Errorf("canonicalRoots() = %v, want %v", got, want) + } +} + +// TestTempDirRootsBoundTheOrphanArm checks that the deleted-cwd bound +// actually covers the directory this process's own t.TempDir() lands in — +// if it did not, the arm would silently never fire. +func TestTempDirRootsBoundTheOrphanArm(t *testing.T) { + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty; the deleted-cwd arm could never fire") + } + dir := t.TempDir() + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + resolved = dir + } + if !underAnyRoot(dir, roots) && !underAnyRoot(resolved, roots) { + t.Errorf("t.TempDir() %q (resolved %q) is under none of tempDirRoots() %v", dir, resolved, roots) + } + if underAnyRoot("/Users/dev/project/.beads/dolt", roots) { + t.Errorf("a production workspace path matched tempDirRoots() %v", roots) + } +} + +// TestTempDirRootsRejectsOverbroadTMPDIR covers the bound on the bound. +// TMPDIR is an ordinary environment variable, so os.TempDir() can be handed +// "/" or a home directory; either would put every workspace on the box back +// inside the deleted-cwd arm, which is precisely what tempDirRoots exists to +// prevent. /tmp and the real per-user temp dir must survive that filter. +func TestTempDirRootsRejectsOverbroadTMPDIR(t *testing.T) { + // A real, non-temp home. The ambient one cannot be trusted for these + // expectations: CI (scripts/ci/lib/test-env.sh) runs the suite with HOME + // under /tmp, where "a workspace under the home directory" and "a path + // under /tmp" are the same path. + const home = "/home/beads-fixture" + t.Setenv("HOME", home) + + t.Run("normal TMPDIR keeps a usable root", func(t *testing.T) { + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty for the ambient TMPDIR") + } + for _, root := range roots { + if root == string(filepath.Separator) { + t.Errorf("tempDirRoots() = %v, which includes the filesystem root", roots) + } + if isUnderDir(filepath.Clean(home), root) { + t.Errorf("tempDirRoots() root %q contains the home directory %q", root, home) + } + } + }) + + t.Run("TMPDIR=/ is dropped", func(t *testing.T) { + t.Setenv("TMPDIR", "/") + roots := tempDirRoots() + for _, root := range roots { + if filepath.Clean(root) == string(filepath.Separator) { + t.Fatalf("tempDirRoots() = %v, want the filesystem root dropped", roots) + } + } + // The hardcoded /tmp fallback still has to come through, or the + // deleted-cwd arm would quietly stop working. + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v, want /tmp still covered", roots) + } + if underAnyRoot(filepath.Join(home, "project", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v still covers a path under the home directory", roots) + } + }) + + t.Run("TMPDIR=$HOME is dropped", func(t *testing.T) { + t.Setenv("TMPDIR", home) + roots := tempDirRoots() + if underAnyRoot(filepath.Join(home, "project", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v covers a workspace under the home directory", roots) + } + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v, want /tmp still covered", roots) + } + }) + + // The CI shape: HOME is a throwaway directory under /tmp. /tmp contains + // it, and must survive anyway — this is the case that emptied + // tempDirRoots on every Linux runner and disabled the arm there. + t.Run("sandbox HOME under /tmp keeps /tmp", func(t *testing.T) { + sandbox := "/tmp/beads-test-env-abc123/home" + t.Setenv("HOME", sandbox) + roots := tempDirRoots() + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v with HOME=%s, want /tmp still covered", roots, sandbox) + } + for _, root := range roots { + if filepath.Clean(root) == string(filepath.Separator) { + t.Errorf("tempDirRoots() = %v, which includes the filesystem root", roots) + } + } + }) + + // TMPDIR=$HOME with a sandbox home: the root IS the home, so it is + // dropped even though a sandbox home is otherwise ignored. + t.Run("TMPDIR=sandbox HOME is dropped", func(t *testing.T) { + sandbox := "/tmp/beads-test-env-abc123/home" + t.Setenv("HOME", sandbox) + t.Setenv("TMPDIR", sandbox) + for _, root := range tempDirRoots() { + if filepath.Clean(root) == sandbox { + t.Errorf("tempDirRoots() kept %q, the sandbox home itself", root) + } + } + }) +} + +// TestIsCredibleTempRoot tables the predicate directly, including the shapes +// no environment on this box can produce. +func TestIsCredibleTempRoot(t *testing.T) { + const home = "/Users/dev" + cases := []struct { + root string + want bool + }{ + {"/tmp", true}, + {"/private/tmp", true}, + {"/var/folders/c1/xx/T", true}, + {"/var/folders/c1/xx/T/beads-bd-tests-1", true}, + {"/", false}, + {"//", false}, + {"", false}, + {".", false}, + {"relative/tmp", false}, + {home, false}, + {home + "/", false}, + {"/Users", false}, + {"/Users/dev/tmp", true}, + {"/Users/other", true}, + } + for _, tc := range cases { + if got := isCredibleTempRoot(tc.root, home); got != tc.want { + t.Errorf("isCredibleTempRoot(%q, %q) = %v, want %v", tc.root, home, got, tc.want) + } + } + + // A sandbox home (CI's HOME under /tmp) does not disqualify the roots + // that contain it; the root that IS the home is still out, and a real + // home under an overbroad TMPDIR still rejects that TMPDIR. + const sandbox = "/tmp/beads-test-env-abc123/home" + sandboxCases := []struct { + root, home string + want bool + }{ + {"/tmp", sandbox, true}, + {"/tmp/beads-test-env-abc123", sandbox, true}, + {sandbox, sandbox, false}, + {sandbox + "/", sandbox, false}, + {"/", sandbox, false}, + {"/home", "/home/runner", false}, + {"/tmp", "/tmp", false}, + } + for _, tc := range sandboxCases { + if got := isCredibleTempRoot(tc.root, tc.home); got != tc.want { + t.Errorf("isCredibleTempRoot(%q, %q) = %v, want %v", tc.root, tc.home, got, tc.want) + } + } + + // With no home known, only the structural rejections apply. + if !isCredibleTempRoot("/Users/dev", "") { + t.Error("isCredibleTempRoot with no home should accept an ordinary absolute path") + } + if isCredibleTempRoot("/", "") { + t.Error("isCredibleTempRoot must reject the filesystem root even with no home") + } +} + +// TestSandboxHomeUnderPerUserTempRoot is the macOS half of the sandbox-home +// carve-out. On Linux the throwaway HOME lands under /tmp; on macOS os.TempDir() +// is a per-user tree under /var/folders, so a suite that pins HOME to a +// t.TempDir() (cmd/bd/test_repo_beads_guard_test.go:128, +// internal/beads/testmain_test.go:50) puts HOME *inside* os.TempDir(). Judging +// sandbox-ness against /tmp alone made that home disqualify os.TempDir() +// itself, tempDirRoots() dropped to [/tmp /private/tmp], and the deleted-cwd +// arm went inert on every Mac (wy-j2zc8q). +func TestSandboxHomeUnderPerUserTempRoot(t *testing.T) { + // The platform table, exercised from any platform: goos is a parameter so + // the darwin row is pinned on Linux CI too, not skipped there. + t.Run("fixed temp roots by platform", func(t *testing.T) { + cases := []struct { + path string + goos string + want bool + }{ + {"/var/folders/zz/qqqqqqqq/T/fake-home", "darwin", true}, + {"/private/var/folders/zz/qqqqqqqq/T/fake-home", "darwin", true}, + {"/var/folders/zz/qqqqqqqq/T/fake-home", "linux", false}, + {"/tmp/beads-test-env-abc123/home", "darwin", true}, + {"/tmp/beads-test-env-abc123/home", "linux", true}, + // The invariant the bound exists for: a real home is never a + // sandbox home, whatever TMPDIR says. + {"/home/runner", "linux", false}, + {"/home/runner", "darwin", false}, + {"/Users/dev", "darwin", false}, + } + for _, tc := range cases { + if got := isUnderFixedTempRoots(tc.path, tc.goos); got != tc.want { + t.Errorf("isUnderFixedTempRoots(%q, %q) = %v, want %v", tc.path, tc.goos, got, tc.want) + } + } + + // The rows above are only host-independent because the darwin table + // spells the symlink-resolved forms out. canonicalRoots would add + // them via EvalSymlinks on a Mac and NOT on Linux, so without the + // literals the /private/… rows pass here and fail on Linux CI — the + // exact split this assertion closes (wy-j2zc8q). + darwinRoots := fixedTempRoots("darwin") + for _, want := range []string{"/tmp", "/private/tmp", "/var/folders", "/private/var/folders"} { + if !slices.Contains(darwinRoots, want) { + t.Errorf("fixedTempRoots(%q) = %v, missing the literal %q", "darwin", darwinRoots, want) + } + } + }) + + // End to end on THIS platform: HOME inside the real os.TempDir() must not + // cost us os.TempDir() as a root. + t.Run("HOME inside the real os.TempDir keeps it a root", func(t *testing.T) { + home := filepath.Join(os.TempDir(), "beads-sandbox-home-probe") + t.Setenv("HOME", home) + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty; the deleted-cwd arm could never fire") + } + if !underAnyRoot(filepath.Join(os.TempDir(), "beads-bd-tests-xyz", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v with HOME=%s, want os.TempDir() %q still covered", + roots, home, os.TempDir()) + } + }) +} diff --git a/internal/doltserver/testmain_integration_test.go b/internal/doltserver/testmain_integration_test.go index 8080af4e2..bab8b059e 100644 --- a/internal/doltserver/testmain_integration_test.go +++ b/internal/doltserver/testmain_integration_test.go @@ -8,8 +8,13 @@ import ( "testing" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" ) +const integrationSuiteRootPrefix = "beads-doltserver-tests-" + +var integrationSuiteTempRoot string + // TestMain covers the integration-tagged tests in this file's package // (lifecycle_integration_test.go, dirty_state_test.go, port_race_test.go, // socket_integration_test.go), which call doltserver.Start directly against @@ -29,21 +34,50 @@ import ( // the case Pdeathsig is meant to protect. See procattr_linux.go for why // this is a narrower, separate flag from BEADS_TEST_MODE. func TestMain(m *testing.M) { + os.Exit(runIntegrationTests(m)) +} + +func runIntegrationTests(m *testing.M) int { + doltserver.SweepDeadSuiteRoots(os.TempDir(), integrationSuiteRootPrefix) + root, err := testutil.PinSuiteTempRoot(integrationSuiteRootPrefix + "*") + if err != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) + return 1 + } + integrationSuiteTempRoot = root + defer os.RemoveAll(root) + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + os.Setenv("BEADS_TEST_MODE", "1") os.Setenv("BEADS_TEST_PDEATHSIG", "1") code := m.Run() - // No suite root passed: each test here owns its own t.TempDir(), not a - // shared package-level root, so this relies solely on the cwdDeleted - // signal (a leaked server's temp dir was already cleaned up out from - // under it) rather than any cwd-under-root match. - killed := doltserver.SweepOrphanedTestServers() - if len(killed) > 0 { - fmt.Fprintf(os.Stderr, "doltserver integration tests: swept %d orphaned dolt sql-server process(es)\n", len(killed)) - } + // Pinning both temporary-directory APIs above gives this sweep ownership + // even after a test has deleted the leaked server's working directory. + killed := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("internal/doltserver (integration)", code, killed) os.Unsetenv("BEADS_TEST_MODE") os.Unsetenv("BEADS_TEST_PDEATHSIG") - os.Exit(code) + return code +} + +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if integrationSuiteTempRoot == "" { + t.Fatal("TestMain did not pin integrationSuiteTempRoot") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, integrationSuiteTempRoot) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, integrationSuiteTempRoot) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, integrationSuiteTempRoot) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, integrationSuiteTempRoot) + } } diff --git a/internal/storage/dbproxy/server/doltserver_test.go b/internal/storage/dbproxy/server/doltserver_test.go index 9cbcfed6d..15b598be8 100644 --- a/internal/storage/dbproxy/server/doltserver_test.go +++ b/internal/storage/dbproxy/server/doltserver_test.go @@ -26,6 +26,15 @@ import ( const stopTimeout = 15 * time.Second +const ( + // backendExitTimeout is how long a fixture waits for the dolt sql-server + // to leave the process table after Stop returned. It is generous on + // purpose: this only decides between "took a while" and "still there", + // and only the second one is a defect. + backendExitTimeout = 30 * time.Second + backendExitPoll = 50 * time.Millisecond +) + func requireDolt(t *testing.T) string { t.Helper() p, err := exec.LookPath("dolt") @@ -70,10 +79,144 @@ func newDoltServer(t *testing.T) (*server.DoltServer, string) { // Close the log file handle before t.TempDir's RemoveAll runs. // On Windows, an open handle prevents directory removal; Stop is // idempotent so this is safe even when the test calls Stop itself. - t.Cleanup(func() { _ = s.Stop(context.Background()) }) + verifiedStopCleanup(t, s, rootDir) return s, rootDir } +// verifiedStopCleanup is the stop the standard fixture registers. It +// stops the server and then asserts the `dolt sql-server` it recorded is +// actually gone. +// +// This cleanup used to be `_ = s.Stop(context.Background())`. Discarding the +// error made a failed stop indistinguishable from a clean one, and the +// t.TempDir() holding the server's working directory is removed AFTER this +// cleanup runs (t.TempDir registers its own cleanup first, so it runs last). +// A server that survives its Stop therefore ends the test as a live daemon +// serving a deleted directory. The suite-scoped post-run sweep catches that +// leak at package granularity; verifying here also names the responsible test +// before its cleanup deletes the evidence (wy-j2zc8q). +// +// The check runs on the clean path too, not only when Stop reports an error: +// "Stop returned nil" and "the server exited" are two different claims, and +// this is the fixture whose job is to hold the second one. It costs one +// identity probe against a record that a clean Stop has already removed. +func verifiedStopCleanup(t *testing.T, s *server.DoltServer, rootDir string) { + t.Helper() + t.Cleanup(func() { + // Read the backend's record BEFORE stopping: a clean Stop removes + // the pid file, so afterwards there is nothing left to verify + // against. + record := backendServerRecord(t, rootDir) + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + defer cancel() + if err := s.Stop(ctx); err != nil { + t.Logf("DoltServer.Stop(%s): %v", rootDir, err) + } + requireBackendExited(t, record) + }) +} + +// backendServerRecord returns the pid file this package's DoltServer wrote +// for its `dolt sql-server` child. nil means there is nothing to verify: no +// server was started, or it shut down cleanly and removed its record. +// +// The whole record is returned, not just the PID: Birth is the process-birth +// token that makes the PID safe to act on at all. +func backendServerRecord(t *testing.T, rootDir string) *pidfile.PidFile { + t.Helper() + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil { + t.Logf("read %s in %s: %v", server.PIDFileName, rootDir, err) + return nil + } + if pf == nil || pf.Pid <= 0 { + return nil + } + return pf +} + +// requireBackendExited fails the test if the recorded dolt sql-server is +// still running after Stop returned, then force-kills it so the leak does not +// outlive the run. +// +// Every step is gated on procid birth identity rather than the bare PID. Stop +// reaps the child, so its PID is reusable the instant it exits; signalling an +// unverified PID here could hit an unrelated process. A record with no birth +// token, or one whose token no longer matches, is left alone: not the server +// we recorded, nothing to report. +func requireBackendExited(t *testing.T, pf *pidfile.PidFile) { + t.Helper() + if pf == nil || pf.Pid <= 0 { + return + } + if pf.Birth == "" { + // A record from before birth tokens were written. There is no safe + // way to ask about that PID, so the check is skipped — say so, rather + // than looking like a silent pass. + t.Logf("backend record for pid %d carries no birth token; skipping the survived-Stop check", pf.Pid) + return + } + token := procid.Token(pf.Birth) + + deadline := time.Now().Add(backendExitTimeout) + for { + same, err := procid.Verify(pf.Pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pf.Pid, err) + return + } + if !same { + // Either the process is gone, or the PID now belongs to + // something else. Either way our server is not running. + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(backendExitPoll) + } + + handle, err := procid.Open(pf.Pid, token) + if err != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Stop by more than %s and could not be opened to kill: %v", + pf.Pid, backendExitTimeout, err) + } + return + } + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Stop by more than %s and could not be killed: %v", + pf.Pid, backendExitTimeout, killErr) + } + return + } + t.Errorf("dolt sql-server pid %d survived Stop by more than %s (force-killed)", pf.Pid, backendExitTimeout) +} + +// backendStillRunning re-checks birth identity after procid.Open or +// Handle.Kill failed, and reports whether the recorded server is verifiably +// still there. +// +// Both calls verify the token themselves and return a plain "does not match +// token" error when it no longer does — which is exactly what a process that +// exited between the wait loop's last poll and the kill attempt produces. +// Without this second look, a server that shut down a few milliseconds late +// would be reported as one that survived Stop entirely. A verify that itself +// errors reports "not running": the run is over, and an inconclusive probe is +// not evidence of a leak. +func backendStillRunning(t *testing.T, pid int, token procid.Token) bool { + t.Helper() + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d) after a failed kill attempt: %v", pid, err) + return false + } + return same +} + func stopWithTimeout(t *testing.T, s *server.DoltServer) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) diff --git a/internal/storage/dbproxy/server/testmain_test.go b/internal/storage/dbproxy/server/testmain_test.go new file mode 100644 index 000000000..82c2c844f --- /dev/null +++ b/internal/storage/dbproxy/server/testmain_test.go @@ -0,0 +1,86 @@ +package server_test + +import ( + "fmt" + "os" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. It is +// deliberately short: this package's unix-socket test builds a socket path +// under t.TempDir(), and sun_path is 108 bytes on linux / 104 on macOS. +const suiteRootPrefix = "beads-dbproxy-server-tests-" + +// suiteTempRoot is the TestMain-owned temp directory that every t.TempDir() +// in this package lands under, so the sweeps have a root they can vouch for. +var suiteTempRoot string + +// TestMain puts this package under the same suite-lifecycle contract as every +// other package that daemonizes a `dolt sql-server`. +// +// This suite starts real servers: newDoltServer (and the fixtures that build +// a DoltServer by hand) spawn `dolt sql-server` with cmd.Dir set to the +// rootDir, which is a t.TempDir(). Until now the package had no TestMain at +// all, which left two holes: +// +// - A run killed by `go test -timeout`, CI cancel, or Ctrl-C skips every +// t.Cleanup, so the server outlives the test binary. Its cwd survives too +// (nothing deleted it), and this package claimed no suite root — so that +// debris was reachable by neither arm of the sweep and simply accumulated. +// Claiming a root here is what makes the NEXT run able to reap it. +// +// - A server this suite leaks on a normal run has its cwd deleted by +// t.TempDir's RemoveAll. A root-scoped post-run sweep detects that leak +// and fails this package without consuming another live suite's evidence. +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. Roots with no owner marker, and + // roots whose owner is still running (a parallel package under + // scripts/test.sh, a second `go test`), are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin TMPDIR under a suite-owned root so every t.TempDir() — including + // each server's rootDir, which is its working directory — is nested under + // something the sweeps may vouch for. + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + + code := m.Run() + + // Best-effort reap of any dolt sql-server still running under this run's + // own root — the backstop for a test whose Cleanup did not get to run. + swept := doltserver.SweepSuiteTestServers(root) + return doltserver.ApplyLeakPolicy("internal/storage/dbproxy/server", code, swept) +} + +// TestTempDirLandsUnderSuiteSweepRoot guards the pinning above: if t.TempDir() +// ever stops landing under suiteTempRoot, the post-run sweep silently loses +// its scope and leaked sql-servers become unreapable again. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + dir := t.TempDir() + if !testutil.PathUnderSuiteRoot(dir, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", dir, suiteTempRoot) + } +} diff --git a/internal/storage/dolt/suite_leak_linux_test.go b/internal/storage/dolt/suite_leak_linux_test.go new file mode 100644 index 000000000..538424153 --- /dev/null +++ b/internal/storage/dolt/suite_leak_linux_test.go @@ -0,0 +1,92 @@ +//go:build linux + +package dolt + +import ( + "context" + "errors" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" +) + +// This process boundary exercises the real TestMain allocation and exit policy, +// not just the selector: an ordinary t.TempDir fixture must belong to the suite. +// A shell blocked on stdin supplies a real process without requiring Dolt. +func TestSuiteFixtureLeakFailsOwningSuite(t *testing.T) { + if os.Getenv("BEADS_SUITE_LEAK_PROBE") == "1" { + cwd := t.TempDir() + cmd := exec.Command("/bin/sh", "-c", "read line", "dolt", "sql-server") + cmd.Dir = cwd + // The parent owns the write end. Closing it also stops the stand-in + // if this child fails before TestMain reaches the sweep. + cmd.Stdin = os.NewFile(3, "leak-probe-stdin") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + go func() { _ = cmd.Wait() }() + // Intentionally omit fixture shutdown. t.TempDir removes cwd before + // m.Run returns, so the TestMain sweep must find the deleted-cwd leak. + return + } + + // A foreign suite's deleted-cwd process must survive the child's sweep. + foreign := exec.Command("/bin/sh", "-c", "read line", "dolt", "sql-server") + foreign.Dir = t.TempDir() + stdin, err := foreign.StdinPipe() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close() }) + if err := foreign.Start(); err != nil { + t.Fatal(err) + } + exited := make(chan struct{}) + go func() { + _ = foreign.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = foreign.Process.Kill() + <-exited + }) + if err := os.Remove(foreign.Dir); err != nil { + t.Fatal(err) + } + + readEnd, writeEnd, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer readEnd.Close() + defer writeEnd.Close() + t.Setenv("BEADS_SUITE_LEAK_PROBE", "1") + t.Setenv("BEADS_TEST_SKIP", "dolt") + t.Setenv(doltserver.AllowLeakEnv, "") + // This child is a complete suite, not one of the schema-init helpers + // which deliberately bypass suite ownership and shutdown. + for _, key := range helperSubprocessSentinels { + t.Setenv(key, "") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSuiteFixtureLeakFailsOwningSuite$", "-test.count=1") + child.ExtraFiles = []*os.File{readEnd} + out, err := child.CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("leaking suite returned %v; want exit 1\n%s", err, out) + } + if !strings.Contains(string(out), "FAIL: internal/storage/dolt leaked 1 dolt sql-server") { + t.Fatalf("suite failed without attributing its fixture leak:\n%s", out) + } + select { + case <-exited: + t.Fatal("child suite consumed the foreign suite's leak evidence") + default: + } +} diff --git a/internal/storage/dolt/suite_temp_test.go b/internal/storage/dolt/suite_temp_test.go new file mode 100644 index 000000000..619d311d1 --- /dev/null +++ b/internal/storage/dolt/suite_temp_test.go @@ -0,0 +1,29 @@ +package dolt + +import ( + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Guard the actual TestMain wiring: both APIs must allocate fixtures under +// the owned root or a failed fixture cleanup can leave a green, leaking suite. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + root := filepath.Dir(os.Getenv(testCircuitBreakerDirEnv)) + if root == "" || root == "." { + t.Fatal("TestMain did not pin its suite temp root") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, root) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, root) + } +} diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index ab4a28018..02f367f74 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -57,22 +57,6 @@ func testMainInner(m *testing.M) int { // separate flag from BEADS_TEST_MODE. os.Setenv("BEADS_TEST_PDEATHSIG", "1") - // Suite-owned root for the orphan-server sweep below. Must never be a - // shared/global temp dir (see SweepOrphanedTestServers) — this one is - // unique to this test run and removed when it exits. - suiteTempRoot, tempRootErr := os.MkdirTemp("", "beads-storage-dolt-tests-*") - if tempRootErr != nil { - fmt.Fprintf(os.Stderr, "FATAL: failed to create suite temp root: %v\n", tempRootErr) - return 1 - } else { - defer os.RemoveAll(suiteTempRoot) - if err := os.Setenv(testCircuitBreakerDirEnv, filepath.Join(suiteTempRoot, "circuit")); err != nil { - fmt.Fprintf(os.Stderr, "FATAL: failed to isolate circuit state: %v\n", err) - return 1 - } - defer os.Unsetenv(testCircuitBreakerDirEnv) - } - // AD-01 (be-c5p): the test/bench harness opens a process-local dolt // sql-server (testcontainer or external port). The new database-name // firewall in dolt.New refuses test-named DBs unless this opt-in is set. @@ -87,8 +71,66 @@ func testMainInner(m *testing.M) int { // which skips the deferred TerminateDoltContainer below and leaks the // container. That load is what pushed the contended GET_LOCK past its // 5s wait in CI shard 3. + // + // It returns BEFORE the suite temp root below for the same reason: + // eight concurrent helpers would each run the dead-root sweep and each + // claim a root of its own, and os.Exit would abandon every one of + // them. A helper needs neither: it starts no server, so it has no + // orphans to sweep and nothing for a later run to reap. + // + // It DOES still need its own circuit-breaker directory. That state is + // one file keyed host:port:db, read-modify-written, and the parent's + // BEADS_TEST_CIRCUIT_DIR reaches every helper through the BEADS_ + // allowlist in integration.FilterEnv — so inheriting it would put all + // eight of TestMultiProcessSchemaInit's helpers on ONE file at once, + // against the same host:port:db. Each helper used to get a fresh + // directory under a suite root of its own; this keeps that isolation + // without the root, the sweep, or the marker. + circuitDir, circuitErr := os.MkdirTemp("", "beads-dolt-helper-circuit-*") + if circuitErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to isolate helper circuit state: %v\n", circuitErr) + return 1 + } + // A helper commonly exits through os.Exit (see above), which skips + // this defer. The directory holds a few bytes of circuit state and + // never a server, and it carries no owner marker, so no sweep will + // ever look at it: the occasional leftover is accepted rather than + // given machinery of its own. + defer os.RemoveAll(circuitDir) + if err := os.Setenv(testCircuitBreakerDirEnv, circuitDir); err != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to isolate helper circuit state: %v\n", err) + return 1 + } + defer os.Unsetenv(testCircuitBreakerDirEnv) return m.Run() } + + // Suite-owned root for the orphan-server sweep below. Must never be a + // shared/global temp dir (see SweepSuiteTestServers) — this one is + // unique to this test run and removed when it exits. + // + // Before claiming one, clear out the roots of earlier runs of this suite + // whose process is gone: a `go test -timeout` panic skips both the defer + // below and the post-Run sweep, so those runs' servers outlive every + // cleanup installed here (wy-j2zc8q). Unclaimed roots, and roots whose + // owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + suiteTempRoot, tempRootErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if tempRootErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to create suite temp root: %v\n", tempRootErr) + return 1 + } else { + defer os.RemoveAll(suiteTempRoot) + if err := doltserver.WriteSuiteOwnerMarker(suiteTempRoot); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", suiteTempRoot, err) + } + if err := os.Setenv(testCircuitBreakerDirEnv, filepath.Join(suiteTempRoot, "circuit")); err != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to isolate circuit state: %v\n", err) + return 1 + } + defer os.Unsetenv(testCircuitBreakerDirEnv) + } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { @@ -129,7 +171,8 @@ func testMainInner(m *testing.M) int { // suite's own temp root (e.g. a SIGKILLed run of the multiprocess // schema init tests, which call doltserver.Start directly) — see // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(suiteTempRoot) + swept := doltserver.SweepSuiteTestServers(suiteTempRoot) + code = doltserver.ApplyLeakPolicy("internal/storage/dolt", code, swept) testServerPort = 0 os.Unsetenv("BEADS_DOLT_PORT") @@ -138,6 +181,10 @@ func testMainInner(m *testing.M) int { return code } +// suiteRootPrefix is testMainInner's PinSuiteTempRoot pattern without its random +// tail. It is what SweepDeadSuiteRoots globs for, so the two must not drift. +const suiteRootPrefix = "beads-storage-dolt-tests-" + // initSharedSchema creates a store on the shared DB, sets config, and commits // so that branches inherit the full schema. func initSharedSchema(port int) error { diff --git a/internal/storage/uow/doltserver_provider_test.go b/internal/storage/uow/doltserver_provider_test.go index 8d84bbd3f..3fa47c201 100644 --- a/internal/storage/uow/doltserver_provider_test.go +++ b/internal/storage/uow/doltserver_provider_test.go @@ -89,11 +89,7 @@ func TestNewDoltServerUOWProvider_HappyPath(t *testing.T) { require.NoError(t, err) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") @@ -134,11 +130,7 @@ func TestNewDoltServerUOWProvider_ConcurrentInstantiation(t *testing.T) { require.NoError(t, err) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") diff --git a/internal/storage/uow/external_doltserver_provider_test.go b/internal/storage/uow/external_doltserver_provider_test.go index 608b084fd..eab4eb81d 100644 --- a/internal/storage/uow/external_doltserver_provider_test.go +++ b/internal/storage/uow/external_doltserver_provider_test.go @@ -74,11 +74,7 @@ func TestNewExternalDoltServerUOWProvider_EndToEnd(t *testing.T) { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -140,11 +136,7 @@ func TestNewExternalDoltServerUOWProvider_ConcurrentInstantiation(t *testing.T) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") external := configfile.ExternalDoltConfig{Host: "127.0.0.1", Port: portInt} @@ -228,11 +220,7 @@ func TestNewExternalDoltServerUOWProvider_FreshInitSelfHealsAfterMidPassFailure( storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") // Fire exactly once, right after migration step 1 (create issues) has @@ -319,11 +307,7 @@ func TestNewExternalDoltServerUOWProvider_PreexistingDirtyDatabaseIsNotHealed(t storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) diff --git a/internal/storage/uow/lostupdate_dolt_test.go b/internal/storage/uow/lostupdate_dolt_test.go index 1f1a3c393..9bf15d1ef 100644 --- a/internal/storage/uow/lostupdate_dolt_test.go +++ b/internal/storage/uow/lostupdate_dolt_test.go @@ -71,11 +71,7 @@ func TestUOW_ConcurrentMergeOps_NoLostUpdate(t *testing.T) { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) diff --git a/internal/storage/uow/shared_server_cleanup_guard_test.go b/internal/storage/uow/shared_server_cleanup_guard_test.go new file mode 100644 index 000000000..83cf96a4e --- /dev/null +++ b/internal/storage/uow/shared_server_cleanup_guard_test.go @@ -0,0 +1,268 @@ +package uow + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// providerConstructors are the calls that make this package provision a +// shared-server workspace: each one starts, or attaches to, a detached +// `dolt sql-server` whose data directory is a t.TempDir(). +var providerConstructors = map[string]bool{ + "NewDoltServerUOWProvider": true, + "NewExternalDoltServerUOWProvider": true, +} + +// shutdownCallers are the only functions allowed to call proxy.Shutdown +// directly. Everything else goes through verifiedShutdownCleanup, which is +// the one that also asserts the daemon is gone. +var shutdownCallers = map[string]bool{ + "verifiedShutdownCleanup": true, + // shutdownOnInterrupt is the SIGINT/SIGTERM handler: it runs in a + // goroutine on its way to os.Exit, where no assertion could be reported + // and no *testing.T is still live to report it to. + "shutdownOnInterrupt": true, +} + +// validationErrorMarker is the ONLY thing that exempts a function from rule +// C: a test whose name ends in it declares, in the one place a reader and a +// linter both see, that it calls a provider constructor purely to watch it +// reject bad arguments — the constructor returns before starting anything, so +// there is no daemon to clean up. +// +// It is a name marker rather than an inference from the body because every +// body-shaped heuristic we tried was dodgeable. Exempting "any function that +// calls require.Error" let +// TestNewExternalDoltServerUOWProvider_PreexistingDirtyDatabaseIsNotHealed — +// which provisions a REAL server, asserts NoError on the constructor, and only +// later asserts an error from a query — delete both cleanup helpers and stay +// green (wy-j2zc8q, mutation G). A marker cannot be reached by accident: a +// fixture has to be renamed to claim it. +const validationErrorMarker = "_ValidationErrors" + +// errorAssertions are the calls that make the marker's claim true. They do not +// grant the exemption (see above); they are checked AGAINST a function that +// claims it, so a renamed-but-provisioning fixture cannot wear the marker +// silently. +var errorAssertions = map[string]bool{ + "Error": true, // require.Error / assert.Error + "ErrorContains": true, +} + +// successAssertions are the calls that make the marker's claim FALSE. A +// constructor that was rejected returns before starting anything, so a +// validation-error fixture has nothing that can succeed: no require.NoError, +// no assert.NoError anywhere in its body. Checking this closes the one dodge +// the marker alone leaves open — renaming a fixture that really does provision +// a server so it wears the marker. +var successAssertions = map[string]bool{ + "NoError": true, // require.NoError / assert.NoError +} + +// TestEveryServerFixtureRegistersVerifiedCleanup is the structural half of +// this package's leak defense. +// +// verifiedShutdownCleanup only helps the fixtures that call it, and the leak +// it exists to stop (wy-j2zc8q) came back three times precisely because the +// cleanup idiom was copy-pasted per fixture: nine near-identical t.Cleanup +// blocks, any of which could be written slightly differently — or omitted — +// in a tenth fixture without anything noticing until a dev box's process +// table showed a detached sql-server serving a directory deleted hours +// earlier. So the rule is checked against this package's own source. +// +// Three rules, over the functions that take a *testing.T (only those can +// register a cleanup at all — a harness METHOD like openProvider reuses a +// root whose owner already registered one, so it is not the place to look): +// +// A. shutdownOnInterrupt and verifiedShutdownCleanup come as a pair. They +// cover the two ways a fixture's server outlives it — a signal, and a +// returning run — and a fixture copied from another with one half +// dropped is the realistic regression. +// B. Only the two sanctioned helpers call proxy.Shutdown; every fixture +// goes through verifiedShutdownCleanup, which also asserts the pid died. +// C. A function that calls a provider constructor registers the cleanup, +// unless its NAME ends in validationErrorMarker — an explicit, greppable +// allowlist rather than an inference from the body, which a fixture that +// provisions a real server and also asserts an error could dodge. +// +// What it does NOT catch: a fixture that provisions a workspace through some +// future path this file does not name. That is what the non-vacuity counters +// at the bottom are for — they fail if a rule stopped matching anything, +// rather than letting the guard go quietly green on a package it no longer +// understands. +func TestEveryServerFixtureRegistersVerifiedCleanup(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { + return strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parse package sources: %v", err) + } + if len(pkgs) == 0 { + t.Fatal("no test sources parsed; every rule below would pass vacuously") + } + + var paired, provisioning, exempted int + for _, pkg := range pkgs { + for path, file := range pkg.Files { + base := filepath.Base(path) + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + name := fn.Name.Name + calls := calledNames(fn) + + // Rule B applies to every function, helper or not. + if calls["Shutdown"] && !shutdownCallers[name] { + t.Errorf("%s: %s calls proxy.Shutdown directly; use verifiedShutdownCleanup(t, storeRootDir), "+ + "which also asserts the dolt sql-server pid is gone (wy-j2zc8q)", base, name) + } + if !takesTestingT(fn) || shutdownCallers[name] { + continue + } + + // Rule A. + switch { + case calls["shutdownOnInterrupt"] && !calls["verifiedShutdownCleanup"]: + t.Errorf("%s: %s guards a store root with shutdownOnInterrupt but never registers "+ + "verifiedShutdownCleanup(t, storeRootDir); its dolt sql-server survives a run that "+ + "ends normally (wy-j2zc8q)", base, name) + case calls["verifiedShutdownCleanup"] && !calls["shutdownOnInterrupt"]: + t.Errorf("%s: %s registers verifiedShutdownCleanup but never calls "+ + "shutdownOnInterrupt(t, storeRootDir); its dolt sql-server survives Ctrl-C (wy-j2zc8q)", + base, name) + case calls["shutdownOnInterrupt"]: + paired++ + } + + // Rule C. + if !callsAny(calls, providerConstructors) { + continue + } + if strings.HasSuffix(name, validationErrorMarker) { + // The marker must not be a lie: a function claiming it + // has to actually assert a rejection, and must assert + // nothing SUCCEEDED — a rejected constructor leaves no + // live server, so there is nothing to NoError over. + if !callsAny(calls, errorAssertions) { + t.Errorf("%s: %s is named %q but never asserts the constructor rejected its "+ + "arguments (no require.Error/ErrorContains); drop the marker and register "+ + "verifiedShutdownCleanup(t, storeRootDir) (wy-j2zc8q)", + base, name, validationErrorMarker) + } + if callsAny(calls, successAssertions) { + t.Errorf("%s: %s wears the %q exemption but calls require.NoError/assert.NoError; "+ + "a fixture with something that can succeed is provisioning, not validating — "+ + "drop the marker and register verifiedShutdownCleanup(t, storeRootDir) (wy-j2zc8q)", + base, name, validationErrorMarker) + } + exempted++ + continue + } + provisioning++ + if !calls["verifiedShutdownCleanup"] { + t.Errorf("%s: %s provisions a shared-server workspace but never calls "+ + "verifiedShutdownCleanup(t, storeRootDir); a detached dolt sql-server will outlive "+ + "the run (wy-j2zc8q)", base, name) + } + } + } + } + + // Non-vacuity. Each rule is an assertion about functions that exist; if + // the fixtures were renamed out from under the name sets above, every + // loop body would stop running and the guard would pass while watching + // nothing. + if paired == 0 { + t.Fatal("no function pairs shutdownOnInterrupt with verifiedShutdownCleanup; rule A matched nothing") + } + if provisioning == 0 { + t.Fatalf("no *testing.T function calls any of %v; rule C is watching constructors this package "+ + "no longer uses", sortedKeys(providerConstructors)) + } + // Note there is deliberately NO non-vacuity floor on `exempted`: if the + // marker stops matching, rule C gets STRICTER, and the validation-error + // fixtures fail it loudly rather than anything going quietly green. The + // census is logged so a shrinking numerator is visible in -v output. + t.Logf("rule A paired %d fixture(s); rule C checked %d provisioning function(s), exempted %d marked %q", + paired, provisioning, exempted, validationErrorMarker) +} + +// takesTestingT reports whether fn accepts a *testing.T. Only such a function +// can register a t.Cleanup, so only such a function can be held to the rules +// above: openProvider and friends are methods on a harness whose constructor +// already owns the root's cleanup. +func takesTestingT(fn *ast.FuncDecl) bool { + if fn.Type.Params == nil { + return false + } + for _, field := range fn.Type.Params.List { + star, ok := field.Type.(*ast.StarExpr) + if !ok { + continue + } + sel, ok := star.X.(*ast.SelectorExpr) + if !ok { + continue + } + pkg, ok := sel.X.(*ast.Ident) + if ok && pkg.Name == "testing" && sel.Sel.Name == "T" { + return true + } + } + return false +} + +// calledNames returns the set of function names called anywhere in fn's body, +// by their final identifier: both `Shutdown` from `proxy.Shutdown(...)` and +// the bare `verifiedShutdownCleanup(...)`. Calls inside closures — the +// t.Run(…, func(t *testing.T){…}) subtests these fixtures use — are included, +// and attributed to the enclosing declaration. +// +// Matching the selector's last segment rather than the full `proxy.Shutdown` +// keeps the guard working if the import is ever aliased, at the price of also +// matching some other package's `Shutdown` — a false positive worth having in +// a rule about not shutting servers down by hand. +func calledNames(fn *ast.FuncDecl) map[string]bool { + names := make(map[string]bool) + ast.Inspect(fn, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch f := call.Fun.(type) { + case *ast.Ident: + names[f.Name] = true + case *ast.SelectorExpr: + names[f.Sel.Name] = true + } + return true + }) + return names +} + +func callsAny(calls, want map[string]bool) bool { + for name := range want { + if calls[name] { + return true + } + } + return false +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/storage/uow/shared_server_cleanup_test.go b/internal/storage/uow/shared_server_cleanup_test.go new file mode 100644 index 000000000..0f12a85a2 --- /dev/null +++ b/internal/storage/uow/shared_server_cleanup_test.go @@ -0,0 +1,175 @@ +package uow + +import ( + "testing" + "time" + + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// verifiedShutdownCleanup is the ONE cleanup every fixture in this package +// that provisions a shared-server workspace registers. It stops the proxy and +// the `dolt sql-server` it manages, and — when the stop did not report clean — +// asserts the recorded server PID is actually gone. +// +// Each of those fixtures used to inline +// +// t.Cleanup(func() { +// if err := proxy.Shutdown(storeRootDir); err != nil { +// t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) +// } +// }) +// +// which stops the server on the happy path and, on the unhappy one, prints a +// line into a passing test's output and moves on. That is how this package +// produced detached sql-servers still serving temp trees that had been +// deleted hours earlier (wy-j2zc8q): `proxy.Shutdown` has several documented +// refusals — an unverifiable process record, a lock it could not acquire +// inside its 5s budget, a spawn still in flight — and every one of them left a +// live daemon behind a green run. Nine copies of the same silent handler also +// meant a tenth fixture could be written without one and nothing would notice. +// +// Pair it with shutdownOnInterrupt, which covers the SIGINT/SIGTERM path that +// skips t.Cleanup entirely. +func verifiedShutdownCleanup(t *testing.T, storeRootDir string) { + t.Helper() + t.Cleanup(func() { + // Read the backend's record BEFORE shutting down: a successful + // proxy.Shutdown removes the pid file, so afterwards there is + // nothing left to verify against. + record := backendServerRecord(t, storeRootDir) + err := proxy.Shutdown(storeRootDir) + if err == nil { + // Shutdown killed the backend AND confirmed its exit + // (waitForRecordedProcessExit) before removing the record. + // There is nothing left to check. + return + } + t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) + requireBackendExited(t, record) + }) +} + +const ( + // backendExitTimeout is how long a test waits for the dolt sql-server to + // leave the process table after proxy.Shutdown failed. It is generous on + // purpose: Shutdown's own budget is already 5s of confirmation plus a 2s + // post-kill minimum, and this only runs on the error path, where the + // interesting outcome is "still there", not "took a while". + backendExitTimeout = 30 * time.Second + backendExitPoll = 50 * time.Millisecond +) + +// backendServerRecord returns the pid file the proxy's child manager wrote +// for the dolt sql-server under rootDir (server.PIDFileName). nil means there +// is nothing to verify: no server was started, or it shut down cleanly and +// removed its record. +// +// The whole record is returned, not just the PID: Birth is the process-birth +// token that makes the PID safe to act on at all. +func backendServerRecord(t *testing.T, rootDir string) *pidfile.PidFile { + t.Helper() + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil { + t.Logf("read %s in %s: %v", server.PIDFileName, rootDir, err) + return nil + } + if pf == nil || pf.Pid <= 0 { + return nil + } + return pf +} + +// requireBackendExited fails the test if the recorded dolt sql-server is +// still running after proxy.Shutdown returned an ERROR, then force-kills it +// so the leak does not outlive the run. +// +// It is deliberately a t.Errorf and not a t.Logf: a surviving server holds +// the temp tree the test is about to delete, which is precisely how this +// package produced sql-servers still serving directories that had been gone +// for hours (wy-j2zc8q). A leak must be visible where it is caused, not +// discovered later by a sweep. +// +// Every step is gated on procid birth identity rather than the bare PID. The +// server is not this process's child, so its PID is reusable the instant it +// exits, and proxy.Shutdown deliberately REFUSES to signal a record it cannot +// verify (ErrUnverifiableProcess) — killing that same PID here would defeat +// the exact protection the package went out of its way to provide. A record +// with no Birth token, or one whose token no longer matches, is therefore +// left alone: not the server we recorded, nothing to report. +func requireBackendExited(t *testing.T, pf *pidfile.PidFile) { + t.Helper() + if pf == nil || pf.Pid <= 0 { + return + } + if pf.Birth == "" { + // A record from before birth tokens were written. There is no safe + // way to ask about that PID, so the check is skipped — say so, rather + // than looking like a silent pass. + t.Logf("backend record for pid %d carries no birth token; skipping the survived-Shutdown check", pf.Pid) + return + } + token := procid.Token(pf.Birth) + + deadline := time.Now().Add(backendExitTimeout) + for { + same, err := procid.Verify(pf.Pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pf.Pid, err) + return + } + if !same { + // Either the process is gone, or the PID now belongs to + // something else. Either way our server is not running. + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(backendExitPoll) + } + + handle, err := procid.Open(pf.Pid, token) + if err != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s and could not be opened to kill: %v", + pf.Pid, backendExitTimeout, err) + } + return + } + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s and could not be killed: %v", + pf.Pid, backendExitTimeout, killErr) + } + return + } + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s (force-killed)", pf.Pid, backendExitTimeout) +} + +// backendStillRunning re-checks birth identity after procid.Open or +// Handle.Kill failed, and reports whether the recorded server is verifiably +// still there. +// +// Both calls verify the token themselves and return a plain "does not match +// token" error when it no longer does — which is exactly what a process that +// exited between the wait loop's last poll and the kill attempt produces, and +// that error is NOT procid.IsProcessGone. Without this second look, a server +// that shut down a few milliseconds late would be reported as one that +// survived Shutdown entirely. A verify that itself errors reports "not +// running": the run is over, and an inconclusive probe is not evidence of a +// leak. +func backendStillRunning(t *testing.T, pid int, token procid.Token) bool { + t.Helper() + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d) after a failed kill attempt: %v", pid, err) + return false + } + return same +} diff --git a/internal/storage/uow/team_server_integration_test.go b/internal/storage/uow/team_server_integration_test.go index 5ce079653..2f1df42fc 100644 --- a/internal/storage/uow/team_server_integration_test.go +++ b/internal/storage/uow/team_server_integration_test.go @@ -39,11 +39,7 @@ func newTeamServerHarness(t *testing.T) *teamServerHarness { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) return &teamServerHarness{ port: portInt, diff --git a/internal/storage/uow/testmain_test.go b/internal/storage/uow/testmain_test.go new file mode 100644 index 000000000..165b8b644 --- /dev/null +++ b/internal/storage/uow/testmain_test.go @@ -0,0 +1,78 @@ +package uow + +import ( + "fmt" + "os" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// suiteRootPrefix is this suite's os.MkdirTemp pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-storage-uow-tests-" + +// suiteTempRoot is the TestMain-owned temp directory that every t.TempDir() +// in this package lands under, so SweepSuiteTestServers has a root it can +// vouch for when it reaps leaked dolt sql-servers. +var suiteTempRoot string + +// TestMain gives this package the suite-lifecycle it was missing. +// +// The integration tests here (newTestUOWProvider) start a real proxied +// `dolt sql-server` whose data directory is a t.TempDir(). Cleanup was purely +// per-test — proxy.Shutdown in a t.Cleanup — and there was no TestMain at +// all, so a run killed by `go test -timeout` (which panics the binary and +// skips every Cleanup) left the server running with its data directory +// deleted out from under it, forever. Observed live on a dev box as a +// TestUOWDependencyEditorContract server still serving a temp dir that had +// been gone for hours (wy-j2zc8q). +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. Roots with no owner marker, and + // roots whose owner is still running (a parallel package under + // scripts/test.sh, a second `go test`), are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin TMPDIR under a suite-owned root so every t.TempDir() — including + // the storeRootDir a provider serves — is nested under something the + // post-run sweep may vouch for. Same pattern as cmd/bd/doctor's TestMain. + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + + code := m.Run() + + // Best-effort reap of any dolt sql-server still running under this run's + // own root — the backstop for a test whose Cleanup did not get to run. + swept := doltserver.SweepSuiteTestServers(root) + return doltserver.ApplyLeakPolicy("internal/storage/uow", code, swept) +} + +// TestTempDirLandsUnderSuiteSweepRoot guards the pinning above: if t.TempDir() +// ever stops landing under suiteTempRoot, the post-run sweep silently loses +// its scope and leaked sql-servers become unreapable again. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + dir := t.TempDir() + if !testutil.PathUnderSuiteRoot(dir, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", dir, suiteTempRoot) + } +} diff --git a/internal/storage/uow/version_reconcile_integration_test.go b/internal/storage/uow/version_reconcile_integration_test.go index 7fb4ebda0..905d93d67 100644 --- a/internal/storage/uow/version_reconcile_integration_test.go +++ b/internal/storage/uow/version_reconcile_integration_test.go @@ -31,11 +31,7 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { require.NoError(t, err) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") diff --git a/internal/testutil/suite_temp.go b/internal/testutil/suite_temp.go index dbe7250aa..53b1860b4 100644 --- a/internal/testutil/suite_temp.go +++ b/internal/testutil/suite_temp.go @@ -2,12 +2,14 @@ package testutil import ( "os" + "path/filepath" + "strings" ) // PinSuiteTempRoot creates a suite-owned temp directory and points // GOTMPDIR, TMPDIR, TMP, and TEMP at it so testing.T.TempDir (Go 1.24+ // uses GOTMPDIR) and os.MkdirTemp land under a root -// SweepOrphanedTestServers can reap. +// SweepSuiteTestServers can reap. // // Call from TestMain before m.Run(). The caller owns cleanup: sweep // first, then RemoveAll the returned path. Never pass os.TempDir() as @@ -26,3 +28,33 @@ func PinSuiteTempRoot(pattern string) (string, error) { } return root, nil } + +// PathUnderSuiteRoot reports whether dir is root or nested under it, +// comparing symlink-resolved paths (macOS hands out /var paths that resolve +// to /private/var). +// +// It backs the TestTempDirLandsUnderSuiteSweepRoot guard that every suite +// pinning a root with PinSuiteTempRoot installs: if t.TempDir() ever stops +// landing under the pinned root, the post-run sweep silently loses its scope +// and leaked sql-servers become unreapable again. The guard lives with each +// suite (it needs that suite's package-level root), but this predicate is +// shared, because it is the piece most likely to need a synchronized fix +// later and per-suite copies drift silently (wy-j2zc8q). +func PathUnderSuiteRoot(dir, root string) bool { + dir = evalOrSelf(dir) + root = evalOrSelf(root) + if dir == root { + return true + } + sep := string(os.PathSeparator) + return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) +} + +// evalOrSelf resolves symlinks in p, falling back to p itself when the path +// cannot be resolved (it may not exist yet, or a parent may be unreadable). +func evalOrSelf(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p +} diff --git a/scripts/repro-dolt-prod-timeouts/main_test.go b/scripts/repro-dolt-prod-timeouts/main_test.go index 11d7138cf..5e8f4f114 100644 --- a/scripts/repro-dolt-prod-timeouts/main_test.go +++ b/scripts/repro-dolt-prod-timeouts/main_test.go @@ -495,6 +495,19 @@ func TestSeedProductionShapeFullSmallIssueCountRealSchema(t *testing.T) { if err := serverCmd.Start(); err != nil { t.Fatal(err) } + // Deliberately exempt from the suite-sweep contract every other package + // that spawns a `dolt sql-server` is wired into (internal/doltserver's + // SweepDeadSuiteRoots + SweepSuiteTestServers + ApplyLeakPolicy). + // This is a manual repro harness, and its cleanup is an unconditional + // Kill + Wait rather than a stop that can fail silently: the server is + // dead before t.TempDir() removes baseDir, so this package cannot produce + // the live-server-with-deleted-cwd state that another suite's post-run + // sweep reaps and fails on. A run killed outright (`go test -timeout`, + // Ctrl-C) does leak one, but with an intact cwd and no claimed suite + // root, so no sweep arm can reach it and no suite is misblamed for it. + // Wiring a TestMain here would buy only the killed-run reap; if that + // becomes worth it, the template is internal/storage/uow/testmain_test.go + // (wy-j2zc8q). t.Cleanup(func() { _ = serverCmd.Process.Kill() _ = serverCmd.Wait()