fix(doltserver): sweep Darwin test servers (wy-s4zdj)

Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Steve Yegge
This commit is contained in:
Steve Yegge 2026-07-25 20:48:10 -07:00
commit 097757b3ec
6 changed files with 277 additions and 6 deletions

View file

@ -0,0 +1,65 @@
//go:build darwin
package main
import (
"errors"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
)
type sweepTestRunner struct {
run func() int
}
func (r sweepTestRunner) Run() int {
return r.run()
}
func TestRunTestsAndSweepReapsOrphanedServer(t *testing.T) {
oldRoot := testTempRoot
testTempRoot = t.TempDir()
t.Cleanup(func() { testTempRoot = oldRoot })
serverDir := filepath.Join(testTempRoot, "orphan", ".beads", "dolt")
if err := os.MkdirAll(serverDir, 0o755); err != nil {
t.Fatalf("mkdir server dir: %v", err)
}
script := filepath.Join(testTempRoot, "dolt")
if err := os.WriteFile(script, []byte("#!/bin/sh\ntrap 'exit 0' TERM\nwhile :; do sleep 1; done\n"), 0o755); err != nil {
t.Fatalf("write fake dolt: %v", err)
}
var cmd *exec.Cmd
runner := sweepTestRunner{run: func() int {
cmd = exec.Command(script, "sql-server")
cmd.Dir = serverDir
if err := cmd.Start(); err != nil {
t.Fatalf("start fake dolt sql-server: %v", err)
}
t.Cleanup(func() { _ = cmd.Process.Kill() })
return 0
}}
if code := runTestsAndSweep(runner); code != 0 {
t.Fatalf("runTestsAndSweep() = %d, want 0", code)
}
if cmd == nil || cmd.Process == nil {
t.Fatal("runner did not start fake dolt sql-server")
}
waitDone := make(chan error, 1)
go func() { waitDone <- cmd.Wait() }()
select {
case err := <-waitDone:
var exitErr *exec.ExitError
if err != nil && !errors.As(err, &exitErr) {
t.Fatalf("fake dolt sql-server exited unexpectedly: %v", err)
}
case <-time.After(3 * time.Second):
t.Fatal("runTestsAndSweep left fake dolt sql-server running")
}
}

View file

@ -43,7 +43,11 @@ func testTempDir(pattern string) (string, error) {
// This is the suite most likely to leak — most e2e tests here run a real
// `bd` binary against a `.beads` dir under testTempRoot with auto-start
// enabled. See gastownhall/beads mybd-q6cz.
func runTestsAndSweep(m *testing.M) int {
type testRunner interface {
Run() int
}
func runTestsAndSweep(m testRunner) int {
code := m.Run()
doltserver.SweepOrphanedTestServers(testTempRoot)
return code

View file

@ -1,6 +1,7 @@
package doltserver
import (
"strconv"
"strings"
)
@ -99,3 +100,45 @@ func isUnderDir(dir, root string) bool {
}
return strings.HasPrefix(dir, root+"/")
}
// gatherPSCandidates parses the output of `ps -axo pid=,command=` and
// resolves the working directory of each dolt sql-server candidate. Darwin
// uses this path because it has no /proc filesystem.
//
// cwdForPID returns the resolved cwd, whether that cwd has been deleted, and
// whether it could be determined. Keeping the command execution outside this
// parser makes the safety-critical selection path deterministic to test.
func gatherPSCandidates(psOutput []byte, cwdForPID func(int) (string, bool, bool)) []serverCandidate {
var candidates []serverCandidate
for _, line := range strings.Split(string(psOutput), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
pidText, cmdline, found := strings.Cut(line, " ")
if !found {
continue
}
pid, err := strconv.Atoi(pidText)
if err != nil || pid <= 0 {
continue
}
cmdline = strings.TrimSpace(cmdline)
if !isDoltServerCmdline(cmdline) {
continue
}
cwd, deleted, ok := cwdForPID(pid)
if !ok {
continue
}
candidates = append(candidates, serverCandidate{
pid: pid,
cmdline: cmdline,
cwd: cwd,
cwdDeleted: deleted,
})
}
return candidates
}

View file

@ -0,0 +1,123 @@
//go:build darwin
package doltserver
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
)
// SweepOrphanedTestServers reaps `dolt sql-server` processes that are
// provably leaked test debris: their working directory has been deleted, or
// sits under one of suiteTempRoots. On Darwin, process candidates come from
// ps and their working directories from lsof because /proc is unavailable.
//
// suiteTempRoots MUST be directories owned by the calling suite alone, never
// a shared/global temp directory. This is best-effort: process-listing errors
// and candidates whose cwd cannot be resolved are ignored.
//
// Returns the PIDs it sent a kill signal to.
func SweepOrphanedTestServers(suiteTempRoots ...string) []int {
candidates := gatherDoltServerCandidates()
pids := selectOrphanTestServerPIDs(candidates, canonicalDarwinRoots(suiteTempRoots))
self := os.Getpid()
var killed []int
for _, pid := range pids {
if pid == self {
continue
}
// Re-read the process command immediately before signaling so a PID
// recycled since candidate collection cannot target an unrelated
// process.
if !isDoltServerProcess(pid) {
continue
}
if err := syscall.Kill(pid, syscall.SIGTERM); err == nil {
killed = append(killed, pid)
}
}
if len(killed) == 0 {
return killed
}
fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed)
time.Sleep(300 * time.Millisecond)
for _, pid := range killed {
// Revalidate again before escalation for the same PID-reuse reason.
if !isDoltServerProcess(pid) {
continue
}
_ = syscall.Kill(pid, syscall.SIGKILL)
}
return killed
}
// canonicalDarwinRoots makes caller roots comparable with lsof output. macOS
// commonly reports a cwd below /private/var while os.MkdirTemp returned the
// equivalent /var path.
func canonicalDarwinRoots(roots []string) []string {
canonical := make([]string, 0, len(roots))
for _, root := range roots {
if root == "" {
continue
}
resolved, err := filepath.EvalSymlinks(root)
if err != nil {
canonical = append(canonical, root)
continue
}
canonical = append(canonical, resolved)
}
return canonical
}
func gatherDoltServerCandidates() []serverCandidate {
out, err := exec.Command("ps", "-axo", "pid=,command=").Output()
if err != nil {
return nil
}
return gatherPSCandidates(out, readDarwinCwd)
}
func isDoltServerProcess(pid int) bool {
out, err := exec.Command("ps", "-p", strconv.Itoa(pid), "-o", "command=").Output()
if err != nil {
return false
}
return isDoltServerCmdline(strings.TrimSpace(string(out)))
}
// readDarwinCwd resolves pid's cwd from lsof's machine-readable field output.
// lsof emits the name as an `n` field after selecting descriptor `cwd`.
func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) {
out, err := exec.Command(
"lsof", "-a", "-p", strconv.Itoa(pid), "-d", "cwd", "-Fn",
).Output()
if err != nil {
return "", false, false
}
for _, line := range strings.Split(string(out), "\n") {
if !strings.HasPrefix(line, "n") {
continue
}
cwd = strings.TrimPrefix(line, "n")
const deletedSuffix = " (deleted)"
if strings.HasSuffix(cwd, deletedSuffix) {
return strings.TrimSuffix(cwd, deletedSuffix), true, true
}
if cwd != "" {
return cwd, false, true
}
}
return "", false, false
}

View file

@ -1,11 +1,10 @@
//go:build !linux
//go:build !linux && !darwin
package doltserver
// SweepOrphanedTestServers is a no-op on non-Linux platforms. The leaked
// dolt sql-server evidence for gastownhall/beads mybd-q6cz is Linux-only
// (/proc-based process/cwd inspection isn't portable), so this stub exists
// only so callers (test TestMains) compile unchanged everywhere.
// SweepOrphanedTestServers is a no-op on platforms where process command
// lines and working directories cannot be inspected by an implementation in
// this package. The stub keeps callers (test TestMains) portable.
func SweepOrphanedTestServers(_ ...string) []int {
return nil
}

View file

@ -144,3 +144,40 @@ func TestIsUnderDir(t *testing.T) {
}
}
}
func TestGatherPSCandidates(t *testing.T) {
psOutput := []byte(`
101 dolt sql-server -H 127.0.0.1 -P 12345
not-a-pid dolt sql-server
102 dolt status
103 /opt/dolt --prof cpu sql-server -P 12346
104 dolt sql-server -P 12347
`)
cwds := map[int]struct {
dir string
deleted bool
ok bool
}{
101: {dir: "/tmp/my-suite/.beads/dolt", ok: true},
103: {dir: "/tmp/deleted-suite/.beads/dolt", deleted: true, ok: true},
104: {ok: false},
}
candidates := gatherPSCandidates(psOutput, func(pid int) (string, bool, bool) {
cwd := cwds[pid]
return cwd.dir, cwd.deleted, cwd.ok
})
wantCandidates := []serverCandidate{
{pid: 101, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite/.beads/dolt"},
{pid: 103, cmdline: "/opt/dolt --prof cpu sql-server -P 12346", cwd: "/tmp/deleted-suite/.beads/dolt", cwdDeleted: true},
}
if !reflect.DeepEqual(candidates, wantCandidates) {
t.Fatalf("gatherPSCandidates() = %#v, want %#v", candidates, wantCandidates)
}
gotPIDs := selectOrphanTestServerPIDs(candidates, []string{"/tmp/my-suite"})
wantPIDs := []int{101, 103}
if !reflect.DeepEqual(gotPIDs, wantPIDs) {
t.Errorf("darwin ps selection path = %v, want %v", gotPIDs, wantPIDs)
}
}