fix(doltserver): prove listener ownership on darwin with lsof and ps (#7232) (#7257)

On macOS listenerOwnership always answered known=false, so once a foreign
process greeted on the port chosen for a new dolt sql-server, Start adopted
that listener as its own (the default log level emits no ready line to
disprove it). TestStart_RecoversWhenEphemeralPortIsTaken has failed on the
Main workflow's macOS job on every push since #7201 (14/14 completed runs);
PR CI has no macOS test job, so no PR saw it.

Darwin has no /proc, so the listener's processes come from lsof and the
process tree from ps, the two tools this package already uses to find a
port's holder and a server's working directory. Each pid lsof lists on the
port is walked up its parent chain; reaching the child proves the listener
is the child's (or a descendant's, for a wrapper that does not exec). Both
reads run only once a greeting has arrived, never on the bare poll.

known is false when either tool fails or lists no listener at all, and
Start then falls back to the greeting as before. A child that no longer
exists is known not to own anything, as on Linux.

The tests that were Linux-only because ownership was only provable there now
run on darwin too, and the darwin file gets its own fake-read table, parser
tests, and a real-tool test with an in-process listener and a listener in a
grandchild process. On this machine the formerly red test passes 10/10
(it failed 40/40 on main).

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bee 2026-10-06 23:24:18 +00:00 • committed by GitHub
commit d4309098e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 294 additions and 11 deletions

View file

@ -7,6 +7,7 @@ go_library(
"doltserver_unix.go",
"doltserver_windows.go",
"gc_config.go",
"listenerowner_darwin.go",
"listenerowner_linux.go",
"listenerowner_other.go",
"logrotate.go",
@ -63,6 +64,7 @@ go_test(
"fdleak_integration_test.go",
"gc_config_test.go",
"lifecycle_integration_test.go",
"listenerowner_darwin_test.go",
"listenerowner_linux_test.go",
"logrotate_test.go",
"managed_server_proof_test.go",

View file

@ -0,0 +1,141 @@
//go:build darwin
package doltserver
import (
"errors"
"os/exec"
"strconv"
"strings"
"syscall"
)
// The two process reads listenerOwnership makes on darwin; tests replace them.
var (
listeningPIDs = lsofListeningPIDs
processParents = psParentPIDs
)
// listenerOwnership reports whether a TCP listener on port belongs to pid or
// one of its descendants. Darwin has no /proc, so the listener's processes
// come from lsof and the process tree from ps, the same two tools this
// package already uses to find a port's holder and a server's working
// directory. Descendants cover a dolt behind a wrapper script that does not
// exec: the listener's ancestor chain then passes through the wrapper, which
// is the child bd launched.
//
// known is false when the tools cannot answer, and the caller then falls back
// to weaker readiness evidence:
// - lsof or ps is missing or fails;
// - no listener on port is listed at all, although one just answered: lsof
// cannot see another user's sockets without privilege, or the connection
// is forwarded elsewhere.
//
// A child that no longer exists is known not to own anything.
//
// Both reads run only once a greeting has arrived on the port, never on the
// bare readiness poll, and together cost on the order of 100ms.
func listenerOwnership(pid, port int) (owned, known bool) {
if !processExists(pid) {
return false, true
}
holders, err := listeningPIDs(port)
if err != nil || len(holders) == 0 {
return false, false
}
parents, err := processParents()
if err != nil || len(parents) == 0 {
return false, false
}
for _, holder := range holders {
if descendsFrom(holder, pid, parents) {
return true, true
}
}
return false, true
}
// processExists is true while pid names a process this user may signal, or
// one that exists but belongs to someone else.
func processExists(pid int) bool {
if pid <= 0 {
return false
}
err := syscall.Kill(pid, 0)
return err == nil || errors.Is(err, syscall.EPERM)
}
// descendsFrom reports whether p is ancestor itself or one of its
// descendants, walking p's parent chain through parents (child -> parent).
// A chain that reaches a pid the snapshot does not list, pid 0, or a cycle
// (a stale snapshot) is not a match.
func descendsFrom(p, ancestor int, parents map[int]int) bool {
seen := map[int]bool{}
for p > 0 && !seen[p] {
if p == ancestor {
return true
}
seen[p] = true
next, ok := parents[p]
if !ok {
return false
}
p = next
}
return false
}
// lsofListeningPIDs lists the processes holding a TCP listener on port.
// -n and -P skip host and port name lookups, which otherwise dominate the
// run time; -t prints one pid per line. lsof exits 1 with no output when
// nothing matches, which is reported as no listener, not as an error, so the
// caller treats both the same way.
func lsofListeningPIDs(port int) ([]int, error) {
out, err := exec.Command("lsof", "-nP", "-t", "-iTCP:"+strconv.Itoa(port), "-sTCP:LISTEN").Output() //nolint:gosec // G702: port is an internal int, not user input
if err != nil && len(out) == 0 {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
return nil, nil
}
return nil, err
}
return parseLsofPIDs(out), nil
}
// parseLsofPIDs reads `lsof -t` output: one pid per line.
func parseLsofPIDs(out []byte) []int {
var pids []int
for _, line := range strings.Split(string(out), "\n") {
if pid, err := strconv.Atoi(strings.TrimSpace(line)); err == nil && pid > 0 {
pids = append(pids, pid)
}
}
return pids
}
// psParentPIDs snapshots every process's parent from `ps -axo pid=,ppid=`.
func psParentPIDs() (map[int]int, error) {
out, err := exec.Command("ps", "-axo", "pid=,ppid=").Output()
if err != nil {
return nil, err
}
return parsePSParents(out), nil
}
// parsePSParents reads `ps -axo pid=,ppid=` output into child -> parent.
func parsePSParents(out []byte) map[int]int {
parents := map[int]int{}
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) != 2 {
continue
}
pid, err1 := strconv.Atoi(fields[0])
ppid, err2 := strconv.Atoi(fields[1])
if err1 != nil || err2 != nil || pid <= 0 {
continue
}
parents[pid] = ppid
}
return parents
}

View file

@ -0,0 +1,140 @@
//go:build darwin
package doltserver
import (
"errors"
"os"
"os/exec"
"strconv"
"testing"
"time"
)
func withProcessReads(t *testing.T, pids func(int) ([]int, error), parents func() (map[int]int, error)) {
t.Helper()
origPIDs, origParents := listeningPIDs, processParents
listeningPIDs, processParents = pids, parents
t.Cleanup(func() { listeningPIDs, processParents = origPIDs, origParents })
}
func TestListenerOwnership_FakeReads(t *testing.T) {
self := os.Getpid()
tree := map[int]int{self: 1, 501: self, 502: 501, 600: 1, 700: 700}
cases := []struct {
name string
pids []int
pidsErr error
parents map[int]int
parentsErr error
owned bool
known bool
}{
{name: "child holds the listener", pids: []int{self}, parents: tree, owned: true, known: true},
{name: "grandchild holds the listener (wrapper)", pids: []int{502}, parents: tree, owned: true, known: true},
{name: "foreign process holds it", pids: []int{600}, parents: tree, owned: false, known: true},
{name: "foreign and own both listed", pids: []int{600, 501}, parents: tree, owned: true, known: true},
{name: "holder missing from the snapshot", pids: []int{999}, parents: tree, owned: false, known: true},
{name: "cyclic snapshot does not loop", pids: []int{700}, parents: tree, owned: false, known: true},
{name: "no listener listed", pids: nil, parents: tree, owned: false, known: false},
{name: "lsof fails", pidsErr: errors.New("lsof: not found"), parents: tree, owned: false, known: false},
{name: "ps fails", pids: []int{self}, parentsErr: errors.New("ps: not found"), owned: false, known: false},
{name: "empty snapshot", pids: []int{self}, parents: map[int]int{}, owned: false, known: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
withProcessReads(t,
func(int) ([]int, error) { return tc.pids, tc.pidsErr },
func() (map[int]int, error) { return tc.parents, tc.parentsErr },
)
owned, known := listenerOwnership(self, 1)
if owned != tc.owned || known != tc.known {
t.Errorf("listenerOwnership = (%v, %v), want (%v, %v)", owned, known, tc.owned, tc.known)
}
})
}
}
func TestListenerOwnership_DeadChildIsKnown(t *testing.T) {
withProcessReads(t,
func(int) ([]int, error) { t.Fatal("listener read on a dead child"); return nil, nil },
func() (map[int]int, error) { t.Fatal("tree read on a dead child"); return nil, nil },
)
if owned, known := listenerOwnership(1<<30, 1); owned || !known {
t.Errorf("listenerOwnership(no such pid) = (%v, %v), want (false, true)", owned, known)
}
}
func TestParseLsofPIDs(t *testing.T) {
got := parseLsofPIDs([]byte("12245\n\n 77360 \nnot-a-pid\n0\n"))
want := []int{12245, 77360}
if len(got) != len(want) {
t.Fatalf("parseLsofPIDs = %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("parseLsofPIDs = %v, want %v", got, want)
}
}
}
func TestParsePSParents(t *testing.T) {
got := parsePSParents([]byte(" 1 0\n 501 1\n12245 501\nbad line here\n 77360 x\n"))
want := map[int]int{1: 0, 501: 1, 12245: 501}
if len(got) != len(want) {
t.Fatalf("parsePSParents = %v, want %v", got, want)
}
for k, v := range want {
if got[k] != v {
t.Fatalf("parsePSParents[%d] = %d, want %d", k, got[k], v)
}
}
}
// TestListenerOwnership_RealTools mirrors the Linux real-/proc test: our own
// in-process listener is ours, a pid that does not exist owns nothing, a
// sibling process's listener is not ours, and a listener in a child process
// we spawn is ours through the tree.
func TestListenerOwnership_RealTools(t *testing.T) {
if _, err := exec.LookPath("lsof"); err != nil {
t.Skip("lsof not on PATH")
}
port := foreignGreeter(t)
if owned, known := listenerOwnership(os.Getpid(), port); !owned || !known {
t.Errorf("listenerOwnership(self, own listener) = (%v, %v), want (true, true)", owned, known)
}
if owned, known := listenerOwnership(1<<30, port); owned || !known {
t.Errorf("listenerOwnership(no such pid) = (%v, %v), want (false, true)", owned, known)
}
// A child that holds its own listener: nc -l on a fresh port, under a
// shell that does not exec it, so the listener is a grandchild.
ncPort := freeLoopbackPort(t)
cmd := exec.Command("sh", "-c", "nc -l 127.0.0.1 "+strconv.Itoa(ncPort)+"; exit 0")
if err := cmd.Start(); err != nil {
t.Skipf("cannot start nc under sh: %v", err)
}
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
deadline := time.Now().Add(5 * time.Second)
for {
pids, err := listeningPIDs(ncPort)
if err == nil && len(pids) > 0 {
break
}
if time.Now().After(deadline) {
t.Skip("nc never listened within 5s")
}
time.Sleep(50 * time.Millisecond)
}
if owned, known := listenerOwnership(cmd.Process.Pid, ncPort); !owned || !known {
t.Errorf("listenerOwnership(sh child, grandchild nc listener) = (%v, %v), want (true, true)", owned, known)
}
if owned, known := listenerOwnership(os.Getpid(), ncPort); !owned || !known {
t.Errorf("listenerOwnership(self, descendant nc listener) = (%v, %v), want (true, true)", owned, known)
}
// The child does not own its parent's listener: the tree is walked up
// from the holder, never down from the pid being asked about.
if owned, known := listenerOwnership(cmd.Process.Pid, port); owned || !known {
t.Errorf("listenerOwnership(sh child, parent's listener) = (%v, %v), want (false, true)", owned, known)
}
}

View file

@ -1,4 +1,4 @@
//go:build !linux
//go:build !linux && !darwin
package doltserver

View file

@ -142,8 +142,8 @@ const (
// own startup, which under load takes longer than any fixed grace period, so
// a greeting only counts once the listener is shown to be the child's: by
// dolt's ready line in the child's output when its log level emits one, or
// else by the listening socket belonging to the child's process tree (Linux,
// via /proc). When the socket is shown to belong to someone else the wait
// else by the listening socket belonging to the child's process tree (Linux
// via /proc, darwin via lsof and ps). When the socket is shown to belong to someone else the wait
// ends at once with ErrPortInUse. Where neither proof is available the
// greeting decides, as before, unless the log level promises a ready line.
// Either way a child that exits, or says its port is taken, ends the wait,

View file

@ -110,8 +110,8 @@ func TestAwaitOwnedListener_ForeignListenerIsNotReady(t *testing.T) {
{name: "ownership says foreign", owner: func(int, int) (bool, bool) { return false, true }},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.owner == nil && runtime.GOOS != "linux" {
t.Skip("listener ownership is only provable on linux")
if tc.owner == nil && runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
t.Skip("listener ownership is only provable on linux and darwin")
}
port := foreignGreeter(t)
logPath := filepath.Join(t.TempDir(), "dolt-server.log")
@ -166,7 +166,7 @@ func TestAwaitOwnedListener_OwnListenerIsReady(t *testing.T) {
}); err != nil {
t.Fatalf("awaitOwnedListener on the child's own listener: %v", err)
}
if runtime.GOOS == "linux" {
if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
if owned, known := listenerOwnership(srv.pid, port); !owned || !known {
t.Errorf("listenerOwnership(child) = (%v, %v), want (true, true)", owned, known)
}
@ -206,8 +206,8 @@ func TestAwaitOwnedListener_ChildExitIsReported(t *testing.T) {
// child's startup (which under load can outlast the ready timeout and turn a
// movable port into a hard failure); it returns ErrPortInUse straight away.
func TestAwaitOwnedListener_ProvenForeignEndsWaitAtOnce(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("listener ownership is only provable on linux")
if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
t.Skip("listener ownership is only provable on linux and darwin")
}
port := foreignGreeter(t)
logPath := filepath.Join(t.TempDir(), "dolt-server.log")
@ -378,15 +378,15 @@ func TestStart_RecoversWhenEphemeralPortIsTaken(t *testing.T) {
if got := calls.Load(); got != 2 {
t.Errorf("allocateEphemeralPort called %d times, want 2 (the taken port, then a fresh one)", got)
}
// On Linux the first child can be killed before it records its launch:
// /proc proves the port foreign as soon as the greeting arrives.
// On Linux and darwin the first child can be killed before it records its launch:
// the process tree proves the port foreign as soon as the greeting arrives.
if got := launchedPorts(t, launches); len(got) == 0 || got[len(got)-1] != fmt.Sprint(state.Port) {
t.Errorf("launches = %v, want the last one on the final port %d", got, state.Port)
}
if got := readPortFile(beadsDir); got != state.Port {
t.Errorf("port file = %d, want %d", got, state.Port)
}
if runtime.GOOS == "linux" {
if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
if owned, known := listenerOwnership(state.PID, state.Port); !owned || !known {
t.Errorf("listener on %d is not owned by the started server (PID %d)", state.Port, state.PID)
}