diff --git a/cmd/bd/shared_server_cleanup_test.go b/cmd/bd/shared_server_cleanup_test.go index 6e6d0c584..914d612c2 100644 --- a/cmd/bd/shared_server_cleanup_test.go +++ b/cmd/bd/shared_server_cleanup_test.go @@ -18,8 +18,9 @@ import ( // runs a subprocess `bd init` with BEADS_DOLT_SHARED_SERVER=1 owes. // // That init takes the shared-global branch in init.go, which calls -// doltserver.Start(sharedDir) — and Start DAEMONIZES: cmd.Process.Release(), -// no Wait, cmd.Dir = /dolt (doltserver.go:1443-1446). The `bd` +// doltserver.Start(sharedDir) — and Start DAEMONIZES: the server is its own +// process group (Setpgid) with cmd.Dir = /dolt, and outlives bd (bd +// only reaps it if it exits while bd is still running). The `bd` // subprocess then exits and the sql-server keeps running, serving a directory // under the suite's temp HOME that TestMain deletes on the way out. Nothing in // those fixtures ever stopped it, so a full cmd/bd run finished with a live diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index d823d3c8e..4c4dc7c00 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -7,6 +7,8 @@ go_library( "doltserver_unix.go", "doltserver_windows.go", "gc_config.go", + "listenerowner_linux.go", + "listenerowner_other.go", "logrotate.go", "manifest_recovery.go", "physical_root.go", @@ -14,6 +16,8 @@ go_library( "procattr_linux.go", "procattr_other_unix.go", "servermode.go", + "startowned.go", + "startupwatch.go", "sweep.go", "sweep_darwin.go", "sweep_linux.go", @@ -54,9 +58,11 @@ go_test( "doltserver_test.go", "doltserver_unix_test.go", "doltserver_windows_test.go", + "fakedolt_test.go", "fdleak_integration_test.go", "gc_config_test.go", "lifecycle_integration_test.go", + "listenerowner_linux_test.go", "logrotate_test.go", "managed_server_proof_test.go", "manifest_recovery_test.go", @@ -67,6 +73,8 @@ go_test( "probe_test.go", "procattr_linux_test.go", "socket_integration_test.go", + "startowned_test.go", + "startupwatch_test.go", "sweep_darwin_test.go", "sweep_reap_unix_test.go", "sweep_suite_linux_test.go", diff --git a/internal/doltserver/doltserver.go b/internal/doltserver/doltserver.go index 7b53ed2d5..252878643 100644 --- a/internal/doltserver/doltserver.go +++ b/internal/doltserver/doltserver.go @@ -246,7 +246,7 @@ func isFalsyBool(s string) bool { return err == nil && !b } -// readyTimeout returns the timeout used by waitForReady when starting the +// readyTimeout returns the timeout awaitOwnedListener uses when starting the // dolt sql-server. Defaults to 10 seconds, but can be overridden via the // BEADS_DOLT_READY_TIMEOUT environment variable (positive integer seconds). // First-run Dolt SQL engine initialization can take ~60s on slower hardware @@ -417,8 +417,10 @@ func maxDoltServers() int { // allocateEphemeralPort asks the OS for a free TCP port on host. // It binds to port 0, reads the assigned port, and closes the listener. // The caller should pass the returned port to dolt sql-server promptly -// to minimize the TOCTOU window. -func allocateEphemeralPort(host string) (int, error) { +// to minimize the TOCTOU window; Start still has to prove the server it +// launched is the one that bound it (see awaitOwnedListener). A var so tests +// can hand Start a port another process holds. +var allocateEphemeralPort = func(host string) (int, error) { ln, err := net.Listen("tcp", net.JoinHostPort(host, "0")) if err != nil { return 0, fmt.Errorf("allocating ephemeral port: %w", err) @@ -1670,6 +1672,11 @@ func startLocked(beadsDir string) (*State, error) { actualPort int lastErr error attempts int + // notReady is set when a launched server neither came up nor + // reported a port conflict (it timed out or exited). + notReady error + notReadyPID int + started *startedServer ) { // Ensure dolt database directory is initialized @@ -1705,6 +1712,13 @@ func startLocked(beadsDir string) (*State, error) { // a fresh port from the OS with retry for TOCTOU races. actualPort = cfg.Port explicitPort := actualPort > 0 + // movable: Start may move to a fresh ephemeral port when the one it + // launched on turns out to be held by another process. True for a + // port bd chose itself: none configured (ephemeral), or bd's own + // port-file record outside shared-server mode (GH#4052: auto-start + // may replace non-authoritative ports). Operator-configured ports and + // the shared-server port stay pinned. + movable := !explicitPort || (cfg.PortSource == PortSourcePortFile && !cfg.PortSharedServer) if explicitPort { // Explicit port: check for conflicts and adopt existing servers. @@ -1742,12 +1756,12 @@ func startLocked(beadsDir string) (*State, error) { pid = 0 lastErr = nil attempts = 1 - if !explicitPort { + if movable { attempts = maxEphemeralPortAttempts } for i := range attempts { - if !explicitPort { + if !explicitPort || i > 0 { p, allocErr := allocateEphemeralPort(cfg.Host) if allocErr != nil { lastErr = allocErr @@ -1765,7 +1779,7 @@ func startLocked(beadsDir string) (*State, error) { cfgBody, cfgErr := buildDoltServerYAMLConfig(cfg.Host, actualPort, cfg.RemotesAPIPort, debug, cfgDir) if cfgErr != nil { lastErr = fmt.Errorf("rendering managed sql-server config: %w", cfgErr) - if !explicitPort { + if movable { continue } break @@ -1777,14 +1791,14 @@ func startLocked(beadsDir string) (*State, error) { absConfigPath, absErr := filepath.Abs(doltServerConfigPath(beadsDir)) if absErr != nil { lastErr = fmt.Errorf("resolving managed sql-server config path: %w", absErr) - if !explicitPort { + if movable { continue } break } if werr := os.WriteFile(absConfigPath, cfgBody, 0600); werr != nil { lastErr = fmt.Errorf("writing managed sql-server config: %w", werr) - if !explicitPort { + if movable { continue } break @@ -1808,34 +1822,77 @@ func startLocked(beadsDir string) (*State, error) { // restarted. os/exec does not close those; mark them first. sanitizeInheritedFDs() - if startErr := cmd.Start(); startErr != nil { + logOffset := logSize(logFile) + srv, startErr := launchServer(cmd) + if startErr != nil { lastErr = startErr - if !explicitPort { + if movable { continue // retry with a new ephemeral port } break } - pid = cmd.Process.Pid - _ = cmd.Process.Release() - - // Quick check: did the process exit immediately (bind failure)? - // Give it a moment to fail on port bind before proceeding. - time.Sleep(200 * time.Millisecond) - if !isProcessAlive(pid) { - lastErr = fmt.Errorf("dolt sql-server exited immediately on port %d (attempt %d/%d)", actualPort, i+1, attempts) - pid = 0 - if !explicitPort { - continue - } + // Record the server before waiting on it, as soon as it exists: + // if bd is interrupted during the wait, the next bd (or bd dolt + // stop / killall) must still find the dolt that holds the + // database lock. Both files, with the real port: IsRunning stops + // a tracked server whose port it cannot determine. + if werr := writeServerStateFiles(beadsDir, srv.pid, actualPort); werr != nil { + srv.killAndWait() + removeServerStateFiles(beadsDir) + lastErr = werr break } + // Wait until this child, not whatever else holds the port, is + // accepting connections. A port another process holds shows up + // as ErrPortInUse: a port bd chose itself moves to a fresh one; + // an operator-configured port stays pinned. + waitErr := awaitOwnedListener(srv, startupProbe{ + host: cfg.Host, + port: actualPort, + logPath: logPath(beadsDir), + logOffset: logOffset, + readyLineLogged: debug, + timeout: readyTimeout(), + }) + if waitErr != nil { + srv.killAndWait() + removeServerStateFiles(beadsDir) + if errors.Is(waitErr, ErrPortInUse) { + if movable { + lastErr = fmt.Errorf("%w (attempt %d/%d)", waitErr, i+1, attempts) + continue + } + lastErr = fmt.Errorf("%w; %s", waitErr, pinnedPortRemedy(cfg, beadsDir, actualPort)) + break + } + notReady, notReadyPID = waitErr, srv.pid + break + } + + started = srv + pid = srv.pid lastErr = nil break } _ = logFile.Close() + if notReady != nil { + if hasJournalCorruption, logErr := logHasCorruptJournalError(logPath(beadsDir)); logErr == nil && hasJournalCorruption { + return nil, fmt.Errorf("server started (PID %d) but not accepting connections on port %d: %w\n\n%s", + notReadyPID, actualPort, notReady, corruptJournalRecoveryHint(beadsDir)) + } + if dirs, detErr := detectCorruptManifest(beadsDir, doltDir); detErr == nil && len(dirs) > 0 { + return nil, fmt.Errorf("server started (PID %d) but not accepting connections on port %d: %w\n"+ + "Corrupt manifest with no recoverable data detected (GH#3290) in:\n %s\n"+ + "Run 'bd doctor --fix' to back up the corrupt database(s) and reinitialize.\nCheck logs: %s", + notReadyPID, actualPort, notReady, strings.Join(dirs, "\n "), logPath(beadsDir)) + } + return nil, fmt.Errorf("server started (PID %d) but not accepting connections on port %d: %w\nCheck logs: %s", + notReadyPID, actualPort, notReady, logPath(beadsDir)) + } + if lastErr != nil { // GH#3290 / bd-6dnrw.6: unclean-shutdown manifest corruption is // detected here but never auto-repaired — reinitializing .dolt is @@ -1851,41 +1908,10 @@ func startLocked(beadsDir string) (*State, error) { } } - // Write PID and port files - if err := os.WriteFile(pidPath(beadsDir), []byte(strconv.Itoa(pid)), 0600); err != nil { - if proc, findErr := os.FindProcess(pid); findErr == nil { - _ = proc.Kill() - } - return nil, fmt.Errorf("writing PID file: %w", err) - } - if err := writePortFile(beadsDir, actualPort); err != nil { - if proc, findErr := os.FindProcess(pid); findErr == nil { - _ = proc.Kill() - } - _ = os.Remove(pidPath(beadsDir)) - return nil, fmt.Errorf("writing port file: %w", err) - } - - // Wait for server to accept connections - if err := waitForReady(cfg.Host, actualPort, readyTimeout()); err != nil { - if proc, findErr := os.FindProcess(pid); findErr == nil { - _ = proc.Kill() - } - _ = os.Remove(pidPath(beadsDir)) - _ = os.Remove(portPath(beadsDir)) - if hasJournalCorruption, logErr := logHasCorruptJournalError(logPath(beadsDir)); logErr == nil && hasJournalCorruption { - return nil, fmt.Errorf("server started (PID %d) but not accepting connections on port %d: %w\n\n%s", - pid, actualPort, err, corruptJournalRecoveryHint(beadsDir)) - } - return nil, fmt.Errorf("server started (PID %d) but not accepting connections on port %d: %w\nCheck logs: %s", - pid, actualPort, err, logPath(beadsDir)) - } + // The PID and port files were written when the server was launched. if err := waitForRemotesAPI(cfg.RemotesAPIPort, readyTimeout()); err != nil { - if proc, findErr := os.FindProcess(pid); findErr == nil { - _ = proc.Kill() - } - _ = os.Remove(pidPath(beadsDir)) - _ = os.Remove(portPath(beadsDir)) + started.killAndWait() + removeServerStateFiles(beadsDir) return nil, fmt.Errorf("server started (PID %d) but remotesapi is not accepting connections on port %d: %w\nCheck logs: %s", pid, cfg.RemotesAPIPort, err, logPath(beadsDir)) } @@ -2280,30 +2306,6 @@ func KillStaleServers(beadsDir string) ([]int, error) { ) } -// waitForReady polls until the server accepts TCP connections AND greets -// with a MySQL handshake. Draining the handshake before closing the probe -// connection makes Close() send TCP FIN instead of RST, which prevents the -// dolt sql-server process from interpreting probe closes as aborted MySQL -// handshakes and crashing (see gastownhall/beads#4132, #4133). -// -// A dial that succeeds but never greets (TCP listener accepting, MySQL -// engine not yet writing) is not treated as ready: this function keeps -// polling until either a greeting arrives or the deadline is reached. -func waitForReady(host string, port int, timeout time.Duration) error { - addr := net.JoinHostPort(host, strconv.Itoa(port)) - deadline := time.Now().Add(timeout) - - for time.Now().Before(deadline) { - greeted, err := ProbeSQLServer("tcp", addr, 500*time.Millisecond) //nolint:gosec // G704: addr is built from internal host+port, not user input - if err == nil && greeted { - return nil - } - time.Sleep(500 * time.Millisecond) - } - - return fmt.Errorf("timeout after %s waiting for server at %s", timeout, addr) -} - func waitForRemotesAPI(port int, timeout time.Duration) error { if port <= 0 { return nil diff --git a/internal/doltserver/doltserver_test.go b/internal/doltserver/doltserver_test.go index 45f76afd8..ef8573fd6 100644 --- a/internal/doltserver/doltserver_test.go +++ b/internal/doltserver/doltserver_test.go @@ -1220,6 +1220,7 @@ func TestDefaultConfigReturnsZeroForStandalone(t *testing.T) { // giving each project a unique port without hash collisions (GH#2098). t.Setenv("GT_ROOT", "") t.Setenv("BEADS_DOLT_SERVER_PORT", "") + isolateUserConfig(t) dir := t.TempDir() cfg := DefaultConfig(dir) @@ -1243,6 +1244,7 @@ func TestDefaultConfigPortFileTakesPrecedence(t *testing.T) { // Port file (written by Start) should take precedence over ephemeral. t.Setenv("GT_ROOT", "") t.Setenv("BEADS_DOLT_SERVER_PORT", "") + isolateUserConfig(t) dir := t.TempDir() if err := writePortFile(dir, 14567); err != nil { @@ -2850,9 +2852,9 @@ func TestWaitForReady(t *testing.T) { } // Spawn a goroutine that delays binding the port. This simulates a - // "slow server" -- the TCP listener is not yet bound when waitForReady + // "slow server" -- the TCP listener is not yet bound when the readiness probe // is first called. Once bound, each accepted connection is sent a fake - // MySQL handshake greeting so waitForReady's post-F7 "must be greeted, + // MySQL handshake greeting so the readiness probe's post-F7 "must be greeted, // not just accepted" check is satisfiable. bindAfter := 200 * time.Millisecond listenerReady := make(chan net.Listener, 1) @@ -2889,14 +2891,14 @@ func TestWaitForReady(t *testing.T) { // the port; "succeeds" runs after. t.Run("times out when server not ready in time", func(t *testing.T) { // 50ms is well under the 200ms bind delay, so this MUST time out. - if err := waitForReady(host, port, 50*time.Millisecond); err == nil { + if err := awaitGreeting(host, port, 50*time.Millisecond); err == nil { t.Errorf("expected timeout error, got nil") } }) t.Run("succeeds when server becomes ready in time", func(t *testing.T) { // 2 seconds is well over the remaining bind delay; gives comfortable margin. - if err := waitForReady(host, port, 2*time.Second); err != nil { + if err := awaitGreeting(host, port, 2*time.Second); err != nil { t.Errorf("expected nil error, got: %v", err) } }) diff --git a/internal/doltserver/doltserver_unix.go b/internal/doltserver/doltserver_unix.go index 03cccf8b4..5e1533c6a 100644 --- a/internal/doltserver/doltserver_unix.go +++ b/internal/doltserver/doltserver_unix.go @@ -160,3 +160,12 @@ func gracefulStop(pid int, timeout time.Duration) error { time.Sleep(100 * time.Millisecond) return nil } + +// killProcessGroup SIGKILLs the process group pgid leads. Start launches +// each dolt sql-server as a group leader (Setpgid), so this reaches a dolt +// that a non-exec wrapper started, too. +func killProcessGroup(pgid int) { + if pgid > 0 { + _ = syscall.Kill(-pgid, syscall.SIGKILL) + } +} diff --git a/internal/doltserver/doltserver_windows.go b/internal/doltserver/doltserver_windows.go index 4fd38b91d..385238e70 100644 --- a/internal/doltserver/doltserver_windows.go +++ b/internal/doltserver/doltserver_windows.go @@ -126,3 +126,6 @@ func gracefulStop(pid int, timeout time.Duration) error { time.Sleep(500 * time.Millisecond) return nil } + +// killProcessGroup is a no-op on Windows; killAndWait kills the child itself. +func killProcessGroup(int) {} diff --git a/internal/doltserver/fakedolt_test.go b/internal/doltserver/fakedolt_test.go new file mode 100644 index 000000000..ab1b43e0e --- /dev/null +++ b/internal/doltserver/fakedolt_test.go @@ -0,0 +1,119 @@ +package doltserver_test + +import ( + "fmt" + "net" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +// fakeDoltEnv, when set, makes this test binary act as a stand-in `dolt` +// (see fakeDolt). TestMain checks it before anything else. Tests reach it +// through a `dolt` shim on PATH that execs this binary; see +// installFakeDolt in startowned_test.go, which repeats these names. +const ( + fakeDoltEnv = "BEADS_TEST_FAKE_DOLT" + fakeDoltDelayEnv = "BEADS_TEST_FAKE_DOLT_DELAY" + // fakeDoltInUseEnv lists ports ("all" for every port) the fake reports + // taken without trying to bind, as if another process grabbed them. + fakeDoltInUseEnv = "BEADS_TEST_FAKE_DOLT_INUSE_PORTS" + // fakeDoltExitEnv makes the fake exit with that status after its + // startup delay instead of binding. + fakeDoltExitEnv = "BEADS_TEST_FAKE_DOLT_EXIT" + // fakeDoltReadyEnv makes the fake log dolt's ready line once bound, as + // dolt does at log level info or debug. + fakeDoltReadyEnv = "BEADS_TEST_FAKE_DOLT_READY" + // fakeDoltLaunchesEnv names a file the fake appends each sql-server + // launch's port to. + fakeDoltLaunchesEnv = "BEADS_TEST_FAKE_DOLT_LAUNCHES" +) + +var fakeDoltConfigPortRe = regexp.MustCompile(`(?m)^\s+port:\s*(\d+)`) + +// fakeDolt answers the dolt invocations doltserver.Start makes. Its +// sql-server behaves like dolt's where port races are concerned: it spends +// $BEADS_TEST_FAKE_DOLT_DELAY on "startup", then binds its port; if the port +// is taken it prints dolt's "Port N already in use." and exits 1, otherwise it +// greets every connection with a few bytes (a stand-in MySQL handshake) until +// killed. It never logs the ready line, like dolt at log_level warning. +func fakeDolt(args []string) int { + if len(args) == 0 { + return 2 + } + switch args[0] { + case "version": + fmt.Println("dolt version 2.1.8") + return 0 + case "config": + fmt.Println("fake") + return 0 + case "init": + if err := os.MkdirAll(".dolt", 0o750); err != nil { + return 1 + } + return 0 + case "sql-server": + default: + return 2 + } + host, port := "127.0.0.1", 0 + for i := 1; i+1 < len(args); i++ { + switch args[i] { + case "--config": + b, err := os.ReadFile(args[i+1]) + if err != nil { + fmt.Println(err) + return 1 + } + if m := fakeDoltConfigPortRe.FindSubmatch(b); m != nil { + port, _ = strconv.Atoi(string(m[1])) + } + case "-P": + port, _ = strconv.Atoi(args[i+1]) + case "-H": + host = args[i+1] + } + } + if port == 0 { + fmt.Println("fake dolt: no port") + return 2 + } + if path := os.Getenv(fakeDoltLaunchesEnv); path != "" { + if f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600); err == nil { //nolint:gosec // G304: test-controlled path + _, _ = fmt.Fprintf(f, "%d\n", port) + _ = f.Close() + } + } + if d, err := time.ParseDuration(os.Getenv(fakeDoltDelayEnv)); err == nil { + time.Sleep(d) + } + if code, err := strconv.Atoi(os.Getenv(fakeDoltExitEnv)); err == nil { + fmt.Println("fake dolt: failing startup on purpose") + return code + } + for _, p := range strings.Split(os.Getenv(fakeDoltInUseEnv), ",") { + if p == "all" || p == strconv.Itoa(port) { + fmt.Printf("Port %d already in use.\n", port) + return 1 + } + } + ln, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(port))) + if err != nil { + fmt.Printf("Port %d already in use.\n", port) + return 1 + } + if os.Getenv(fakeDoltReadyEnv) != "" { + fmt.Println(`time="2026-10-04T12:00:00Z" level=info msg="Server ready. Accepting connections."`) + } + for { + c, err := ln.Accept() + if err != nil { + return 1 + } + _, _ = c.Write([]byte("\x0a5.7.9-fake-dolt\x00")) + _ = c.Close() + } +} diff --git a/internal/doltserver/listenerowner_linux.go b/internal/doltserver/listenerowner_linux.go new file mode 100644 index 000000000..d4be9a917 --- /dev/null +++ b/internal/doltserver/listenerowner_linux.go @@ -0,0 +1,152 @@ +//go:build linux + +package doltserver + +import ( + "bufio" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" +) + +// procRoot is where listenerOwnership reads /proc; tests point it at a fake +// tree. +var procRoot = "/proc" + +// listenerOwnership reports whether a TCP listener on port belongs to pid or +// one of its descendants. Descendants cover a dolt behind a wrapper script +// that does not exec; finding them needs task/*/children +// (CONFIG_PROC_CHILDREN), and without it such a wrapper's dolt is not seen as +// the child's. +// +// known is false when /proc cannot answer, and the caller then falls back to +// weaker readiness evidence: +// - /proc is not this process's PID namespace's (/proc/self is not us); +// - the child's descriptors cannot be read (permissions, hardening); +// - no listener on port is listed at all, although one just answered: +// /proc/net/tcp is incomplete here (WSL1, some sandboxes) or the +// connection is forwarded elsewhere. +// +// A child that no longer exists is known not to own anything. +func listenerOwnership(pid, port int) (owned, known bool) { + if procRoot == "/proc" { + if self, err := os.Readlink("/proc/self"); err != nil || self != strconv.Itoa(os.Getpid()) { + return false, false + } + } + if _, err := os.ReadDir(filepath.Join(procRoot, strconv.Itoa(pid), "fd")); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, true + } + return false, false + } + inodes, err := listeningSocketInodes(port) + if err != nil || len(inodes) == 0 { + return false, false + } + seen := map[int]bool{} + queue := []int{pid} + for len(queue) > 0 { + p := queue[0] + queue = queue[1:] + if p <= 0 || seen[p] { + continue + } + seen[p] = true + if processHoldsSocket(p, inodes) { + return true, true + } + queue = append(queue, childPIDs(p)...) + } + return false, true +} + +// listeningSocketInodes returns the socket inodes of TCP listeners on port, +// from /proc/net/tcp and /proc/net/tcp6. +func listeningSocketInodes(port int) (map[string]bool, error) { + inodes := map[string]bool{} + read := 0 + for _, name := range []string{"tcp", "tcp6"} { + f, err := os.Open(filepath.Join(procRoot, "net", name)) //nolint:gosec // G304: fixed /proc paths + if err != nil { + continue + } + read++ + err = parseProcNetTCPListeners(bufio.NewScanner(f), port, inodes) + _ = f.Close() + if err != nil { + return nil, err + } + } + if read == 0 { + return nil, fmt.Errorf("no %s/net/tcp", procRoot) + } + return inodes, nil +} + +// tcpListenState is st for TCP_LISTEN in /proc/net/tcp. +const tcpListenState = "0A" + +// parseProcNetTCPListeners adds to inodes the inode of every row of a +// /proc/net/tcp{,6} table that is a listener on port. A row is +// "sl local_address rem_address st ... uid timeout inode ...", with +// local_address "HEXADDR:HEXPORT". +func parseProcNetTCPListeners(sc *bufio.Scanner, port int, inodes map[string]bool) error { + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) < 10 || fields[3] != tcpListenState { + continue + } + i := strings.LastIndexByte(fields[1], ':') + if i < 0 { + continue + } + p, err := strconv.ParseUint(fields[1][i+1:], 16, 16) + if err != nil || int(p) != port { + continue + } + if fields[9] != "0" { + inodes[fields[9]] = true + } + } + return sc.Err() +} + +func processHoldsSocket(pid int, inodes map[string]bool) bool { + dir := filepath.Join(procRoot, strconv.Itoa(pid), "fd") + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + link, err := os.Readlink(filepath.Join(dir, e.Name())) + if err != nil || !strings.HasPrefix(link, "socket:[") || !strings.HasSuffix(link, "]") { + continue + } + if inodes[link[len("socket:["):len(link)-1]] { + return true + } + } + return false +} + +func childPIDs(pid int) []int { + tasks, _ := filepath.Glob(filepath.Join(procRoot, strconv.Itoa(pid), "task", "*", "children")) + var out []int + for _, t := range tasks { + b, err := os.ReadFile(t) //nolint:gosec // G304: path is built from /proc and a pid + if err != nil { + continue + } + for _, f := range strings.Fields(string(b)) { + if c, err := strconv.Atoi(f); err == nil { + out = append(out, c) + } + } + } + return out +} diff --git a/internal/doltserver/listenerowner_linux_test.go b/internal/doltserver/listenerowner_linux_test.go new file mode 100644 index 000000000..adf97cb0e --- /dev/null +++ b/internal/doltserver/listenerowner_linux_test.go @@ -0,0 +1,133 @@ +//go:build linux + +package doltserver + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// fakeProc builds a /proc tree with one listener (inode 4242) on port and a +// process pid whose fd table holds sockets for inodes. +func fakeProc(t *testing.T, port, pid int, tcpRows bool, inodes ...string) string { + t.Helper() + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "net"), 0o750); err != nil { + t.Fatal(err) + } + table := " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + if tcpRows { + table += fmt.Sprintf(" 0: 0100007F:%04X 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 4242 1 0 100 0 0 10 0\n", port) + } + if err := os.WriteFile(filepath.Join(root, "net", "tcp"), []byte(table), 0o600); err != nil { + t.Fatal(err) + } + fd := filepath.Join(root, fmt.Sprint(pid), "fd") + if err := os.MkdirAll(fd, 0o750); err != nil { + t.Fatal(err) + } + for i, ino := range inodes { + if err := os.Symlink("socket:["+ino+"]", filepath.Join(fd, fmt.Sprint(i+3))); err != nil { + t.Fatal(err) + } + } + return root +} + +func withProcRoot(t *testing.T, root string) { + t.Helper() + orig := procRoot + procRoot = root + t.Cleanup(func() { procRoot = orig }) +} + +func TestListenerOwnership_FakeProc(t *testing.T) { + const port, pid = 41234, 777 + for _, tc := range []struct { + name string + rows bool + inodes []string + pid int + unreadableFD bool + owned, known bool + }{ + {name: "child holds the listener", rows: true, inodes: []string{"4242"}, pid: pid, owned: true, known: true}, + {name: "listener belongs to someone else", rows: true, inodes: []string{"9999"}, pid: pid, known: true}, + // A child that has exited (and been reaped) owns nothing; this must + // not read as "unknown", which would accept a foreign greeting. + {name: "child gone from /proc", rows: true, pid: pid + 1, known: true}, + // Something answered but /proc lists no listener: /proc is not + // telling the whole story, so fall back rather than call it foreign. + {name: "no listener row", rows: false, inodes: []string{"4242"}, pid: pid}, + {name: "fd table unreadable", rows: true, inodes: []string{"4242"}, pid: pid, unreadableFD: true}, + } { + t.Run(tc.name, func(t *testing.T) { + root := fakeProc(t, port, pid, tc.rows, tc.inodes...) + if tc.unreadableFD { + if os.Geteuid() == 0 { + t.Skip("root reads a mode-000 directory") + } + fd := filepath.Join(root, fmt.Sprint(pid), "fd") + if err := os.Chmod(fd, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(fd, 0o750) }) + } + withProcRoot(t, root) + owned, known := listenerOwnership(tc.pid, port) + if owned != tc.owned || known != tc.known { + t.Errorf("listenerOwnership = (%v, %v), want (%v, %v)", owned, known, tc.owned, tc.known) + } + }) + } +} + +func TestParseProcNetTCPListeners(t *testing.T) { + table := ` sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 0100007F:A0D2 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 111 1 0 100 0 0 10 0 + 1: 0100007F:A0D2 0100007F:C000 01 00000000:00000000 00:00000000 00000000 1000 0 222 1 0 100 0 0 10 0 + 2: 00000000:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 333 1 0 100 0 0 10 0 +` + for _, tc := range []struct { + name string + table string + port int + want []string + }{ + {name: "listener only, not the established row", table: table, port: 0xA0D2, want: []string{"111"}}, + {name: "other port", table: table, port: 0x1F90, want: []string{"333"}}, + {name: "empty table", table: "", port: 0xA0D2}, + {name: "header only", table: strings.SplitN(table, "\n", 2)[0] + "\n", port: 0xA0D2}, + } { + t.Run(tc.name, func(t *testing.T) { + got := map[string]bool{} + if err := parseProcNetTCPListeners(bufio.NewScanner(strings.NewReader(tc.table)), tc.port, got); err != nil { + t.Fatal(err) + } + if len(got) != len(tc.want) { + t.Fatalf("inodes = %v, want %v", got, tc.want) + } + for _, w := range tc.want { + if !got[w] { + t.Errorf("missing inode %s in %v", w, got) + } + } + }) + } +} + +// TestListenerOwnership_RealProc runs against the real /proc: this process +// owns its own listener, and a pid that does not exist owns nothing (known). +func TestListenerOwnership_RealProc(t *testing.T) { + port := foreignGreeter(t) + if owned, known := listenerOwnership(os.Getpid(), port); !owned || !known { + t.Errorf("listenerOwnership(self, own listener) = (%v, %v), want (true, true)", owned, known) + } + if owned, known := listenerOwnership(1<<30, port); owned || !known { + t.Errorf("listenerOwnership(no such pid) = (%v, %v), want (false, true)", owned, known) + } +} diff --git a/internal/doltserver/listenerowner_other.go b/internal/doltserver/listenerowner_other.go new file mode 100644 index 000000000..5194e01b0 --- /dev/null +++ b/internal/doltserver/listenerowner_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package doltserver + +// listenerOwnership cannot tell which process holds a listener on this +// platform without shelling out on every readiness poll, so it reports +// known == false and Start falls back to the greeting plus the child's own +// output and exit status. +func listenerOwnership(pid, port int) (owned, known bool) { + return false, false +} diff --git a/internal/doltserver/portsources_test.go b/internal/doltserver/portsources_test.go index 8c424c5ca..999696233 100644 --- a/internal/doltserver/portsources_test.go +++ b/internal/doltserver/portsources_test.go @@ -9,12 +9,30 @@ import ( "github.com/steveyegge/beads/internal/config" ) +// isolateUserConfig points the user-level config locations (HOME, XDG, and +// USERPROFILE) at an empty directory. config.Initialize merges the +// user-level configs (~/.config/bd/config.yaml, legacy ~/.beads/config.yaml), +// and the integration TestMain does not redirect HOME, so without this an +// operator's dolt.shared-server: true switches DefaultConfig to the shared +// server's state dir and its port file. It also drops any config an earlier +// test in the run loaded (from the real HOME), and does so again afterwards. +func isolateUserConfig(t *testing.T) { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) +} + // Asserts the precedence chain against DefaultConfig itself by layering each // source on top of the previous ones: env > port file > Dolt server // config.yaml > Beads config.yaml > metadata.json (GH#4511). func TestDefaultConfigPrecedenceChain(t *testing.T) { t.Setenv("BEADS_DOLT_SERVER_PORT", "") t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + isolateUserConfig(t) beadsDir := t.TempDir() @@ -81,6 +99,7 @@ func TestDefaultConfigPrecedenceChain(t *testing.T) { func TestDefaultConfigPortSource(t *testing.T) { t.Setenv("BEADS_DOLT_SERVER_PORT", "") t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + isolateUserConfig(t) beadsDir := t.TempDir() diff --git a/internal/doltserver/probe_test.go b/internal/doltserver/probe_test.go index b2b9ea16b..f0e6e235c 100644 --- a/internal/doltserver/probe_test.go +++ b/internal/doltserver/probe_test.go @@ -2,6 +2,7 @@ package doltserver import ( "net" + "os" "sync/atomic" "testing" "time" @@ -12,7 +13,19 @@ import ( // payload bytes) used by these tests to simulate a dolt sql-server greeting. var fakeMySQLGreeting = []byte{0x08, 0x00, 0x00, 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a} -// TestWaitForReadyEmitsRST proves waitForReady connects exactly once against +// awaitGreeting runs awaitOwnedListener's greeting probe against host:port +// with ownership taken as proven, for the tests below that exercise only the +// probe (greeting required, FIN not RST, re-polling). The "child" is this +// process, which never exits during the test. +func awaitGreeting(host string, port int, timeout time.Duration) error { + srv := &startedServer{pid: os.Getpid(), exited: make(chan struct{})} + return awaitOwnedListener(srv, startupProbe{ + host: host, port: port, timeout: timeout, + owner: func(int, int) (bool, bool) { return true, true }, + }) +} + +// TestWaitForReadyEmitsRST proves the readiness probe (awaitOwnedListener) connects exactly once against // a server that greets immediately, and that the probe connection is drained // before Close() (so the TCP stack sends FIN, not RST). See // gastownhall/beads#4132, #4133. @@ -42,22 +55,22 @@ func TestWaitForReadyEmitsRST(t *testing.T) { time.Sleep(50 * time.Millisecond) // let goroutine bind - if err := waitForReady("127.0.0.1", port, 5*time.Second); err != nil { - t.Fatalf("waitForReady: %v", err) + if err := awaitGreeting("127.0.0.1", port, 5*time.Second); err != nil { + t.Fatalf("the readiness probe: %v", err) } time.Sleep(100 * time.Millisecond) // let the accept goroutine finish counting got := connects.Load() if got != 1 { - t.Errorf("waitForReady made %d TCP connections (expected 1)", got) + t.Errorf("the readiness probe made %d TCP connections (expected 1)", got) } - t.Logf("waitForReady made %d TCP connections (drained, no RST)", got) + t.Logf("the readiness probe made %d TCP connections (drained, no RST)", got) } // TestWaitForReadyRepollsUntilGreeted is the F7 regression test: a listener // that accepts TCP connections but never writes anything is a "TCP-accepting -// but mute" server — not ready. waitForReady must not treat dial success +// but mute" server — not ready. the readiness probe must not treat dial success // alone as readiness; it must keep polling until a greeting arrives or the // deadline passes. func TestWaitForReadyRepollsUntilGreeted(t *testing.T) { @@ -85,12 +98,12 @@ func TestWaitForReadyRepollsUntilGreeted(t *testing.T) { } }() - err = waitForReady("127.0.0.1", port, 700*time.Millisecond) + err = awaitGreeting("127.0.0.1", port, 700*time.Millisecond) if err == nil { - t.Fatalf("waitForReady returned nil for a mute (accepting, never-greeting) server; want a timeout error") + t.Fatalf("the readiness probe returned nil for a mute (accepting, never-greeting) server; want a timeout error") } if got := accepts.Load(); got < 2 { - t.Errorf("waitForReady accepted only %d connection(s) before giving up; want re-polling (>=2) since dial-success-without-greeting must not be treated as ready", got) + t.Errorf("the readiness probe accepted only %d connection(s) before giving up; want re-polling (>=2) since dial-success-without-greeting must not be treated as ready", got) } }) @@ -114,7 +127,7 @@ func TestWaitForReadyRepollsUntilGreeted(t *testing.T) { defer c.Close() if attempt < 2 { // First accept(s): mute, so the probe times out its read - // and waitForReady must re-poll instead of declaring victory. + // and the readiness probe must re-poll instead of declaring victory. time.Sleep(600 * time.Millisecond) return } @@ -124,11 +137,11 @@ func TestWaitForReadyRepollsUntilGreeted(t *testing.T) { } }() - if err := waitForReady("127.0.0.1", port, 5*time.Second); err != nil { - t.Fatalf("waitForReady: %v", err) + if err := awaitGreeting("127.0.0.1", port, 5*time.Second); err != nil { + t.Fatalf("the readiness probe: %v", err) } if got := callCount.Load(); got < 2 { - t.Errorf("expected waitForReady to re-poll past the mute accept, got only %d accept(s)", got) + t.Errorf("expected the readiness probe to re-poll past the mute accept, got only %d accept(s)", got) } }) } diff --git a/internal/doltserver/startowned.go b/internal/doltserver/startowned.go new file mode 100644 index 000000000..ff4454892 --- /dev/null +++ b/internal/doltserver/startowned.go @@ -0,0 +1,261 @@ +package doltserver + +import ( + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "strconv" + "time" +) + +// startedServer is a dolt sql-server child Start launched. A goroutine waits +// on it, which both reports its exit and reaps it: without that, a child that +// exits while bd is still running stays a zombie, and a zombie still answers +// kill(pid, 0), so Start could not see that its server had died. +type startedServer struct { + pid int + proc *os.Process + exited chan struct{} +} + +func launchServer(cmd *exec.Cmd) (*startedServer, error) { + if err := cmd.Start(); err != nil { + return nil, err + } + s := &startedServer{pid: cmd.Process.Pid, proc: cmd.Process, exited: make(chan struct{})} + go func() { + _, _ = s.proc.Wait() + close(s.exited) + }() + return s, nil +} + +func (s *startedServer) hasExited() bool { + select { + case <-s.exited: + return true + default: + return false + } +} + +// killExitWait bounds how long killAndWait waits for a killed child to exit. +const killExitWait = 5 * time.Second + +// killAndWait kills the child (and, on unix, the process group it leads, so +// a dolt behind a wrapper that did not exec goes too) and waits briefly for +// it to exit. The wait matters before a retry: the next attempt opens the +// same database, and the killed child holds dolt's database lock until the +// kernel has finished its exit. +func (s *startedServer) killAndWait() { + if s == nil { + return + } + if !s.hasExited() { + killProcessGroup(s.pid) + } + _ = s.proc.Kill() + select { + case <-s.exited: + case <-time.After(killExitWait): + } +} + +// writeServerStateFiles records a launched server in the PID and port files. +func writeServerStateFiles(beadsDir string, pid, port int) error { + if err := os.WriteFile(pidPath(beadsDir), []byte(strconv.Itoa(pid)), 0600); err != nil { + return fmt.Errorf("writing PID file: %w", err) + } + if err := writePortFile(beadsDir, port); err != nil { + _ = os.Remove(pidPath(beadsDir)) + return fmt.Errorf("writing port file: %w", err) + } + return nil +} + +func removeServerStateFiles(beadsDir string) { + _ = os.Remove(pidPath(beadsDir)) + _ = os.Remove(portPath(beadsDir)) +} + +// pinnedPortRemedy says why Start did not move off port, which another +// process holds, and what the operator can do, worded by where the port came +// from. +func pinnedPortRemedy(cfg *Config, beadsDir string, port int) string { + switch { + case cfg.PortSource == PortSourceSharedServerDefault: + return fmt.Sprintf("port %d is the shared-server default port, which every project on this host dials, so bd will not move it: "+ + "free it, or configure a different shared-server port with BEADS_DOLT_SERVER_PORT or dolt.port", port) + case cfg.PortSource == PortSourcePortFile: + return fmt.Sprintf("port %d is the shared server's recorded port (%s), which other projects dial, so bd will not move it: "+ + "free it, or stop the shared server and delete that file", port, portPath(beadsDir)) + default: + src := string(cfg.PortSource) + for _, ps := range portSources { + if ps.source == cfg.PortSource { + src = ps.label + } + } + switch cfg.PortSource { + case PortSourceCallerExplicit: + src = "an explicit port setting (e.g. bd init --server-port)" + case PortSourceUnset: + src = "an explicit setting" + } + return fmt.Sprintf("port %d comes from %s, which bd will not override: free it, or configure a different one with: bd dolt set port ", port, src) + } +} + +// startupProbe describes how awaitOwnedListener decides that the child it +// launched is the dolt sql-server answering on host:port. +type startupProbe struct { + host string + port int + // logPath and logOffset locate the child's output: it writes straight to + // the server log (it outlives bd, so it cannot write to a pipe into bd), + // and everything after logOffset is this child's. + logPath string + logOffset int64 + // readyLineLogged is true when the child's log level lets dolt log + // DoltReadyLine (debug mode); the default warning level does not. + readyLineLogged bool + timeout time.Duration + // owner is listenerOwnership; tests replace it. + owner func(pid, port int) (owned, known bool) +} + +const ( + startupProbeDialTimeout = 500 * time.Millisecond + startupProbeInterval = 250 * time.Millisecond +) + +// awaitOwnedListener waits until the dolt sql-server srv is accepting +// connections on its port. +// +// A MySQL greeting on the port is not enough on its own: the port is chosen +// before dolt binds it, and if another process (another dolt, say) takes it +// in between, the greeting comes from that process while this child logs +// "Port N already in use." and exits. dolt only reaches that check after its +// own startup, which under load takes longer than any fixed grace period, so +// a greeting only counts once the listener is shown to be the child's: by +// dolt's ready line in the child's output when its log level emits one, or +// else by the listening socket belonging to the child's process tree (Linux, +// via /proc). When the socket is shown to belong to someone else the wait +// ends at once with ErrPortInUse. Where neither proof is available the +// greeting decides, as before, unless the log level promises a ready line. +// Either way a child that exits, or says its port is taken, ends the wait, +// with ErrPortInUse in the latter case. +func awaitOwnedListener(srv *startedServer, p startupProbe) error { + addr := net.JoinHostPort(p.host, strconv.Itoa(p.port)) + owner := p.owner + if owner == nil { + owner = listenerOwnership + } + watch := NewStartupWatch(nil, p.port) + tail := logTail{path: p.logPath, off: p.logOffset} + deadline := time.Now().Add(p.timeout) + answeredUnproven := false + for { + if err := childStartupFailure(srv, &tail, watch, addr); err != nil { + return err + } + greeted, _ := ProbeSQLServer("tcp", addr, startupProbeDialTimeout) //nolint:gosec // G704: addr is built from internal host+port, not user input + if greeted { + // Re-read the output before judging: a child that lost the port + // may have said so (and been reaped) since the check above. + if err := childStartupFailure(srv, &tail, watch, addr); err != nil { + return err + } + owned, known := watch.IsReady(), true + if !owned { + owned, known = owner(srv.pid, p.port) + } + switch { + case owned || (!known && !p.readyLineLogged): + if err := childStartupFailure(srv, &tail, watch, addr); err != nil { + return err + } + return nil + case known: + // Proven: the listener belongs to another process. The child + // cannot bind this port; do not wait for it to find out. + return fmt.Errorf("%w: another process (not dolt sql-server PID %d) holds %s", ErrPortInUse, srv.pid, addr) + default: + answeredUnproven = true + } + } + if time.Now().After(deadline) { + return startupTimeout(p, addr, srv.pid, answeredUnproven) + } + wait := startupProbeInterval + if left := time.Until(deadline); left < wait { + wait = left + } + select { + case <-srv.exited: + case <-time.After(wait): + if time.Now().After(deadline) { + return startupTimeout(p, addr, srv.pid, answeredUnproven) + } + } + } +} + +func startupTimeout(p startupProbe, addr string, pid int, answeredUnproven bool) error { + if answeredUnproven { + return fmt.Errorf("timeout after %s: something answered at %s, but dolt sql-server (PID %d) never logged %q, "+ + "which proves the listener is its own at log level debug; if a wrapper filters dolt's output, turn off debug mode (BEADS_DOLT_DEBUG, dolt.debug)", + p.timeout, addr, pid, DoltReadyLine) + } + return fmt.Errorf("timeout after %s waiting for server at %s", p.timeout, addr) +} + +// childStartupFailure reads the child's latest output and returns why srv +// cannot become ready, or nil while it still may. The output is read every +// time, not only once the exit has been observed: the reaper goroutine can +// lag the child's death, and the port-in-use line is already in the log. +func childStartupFailure(srv *startedServer, tail *logTail, watch *StartupWatch, addr string) error { + exited := srv.hasExited() + tail.feed(watch) + if watch.SawPortInUse() { + return fmt.Errorf("%w: dolt sql-server (PID %d) could not bind %s", ErrPortInUse, srv.pid, addr) + } + if exited { + return errors.New("dolt sql-server (PID " + strconv.Itoa(srv.pid) + ") exited before accepting connections") + } + return nil +} + +// logTail reads what has been appended to a file since off. +type logTail struct { + path string + off int64 +} + +func (t *logTail) feed(w io.Writer) { + f, err := os.Open(t.path) + if err != nil { + return + } + defer func() { _ = f.Close() }() + if st, err := f.Stat(); err == nil && st.Size() < t.off { + // Truncated underneath us (an external copytruncate rotation): + // everything now in the file is new. + t.off = 0 + } + n, _ := io.Copy(w, io.NewSectionReader(f, t.off, 1<<62)) + t.off += n +} + +// logSize returns the current size of f, the offset a newly launched child's +// output will start at. +func logSize(f *os.File) int64 { + st, err := f.Stat() + if err != nil { + return 0 + } + return st.Size() +} diff --git a/internal/doltserver/startowned_test.go b/internal/doltserver/startowned_test.go new file mode 100644 index 000000000..f672b940e --- /dev/null +++ b/internal/doltserver/startowned_test.go @@ -0,0 +1,549 @@ +//go:build !windows + +package doltserver + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" +) + +// These names match fakedolt_test.go, which TestMain dispatches to. +const ( + testFakeDoltEnv = "BEADS_TEST_FAKE_DOLT" + testFakeDoltDelayEnv = "BEADS_TEST_FAKE_DOLT_DELAY" + testFakeDoltInUseEnv = "BEADS_TEST_FAKE_DOLT_INUSE_PORTS" + testFakeDoltExitEnv = "BEADS_TEST_FAKE_DOLT_EXIT" + testFakeDoltReadyEnv = "BEADS_TEST_FAKE_DOLT_READY" + testFakeDoltLaunchesEnv = "BEADS_TEST_FAKE_DOLT_LAUNCHES" + // testFakeDoltDelay is the fake dolt's startup time. It is longer than + // the 200ms "did it exit immediately?" check Start used to rely on, which + // is the window that let a foreign listener's greeting pass as ready. + testFakeDoltDelay = "700ms" +) + +// foreignGreeter listens on a free loopback port and greets every connection +// the way a MySQL server does, standing in for another process (another dolt) +// that took the port Start chose. +func foreignGreeter(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + _, _ = c.Write([]byte("\x0a5.7.9-foreign\x00")) + _ = c.Close() + } + }() + return ln.Addr().(*net.TCPAddr).Port +} + +// startFakeSQLServer launches this test binary as a fake `dolt sql-server -P +// port` (see fakeDolt) writing to logPath, and returns it with the log offset +// its output starts at. +func startFakeSQLServer(t *testing.T, port int, logPath string, extraEnv ...string) (*startedServer, int64) { + return startFakeSQLServerDelay(t, port, logPath, testFakeDoltDelay, extraEnv...) +} + +func startFakeSQLServerDelay(t *testing.T, port int, logPath, delay string, extraEnv ...string) (*startedServer, int64) { + t.Helper() + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + defer logFile.Close() + _, _ = logFile.WriteString("earlier run: Port " + fmt.Sprint(port+1) + " already in use.\n") + cmd := exec.Command(os.Args[0], "sql-server", "-H", "127.0.0.1", "-P", fmt.Sprint(port), "--loglevel=warning") + cmd.Env = append(append(os.Environ(), testFakeDoltEnv+"=1", testFakeDoltDelayEnv+"="+delay), extraEnv...) + cmd.Stdout = logFile + cmd.Stderr = logFile + off := logSize(logFile) + srv, err := launchServer(cmd) + if err != nil { + t.Fatal(err) + } + t.Cleanup(srv.killAndWait) + return srv, off +} + +func freeLoopbackPort(t *testing.T) int { + t.Helper() + p, err := allocateEphemeralPort("127.0.0.1") + if err != nil { + t.Fatal(err) + } + return p +} + +// TestAwaitOwnedListener_ForeignListenerIsNotReady: when another process +// greets on the port before the launched server reaches its bind, the wait +// must not report ready; it ends with ErrPortInUse (at once when ownership +// is proven foreign, else once the child says so). +// The old check (greeting only, after a 200ms exit check) accepted the +// foreign greeting immediately. +func TestAwaitOwnedListener_ForeignListenerIsNotReady(t *testing.T) { + for _, tc := range []struct { + name string + owner func(pid, port int) (bool, bool) + }{ + // The platform check; on Linux this is the real /proc lookup. + {name: "platform ownership check", owner: nil}, + // Where ownership is unknown, the child's own port-in-use report + // still ends the wait, as long as it arrives before a greeting is + // accepted. Model a known-not-owned answer so the test is + // deterministic on every unix. + {name: "ownership says foreign", owner: func(int, int) (bool, bool) { return false, true }}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.owner == nil && runtime.GOOS != "linux" { + t.Skip("listener ownership is only provable on linux") + } + port := foreignGreeter(t) + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + srv, off := startFakeSQLServer(t, port, logPath) + + err := awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, logOffset: off, + timeout: 20 * time.Second, owner: tc.owner, + }) + if !errors.Is(err, ErrPortInUse) { + t.Fatalf("awaitOwnedListener = %v, want ErrPortInUse (a foreign greeting must not count as ready)", err) + } + }) + } +} + +// TestAwaitOwnedListener_ImmediatePortInUseExit pins the fast-exit race the +// proxied launcher hit (#7184's follow-up): a dolt that finds its port taken +// can exit within milliseconds, before Start looks at it at all. Start runs +// nothing fallible between launching the child and this wait, and the wait +// drains the child's log after seeing it exit, so the exit is still +// classified as ErrPortInUse (recoverable), never as a generic failure. +func TestAwaitOwnedListener_ImmediatePortInUseExit(t *testing.T) { + for i := 0; i < 5; i++ { + ln, err := net.Listen("tcp", "127.0.0.1:0") // mute holder, like the port hog + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + srv, off := startFakeSQLServerDelay(t, port, logPath, "0s") + <-srv.exited // the child is gone before the wait starts + err = awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, logOffset: off, timeout: 20 * time.Second, + }) + _ = ln.Close() + if !errors.Is(err, ErrPortInUse) { + t.Fatalf("attempt %d: awaitOwnedListener = %v, want ErrPortInUse", i, err) + } + } +} + +// TestAwaitOwnedListener_OwnListenerIsReady is the positive control: the +// launched server's own listener is accepted, and an earlier run's +// port-in-use line in the same log (before the offset) is ignored. +func TestAwaitOwnedListener_OwnListenerIsReady(t *testing.T) { + port := freeLoopbackPort(t) + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + srv, off := startFakeSQLServer(t, port, logPath) + if err := awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, logOffset: off, timeout: 20 * time.Second, + }); err != nil { + t.Fatalf("awaitOwnedListener on the child's own listener: %v", err) + } + if runtime.GOOS == "linux" { + if owned, known := listenerOwnership(srv.pid, port); !owned || !known { + t.Errorf("listenerOwnership(child) = (%v, %v), want (true, true)", owned, known) + } + other := foreignGreeter(t) + if owned, known := listenerOwnership(srv.pid, other); owned || !known { + t.Errorf("listenerOwnership(child, foreign port) = (%v, %v), want (false, true)", owned, known) + } + } +} + +// TestAwaitOwnedListener_ChildExitIsReported: a child that exits without a +// port conflict ends the wait promptly instead of running out the timeout. +// The old check could not see this at all past its first 200ms: the child +// was released unreaped, and a zombie still answers kill(pid, 0). +func TestAwaitOwnedListener_ChildExitIsReported(t *testing.T) { + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + cmd := exec.Command(os.Args[0], "sql-server") // no port: the fake exits 2 + cmd.Env = append(os.Environ(), testFakeDoltEnv+"=1") + srv, err := launchServer(cmd) + if err != nil { + t.Fatal(err) + } + start := time.Now() + err = awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: freeLoopbackPort(t), logPath: logPath, timeout: 20 * time.Second, + }) + if err == nil || errors.Is(err, ErrPortInUse) || !strings.Contains(err.Error(), "exited before accepting connections") { + t.Fatalf("awaitOwnedListener = %v, want an exited-before-ready error", err) + } + if time.Since(start) > 10*time.Second { + t.Errorf("exit took %s to notice", time.Since(start)) + } +} + +// TestAwaitOwnedListener_ProvenForeignEndsWaitAtOnce: when /proc proves the +// greeting came from another process, the wait does not sit out a slow +// child's startup (which under load can outlast the ready timeout and turn a +// movable port into a hard failure); it returns ErrPortInUse straight away. +func TestAwaitOwnedListener_ProvenForeignEndsWaitAtOnce(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("listener ownership is only provable on linux") + } + port := foreignGreeter(t) + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + srv, off := startFakeSQLServerDelay(t, port, logPath, "60s") + start := time.Now() + err := awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, logOffset: off, timeout: 30 * time.Second, + }) + if !errors.Is(err, ErrPortInUse) { + t.Fatalf("awaitOwnedListener = %v, want ErrPortInUse", err) + } + if d := time.Since(start); d > 10*time.Second { + t.Errorf("took %s; a proven-foreign listener must end the wait without waiting for the child", d) + } +} + +// TestAwaitOwnedListener_ReadsLogBeforeExitIsSeen: the child's port-in-use +// line counts as soon as it is in the log, even before bd has observed the +// child's exit (the reaper goroutine can lag it under load). Here the +// "child" never exits at all. +func TestAwaitOwnedListener_ReadsLogBeforeExitIsSeen(t *testing.T) { + port := foreignGreeter(t) + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + if err := os.WriteFile(logPath, []byte(fmt.Sprintf("Port %d already in use.\n", port)), 0o600); err != nil { + t.Fatal(err) + } + srv := &startedServer{pid: os.Getpid(), exited: make(chan struct{})} + err := awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, timeout: 5 * time.Second, + owner: func(int, int) (bool, bool) { return false, false }, // unknown: the greeting alone would pass + }) + if !errors.Is(err, ErrPortInUse) { + t.Fatalf("awaitOwnedListener = %v, want ErrPortInUse", err) + } +} + +// TestAwaitOwnedListener_ReadyLine covers the debug-mode proof: with a log +// level that emits dolt's ready line, the line proves ownership when /proc +// cannot, and without it an unproven greeting is never accepted. +func TestAwaitOwnedListener_ReadyLine(t *testing.T) { + unknown := func(int, int) (bool, bool) { return false, false } + t.Run("ready line proves ownership", func(t *testing.T) { + port := freeLoopbackPort(t) + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + srv, off := startFakeSQLServer(t, port, logPath, testFakeDoltReadyEnv+"=1") + if err := awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, logOffset: off, + readyLineLogged: true, timeout: 20 * time.Second, owner: unknown, + }); err != nil { + t.Fatalf("awaitOwnedListener with the ready line logged: %v", err) + } + }) + t.Run("no ready line, unknown ownership: not accepted", func(t *testing.T) { + port := freeLoopbackPort(t) + logPath := filepath.Join(t.TempDir(), "dolt-server.log") + srv, off := startFakeSQLServerDelay(t, port, logPath, "0s") + err := awaitOwnedListener(srv, startupProbe{ + host: "127.0.0.1", port: port, logPath: logPath, logOffset: off, + readyLineLogged: true, timeout: 8 * time.Second, owner: unknown, + }) + if err == nil || !strings.Contains(err.Error(), DoltReadyLine) { + t.Fatalf("awaitOwnedListener = %v, want a timeout naming the missing ready line", err) + } + }) +} + +// installFakeDolt puts a `dolt` on PATH that runs this test binary as +// fakeDolt. +func installFakeDolt(t *testing.T) { + t.Helper() + dir := t.TempDir() + shim := "#!/bin/sh\nexec " + shellQuote(os.Args[0]) + " \"$@\"\n" + if err := os.WriteFile(filepath.Join(dir, "dolt"), []byte(shim), 0o700); err != nil { //nolint:gosec // G306: the shim must be executable + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv(testFakeDoltEnv, "1") + t.Setenv(testFakeDoltDelayEnv, testFakeDoltDelay) +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// fakeStartWorkspace prepares a per-project workspace for the real Start +// with the fake dolt on PATH and the user config isolated, and returns its +// beads dir and the file the fake records each launch's port in. +func fakeStartWorkspace(t *testing.T) (beadsDir, launches string) { + t.Helper() + installFakeDolt(t) + isolateUserConfig(t) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "0") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_READY_TIMEOUT", "20") + launches = filepath.Join(t.TempDir(), "launches") + t.Setenv(testFakeDoltLaunchesEnv, launches) + beadsDir = filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + return beadsDir, launches +} + +func launchedPorts(t *testing.T, launches string) []string { + t.Helper() + b, err := os.ReadFile(launches) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return strings.Fields(string(b)) +} + +// startForTest runs Start and makes sure whatever it left running dies with +// the test. +func startForTest(t *testing.T, beadsDir string) (*State, error) { + t.Helper() + state, err := Start(beadsDir) + if state != nil && state.PID > 0 { + pid := state.PID + t.Cleanup(func() { + if p, findErr := os.FindProcess(pid); findErr == nil { + _ = p.Kill() + } + }) + } + return state, err +} + +func assertNoStateFiles(t *testing.T, beadsDir string) { + t.Helper() + for _, p := range []string{pidPath(beadsDir), portPath(beadsDir)} { + if _, err := os.Stat(p); err == nil { + t.Errorf("%s left behind after a failed start", filepath.Base(p)) + } + } +} + +// TestStart_RecoversWhenEphemeralPortIsTaken drives the real Start with a +// fake dolt whose first ephemeral port is held by a foreign greeter that took +// it between allocation and bind. Start must not adopt the foreign listener; +// it must move to a fresh port and come up there. Before the fix it returned +// success on the foreign port. +func TestStart_RecoversWhenEphemeralPortIsTaken(t *testing.T) { + beadsDir, launches := fakeStartWorkspace(t) + if cfg := DefaultConfig(beadsDir); cfg.Port != 0 || cfg.Mode != ServerModeOwned { + t.Fatalf("workspace resolves port %d (%s), mode %v; want the ephemeral owned path", cfg.Port, cfg.PortSource, cfg.Mode) + } + foreign := foreignGreeter(t) + + var calls atomic.Int32 + orig := allocateEphemeralPort + allocateEphemeralPort = func(host string) (int, error) { + if calls.Add(1) == 1 { + return foreign, nil + } + return orig(host) + } + t.Cleanup(func() { allocateEphemeralPort = orig }) + + state, err := startForTest(t, beadsDir) + if err != nil { + log, _ := os.ReadFile(logPath(beadsDir)) + t.Fatalf("Start: %v\nlog:\n%s", err, log) + } + if state.Port == foreign { + t.Fatalf("Start adopted port %d, which a foreign process holds", foreign) + } + if got := calls.Load(); got != 2 { + t.Errorf("allocateEphemeralPort called %d times, want 2 (the taken port, then a fresh one)", got) + } + // On Linux the first child can be killed before it records its launch: + // /proc proves the port foreign as soon as the greeting arrives. + if got := launchedPorts(t, launches); len(got) == 0 || got[len(got)-1] != fmt.Sprint(state.Port) { + t.Errorf("launches = %v, want the last one on the final port %d", got, state.Port) + } + if got := readPortFile(beadsDir); got != state.Port { + t.Errorf("port file = %d, want %d", got, state.Port) + } + if runtime.GOOS == "linux" { + if owned, known := listenerOwnership(state.PID, state.Port); !owned || !known { + t.Errorf("listener on %d is not owned by the started server (PID %d)", state.Port, state.PID) + } + } +} + +// TestStart_RecordsServerWhileStarting: the PID and port files exist, with +// the real port, while Start waits for the server. A bd interrupted during +// that wait must leave the server findable by the next bd, bd dolt stop and +// killall, since it holds the database lock. +func TestStart_RecordsServerWhileStarting(t *testing.T) { + beadsDir, _ := fakeStartWorkspace(t) + t.Setenv(testFakeDoltDelayEnv, "3s") + + type seen struct{ pid, port int } + during := make(chan seen, 1) + stop := make(chan struct{}) + go func() { + for { + select { + case <-stop: + return + case <-time.After(20 * time.Millisecond): + } + b, err := os.ReadFile(pidPath(beadsDir)) + if err != nil { + continue + } + var pid int + _, _ = fmt.Sscan(string(b), &pid) + if port := readPortFile(beadsDir); pid > 0 && port > 0 { + during <- seen{pid, port} + return + } + } + }() + state, err := startForTest(t, beadsDir) + close(stop) + if err != nil { + t.Fatalf("Start: %v", err) + } + select { + case got := <-during: + if got.pid != state.PID || got.port != state.Port { + t.Errorf("state files during startup = pid %d port %d, want pid %d port %d", got.pid, got.port, state.PID, state.Port) + } + default: + t.Fatal("no PID+port files appeared while Start waited for the server") + } +} + +// TestStart_RemovesStateFilesAfterFailedStart: a server that never comes up +// is killed and its PID and port files are removed, on the notReady path. +func TestStart_RemovesStateFilesAfterFailedStart(t *testing.T) { + beadsDir, _ := fakeStartWorkspace(t) + t.Setenv(testFakeDoltDelayEnv, "1s") + t.Setenv(testFakeDoltExitEnv, "3") + + sawFiles := make(chan struct{}, 1) + stop := make(chan struct{}) + go func() { + for { + select { + case <-stop: + return + case <-time.After(20 * time.Millisecond): + } + if _, err := os.Stat(pidPath(beadsDir)); err == nil && readPortFile(beadsDir) > 0 { + sawFiles <- struct{}{} + return + } + } + }() + _, err := startForTest(t, beadsDir) + close(stop) + if err == nil || !strings.Contains(err.Error(), "exited before accepting connections") { + t.Fatalf("Start = %v, want an exited-before-ready failure", err) + } + select { + case <-sawFiles: + default: + t.Error("state files never appeared during the failed attempt") + } + assertNoStateFiles(t, beadsDir) +} + +// TestStart_PinnedPortIsNotMoved: an operator-configured port that another +// process takes after Start's pre-check fails with ErrPortInUse after a +// single launch, names where the port came from, and leaves no state files. +func TestStart_PinnedPortIsNotMoved(t *testing.T) { + beadsDir, launches := fakeStartWorkspace(t) + port := freeLoopbackPort(t) + t.Setenv("BEADS_DOLT_SERVER_PORT", fmt.Sprint(port)) + t.Setenv(testFakeDoltInUseEnv, "all") // the holder appears after reclaimPort + + _, err := startForTest(t, beadsDir) + if !errors.Is(err, ErrPortInUse) { + t.Fatalf("Start = %v, want ErrPortInUse", err) + } + if got := launchedPorts(t, launches); len(got) != 1 || got[0] != fmt.Sprint(port) { + t.Errorf("launches = %v, want exactly one, on the pinned port %d", got, port) + } + for _, want := range []string{"Environment variable (BEADS_DOLT_SERVER_PORT)", "bd dolt set port"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error does not mention %q: %v", want, err) + } + } + assertNoStateFiles(t, beadsDir) +} + +// TestStart_PortFilePortMovesOutsideSharedMode: bd's own port-file record is +// a port bd chose, so outside shared-server mode it moves like an ephemeral +// one when another process holds it (GH#4052). +func TestStart_PortFilePortMovesOutsideSharedMode(t *testing.T) { + beadsDir, launches := fakeStartWorkspace(t) + recorded := freeLoopbackPort(t) + if err := writePortFile(beadsDir, recorded); err != nil { + t.Fatal(err) + } + if cfg := DefaultConfig(beadsDir); cfg.Port != recorded || cfg.PortSource != PortSourcePortFile { + t.Fatalf("DefaultConfig = port %d source %q, want %d from the port file", cfg.Port, cfg.PortSource, recorded) + } + t.Setenv(testFakeDoltInUseEnv, fmt.Sprint(recorded)) + + state, err := startForTest(t, beadsDir) + if err != nil { + t.Fatalf("Start: %v", err) + } + if state.Port == recorded { + t.Fatalf("Start stayed on the taken port %d", recorded) + } + if got := launchedPorts(t, launches); len(got) != 2 || got[0] != fmt.Sprint(recorded) { + t.Errorf("launches = %v, want the recorded port %d and then a fresh one", got, recorded) + } + if got := readPortFile(beadsDir); got != state.Port { + t.Errorf("port file = %d, want the new port %d", got, state.Port) + } +} + +// TestPinnedPortRemedy pins the wording per port source. +func TestPinnedPortRemedy(t *testing.T) { + dir := t.TempDir() + for _, tc := range []struct { + src PortSource + want []string + }{ + {PortSourceEnv, []string{"BEADS_DOLT_SERVER_PORT", "bd dolt set port"}}, + {PortSourceConfigYaml, []string{"dolt.port", "bd dolt set port"}}, + {PortSourceCallerExplicit, []string{"--server-port"}}, + {PortSourceSharedServerDefault, []string{"shared-server default port", "BEADS_DOLT_SERVER_PORT"}}, + {PortSourcePortFile, []string{"shared server's recorded port", portPath(dir)}}, + } { + got := pinnedPortRemedy(&Config{PortSource: tc.src}, dir, 3308) + for _, w := range tc.want { + if !strings.Contains(got, w) { + t.Errorf("%s: %q does not mention %q", tc.src, got, w) + } + } + } +} diff --git a/internal/doltserver/startupwatch.go b/internal/doltserver/startupwatch.go new file mode 100644 index 000000000..bc4b2cf5d --- /dev/null +++ b/internal/doltserver/startupwatch.go @@ -0,0 +1,133 @@ +package doltserver + +import ( + "bytes" + "errors" + "fmt" + "io" + "sync" + + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" +) + +// This file holds the startup signals both bd-managed dolt sql-server +// launchers read from the child's own output: the plain server-mode Start in +// this package and the proxied DoltServer in internal/storage/dbproxy/server. +// A dial (or even a MySQL greeting) on the chosen port is not proof that the +// child bound it: the port is picked before dolt binds it, and another +// process can take it in between. dolt then logs that the port is in use and +// exits, while the dial reaches the other process. + +// ErrPortInUse reports that a dolt sql-server bd launched exited before it +// was ready because another process holds its SQL listener port. +var ErrPortInUse = errors.New("dolt sql-server listener port is already in use") + +// DoltReadyLine is what dolt sql-server (go-mysql-server) logs at info level +// once its listener is bound and accepting. Seeing it on the output of the +// process bd launched proves the port belongs to that process. +const DoltReadyLine = "Server ready. Accepting connections." + +// doltPortInUseTexts are the ways a dolt sql-server reports that port, its SQL +// listener port, is taken: +// - dolt's own pre-check: "Port 3306 already in use." +// - go-mysql-server's pre-check: "Port 127.0.0.1:3306 already in use." +// - the bind losing the race after both checks, as Go's net error: +// "listen tcp 127.0.0.1:3306: bind: address already in use", or on +// Windows "...:3306: bind: Only one usage of each socket address ...". +// +// The port is part of every form, so a different listener dolt fails to bind +// (metrics, MCP) is not mistaken for the SQL port, nor is dolt's non-fatal +// unix-socket warning ("... is already in use"). dolt prints the pre-check +// message whatever its log level, so it is visible at log_level warning too. +func doltPortInUseTexts(port int) [][]byte { + return [][]byte{ + []byte(fmt.Sprintf("Port %d already in use", port)), + []byte(fmt.Sprintf(":%d already in use", port)), + []byte(fmt.Sprintf(":%d: bind: address already in use", port)), + []byte(fmt.Sprintf(":%d: bind: Only one usage of each socket address", port)), + } +} + +// LogLevelEmitsReadyLine reports whether dolt logs DoltReadyLine (an info +// message) at level. +func LogLevelEmitsReadyLine(level servercfg.LogLevel) bool { + switch level { + case servercfg.LogLevel_Trace, servercfg.LogLevel_Debug, servercfg.LogLevel_Info: + return true + } + return false +} + +// StartupWatch receives a dolt sql-server's stdout and stderr. It forwards +// everything to dst (if any) and, until the ready line shows up, scans the +// output for that line and for a conflict on the SQL listener port. +type StartupWatch struct { + dst io.Writer + + inUse [][]byte + + mu sync.Mutex + tail []byte + scanning bool + portInUse bool + ready chan struct{} +} + +// startupWatchTail is how much recent output the watcher keeps for matching, +// so a marker split across two writes is still found. +const startupWatchTail = 4096 + +// NewStartupWatch returns a watcher for a dolt sql-server whose SQL listener +// is port. dst may be nil. +func NewStartupWatch(dst io.Writer, port int) *StartupWatch { + return &StartupWatch{dst: dst, inUse: doltPortInUseTexts(port), scanning: true, ready: make(chan struct{})} +} + +// Write never fails: a log file that cannot be written must not break the +// pipe the dolt sql-server writes to. +func (w *StartupWatch) Write(p []byte) (int, error) { + if w.dst != nil { + _, _ = w.dst.Write(p) + } + w.mu.Lock() + defer w.mu.Unlock() + if !w.scanning { + return len(p), nil + } + w.tail = append(w.tail, p...) + for _, text := range w.inUse { + if bytes.Contains(w.tail, text) { + w.portInUse = true + } + } + if bytes.Contains(w.tail, []byte(DoltReadyLine)) { + w.scanning = false + w.tail = nil + close(w.ready) + return len(p), nil + } + if over := len(w.tail) - startupWatchTail; over > 0 { + w.tail = append(w.tail[:0], w.tail[over:]...) + } + return len(p), nil +} + +// ReadyCh is closed once the ready line has been seen. +func (w *StartupWatch) ReadyCh() <-chan struct{} { return w.ready } + +// IsReady reports whether the ready line has been seen. +func (w *StartupWatch) IsReady() bool { + select { + case <-w.ready: + return true + default: + return false + } +} + +// SawPortInUse reports whether the output said the SQL listener port is taken. +func (w *StartupWatch) SawPortInUse() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.portInUse +} diff --git a/internal/doltserver/startupwatch_test.go b/internal/doltserver/startupwatch_test.go new file mode 100644 index 000000000..9bdffe1fb --- /dev/null +++ b/internal/doltserver/startupwatch_test.go @@ -0,0 +1,84 @@ +package doltserver + +import ( + "bytes" + "strings" + "testing" +) + +// writeChunks feeds s to a new watcher for port in the given chunk sizes +// (the last size repeats) and returns the watcher and what reached dst. +func writeChunks(t *testing.T, port int, s string, sizes ...int) (*StartupWatch, string) { + t.Helper() + var dst bytes.Buffer + w := NewStartupWatch(&dst, port) + b := []byte(s) + for i := 0; len(b) > 0; i++ { + n := sizes[len(sizes)-1] + if i < len(sizes) { + n = sizes[i] + } + if n > len(b) { + n = len(b) + } + got, err := w.Write(b[:n]) + if err != nil || got != n { + t.Fatalf("Write = (%d, %v), want (%d, nil)", got, err, n) + } + b = b[n:] + } + return w, dst.String() +} + +func TestStartupWatch(t *testing.T) { + ready := `time="2026-10-04T05:45:58Z" level=info msg="Server ready. Accepting connections."` + "\n" + pad := strings.Repeat("x", 5000) + "\n" + for _, tc := range []struct { + name string + port int + out string + sizes []int + ready bool + portInUse bool + }{ + {name: "ready line in one write", port: 3306, out: ready, sizes: []int{1 << 20}, ready: true}, + {name: "ready line split across writes", port: 3306, out: ready, sizes: []int{40, 30, 1 << 20}, ready: true}, + {name: "ready line one byte per write", port: 3306, out: ready, sizes: []int{1}, ready: true}, + {name: "ready line with CRLF", port: 3306, out: strings.ReplaceAll(ready, "\n", "\r\n"), sizes: []int{7}, ready: true}, + {name: "ready line after more than the tail of output", port: 3306, out: pad + ready, sizes: []int{1000}, ready: true}, + {name: "MCP ready line is not the SQL ready line", port: 3306, out: "Dolt MCP server ready. Accepting connections.\n", sizes: []int{1 << 20}}, + {name: "dolt pre-check", port: 8000, out: "Port 8000 already in use.\n", sizes: []int{5}, portInUse: true}, + {name: "dolt pre-check at log_level warning", port: 39847, out: "Starting server with Config HP=\"127.0.0.1:39847\"|T=\"28800000\"|R=\"false\"|L=\"warning\"\nPort 39847 already in use.\n", sizes: []int{1 << 20}, portInUse: true}, + {name: "dolt pre-check for another port", port: 80, out: "Port 8000 already in use.\n", sizes: []int{1 << 20}}, + {name: "go-mysql-server pre-check", port: 41234, out: "Port 127.0.0.1:41234 already in use.\n", sizes: []int{3}, portInUse: true}, + {name: "kernel bind failure", port: 41234, out: "listen tcp 127.0.0.1:41234: bind: address already in use\n", sizes: []int{1}, portInUse: true}, + {name: "windows bind failure", port: 41234, out: "listen tcp 127.0.0.1:41234: bind: Only one usage of each socket address (protocol/network address/port) is normally permitted.\r\n", sizes: []int{9}, portInUse: true}, + {name: "another listener's bind failure", port: 41234, out: "listen tcp :9091: bind: address already in use\n", sizes: []int{1 << 20}}, + {name: "unix socket warning", port: 41234, out: "unix socket set up failed: bind address at given unix socket path is already in use\n", sizes: []int{1 << 20}}, + {name: "port-in-use after the tail of output", port: 41234, out: pad + "Port 41234 already in use.\n", sizes: []int{999}, portInUse: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w, got := writeChunks(t, tc.port, tc.out, tc.sizes...) + if got != tc.out { + t.Errorf("every byte must reach the log unchanged: got %q", got) + } + if w.IsReady() != tc.ready { + t.Errorf("IsReady = %v, want %v", w.IsReady(), tc.ready) + } + if w.SawPortInUse() != tc.portInUse { + t.Errorf("SawPortInUse = %v, want %v", w.SawPortInUse(), tc.portInUse) + } + }) + } +} + +func TestStartupWatchNilLog(t *testing.T) { + w := NewStartupWatch(nil, 3306) + n, err := w.Write([]byte("Server ready. Accepting connections.\n")) + if err != nil || n != 37 { + t.Fatalf("Write = (%d, %v), want (37, nil)", n, err) + } + if !w.IsReady() { + t.Fatal("ready line not seen") + } +} diff --git a/internal/doltserver/testmain_integration_test.go b/internal/doltserver/testmain_integration_test.go index bab8b059e..9a8c83745 100644 --- a/internal/doltserver/testmain_integration_test.go +++ b/internal/doltserver/testmain_integration_test.go @@ -34,6 +34,9 @@ var integrationSuiteTempRoot string // the case Pdeathsig is meant to protect. See procattr_linux.go for why // this is a narrower, separate flag from BEADS_TEST_MODE. func TestMain(m *testing.M) { + if os.Getenv(fakeDoltEnv) != "" { + os.Exit(fakeDolt(os.Args[1:])) + } os.Exit(runIntegrationTests(m)) } diff --git a/internal/doltserver/testmain_test.go b/internal/doltserver/testmain_test.go index 14f5981a5..ba078537f 100644 --- a/internal/doltserver/testmain_test.go +++ b/internal/doltserver/testmain_test.go @@ -36,6 +36,9 @@ const suiteRootPrefix = "beads-doltserver-tests-" // Integration-tagged runs use the stronger suite-root TestMain in // testmain_integration_test.go instead. func TestMain(m *testing.M) { + if os.Getenv(fakeDoltEnv) != "" { + os.Exit(fakeDolt(os.Args[1:])) + } os.Exit(runTests(m)) } diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index eb21ed9c1..55b4c5769 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -1,14 +1,12 @@ package server import ( - "bytes" "context" "crypto/sha256" "database/sql" "encoding/hex" "errors" "fmt" - "io" "net" "os" "os/exec" @@ -16,7 +14,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "github.com/cenkalti/backoff/v4" @@ -72,7 +69,7 @@ const RuntimeConfigFileName = "proxy-child.runtime-config.yaml" // it was ready because another process holds its listener port. Start // returns it, wrapped, when no PortConflictPolicy is set, when the policy // declines, or after maxStartPortAttempts ports. -var ErrPortInUse = errors.New("dolt sql-server listener port is already in use") +var ErrPortInUse = doltserver.ErrPortInUse // PortConflictPolicy decides whether Start may run the server on another port // when the port configPath names (inUsePort) is held by another process. It @@ -82,33 +79,6 @@ var ErrPortInUse = errors.New("dolt sql-server listener port is already in use") // are its own to move again without asking. type PortConflictPolicy func(configPath string, inUsePort int) error -// doltReadyLine is what dolt sql-server (go-mysql-server) logs at info level -// once its listener is bound and accepting. Seeing it on the stdout/stderr of -// the process Start launched is what proves the port belongs to that process: -// a dial alone also succeeds against another process that took the port, and -// in that case dolt logs "Port N already in use." and exits instead. -const doltReadyLine = "Server ready. Accepting connections." - -// doltPortInUseTexts are the ways a dolt sql-server reports that port, its SQL -// listener port, is taken: -// - dolt's own pre-check: "Port 3306 already in use." -// - go-mysql-server's pre-check: "Port 127.0.0.1:3306 already in use." -// - the bind losing the race after both checks, as Go's net error: -// "listen tcp 127.0.0.1:3306: bind: address already in use", or on -// Windows "...:3306: bind: Only one usage of each socket address ...". -// -// The port is part of every form, so a different listener dolt fails to bind -// (metrics, MCP) is not mistaken for the SQL port, nor is dolt's non-fatal -// unix-socket warning ("... is already in use"). -func doltPortInUseTexts(port int) [][]byte { - return [][]byte{ - []byte(fmt.Sprintf("Port %d already in use", port)), - []byte(fmt.Sprintf(":%d already in use", port)), - []byte(fmt.Sprintf(":%d: bind: address already in use", port)), - []byte(fmt.Sprintf(":%d: bind: Only one usage of each socket address", port)), - } -} - // captureBirth is procid.Capture; tests replace it to make the capture lose // the race with a child that exits at once. var captureBirth = procid.Capture @@ -527,7 +497,7 @@ func (s *DoltServer) startAttempt(ctx context.Context, prepare bool) error { s.eg, s.egCtx, s.cancel = nil, nil, nil cancel() lock.Unlock() - if watch.sawPortInUse() { + if watch.SawPortInUse() { s.pid = 0 return fmt.Errorf("server: DoltServer.Start: %w", s.exitedBeforeReady(watch)) } @@ -597,8 +567,8 @@ func (s *DoltServer) startAttempt(ctx context.Context, prepare bool) error { // dolt only reaches its port check after its own startup, which under load // can take longer than any grace period, and until then a foreign listener // answers exactly like a ready server. -func (s *DoltServer) waitReady(ctx context.Context, watch *startupWatch) error { - needReadyLine := logLevelEmitsReadyLine(s.config.LogLevel()) +func (s *DoltServer) waitReady(ctx context.Context, watch *doltserver.StartupWatch) error { + needReadyLine := doltserver.LogLevelEmitsReadyLine(s.config.LogLevel()) deadline := time.Now().Add(startReadyTimeout) var lastErr error answered := false @@ -615,7 +585,7 @@ func (s *DoltServer) waitReady(ctx context.Context, watch *startupWatch) error { answered = true } switch { - case err == nil && (!needReadyLine || watch.isReady()): + case err == nil && (!needReadyLine || watch.IsReady()): if s.egCtx.Err() != nil { return s.exitedBeforeReady(watch) } @@ -627,14 +597,14 @@ func (s *DoltServer) waitReady(ctx context.Context, watch *startupWatch) error { } if time.Now().After(deadline) { - if needReadyLine && !watch.isReady() { + if needReadyLine && !watch.IsReady() { what := "and nothing answered on " + s.listenAddr() if answered { what = "although something answered on " + s.listenAddr() } return fmt.Errorf("dolt sql-server (pid %d) never logged %q within %s %s; readiness needs that line on dolt's own output: "+ "if a wrapper filters dolt's stdout/stderr or this dolt words it differently, set log_level to warning in %s to fall back to dial-only readiness", - s.pid, doltReadyLine, startReadyTimeout, what, s.configPath) + s.pid, doltserver.DoltReadyLine, startReadyTimeout, what, s.configPath) } return fmt.Errorf("listener not ready after %s: %w", startReadyTimeout, lastErr) } @@ -644,7 +614,7 @@ func (s *DoltServer) waitReady(ctx context.Context, watch *startupWatch) error { return ctx.Err() case <-s.egCtx.Done(): return s.exitedBeforeReady(watch) - case <-watch.readyCh(): + case <-watch.ReadyCh(): case <-time.After(startReadyPollInterval): } } @@ -661,94 +631,19 @@ func (s *DoltServer) listenAddr() string { // supervising goroutine's cmd.Wait returns only after the output copy has // finished, so by the time egCtx is done the watcher has seen everything the // child wrote. -func (s *DoltServer) exitedBeforeReady(watch *startupWatch) error { - if watch.sawPortInUse() { +func (s *DoltServer) exitedBeforeReady(watch *doltserver.StartupWatch) error { + if watch.SawPortInUse() { return fmt.Errorf("%w: dolt sql-server could not bind %s and exited", ErrPortInUse, s.listenAddr()) } return errors.New("dolt sql-server exited before listener became ready") } -// logLevelEmitsReadyLine reports whether dolt logs doltReadyLine (an info -// message) at level. -func logLevelEmitsReadyLine(level servercfg.LogLevel) bool { - switch level { - case servercfg.LogLevel_Trace, servercfg.LogLevel_Debug, servercfg.LogLevel_Info: - return true +// newStartupWatch watches the child's output, forwarding it to logFile. +func newStartupWatch(logFile *os.File, port int) *doltserver.StartupWatch { + if logFile == nil { + return doltserver.NewStartupWatch(nil, port) } - return false -} - -// startupWatch receives the dolt sql-server's stdout and stderr. It forwards -// everything to the log file (if any) and, until the ready line shows up, -// scans the output for that line and for a port conflict. -type startupWatch struct { - dst io.Writer - - inUse [][]byte - - mu sync.Mutex - tail []byte - scanning bool - portInUse bool - ready chan struct{} -} - -// startupWatchTail is how much recent output the watcher keeps for matching, -// so a marker split across two writes is still found. -const startupWatchTail = 4096 - -func newStartupWatch(logFile *os.File, port int) *startupWatch { - w := &startupWatch{inUse: doltPortInUseTexts(port), scanning: true, ready: make(chan struct{})} - if logFile != nil { - w.dst = logFile - } - return w -} - -// Write never fails: a log file that cannot be written must not break the -// pipe the dolt sql-server writes to. -func (w *startupWatch) Write(p []byte) (int, error) { - if w.dst != nil { - _, _ = w.dst.Write(p) - } - w.mu.Lock() - defer w.mu.Unlock() - if !w.scanning { - return len(p), nil - } - w.tail = append(w.tail, p...) - for _, text := range w.inUse { - if bytes.Contains(w.tail, text) { - w.portInUse = true - } - } - if bytes.Contains(w.tail, []byte(doltReadyLine)) { - w.scanning = false - w.tail = nil - close(w.ready) - return len(p), nil - } - if over := len(w.tail) - startupWatchTail; over > 0 { - w.tail = append(w.tail[:0], w.tail[over:]...) - } - return len(p), nil -} - -func (w *startupWatch) readyCh() <-chan struct{} { return w.ready } - -func (w *startupWatch) isReady() bool { - select { - case <-w.ready: - return true - default: - return false - } -} - -func (w *startupWatch) sawPortInUse() bool { - w.mu.Lock() - defer w.mu.Unlock() - return w.portInUse + return doltserver.NewStartupWatch(logFile, port) } func (s *DoltServer) Stop(ctx context.Context) error { diff --git a/internal/storage/dbproxy/server/doltserver_internal_test.go b/internal/storage/dbproxy/server/doltserver_internal_test.go index 5b06d7aff..195297bdb 100644 --- a/internal/storage/dbproxy/server/doltserver_internal_test.go +++ b/internal/storage/dbproxy/server/doltserver_internal_test.go @@ -1,7 +1,6 @@ package server import ( - "bytes" "context" "errors" "fmt" @@ -9,7 +8,6 @@ import ( "os" "path/filepath" "runtime" - "strings" "testing" "time" @@ -18,73 +16,6 @@ import ( "github.com/stretchr/testify/require" ) -// writeChunks feeds s to a new watcher for port in the given chunk sizes -// (the last size repeats) and returns the watcher and what reached dst. -func writeChunks(t *testing.T, port int, s string, sizes ...int) (*startupWatch, string) { - t.Helper() - var dst bytes.Buffer - w := newStartupWatch(nil, port) - w.dst = &dst - b := []byte(s) - for i := 0; len(b) > 0; i++ { - n := sizes[len(sizes)-1] - if i < len(sizes) { - n = sizes[i] - } - if n > len(b) { - n = len(b) - } - got, err := w.Write(b[:n]) - require.NoError(t, err) - require.Equal(t, n, got) - b = b[n:] - } - return w, dst.String() -} - -func TestStartupWatch(t *testing.T) { - ready := `time="2026-10-04T05:45:58Z" level=info msg="Server ready. Accepting connections."` + "\n" - pad := strings.Repeat("x", 5000) + "\n" - for _, tc := range []struct { - name string - port int - out string - sizes []int - ready bool - portInUse bool - }{ - {name: "ready line in one write", port: 3306, out: ready, sizes: []int{1 << 20}, ready: true}, - {name: "ready line split across writes", port: 3306, out: ready, sizes: []int{40, 30, 1 << 20}, ready: true}, - {name: "ready line one byte per write", port: 3306, out: ready, sizes: []int{1}, ready: true}, - {name: "ready line with CRLF", port: 3306, out: strings.ReplaceAll(ready, "\n", "\r\n"), sizes: []int{7}, ready: true}, - {name: "ready line after more than the tail of output", port: 3306, out: pad + ready, sizes: []int{1000}, ready: true}, - {name: "MCP ready line is not the SQL ready line", port: 3306, out: "Dolt MCP server ready. Accepting connections.\n", sizes: []int{1 << 20}}, - {name: "dolt pre-check", port: 8000, out: "Port 8000 already in use.\n", sizes: []int{5}, portInUse: true}, - {name: "dolt pre-check for another port", port: 80, out: "Port 8000 already in use.\n", sizes: []int{1 << 20}}, - {name: "go-mysql-server pre-check", port: 41234, out: "Port 127.0.0.1:41234 already in use.\n", sizes: []int{3}, portInUse: true}, - {name: "kernel bind failure", port: 41234, out: "listen tcp 127.0.0.1:41234: bind: address already in use\n", sizes: []int{1}, portInUse: true}, - {name: "windows bind failure", port: 41234, out: "listen tcp 127.0.0.1:41234: bind: Only one usage of each socket address (protocol/network address/port) is normally permitted.\r\n", sizes: []int{9}, portInUse: true}, - {name: "another listener's bind failure", port: 41234, out: "listen tcp :9091: bind: address already in use\n", sizes: []int{1 << 20}}, - {name: "unix socket warning", port: 41234, out: "unix socket set up failed: bind address at given unix socket path is already in use\n", sizes: []int{1 << 20}}, - {name: "port-in-use after the tail of output", port: 41234, out: pad + "Port 41234 already in use.\n", sizes: []int{999}, portInUse: true}, - } { - t.Run(tc.name, func(t *testing.T) { - w, got := writeChunks(t, tc.port, tc.out, tc.sizes...) - assert.Equal(t, tc.out, got, "every byte must reach the log unchanged") - assert.Equal(t, tc.ready, w.isReady()) - assert.Equal(t, tc.portInUse, w.sawPortInUse()) - }) - } -} - -func TestStartupWatchNilLog(t *testing.T) { - w := newStartupWatch(nil, 3306) - n, err := w.Write([]byte("Server ready. Accepting connections.\n")) - require.NoError(t, err) - assert.Equal(t, 37, n) - assert.True(t, w.isReady()) -} - // newFakeDoltServer builds a DoltServer whose dolt is this test binary in // fake mode (see fakeDolt in testmain_test.go) and whose config names port at // log_level info.