diff --git a/cmd/bd/db_proxy_child.go b/cmd/bd/db_proxy_child.go index 474981970..acab78f2b 100644 --- a/cmd/bd/db_proxy_child.go +++ b/cmd/bd/db_proxy_child.go @@ -80,7 +80,14 @@ not intended to be invoked directly by users.`, func newDatabaseServer(backend proxy.Backend, rootDir, configPath, logPath, doltBin, database string, external configfile.ExternalDoltConfig) (server.DatabaseServer, error) { switch backend { case proxy.BackendLocalServer: - return server.NewDoltServer(doltBin, rootDir, configPath, logPath, 0, database) + s, err := server.NewDoltServer(doltBin, rootDir, configPath, logPath, 0, database) + if err != nil { + return nil, err + } + // A Beads-generated config's port was picked before dolt binds it; + // if another process got there first, run on a free port instead. + s.SetPortConflictPolicy(managedPortConflictPolicy) + return s, nil case proxy.BackendExternal: return server.NewExternalDoltServer(external) case proxy.BackendLocalSharedServer: diff --git a/cmd/bd/proxied_local_lifecycle_linux_test.go b/cmd/bd/proxied_local_lifecycle_linux_test.go index 1a3f02b2a..fd5c0e7d1 100644 --- a/cmd/bd/proxied_local_lifecycle_linux_test.go +++ b/cmd/bd/proxied_local_lifecycle_linux_test.go @@ -46,12 +46,15 @@ import ( "fmt" "net/netip" "os" + "path/filepath" "strconv" "strings" "testing" "time" "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" + + "github.com/steveyegge/beads/internal/storage/dbproxy/server" ) func TestManagedLocalProxiedLifecycleSmoke(t *testing.T) { @@ -184,6 +187,20 @@ func assertManagedConfigLoopback(t *testing.T, p proxiedProject, backendPort int if got := cfg.Host(); got != "127.0.0.1" { t.Errorf("generated backend listener host: got %q, want %q", got, "127.0.0.1") } + // When another process held the generated port at startup, the proxy + // child runs dolt from a runtime copy with only the port moved; the + // generated file keeps its own port. + runtimePath := filepath.Join(p.proxyRoot, server.RuntimeConfigFileName) + if runtimeBody, err := os.ReadFile(runtimePath); err == nil { + runtimeCfg, err := servercfg.NewYamlConfig(runtimeBody) + if err != nil { + t.Fatalf("parse runtime backend config %s: %v", runtimePath, err) + } + if got := runtimeCfg.Port(); got != backendPort { + t.Errorf("runtime backend port %d disagrees with proxy-child.pid port %d", got, backendPort) + } + return + } if got := cfg.Port(); got != backendPort { t.Errorf("generated backend port %d disagrees with proxy-child.pid port %d", got, backendPort) } diff --git a/cmd/bd/proxied_server.go b/cmd/bd/proxied_server.go index 9b4591db9..6e730aa3a 100644 --- a/cmd/bd/proxied_server.go +++ b/cmd/bd/proxied_server.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" @@ -337,11 +338,109 @@ func renderProxiedServerConfig(port int) ([]byte, error) { if err != nil { return nil, err } - return append([]byte(managedProxiedServerConfigMarker), body...), nil + out := []byte(managedProxiedServerConfigMarker + managedPortRecordLine(port)) + return append(out, body...), nil } const proxiedServerListenerHost = "127.0.0.1" +// managedPortRecordPrefix starts the comment line, right after the managed +// marker, that records the listener port Beads chose. The dolt sql-server +// binds that port only after the proxy child starts it, so another process +// can take it first; the record lets the child tell a port Beads chose from +// one an operator set (see managedPortConflictPolicy). The file itself is +// still never rewritten: a moved port lives only in the proxy child's runtime +// config (server.RuntimeConfigFileName). +const managedPortRecordPrefix = "# beads-chosen-listener-port: " + +func managedPortRecordLine(port int) string { + return fmt.Sprintf("%s%d (Beads chose this port. If another process holds it when the server starts, Beads runs the server on a free port for that run, without editing this file. To pin a port, set listener.port to another value or delete this line.)\n", + managedPortRecordPrefix, port) +} + +// renderLegacyProxiedServerConfig is renderProxiedServerConfig as it was +// before the port record existed: the marker followed directly by the YAML. +func renderLegacyProxiedServerConfig(port int) ([]byte, error) { + body, err := renderProxiedServerConfig(port) + if err != nil { + return nil, err + } + return append([]byte(managedProxiedServerConfigMarker), body[len(managedProxiedServerConfigMarker)+len(managedPortRecordLine(port)):]...), nil +} + +// managedPortConflictPolicy is the managed-local backend's +// server.PortConflictPolicy. It lets the proxy child run the dolt sql-server +// on a free port (through a runtime config; configPath is never modified) +// only when configPath's listener.port is a port Beads chose: +// - the file carries the managed marker and the port record, and +// listener.port still equals the recorded port; or +// - the file is byte-identical to a config Beads generated before the +// record existed (any hand edit makes the bytes differ). +// +// Anything else (an operator's own config, a hand-set port, a hand-edited +// legacy config) is pinned, and the error says what to do. +func managedPortConflictPolicy(configPath string, inUsePort int) error { + body, err := os.ReadFile(configPath) // #nosec G304 -- the proxy child's own --config path + if err != nil { + return fmt.Errorf("read %s: %w", configPath, err) + } + remedy := fmt.Sprintf("free port %d, or set listener.port in %s to a free port", inUsePort, configPath) + if !isManagedProxiedServerConfig(body) { + return fmt.Errorf("%s is not a Beads-generated config, so its port is pinned; %s", configPath, remedy) + } + remedy += ", or delete " + configPath + " so Beads regenerates it" + cfg, err := servercfg.NewYamlConfig(body) + if err != nil { + return fmt.Errorf("parse %s: %w", configPath, err) + } + current := cfg.Port() + if current != inUsePort { + return fmt.Errorf("%s: listener.port is %d, not the in-use port %d; %s", configPath, current, inUsePort, remedy) + } + if recorded, _, ok := cutManagedPortRecord(body[len(managedProxiedServerConfigMarker):]); ok { + if current != recorded { + return fmt.Errorf("%s: listener.port %d was set by hand (Beads chose %d), so it is pinned; %s", configPath, current, recorded, remedy) + } + return nil + } + legacy, err := renderLegacyProxiedServerConfig(current) + if err != nil { + return fmt.Errorf("render legacy config: %w", err) + } + if bytes.Equal(body, legacy) { + return nil + } + return fmt.Errorf("%s was edited by hand and has no %q record, so its listener.port is pinned; %s", + configPath, strings.TrimSpace(managedPortRecordPrefix), remedy) +} + +// cutManagedPortRecord splits the port record line off the front of rest +// (the managed config after its marker) and returns the recorded port and +// the remaining YAML. The digits must end at a space or the line end and +// name a valid TCP port. +func cutManagedPortRecord(rest []byte) (port int, yamlBody []byte, ok bool) { + if !bytes.HasPrefix(rest, []byte(managedPortRecordPrefix)) { + return 0, nil, false + } + line, after, found := bytes.Cut(rest, []byte("\n")) + if !found { + return 0, nil, false + } + digits := line[len(managedPortRecordPrefix):] + end := 0 + for end < len(digits) && digits[end] >= '0' && digits[end] <= '9' { + end++ + } + if end == 0 || (end < len(digits) && digits[end] != ' ') { + return 0, nil, false + } + port, err := strconv.Atoi(string(digits[:end])) + if err != nil || port < 1 || port > 65535 { + return 0, nil, false + } + return port, after, true +} + // TODO: this needs to return a dolt server uow provider as the global // uow provider used by all commands diff --git a/cmd/bd/proxied_server_test.go b/cmd/bd/proxied_server_test.go index 645864bfc..154919725 100644 --- a/cmd/bd/proxied_server_test.go +++ b/cmd/bd/proxied_server_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "os" @@ -13,6 +14,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1225,3 +1227,236 @@ func TestResolveProxiedServerLogPath_FollowsCustomRoot(t *testing.T) { assert.False(t, isCustom) }) } + +func TestRenderProxiedServerConfig_RecordsChosenPort(t *testing.T) { + body, err := renderProxiedServerConfig(54321) + require.NoError(t, err) + require.True(t, isManagedProxiedServerConfig(body)) + port, _, ok := cutManagedPortRecord(body[len(managedProxiedServerConfigMarker):]) + require.True(t, ok, "generated config must carry the chosen-port record") + assert.Equal(t, 54321, port) +} + +func TestRenderLegacyProxiedServerConfig_IsMarkerPlusYAML(t *testing.T) { + legacy, err := renderLegacyProxiedServerConfig(40001) + require.NoError(t, err) + require.True(t, isManagedProxiedServerConfig(legacy)) + _, _, ok := cutManagedPortRecord(legacy[len(managedProxiedServerConfigMarker):]) + assert.False(t, ok, "the legacy form has no port record") + cfg, err := servercfg.NewYamlConfig(legacy) + require.NoError(t, err) + assert.Equal(t, 40001, cfg.Port()) +} + +func TestCutManagedPortRecord(t *testing.T) { + for _, tc := range []struct { + line string + port int + ok bool + }{ + {line: managedPortRecordPrefix + "40001 (Beads chose this port.)\n", port: 40001, ok: true}, + {line: managedPortRecordPrefix + "40001\n", port: 40001, ok: true}, + {line: managedPortRecordPrefix + "4000abc\n"}, + {line: managedPortRecordPrefix + "0\n"}, + {line: managedPortRecordPrefix + "70000\n"}, + {line: managedPortRecordPrefix + "\n"}, + {line: managedPortRecordPrefix + "40001"}, + {line: "log_level: info\n"}, + } { + port, _, ok := cutManagedPortRecord([]byte(tc.line + "log_level: info\n")) + if tc.line == managedPortRecordPrefix+"40001" { + // No newline before the YAML in this case: the record runs into it. + port, _, ok = cutManagedPortRecord([]byte(tc.line)) + } + assert.Equal(t, tc.ok, ok, "%q", tc.line) + if tc.ok { + assert.Equal(t, tc.port, port, "%q", tc.line) + } + } +} + +func TestManagedPortConflictPolicy_AllowsBeadsChosenPort(t *testing.T) { + for _, tc := range []struct { + name string + body func(t *testing.T) []byte + }{ + {name: "generated config", body: func(t *testing.T) []byte { + b, err := renderProxiedServerConfig(40001) + require.NoError(t, err) + return b + }}, + {name: "generated config edited elsewhere, port unchanged", body: func(t *testing.T) []byte { + b, err := renderProxiedServerConfig(40001) + require.NoError(t, err) + return []byte(strings.Replace(string(b), "log_level: info", "log_level: debug", 1)) + }}, + {name: "generated config with an environment placeholder, port unchanged", body: func(t *testing.T) []byte { + // The server copies the raw text, so the placeholder's value never + // lands in the runtime config. + b, err := renderProxiedServerConfig(40001) + require.NoError(t, err) + return []byte(strings.Replace(string(b), "log_level: info", "log_level: ${BD_TEST_LOG_LEVEL}", 1)) + }}, + {name: "unedited config generated before the record existed", body: func(t *testing.T) []byte { + b, err := renderLegacyProxiedServerConfig(40001) + require.NoError(t, err) + return b + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BD_TEST_LOG_LEVEL", "info") + path := filepath.Join(t.TempDir(), "config.yaml") + body := tc.body(t) + require.NoError(t, os.WriteFile(path, body, 0o600)) + require.NoError(t, managedPortConflictPolicy(path, 40001)) + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, string(body), string(after), "the policy must never modify the config") + }) + } +} + +func TestManagedPortConflictPolicy_PinsOtherPorts(t *testing.T) { + for _, tc := range []struct { + name string + body func(t *testing.T) []byte + inUse int + errHas []string + }{ + { + name: "hand-set port", + body: func(t *testing.T) []byte { + b, err := renderProxiedServerConfig(40001) + require.NoError(t, err) + // The YAML key, not the "...-listener-port: 40001" record above it. + return []byte(strings.Replace(string(b), "\n port: 40001", "\n port: 40002", 1)) + }, + inUse: 40002, + errHas: []string{"set by hand", "free port 40002", "delete"}, + }, + { + name: "hand-edited config generated before the record existed", + body: func(t *testing.T) []byte { + b, err := renderLegacyProxiedServerConfig(40001) + require.NoError(t, err) + return []byte(strings.Replace(string(b), "log_level: info", "log_level: debug", 1)) + }, + inUse: 40001, + errHas: []string{"edited by hand", "pinned", "delete"}, + }, + { + name: "operator config", + body: func(t *testing.T) []byte { + return []byte("listener:\n host: 127.0.0.1\n port: 40001\n") + }, + inUse: 40001, + errHas: []string{"not a Beads-generated config", "set listener.port in"}, + }, + { + name: "in-use port is not the config's port", + body: func(t *testing.T) []byte { + b, err := renderProxiedServerConfig(40001) + require.NoError(t, err) + return b + }, + inUse: 40009, + errHas: []string{"not the in-use port"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BD_TEST_LOG_LEVEL", "info") + path := filepath.Join(t.TempDir(), "config.yaml") + body := tc.body(t) + require.NoError(t, os.WriteFile(path, body, 0o600)) + err := managedPortConflictPolicy(path, tc.inUse) + require.Error(t, err) + for _, want := range tc.errHas { + assert.Contains(t, err.Error(), want) + } + }) + } +} + +// legacyGeneratedConfigV13 is, byte for byte, the config.yaml that bd v1.3.0 +// and v1.3.1 generate for port 40001 (marker plus yaml.Marshal of the +// rendered servercfg.YAMLConfig, before the port record existed). It is +// frozen here so a dolt or yaml upgrade that changes Marshal output cannot +// silently stop existing workspaces from matching the legacy form. +const legacyGeneratedConfigV13 = "# Generated by Beads on first run - safe to delete (will be regenerated). " + + "Never rewritten afterward; hand edits are preserved. See gastownhall/beads#4986.\n" + + "log_level: info\nbehavior:\n auto_gc_behavior:\n archive_level: 0\nlistener:\n host: 127.0.0.1\n port: 40001\nuser_session_vars: []\njwks: []\n" + +func TestManagedPortConflictPolicy_LegacyV13GoldenIsEligible(t *testing.T) { + legacy, err := renderLegacyProxiedServerConfig(40001) + require.NoError(t, err) + assert.Equal(t, legacyGeneratedConfigV13, string(legacy), "legacy render drifted from what v1.3.x wrote") + + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(legacyGeneratedConfigV13), 0o600)) + require.NoError(t, managedPortConflictPolicy(path, 40001)) +} + +// TestManagedPortConflictPolicy_ConsecutiveCollisions drives a real +// server.DoltServer with bd's real policy through repeated port conflicts: +// the stand-in dolt reports its port taken on every launch, so every move +// Start makes collides again. The policy must be consulted about the +// operator's port only, so Start keeps moving until it gives up after its +// port budget, rather than being refused on the second move because the +// runtime port is not config.yaml's port. config.yaml must stay unchanged. +func TestManagedPortConflictPolicy_ConsecutiveCollisions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stand-in dolt is a shell script") + } + dir := t.TempDir() + fakeDolt := filepath.Join(dir, "dolt") + launches := filepath.Join(dir, "launches") + script := `#!/bin/sh +case "$1" in +config) echo fake; exit 0 ;; +init) exit 0 ;; +sql-server) + cfg="$3" + port=$(sed -n 's/^ *port: *\([0-9][0-9]*\).*/\1/p' "$cfg" | head -n 1) + echo "$port" >> "` + launches + `" + echo "Port $port already in use." >&2 + exit 1 ;; +esac +exit 2 +` + require.NoError(t, os.WriteFile(fakeDolt, []byte(script), 0o755)) + + root := filepath.Join(dir, "root") + require.NoError(t, os.MkdirAll(root, 0o755)) + cfgPath := filepath.Join(root, "config.yaml") + body, err := renderProxiedServerConfig(40001) + require.NoError(t, err) + require.NoError(t, os.WriteFile(cfgPath, body, 0o600)) + + s, err := server.NewDoltServer(fakeDolt, root, cfgPath, filepath.Join(dir, "server.log"), 0, "") + require.NoError(t, err) + s.SetPortConflictPolicy(managedPortConflictPolicy) + t.Cleanup(func() { _ = s.Stop(context.Background()) }) + + err = s.Start(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, server.ErrPortInUse) + assert.Contains(t, err.Error(), "gave up after 5 ports", "the policy must not refuse the second and later moves") + assert.NotContains(t, err.Error(), "not the in-use port") + + got, err := os.ReadFile(launches) + require.NoError(t, err) + ports := strings.Fields(string(got)) + require.Len(t, ports, 5, "one launch per port: %v", ports) + assert.Equal(t, "40001", ports[0], "the first launch uses config.yaml's port") + seen := map[string]bool{} + for _, p := range ports { + assert.False(t, seen[p], "port %s launched twice: %v", p, ports) + seen[p] = true + } + + after, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.Equal(t, string(body), string(after), "config.yaml must never be modified") + _, err = os.Stat(filepath.Join(root, server.RuntimeConfigFileName)) + assert.True(t, os.IsNotExist(err), "a failed Start must not leave a runtime config") +} diff --git a/internal/storage/dbproxy/proxy/endpoint.go b/internal/storage/dbproxy/proxy/endpoint.go index 5d4b5d497..8014f4d86 100644 --- a/internal/storage/dbproxy/proxy/endpoint.go +++ b/internal/storage/dbproxy/proxy/endpoint.go @@ -171,8 +171,11 @@ var errStartInterrupted = errors.New("proxy startup interrupted by concurrent sh func PickFreePort() (int, error) { // The managed proxy no longer uses this bind-close allocator: its child // binds port 0 and publishes the kernel-assigned port. The remaining - // production caller allocates the Dolt config port; that race requires - // the managed-config ownership/retry contract deferred to the PR-C RFC. + // production caller allocates the Dolt config port. Another process can + // take that port before dolt binds it; server.DoltServer detects that + // (dolt's own ready line, not a bare dial, proves readiness) and, for a + // Beads-chosen port, retries on a fresh port through a runtime copy of + // the config; the config file itself is never rewritten. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return 0, err diff --git a/internal/storage/dbproxy/proxy/server.go b/internal/storage/dbproxy/proxy/server.go index c15d09dcc..d3fe92812 100644 --- a/internal/storage/dbproxy/proxy/server.go +++ b/internal/storage/dbproxy/proxy/server.go @@ -539,6 +539,15 @@ func waitForServerReady(ctx context.Context, s server.DatabaseServer, timeout ti return err } _ = conn.Close() + // The dial proves something answers on the backend's address, not + // that the backend does. A local backend's Start has already proved + // its own process owns the port (see server.DoltServer.waitReady), + // and while it runs nobody else can bind it, so re-checking Running + // after the dial closes the gap where the backend exited in between + // and another process took the port. + if !s.Running(ctx) { + return errors.New("database server exited during readiness check") + } return nil }, backoff.WithContext(bo, ctx)) } diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index 4b5c883ef..fb64d1fc3 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -30,6 +30,7 @@ go_test( name = "server_test", srcs = [ "doltserver_exit_unix_test.go", + "doltserver_internal_test.go", "doltserver_test.go", "external_dolt_server_test.go", "testmain_test.go", diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index 0cd1a8a4b..eb21ed9c1 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -1,18 +1,22 @@ package server import ( + "bytes" "context" "crypto/sha256" "database/sql" "encoding/hex" "errors" "fmt" + "io" "net" "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" + "sync" "time" "github.com/cenkalti/backoff/v4" @@ -43,12 +47,93 @@ const ( // reported as running forever. var errBackendExited = errors.New("dolt sql-server exited") +// startReadyTimeout is a var so tests can shorten it. +var startReadyTimeout = 30 * time.Second + const ( - startReadyTimeout = 30 * time.Second startReadyPollInterval = 50 * time.Millisecond startReadyDialTimeout = 250 * time.Millisecond ) +// maxStartPortAttempts bounds how many listener ports one Start tries when +// its PortConflictPolicy lets it move off ports another process holds. +const maxStartPortAttempts = 5 + +// RuntimeConfigFileName is the config Start writes in the root directory when +// it runs the dolt sql-server on a different port than the operator config +// names (see SetPortConflictPolicy). It is the operator config with only +// listener.port changed, it exists only while that server runs, and the +// operator config itself is never modified. dolt sql-server has no +// command-line port override: with --config, -P/--port is silently ignored +// (verified against dolt 2.1.8 and 2.2.0), so the override has to be a file. +const RuntimeConfigFileName = "proxy-child.runtime-config.yaml" + +// ErrPortInUse reports that the dolt sql-server Start launched exited before +// it was ready because another process holds its listener port. Start +// returns it, wrapped, when no PortConflictPolicy is set, when the policy +// declines, or after maxStartPortAttempts ports. +var ErrPortInUse = errors.New("dolt sql-server listener port is already in use") + +// PortConflictPolicy decides whether Start may run the server on another port +// when the port configPath names (inUsePort) is held by another process. It +// returns nil to allow it, or an error saying why the port is pinned and what +// the operator can do; Start then fails with ErrPortInUse and that reason. +// Start asks once per Start, on the first conflict; ports it picked itself +// are its own to move again without asking. +type PortConflictPolicy func(configPath string, inUsePort int) error + +// doltReadyLine is what dolt sql-server (go-mysql-server) logs at info level +// once its listener is bound and accepting. Seeing it on the stdout/stderr of +// the process Start launched is what proves the port belongs to that process: +// a dial alone also succeeds against another process that took the port, and +// in that case dolt logs "Port N already in use." and exits instead. +const doltReadyLine = "Server ready. Accepting connections." + +// doltPortInUseTexts are the ways a dolt sql-server reports that port, its SQL +// listener port, is taken: +// - dolt's own pre-check: "Port 3306 already in use." +// - go-mysql-server's pre-check: "Port 127.0.0.1:3306 already in use." +// - the bind losing the race after both checks, as Go's net error: +// "listen tcp 127.0.0.1:3306: bind: address already in use", or on +// Windows "...:3306: bind: Only one usage of each socket address ...". +// +// The port is part of every form, so a different listener dolt fails to bind +// (metrics, MCP) is not mistaken for the SQL port, nor is dolt's non-fatal +// unix-socket warning ("... is already in use"). +func doltPortInUseTexts(port int) [][]byte { + return [][]byte{ + []byte(fmt.Sprintf("Port %d already in use", port)), + []byte(fmt.Sprintf(":%d already in use", port)), + []byte(fmt.Sprintf(":%d: bind: address already in use", port)), + []byte(fmt.Sprintf(":%d: bind: Only one usage of each socket address", port)), + } +} + +// captureBirth is procid.Capture; tests replace it to make the capture lose +// the race with a child that exits at once. +var captureBirth = procid.Capture + +// exitedOnItsOwn reports whether a reaped child ended by itself rather than +// by the Kill failSpawn sends. A killed child reports exit code -1 on unix; +// Windows reports an ordinary code for a killed process, so there an exit +// cannot be told from a kill and this reports false. +func exitedOnItsOwn(ps *os.ProcessState) bool { + return ps != nil && runtime.GOOS != "windows" && ps.ExitCode() >= 0 +} + +// pickFreePort returns a loopback port that was free a moment ago. Tests +// replace it. (proxy.PickFreePort is the same allocator; this package cannot +// import proxy.) +var pickFreePort = func() (int, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + port := ln.Addr().(*net.TCPAddr).Port + _ = ln.Close() + return port, nil +} + type DoltServer struct { id string doltBinExec string @@ -59,10 +144,16 @@ type DoltServer struct { keepAlivePeriod time.Duration logFile *os.File - eg *errgroup.Group - egCtx context.Context - cancel context.CancelFunc - pid int + // portPolicy, when set, lets Start recover from a listener port another + // process holds. See SetPortConflictPolicy. + portPolicy PortConflictPolicy + // launchConfigPath is the --config the current attempt passes to dolt: + // configPath, or the runtime config when Start moved the port. + launchConfigPath string + eg *errgroup.Group + egCtx context.Context + cancel context.CancelFunc + pid int } var _ DatabaseServer = (*DoltServer)(nil) @@ -222,29 +313,169 @@ func (s *DoltServer) doltInitWithRetries(ctx context.Context) error { return backoff.Retry(op, backoff.WithMaxRetries(backoff.WithContext(bo, ctx), maxRetries)) } +// SetPortConflictPolicy installs fn to recover from a listener port another +// process holds. When the dolt sql-server Start launched exits because its +// port is in use and fn allows it, Start picks a free port, writes +// RuntimeConfigFileName (the config with only listener.port changed), and +// launches again from that file, up to maxStartPortAttempts ports. The +// operator config is never modified. Without a policy Start returns +// ErrPortInUse. Call it before Start. +func (s *DoltServer) SetPortConflictPolicy(fn PortConflictPolicy) { + s.portPolicy = fn +} + +func (s *DoltServer) runtimeConfigPath() string { + return filepath.Join(s.rootDir, RuntimeConfigFileName) +} + func (s *DoltServer) Start(ctx context.Context) error { if s.eg != nil || s.egCtx != nil { return fmt.Errorf("server: DoltServer.Start: server already started") } + // A runtime config left by an earlier run is stale: every Start begins + // from the operator config's own port. + _ = os.Remove(s.runtimeConfigPath()) + cfg, err := servercfg.YamlConfigFromFile(filesys.LocalFS, s.configPath) + if err != nil { + return fmt.Errorf("server: DoltServer.Start: parse config %q: %w", s.configPath, err) + } + s.config = cfg + s.launchConfigPath = s.configPath + err = s.startWithPortRecovery(ctx) + if err != nil { + _ = os.Remove(s.runtimeConfigPath()) + } + return err +} +func (s *DoltServer) startWithPortRecovery(ctx context.Context) error { + // The policy judges the operator config, so it is asked once, about the + // port that config names. Later conflicts are on ports Start picked + // itself, which are Start's to move again. + operatorPort := s.config.Port() + for attempt := 1; ; attempt++ { + err := s.startAttempt(ctx, attempt == 1) + if err == nil || !errors.Is(err, ErrPortInUse) { + return err + } + inUse := s.config.Port() + if s.portPolicy == nil { + return fmt.Errorf("%w; %s", err, portInUseRemedy(s.configPath, inUse)) + } + if attempt >= maxStartPortAttempts { + return fmt.Errorf("%w (gave up after %d ports, the last %d; something on this host keeps taking them)", err, attempt, inUse) + } + if attempt == 1 { + if perr := s.portPolicy(s.configPath, operatorPort); perr != nil { + return fmt.Errorf("%w; not moving off port %d: %v", err, inUse, perr) + } + } + if rerr := s.useRuntimePort(inUse); rerr != nil { + return fmt.Errorf("%w; could not move off port %d: %v", err, inUse, rerr) + } + } +} + +// portInUseRemedy says what an operator can do about a pinned port. +func portInUseRemedy(configPath string, port int) string { + return fmt.Sprintf("free port %d, or set listener.port in %s to a free port", port, configPath) +} + +// useRuntimePort writes RuntimeConfigFileName: the operator config with +// listener.port moved to a fresh port (never inUse), and makes it the config +// the next attempt launches with and the server dials. +func (s *DoltServer) useRuntimePort(inUse int) error { + // Parse the raw bytes, not YamlConfigFromFile's env-interpolated text: + // dolt interpolates the runtime file when it reads it, so placeholders + // (and "$$" escapes) must reach it unexpanded, and their values never + // land on disk. + raw, err := os.ReadFile(s.configPath) + if err != nil { + return fmt.Errorf("re-read %s: %w", s.configPath, err) + } + yc, err := servercfg.NewYamlConfig(raw) + if err != nil { + return fmt.Errorf("parse %s: %w", s.configPath, err) + } + port := inUse + for port == inUse { + if port, err = pickFreePort(); err != nil { + return fmt.Errorf("pick free port: %w", err) + } + } + rcfg := *yc + rcfg.ListenerConfig.PortNumber = &port + body := []byte("# Written by Beads: " + s.configPath + " with listener.port moved to a free port for this run\n" + + "# because another process held " + strconv.Itoa(inUse) + ". Removed when the server stops; edit the file above, not this one.\n" + + rcfg.String()) + cfg, err := servercfg.NewYamlConfig(body) + if err != nil { + return fmt.Errorf("render runtime config: %w", err) + } + if cfg.Port() != port { + return fmt.Errorf("render runtime config: port %d, want %d", cfg.Port(), port) + } + if err := writeFileAtomic(s.runtimeConfigPath(), body); err != nil { + return fmt.Errorf("write %s: %w", s.runtimeConfigPath(), err) + } + // Dial and DSN need the values dolt will see, so read the file back the + // way dolt does, with interpolation. + effective, err := servercfg.YamlConfigFromFile(filesys.LocalFS, s.runtimeConfigPath()) + if err != nil { + return fmt.Errorf("re-read %s: %w", s.runtimeConfigPath(), err) + } + s.config = effective + s.launchConfigPath = s.runtimeConfigPath() + return nil +} + +// writeFileAtomic writes body to path (mode 0600) through a temp file and a +// rename, so dolt never reads a partial config. +func writeFileAtomic(path string, body []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { _ = os.Remove(tmpName) }() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(body); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpName, path) +} + +// startAttempt launches dolt sql-server once and waits until it is ready. +// Only the first attempt runs the dolt config/init preparation; a retry after +// a port conflict reuses it. +func (s *DoltServer) startAttempt(ctx context.Context, prepare bool) error { lock, err := util.TryLock(filepath.Join(s.rootDir, LockFileName)) if err != nil { return fmt.Errorf("server: DoltServer.Start: acquire %s: %w", LockFileName, err) } - if err := s.doltConfigure(ctx); err != nil { - lock.Unlock() - return err - } + if prepare { + if err := s.doltConfigure(ctx); err != nil { + lock.Unlock() + return err + } - if err := s.doltInitWithRetries(ctx); err != nil { - lock.Unlock() - return err + if err := s.doltInitWithRetries(ctx); err != nil { + lock.Unlock() + return err + } } args := []string{ "sql-server", - "--config", s.configPath, + "--config", s.launchConfigPath, } managedCtx, cancel := context.WithCancel(context.Background()) @@ -256,10 +487,18 @@ func (s *DoltServer) Start(ctx context.Context) error { cmd := exec.CommandContext(managedCtx, s.doltBinExec, args...) cmd.Dir = s.rootDir cmd.Stdin = nil - if s.logFile != nil { - cmd.Stdout = s.logFile - cmd.Stderr = s.logFile - } + // Both streams go through one watcher, which forwards to the log file + // and spots the ready line and a port conflict for waitReady. + // + // The streams are a pipe into this process rather than the log file + // itself, so a dolt sql-server whose proxy died takes SIGPIPE on its next + // write; the proxy's shutdown and force-stop paths reap such a backend + // anyway. WaitDelay keeps a grandchild that inherited the pipe from + // holding cmd.Wait (and so Stop) open. + watch := newStartupWatch(s.logFile, s.config.Port()) + cmd.Stdout = watch + cmd.Stderr = watch + cmd.WaitDelay = 5 * time.Second // The proxied server runs CALL DOLT_PUSH/FETCH in-process; see // doltserver.ServerSpawnEnv for the guards it needs (GH#4272). @@ -273,23 +512,40 @@ func (s *DoltServer) Start(ctx context.Context) error { } s.pid = cmd.Process.Pid - birth, err := procid.Capture(s.pid) - if err != nil { + // failSpawn tears down a child that Start gave up on before handing it + // to the supervising goroutine. A dolt sql-server that finds its port + // taken can exit within milliseconds, before the identity capture or + // pid record below gets to it, and then those steps fail only because + // the child is already gone. So once the child is reaped (cmd.Wait also + // drains its output into the watcher), its own exit decides the error: + // ErrPortInUse when it reported its port taken, so Start's normal + // recovery runs; "exited before ready" when it exited by itself; the + // step's error only when the child was still alive and we killed it. + failSpawn := func(step string, stepErr error) error { _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - s.eg, s.egCtx, s.cancel, s.pid = nil, nil, nil, 0 + _ = cmd.Wait() + s.eg, s.egCtx, s.cancel = nil, nil, nil cancel() lock.Unlock() - return fmt.Errorf("server: DoltServer.Start: capture child birth identity: %w", err) + if watch.sawPortInUse() { + s.pid = 0 + return fmt.Errorf("server: DoltServer.Start: %w", s.exitedBeforeReady(watch)) + } + if exitedOnItsOwn(cmd.ProcessState) { + code := cmd.ProcessState.ExitCode() + s.pid = 0 + return fmt.Errorf("server: DoltServer.Start: dolt sql-server exited (status %d) before startup completed (%s: %v)", code, step, stepErr) + } + s.pid = 0 + return fmt.Errorf("server: DoltServer.Start: %s: %w", step, stepErr) + } + birth, err := captureBirth(s.pid) + if err != nil { + return failSpawn("capture child birth identity", err) } rootID, err := identity.RootID(s.rootDir) if err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - s.eg, s.egCtx, s.cancel, s.pid = nil, nil, nil, 0 - cancel() - lock.Unlock() - return fmt.Errorf("server: DoltServer.Start: resolve proxy root identity: %w", err) + return failSpawn("resolve proxy root identity", err) } if err := pidfile.Write(s.rootDir, PIDFileName, pidfile.PidFile{ @@ -300,12 +556,7 @@ func (s *DoltServer) Start(ctx context.Context) error { Birth: string(birth), RootID: rootID, }); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - s.eg, s.egCtx, s.cancel, s.pid = nil, nil, nil, 0 - cancel() - lock.Unlock() - return fmt.Errorf("server: DoltServer.Start: write pidfile: %w", err) + return failSpawn("write pidfile", err) } eg.Go(func() error { @@ -316,7 +567,7 @@ func (s *DoltServer) Start(ctx context.Context) error { return errBackendExited }) - if err := s.waitReady(ctx); err != nil { + if err := s.waitReady(ctx, watch); err != nil { cancel() _ = s.eg.Wait() s.eg, s.egCtx, s.cancel, s.pid = nil, nil, nil, 0 @@ -326,11 +577,34 @@ func (s *DoltServer) Start(ctx context.Context) error { return nil } -func (s *DoltServer) waitReady(ctx context.Context) error { +// waitReady waits until the dolt sql-server this Start launched is accepting +// connections on its configured listener. +// +// A successful dial is not enough on its own: the port is chosen before dolt +// binds it, and if another process takes it in between, the dial reaches that +// process while dolt fails to bind and exits. So when the config's log level +// lets dolt log its ready line (info or more verbose, which every +// Beads-generated config uses), the dial only counts after that line has +// appeared on this child's own output. Under a quieter log level there is no +// such signal and the dial alone decides, as before; that is also the escape +// hatch for a dolt whose output never carries the line (a wrapper that +// filters it, a reworded release), which otherwise fails after +// startReadyTimeout with an error naming the missing line. Either way, a +// child that exits first is reported, as ErrPortInUse when it said its port +// was taken. +// +// There is deliberately no "answered for a while, accept anyway" fallback: +// dolt only reaches its port check after its own startup, which under load +// can take longer than any grace period, and until then a foreign listener +// answers exactly like a ready server. +func (s *DoltServer) waitReady(ctx context.Context, watch *startupWatch) error { + needReadyLine := logLevelEmitsReadyLine(s.config.LogLevel()) deadline := time.Now().Add(startReadyTimeout) + var lastErr error + answered := false for { if s.egCtx.Err() != nil { - return errors.New("dolt sql-server exited before listener became ready") + return s.exitedBeforeReady(watch) } dctx, dcancel := context.WithTimeout(ctx, startReadyDialTimeout) @@ -338,23 +612,145 @@ func (s *DoltServer) waitReady(ctx context.Context) error { dcancel() if err == nil { _ = conn.Close() + answered = true + } + switch { + case err == nil && (!needReadyLine || watch.isReady()): + if s.egCtx.Err() != nil { + return s.exitedBeforeReady(watch) + } return nil + case err != nil: + lastErr = err + default: + lastErr = nil } if time.Now().After(deadline) { - return fmt.Errorf("listener not ready after %s: %w", startReadyTimeout, err) + if needReadyLine && !watch.isReady() { + what := "and nothing answered on " + s.listenAddr() + if answered { + what = "although something answered on " + s.listenAddr() + } + return fmt.Errorf("dolt sql-server (pid %d) never logged %q within %s %s; readiness needs that line on dolt's own output: "+ + "if a wrapper filters dolt's stdout/stderr or this dolt words it differently, set log_level to warning in %s to fall back to dial-only readiness", + s.pid, doltReadyLine, startReadyTimeout, what, s.configPath) + } + return fmt.Errorf("listener not ready after %s: %w", startReadyTimeout, lastErr) } select { case <-ctx.Done(): return ctx.Err() case <-s.egCtx.Done(): - return errors.New("dolt sql-server exited before listener became ready") + return s.exitedBeforeReady(watch) + case <-watch.readyCh(): case <-time.After(startReadyPollInterval): } } } +func (s *DoltServer) listenAddr() string { + if sock := s.config.Socket(); sock != "" { + return sock + } + return net.JoinHostPort(s.config.Host(), strconv.Itoa(s.config.Port())) +} + +// exitedBeforeReady is waitReady's error for a child that exited first. The +// supervising goroutine's cmd.Wait returns only after the output copy has +// finished, so by the time egCtx is done the watcher has seen everything the +// child wrote. +func (s *DoltServer) exitedBeforeReady(watch *startupWatch) error { + if watch.sawPortInUse() { + return fmt.Errorf("%w: dolt sql-server could not bind %s and exited", ErrPortInUse, s.listenAddr()) + } + return errors.New("dolt sql-server exited before listener became ready") +} + +// logLevelEmitsReadyLine reports whether dolt logs doltReadyLine (an info +// message) at level. +func logLevelEmitsReadyLine(level servercfg.LogLevel) bool { + switch level { + case servercfg.LogLevel_Trace, servercfg.LogLevel_Debug, servercfg.LogLevel_Info: + return true + } + return false +} + +// startupWatch receives the dolt sql-server's stdout and stderr. It forwards +// everything to the log file (if any) and, until the ready line shows up, +// scans the output for that line and for a port conflict. +type startupWatch struct { + dst io.Writer + + inUse [][]byte + + mu sync.Mutex + tail []byte + scanning bool + portInUse bool + ready chan struct{} +} + +// startupWatchTail is how much recent output the watcher keeps for matching, +// so a marker split across two writes is still found. +const startupWatchTail = 4096 + +func newStartupWatch(logFile *os.File, port int) *startupWatch { + w := &startupWatch{inUse: doltPortInUseTexts(port), scanning: true, ready: make(chan struct{})} + if logFile != nil { + w.dst = logFile + } + return w +} + +// Write never fails: a log file that cannot be written must not break the +// pipe the dolt sql-server writes to. +func (w *startupWatch) Write(p []byte) (int, error) { + if w.dst != nil { + _, _ = w.dst.Write(p) + } + w.mu.Lock() + defer w.mu.Unlock() + if !w.scanning { + return len(p), nil + } + w.tail = append(w.tail, p...) + for _, text := range w.inUse { + if bytes.Contains(w.tail, text) { + w.portInUse = true + } + } + if bytes.Contains(w.tail, []byte(doltReadyLine)) { + w.scanning = false + w.tail = nil + close(w.ready) + return len(p), nil + } + if over := len(w.tail) - startupWatchTail; over > 0 { + w.tail = append(w.tail[:0], w.tail[over:]...) + } + return len(p), nil +} + +func (w *startupWatch) readyCh() <-chan struct{} { return w.ready } + +func (w *startupWatch) isReady() bool { + select { + case <-w.ready: + return true + default: + return false + } +} + +func (w *startupWatch) sawPortInUse() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.portInUse +} + func (s *DoltServer) Stop(ctx context.Context) error { gcErr := s.runShutdownGC(ctx) if gcErr != nil { @@ -388,6 +784,9 @@ func (s *DoltServer) Stop(ctx context.Context) error { rmErr = pidfile.Remove(s.rootDir, PIDFileName) s.pid = 0 } + if err := os.Remove(s.runtimeConfigPath()); err != nil && !os.IsNotExist(err) { + rmErr = errors.Join(rmErr, err) + } if rmErr != nil { rmErr = fmt.Errorf("server: DoltServer.Stop: remove pidfile: %w", rmErr) } diff --git a/internal/storage/dbproxy/server/doltserver_internal_test.go b/internal/storage/dbproxy/server/doltserver_internal_test.go new file mode 100644 index 000000000..5b06d7aff --- /dev/null +++ b/internal/storage/dbproxy/server/doltserver_internal_test.go @@ -0,0 +1,300 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/procid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeChunks feeds s to a new watcher for port in the given chunk sizes +// (the last size repeats) and returns the watcher and what reached dst. +func writeChunks(t *testing.T, port int, s string, sizes ...int) (*startupWatch, string) { + t.Helper() + var dst bytes.Buffer + w := newStartupWatch(nil, port) + w.dst = &dst + b := []byte(s) + for i := 0; len(b) > 0; i++ { + n := sizes[len(sizes)-1] + if i < len(sizes) { + n = sizes[i] + } + if n > len(b) { + n = len(b) + } + got, err := w.Write(b[:n]) + require.NoError(t, err) + require.Equal(t, n, got) + b = b[n:] + } + return w, dst.String() +} + +func TestStartupWatch(t *testing.T) { + ready := `time="2026-10-04T05:45:58Z" level=info msg="Server ready. Accepting connections."` + "\n" + pad := strings.Repeat("x", 5000) + "\n" + for _, tc := range []struct { + name string + port int + out string + sizes []int + ready bool + portInUse bool + }{ + {name: "ready line in one write", port: 3306, out: ready, sizes: []int{1 << 20}, ready: true}, + {name: "ready line split across writes", port: 3306, out: ready, sizes: []int{40, 30, 1 << 20}, ready: true}, + {name: "ready line one byte per write", port: 3306, out: ready, sizes: []int{1}, ready: true}, + {name: "ready line with CRLF", port: 3306, out: strings.ReplaceAll(ready, "\n", "\r\n"), sizes: []int{7}, ready: true}, + {name: "ready line after more than the tail of output", port: 3306, out: pad + ready, sizes: []int{1000}, ready: true}, + {name: "MCP ready line is not the SQL ready line", port: 3306, out: "Dolt MCP server ready. Accepting connections.\n", sizes: []int{1 << 20}}, + {name: "dolt pre-check", port: 8000, out: "Port 8000 already in use.\n", sizes: []int{5}, portInUse: true}, + {name: "dolt pre-check for another port", port: 80, out: "Port 8000 already in use.\n", sizes: []int{1 << 20}}, + {name: "go-mysql-server pre-check", port: 41234, out: "Port 127.0.0.1:41234 already in use.\n", sizes: []int{3}, portInUse: true}, + {name: "kernel bind failure", port: 41234, out: "listen tcp 127.0.0.1:41234: bind: address already in use\n", sizes: []int{1}, portInUse: true}, + {name: "windows bind failure", port: 41234, out: "listen tcp 127.0.0.1:41234: bind: Only one usage of each socket address (protocol/network address/port) is normally permitted.\r\n", sizes: []int{9}, portInUse: true}, + {name: "another listener's bind failure", port: 41234, out: "listen tcp :9091: bind: address already in use\n", sizes: []int{1 << 20}}, + {name: "unix socket warning", port: 41234, out: "unix socket set up failed: bind address at given unix socket path is already in use\n", sizes: []int{1 << 20}}, + {name: "port-in-use after the tail of output", port: 41234, out: pad + "Port 41234 already in use.\n", sizes: []int{999}, portInUse: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w, got := writeChunks(t, tc.port, tc.out, tc.sizes...) + assert.Equal(t, tc.out, got, "every byte must reach the log unchanged") + assert.Equal(t, tc.ready, w.isReady()) + assert.Equal(t, tc.portInUse, w.sawPortInUse()) + }) + } +} + +func TestStartupWatchNilLog(t *testing.T) { + w := newStartupWatch(nil, 3306) + n, err := w.Write([]byte("Server ready. Accepting connections.\n")) + require.NoError(t, err) + assert.Equal(t, 37, n) + assert.True(t, w.isReady()) +} + +// newFakeDoltServer builds a DoltServer whose dolt is this test binary in +// fake mode (see fakeDolt in testmain_test.go) and whose config names port at +// log_level info. +func newFakeDoltServer(t *testing.T, mode string, port int) (*DoltServer, string) { + t.Helper() + t.Setenv("BEADS_TEST_FAKE_DOLT", mode) + rootDir := t.TempDir() + cfg := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(cfg, []byte(fmt.Sprintf("log_level: info\nlistener:\n host: 127.0.0.1\n port: %d\n", port)), 0o600)) + s, err := NewDoltServer(os.Args[0], rootDir, cfg, filepath.Join(t.TempDir(), "server.log"), 0, "") + require.NoError(t, err) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _ = s.Stop(ctx) + }) + return s, rootDir +} + +func freeTestPort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +// TestDoltServer_Start_GivesUpAfterMaxPorts pins the bound on recovery: a +// dolt that reports its port taken on every launch ends Start after +// maxStartPortAttempts ports with ErrPortInUse naming the last one, and no +// runtime config is left behind. +func TestDoltServer_Start_GivesUpAfterMaxPorts(t *testing.T) { + s, rootDir := newFakeDoltServer(t, "inuse", freeTestPort(t)) + asked := 0 + operatorPort := s.config.Port() + s.SetPortConflictPolicy(func(_ string, inUsePort int) error { + asked++ + assert.Equal(t, operatorPort, inUsePort, "the policy is asked about the operator config's port") + return nil + }) + + err := s.Start(context.Background()) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrPortInUse), "got %v", err) + assert.Contains(t, err.Error(), fmt.Sprintf("gave up after %d ports", maxStartPortAttempts)) + assert.Equal(t, 1, asked, "the policy judges the operator config's port, so it is asked once") + _, serr := os.Stat(filepath.Join(rootDir, RuntimeConfigFileName)) + assert.True(t, os.IsNotExist(serr), "a failed Start must not leave a runtime config") +} + +// TestDoltServer_Start_MissingReadyLineIsDiagnosed pins SF4: a dolt that +// answers on its port but never logs the ready line is not accepted, and the +// timeout error names the missing line and the log_level escape hatch. +func TestDoltServer_Start_MissingReadyLineIsDiagnosed(t *testing.T) { + old := startReadyTimeout + // The fake is this test binary, which takes a few seconds to start. + startReadyTimeout = 30 * time.Second + t.Cleanup(func() { startReadyTimeout = old }) + + s, _ := newFakeDoltServer(t, "silent", freeTestPort(t)) + err := s.Start(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "never logged \"Server ready. Accepting connections.\"") + assert.Contains(t, err.Error(), "although something answered") + assert.Contains(t, err.Error(), "log_level to warning") +} + +// TestDoltServer_Start_FakeReadyLine is the positive control for the fake. +func TestDoltServer_Start_FakeReadyLine(t *testing.T) { + s, _ := newFakeDoltServer(t, "ready", freeTestPort(t)) + require.NoError(t, s.Start(context.Background())) + assert.True(t, s.Running(context.Background())) +} + +// TestUseRuntimePort_CopiesRawText pins that the runtime config carries the +// operator config's environment placeholders and "$$" escapes unexpanded: +// dolt interpolates the runtime file when it reads it, so expanding them here +// would expand them twice and put their values on disk. The server's own +// view (Dial, DSN) still uses the interpolated values. +func TestUseRuntimePort_CopiesRawText(t *testing.T) { + t.Setenv("BD_TEST_HOST", "127.0.0.1") + rootDir := t.TempDir() + cfg := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(cfg, []byte("log_level: info\ndata_dir: \"a$$b\"\nlistener:\n host: ${BD_TEST_HOST}\n port: 40001\n"), 0o600)) + s, err := NewDoltServer(os.Args[0], rootDir, cfg, "", 0, "") + require.NoError(t, err) + require.Equal(t, "127.0.0.1", s.config.Host()) + + require.NoError(t, s.useRuntimePort(40001)) + + body, err := os.ReadFile(filepath.Join(rootDir, RuntimeConfigFileName)) + require.NoError(t, err) + assert.Contains(t, string(body), "${BD_TEST_HOST}", "placeholder must be copied, not expanded") + assert.Contains(t, string(body), "a$$b", "a $$ escape must be copied, not unescaped") + assert.Equal(t, "127.0.0.1", s.config.Host(), "the server dials the interpolated host") + assert.NotEqual(t, 40001, s.config.Port()) + assert.Equal(t, filepath.Join(rootDir, RuntimeConfigFileName), s.launchConfigPath) + info, err := os.Stat(filepath.Join(rootDir, RuntimeConfigFileName)) + require.NoError(t, err) + if runtime.GOOS != "windows" { + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + } +} + +// newScriptDoltServer builds a DoltServer whose dolt is a shell script that +// answers `config`/`init` and runs sqlServer (shell) for `sql-server`, with +// $port set to the --config file's listener.port. Shell scripts start in +// milliseconds, unlike the fake-mode test binary, which is what the +// fast-exit cases need. +func newScriptDoltServer(t *testing.T, sqlServer string) *DoltServer { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("stand-in dolt is a shell script") + } + dir := t.TempDir() + bin := filepath.Join(dir, "dolt") + script := "#!/bin/sh\ncase \"$1\" in\nconfig) echo fake; exit 0 ;;\ninit) exit 0 ;;\nsql-server)\n" + + " port=$(sed -n 's/^ *port: *\\([0-9][0-9]*\\).*/\\1/p' \"$3\" | head -n 1)\n" + + sqlServer + "\n ;;\nesac\nexit 2\n" + require.NoError(t, os.WriteFile(bin, []byte(script), 0o755)) + cfg := filepath.Join(dir, "config.yaml") + require.NoError(t, os.WriteFile(cfg, []byte(fmt.Sprintf("log_level: info\nlistener:\n host: 127.0.0.1\n port: %d\n", freeTestPort(t))), 0o600)) + root := filepath.Join(dir, "root") + require.NoError(t, os.MkdirAll(root, 0o755)) + s, err := NewDoltServer(bin, root, cfg, filepath.Join(dir, "server.log"), 0, "") + require.NoError(t, err) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _ = s.Stop(ctx) + }) + return s +} + +// captureAfterChildGone makes the birth-identity capture lose the race the +// way a real fast-exiting dolt can: it waits long enough for the stand-in +// to have exited, then fails as procid does for a dead pid. +func captureAfterChildGone(t *testing.T) { + t.Helper() + old := captureBirth + captureBirth = func(pid int) (procid.Token, error) { + time.Sleep(300 * time.Millisecond) + return "", fmt.Errorf("procid: process is no longer running: no such process") + } + t.Cleanup(func() { captureBirth = old }) +} + +// TestDoltServer_Start_FastPortInUseExitBeatsIdentityCapture pins that a +// child which reports its port taken and exits before Start captured its +// identity is still classified as ErrPortInUse (and so still recoverable), +// not reported as a fatal identity-capture failure. +func TestDoltServer_Start_FastPortInUseExitBeatsIdentityCapture(t *testing.T) { + captureAfterChildGone(t) + s := newScriptDoltServer(t, ` echo "Port $port already in use." >&2; exit 1`) + err := s.Start(context.Background()) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrPortInUse), "got %v", err) + assert.NotContains(t, err.Error(), "capture child birth identity") + + // With a policy, every move hits the same fast exit and Start keeps + // recovering until its port budget runs out. + s2 := newScriptDoltServer(t, ` echo "Port $port already in use." >&2; exit 1`) + s2.SetPortConflictPolicy(func(string, int) error { return nil }) + err = s2.Start(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), fmt.Sprintf("gave up after %d ports", maxStartPortAttempts)) +} + +// TestDoltServer_Start_FastOtherExitBeatsIdentityCapture: a child that exits +// on its own for another reason is reported as having exited, with its +// status, not as an identity-capture failure. +func TestDoltServer_Start_FastOtherExitBeatsIdentityCapture(t *testing.T) { + captureAfterChildGone(t) + s := newScriptDoltServer(t, ` echo "some other startup failure" >&2; exit 3`) + err := s.Start(context.Background()) + require.Error(t, err) + assert.False(t, errors.Is(err, ErrPortInUse), "got %v", err) + assert.Contains(t, err.Error(), "exited (status 3) before startup completed") +} + +// TestDoltServer_Start_IdentityCaptureFailsWhileChildAlive: when the child is +// alive and the capture fails anyway, that failure is fatal, and the child is +// killed rather than left running. +func TestDoltServer_Start_IdentityCaptureFailsWhileChildAlive(t *testing.T) { + old := captureBirth + captureBirth = func(int) (procid.Token, error) { return "", errors.New("capture broke") } + t.Cleanup(func() { captureBirth = old }) + + s := newScriptDoltServer(t, ` exec sleep 30`) + done := make(chan error, 1) + go func() { done <- s.Start(context.Background()) }() + select { + case err := <-done: + require.Error(t, err) + assert.False(t, errors.Is(err, ErrPortInUse), "got %v", err) + assert.Contains(t, err.Error(), "capture child birth identity: capture broke") + case <-time.After(20 * time.Second): + t.Fatal("Start did not return: the live child was not killed") + } + assert.False(t, s.Running(context.Background())) +} + +// TestDoltServer_Start_ImmediateExitRealCapture is the same fast port-in-use +// exit with the real identity capture: whichever of the capture and the +// child's exit wins, the result is ErrPortInUse. +func TestDoltServer_Start_ImmediateExitRealCapture(t *testing.T) { + for i := 0; i < 10; i++ { + s := newScriptDoltServer(t, ` echo "Port $port already in use." >&2; exit 1`) + err := s.Start(context.Background()) + require.Error(t, err) + require.True(t, errors.Is(err, ErrPortInUse), "attempt %d: got %v", i, err) + } +} diff --git a/internal/storage/dbproxy/server/doltserver_test.go b/internal/storage/dbproxy/server/doltserver_test.go index 399d21a14..1b222c3fe 100644 --- a/internal/storage/dbproxy/server/doltserver_test.go +++ b/internal/storage/dbproxy/server/doltserver_test.go @@ -641,3 +641,166 @@ func TestDoltServer_DoltInit_Idempotent(t *testing.T) { require.NoError(t, s.Start(ctx), "Start against pre-initialized rootDir should succeed") t.Cleanup(func() { stopWithTimeout(t, s) }) } + +// holdPort occupies a loopback port with a listener that accepts and then +// ignores connections: a stand-in for another process that took the port the +// config names before dolt could bind it. It returns the port. +func holdPort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + var mu sync.Mutex + var conns []net.Conn + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + } + }() + t.Cleanup(func() { + _ = ln.Close() + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + _ = c.Close() + } + }) + return ln.Addr().(*net.TCPAddr).Port +} + +// newDoltServerOnPort is newDoltServer with the listener port chosen by the +// caller. It returns the server, its root dir and its config path. +func newDoltServerOnPort(t *testing.T, port int) (*server.DoltServer, string, string) { + t.Helper() + bin := requireDolt(t) + t.Setenv("HOME", t.TempDir()) + rootDir := t.TempDir() + cfg := writeConfig(t, port) + log := filepath.Join(t.TempDir(), "server.log") + s, err := server.NewDoltServer(bin, rootDir, cfg, log, 0, "") + require.NoError(t, err) + verifiedStopCleanup(t, s, rootDir) + return s, rootDir, cfg +} + +// TestDoltServer_Start_ForeignListenerIsNotReady pins that a port another +// process holds is not mistaken for the started server. A dial to the port +// succeeds, but the dolt sql-server Start launched cannot bind it and exits; +// Start used to accept the dial and report a running server that was already +// gone. +func TestDoltServer_Start_ForeignListenerIsNotReady(t *testing.T) { + held := holdPort(t) + s, rootDir, _ := newDoltServerOnPort(t, held) + ctx := context.Background() + + err := s.Start(ctx) + require.Error(t, err, "Start must not report ready against another process's listener") + assert.ErrorIs(t, err, server.ErrPortInUse) + assert.False(t, s.Running(ctx)) + left, rerr := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, rerr) + assert.Nil(t, left, "a failed Start must not leave a backend pid record") +} + +// TestDoltServer_Start_NoPolicyNamesRemedy verifies the ErrPortInUse a +// server without a PortConflictPolicy returns tells the operator what to do. +func TestDoltServer_Start_NoPolicyNamesRemedy(t *testing.T) { + held := holdPort(t) + s, _, cfgPath := newDoltServerOnPort(t, held) + err := s.Start(context.Background()) + require.ErrorIs(t, err, server.ErrPortInUse) + assert.Contains(t, err.Error(), fmt.Sprintf("free port %d, or set listener.port in %s", held, cfgPath)) +} + +// TestDoltServer_Start_MovesOffPortHeldByAnotherProcess verifies Start +// recovers when its PortConflictPolicy allows it: the server comes up on a +// fresh port through the runtime config, the operator config is left +// byte-for-byte alone, and the runtime config goes away with the server. +func TestDoltServer_Start_MovesOffPortHeldByAnotherProcess(t *testing.T) { + held := holdPort(t) + s, rootDir, cfgPath := newDoltServerOnPort(t, held) + ctx := context.Background() + before, err := os.ReadFile(cfgPath) + require.NoError(t, err) + + var calls []int + s.SetPortConflictPolicy(func(configPath string, inUsePort int) error { + calls = append(calls, inUsePort) + assert.Equal(t, cfgPath, configPath) + return nil + }) + + require.NoError(t, s.Start(ctx)) + t.Cleanup(func() { stopWithTimeout(t, s) }) + assert.Equal(t, []int{held}, calls, "policy must be asked once, about the held port") + + after, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.Equal(t, string(before), string(after), "the operator config must never be modified") + + pf, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + require.NotNil(t, pf) + assert.NotEqual(t, held, pf.Port, "backend record must advertise the port the server actually uses") + runtimePath := filepath.Join(rootDir, server.RuntimeConfigFileName) + runtimeBody, err := os.ReadFile(runtimePath) + require.NoError(t, err, "a moved server runs from the runtime config") + assert.Contains(t, string(runtimeBody), fmt.Sprintf("port: %d", pf.Port)) + + qctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + db, err := sql.Open("mysql", s.DSN(ctx, "", "root", "")) + require.NoError(t, err) + var got int + require.NoError(t, db.QueryRowContext(qctx, "SELECT 1").Scan(&got), "DSN must reach the server on its new port") + assert.Equal(t, 1, got) + require.NoError(t, db.Close()) + + require.NoError(t, s.Stop(ctx)) + _, err = os.Stat(runtimePath) + assert.True(t, os.IsNotExist(err), "Stop must remove the runtime config, got %v", err) +} + +// TestDoltServer_Start_PolicyDeclines verifies a policy that pins the port +// ends Start with ErrPortInUse carrying the policy's reason. +func TestDoltServer_Start_PolicyDeclines(t *testing.T) { + held := holdPort(t) + s, rootDir, _ := newDoltServerOnPort(t, held) + ctx := context.Background() + s.SetPortConflictPolicy(func(string, int) error { + return errors.New("port is pinned") + }) + + err := s.Start(ctx) + require.Error(t, err) + assert.ErrorIs(t, err, server.ErrPortInUse) + assert.Contains(t, err.Error(), "port is pinned") + assert.False(t, s.Running(ctx)) + _, serr := os.Stat(filepath.Join(rootDir, server.RuntimeConfigFileName)) + assert.True(t, os.IsNotExist(serr), "a declined move must not leave a runtime config") +} + +// TestDoltServer_Start_QuietLogLevelUsesDialReadiness pins the fallback: at +// log_level warning dolt never logs its (info-level) ready line, so Start +// must accept a dial alone rather than wait for a line that cannot come. +func TestDoltServer_Start_QuietLogLevelUsesDialReadiness(t *testing.T) { + bin := requireDolt(t) + t.Setenv("HOME", t.TempDir()) + rootDir := t.TempDir() + cfg := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(cfg, []byte(fmt.Sprintf("log_level: warning\nlistener:\n host: 127.0.0.1\n port: %d\n", freePort(t))), 0o600)) + s, err := server.NewDoltServer(bin, rootDir, cfg, filepath.Join(t.TempDir(), "server.log"), 0, "") + require.NoError(t, err) + verifiedStopCleanup(t, s, rootDir) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + require.NoError(t, s.Start(ctx)) + t.Cleanup(func() { stopWithTimeout(t, s) }) + assert.True(t, s.Running(ctx)) +} diff --git a/internal/storage/dbproxy/server/testmain_test.go b/internal/storage/dbproxy/server/testmain_test.go index 82c2c844f..835bfddac 100644 --- a/internal/storage/dbproxy/server/testmain_test.go +++ b/internal/storage/dbproxy/server/testmain_test.go @@ -2,7 +2,9 @@ package server_test import ( "fmt" + "net" "os" + "regexp" "testing" "github.com/steveyegge/beads/internal/doltserver" @@ -37,9 +39,78 @@ var suiteTempRoot string // t.TempDir's RemoveAll. A root-scoped post-run sweep detects that leak // and fails this package without consuming another live suite's evidence. func TestMain(m *testing.M) { + if mode := os.Getenv(fakeDoltEnv); mode != "" { + os.Exit(fakeDolt(mode, os.Args[1:])) + } os.Exit(testMainInner(m)) } +// fakeDoltEnv, when set, makes this test binary act as a stand-in `dolt` +// (see fakeDolt) so tests can drive DoltServer through startup behaviors a +// real dolt cannot be made to show on demand. The internal tests set it and +// pass os.Args[0] as the dolt binary. +const fakeDoltEnv = "BEADS_TEST_FAKE_DOLT" + +var fakeDoltPortRe = regexp.MustCompile(`(?m)^\s+port:\s*(\d+)`) + +// fakeDolt answers the dolt invocations DoltServer.Start makes. For +// `sql-server --config ` it reads listener.port from the file and, by +// mode: +// - "inuse": reports dolt's own port-in-use error and exits 1; +// - "silent": listens but never logs the ready line; +// - "ready": listens and logs the ready line. +func fakeDolt(mode string, args []string) int { + if len(args) == 0 { + return 2 + } + switch args[0] { + case "config": + fmt.Println("fake") + return 0 + case "init": + return 0 + case "sql-server": + default: + return 2 + } + var cfgPath string + for i := 0; i+1 < len(args); i++ { + if args[i] == "--config" { + cfgPath = args[i+1] + } + } + body, err := os.ReadFile(cfgPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + m := fakeDoltPortRe.FindSubmatch(body) + if m == nil { + fmt.Fprintln(os.Stderr, "no listener.port in", cfgPath) + return 1 + } + port := string(m[1]) + if mode == "inuse" { + fmt.Fprintf(os.Stderr, "Port %s already in use.\n", port) + return 1 + } + ln, err := net.Listen("tcp", "127.0.0.1:"+port) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + if mode == "ready" { + fmt.Fprintln(os.Stderr, `level=info msg="Server ready. Accepting connections."`) + } + for { + c, err := ln.Accept() + if err != nil { + return 0 + } + defer c.Close() + } +} + func testMainInner(m *testing.M) int { // Clear out the roots of earlier runs of this suite whose process is // gone, before claiming one of our own. Roots with no owner marker, and