From 4c63e15107406bdbf3aa6658fb46e52207643d7c Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Mon, 27 Jul 2026 22:29:11 -0700 Subject: [PATCH] fix(dbproxy): race-free proxy port allocation and managed loopback listener policy (#5024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dbproxy): add procid birth-token package and pidfile schema v2 fields Foundation for verified proxy/backend adoption and safe stale-process cleanup (mybd-psxg.5, PR-A stage 1): internal/procid provides versioned process-birth tokens (Linux boot_id+starttime, Windows creation FILETIME, Darwin p_starttime) with TOCTOU-safe signaling handles (pidfd on Linux, re-verified process handle on Windows); pidfile gains additive schema-v2 fields (schema, kind, birth, root_id, control_port) and sentinel-error validation. No behavior changes; existing v1 pidfiles still parse. The procid Token/Verify shape intentionally matches the API requested in gastownhall/beads#4513 so internal/doltserver can adopt it later. Agent-Signature: codex-gpt-5.6-terra-medium on behalf of matt wilkie Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WZHiYc3wjcdy1wrUP2JiQi * feat(dbproxy): publish verifiable proxy identity (secret, control handshake, v2 pidfiles) Stage 2 of mybd-psxg.5 (PR-A): the managed proxy now proves who it is. New leaf package internal/storage/dbproxy/identity provides the workspace RootID (sha256 of the resolved root path), a rotated 0600 proxy.secret, and the IDENT control protocol (client Identify + IdentReply). The proxy opens a loopback control listener that answers only to the workspace secret, and both pidfile writers publish schema-v2 records (kind, birth token, root_id, control_port) captured via procid. A proxy or backend that cannot prove its own identity fails startup instead of publishing. Adoption, cleanup, and kill behavior are unchanged in this stage; the consumers land next so each step stays reviewable. Agent-Signature: codex-gpt-5.6-terra-high on behalf of matt wilkie Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WZHiYc3wjcdy1wrUP2JiQi * fix(dbproxy): verified adoption, fail-closed cleanup, and handle-safe kills Stage 3 of mybd-psxg.5 (PR-A), the security core. Endpoint discovery now classifies records with typed statuses instead of adopting any listener that answers a TCP dial: adoption requires a schema-v2 pidfile, a live birth-token match, the authenticated IDENT handshake, and root/pid/birth/ port cross-checks. Legacy and malformed records fail closed (quarantined under proxy.lock, never adopted, never killed); quarantines are timestamped renames with a 30-day sweep, so no forensic record is ever unlinked. Every kill site (spawn-path backend cleanup and bd dolt stop's Shutdown) signals only through a procid.Handle opened against the recorded birth token, refusing unverifiable live processes with an actionable error. Orphaned Dolt backends left by a SIGKILLed proxy supervisor are now detected regardless of child-lock state and reaped only after verification. A proxy.spawn marker plus a persistent stop epoch closes the window where bd dolt stop could report success while a fork already in flight published a listener afterwards. Agent-Signature: codex-gpt-5.6-sol-xhigh on behalf of matt wilkie Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WZHiYc3wjcdy1wrUP2JiQi * fix(dbproxy): harden identity fixes from cross-vendor adversarial review Fixes every accepted finding from paired independent reviews of the first three stages (reports in the coordination repo, 2026-07-24): - darwin: classify wrapped ESRCH and the zero-length/EIO missing-pid sysctl result as process-gone; stale pidfiles self-heal again. - fallback signaling (darwin, non-pidfd linux): a target that exits after a fatal signal is a successful kill, not an error. - Shutdown verifies root_id on the proxy record too (copied-workspace kill hazard); positional bool pair replaced by killRecordChecks. - birth-token mismatch now classifies as provably-dead everywhere; dead legacy/malformed records quarantine and self-heal; refusal messages name the record file and manual remedy. - discovery-time verify/secret/root errors map to a non-adopting status routed through quarantine-under-lock instead of wedging before the lock probe. - boot_id read failures are environmental errors, never process-gone (closes the one fail-open path found in review). - ForceStopUnverified: executable-identity-checked operator escape for pre-upgrade/unverifiable records, quarantines after, reports truthfully; CLI wiring intentionally deferred. - handshake replies are HMAC'd against a client nonce; constant-time secret compare; bounded accept failures and handshake concurrency; migration purge covers proxy.secret and quarantine files; spawn marker cleared via defer; unix-only constraints on sleep tests. Agent-Signature: codex-gpt-5.6-sol-xhigh on behalf of matt wilkie Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WZHiYc3wjcdy1wrUP2JiQi * feat(dbproxy): wire bd dolt stop --force and prove the lifecycle lane end to end Stage 4 of mybd-psxg.5 (PR-A). bd dolt stop gains --force, routed through proxy.ForceStopUnverified only after the backend is confirmed stopped: it verifies executable identity before signaling, quarantines the record afterward, refuses foreign executables and verifiable v2 records, and reports text/JSON truthfully. The managed-local lane (BEADS_TEST_PROXIED_LOCAL=1) now exercises the whole security model against a real dolt sql-server: foreign-listener rejection, reused-PID no-kill, legacy fail-closed (free lock, held lock refusing a foreign executable, held lock force-stopping a genuine bd), orphan-backend reaping after a SIGKILLed supervisor, 5-process cold-start convergence, and stop/start interleaving. Windows/macOS lane equivalents are defined in the lane header, implementation staged later. Agent-Signature: codex-gpt-5.6-sol-high on behalf of matt wilkie Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WZHiYc3wjcdy1wrUP2JiQi * fix(dbproxy): close cross-vendor review findings — handle lifetimes, pidfd races, force-stop scoping - B1: cleanupOrphanBackend closes the verified handle before waiting for exit, mirroring shutdown.go; an open Windows handle kept the dead PID allocated and made every orphan-backend recovery time out. - B2: Windows procid checks GetExitCodeProcess after OpenProcess, so a terminated process kept resolvable by a third-party handle no longer verifies; "Verify == true implies running" now holds on all platforms. - B3: Linux procid.Open opens the pidfd BEFORE token verification; the pidfd pins the PID number, closing the reuse window where a later Signal could kill an unrelated replacement process. - B4: PidfdSendSignal ESRCH on a fatal signal is treated as successful termination, matching the fallback path (same for Windows terminate). - B5: ForceStopUnverified now covers the backend (proxy-child) record with the same safety checks, and CanForceStopUnverified accepts refusals where the backend is also (or only) unverifiable — the paired-legacy pre-v2 topology was locked out of its own advertised recovery. bd dolt stop --force reports the backend record too. - B6a: an unverified live PID is only signaled when its command line ties it to this workspace (Linux /proc cmdline, Darwin ps args); Windows cannot read another process's command line and now refuses, which beats basename-only killing. Clear errors name the manual remedy. - B6b: one stable OS handle (Linux pidfd, Windows process handle) spans inspection and signaling, removing the inspect/kill TOCTOU. - S1: control-listener accept errors are non-fatal — log and retry with capped exponential backoff instead of tearing down a healthy proxy. - S2: requireUnverifiableRecord surfaces RootID resolution errors instead of failing open into the destructive branch. - S3: killSpawnedChild falls back to cmd.Process.Kill for our own un-reaped child when no verified handle exists, instead of leaking it. - S4: adoption handshake accepts reply schema v2 or newer, matching pidfile.ValidateV2 forward-compat policy. - S5: errors.Is for ERROR_INVALID_PARAMETER. - Tests: pidfd ESRCH-after-exit and Open-after-reap classification; paired-legacy force recovery end to end; foreign-workspace bd process refused; Windows Verify-after-terminate (compile-checked); schema-v3 handshake adoption; accept-loop survival under persistent errors. Agent-Signature: claude-fable-5-medium on behalf of maphew * fix(dbproxy): race-free proxy port allocation and managed loopback listener policy Companion to the identity PR (stacks on it). The parent no longer pre-picks the proxy data port: port 0 flows to the child, which binds 127.0.0.1:0 and publishes the OS-assigned port through its verified pidfile, eliminating the bind-close-launch race; explicitly configured ports keep fixed-port failure semantics. PickFreePort survives only for the dolt-config path, annotated for the follow-up RFC on backend port collisions. Managed dolt configs are now validated by a single numeric-loopback listener policy at init and at runtime resolution, covering all config paths (generated, BEADS_PROXIED_SERVER_CONFIG, sidecar, pre-marker, hand-edited marker, interpolation fallback). Hostnames including localhost are rejected with remediation text naming the --proxied-server-external-* flags; an omitted host is accepted because dolt servercfg defaults it. There is deliberately no bypass env: managed mode assumes loopback for its root/empty-password account. External proxied topology is exempt by construction. Agent-Signature: codex-gpt-5.6-sol-high on behalf of matt wilkie Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WZHiYc3wjcdy1wrUP2JiQi * fix(dbproxy): whole-config managed listener policy, epoch-fenced startup, diagnosis fixes Review findings from two independent passes over the listener-policy commit (gastownhall/beads#5024), rebased onto the rewritten #5013 head: - F1: the managed policy now covers the whole sql-server YAML, not just listener.host. validateManagedListenerHost is renamed validateManagedServerConfigPolicy and additionally rejects listener.socket (incl. the empty-string form, which maps to Dolt's default socket path), remotesapi.port, and any cluster block — all of which open non-loopback listeners for the root/empty-password child. - F2: managed mode requires an explicit numeric loopback listener.host. An omitted host is no longer trusted: servercfg resolves it to "localhost", which CheckForUnixSocket also treats as license to open the shared-namespace default unix socket. This removes the dead net.ParseIP("localhost") branch and the omitted/explicit asymmetry. - F3: one interpolation-aware policy check instead of five scattered raw-bytes ones. ensureProxiedServerConfig now delegates resolution to resolveOrCreateProxiedServerConfig and runs a single validateProxiedServerConfig (YamlConfigFromFile, ${VAR}-expanding) on the resolved path, so a marker config with an interpolated host is judged on what it expands to. - F4: the proxy child re-checks the stop epoch immediately before publishing proxy.pid and aborts if it advanced (the spawn marker is cleared at lock acquisition, so a `bd dolt stop` racing a slow backend start otherwise has no record to wait on). The parent's observed epoch is threaded through a new db-proxy-child --stop-epoch flag into ProxyOpts.StopEpoch. - F5a/b: child-exit diagnosis distinguishes a LockHeldExitCode spawn race from real failures, points at the child's log path otherwise, and timeout messages say "its OS-assigned port" instead of "port 0". - F5c/d: validateManagedProxiedServerConfigAtInit now honors --proxied-server-root-path (the sidecar is not persisted yet at validation time) and drops the redundant explicit-config re-validation already done at flag parse. - F5e: --proxied-server-config-path flag help states the hard managed constraint (docs/cli-reference regenerates from the release pin, so only the source help string changes here). - F5f: the listener-policy lifecycle test documents its accepted PickFreePort EADDRINUSE flake window. Tests: whole-config policy table (socket/remotesapi/cluster/omitted/ localhost/loopback forms), marker ${VAR} host interpolation accept and reject, root-path-flag validation at init (unit + gated integration), and a deterministic stop-epoch-advance-during-startup abort test. Agent-Signature: claude-fable-5-medium on behalf of maphew --------- Co-authored-by: Claude Fable 5 --- cmd/bd/db_proxy_child.go | 3 + cmd/bd/init.go | 8 +- cmd/bd/proxied_local_helpers_test.go | 3 + cmd/bd/proxied_local_lifecycle_linux_test.go | 3 + ...roxied_local_listener_policy_linux_test.go | 158 +++++++++ cmd/bd/proxied_server.go | 121 ++++++- cmd/bd/proxied_server_test.go | 304 ++++++++++++++++++ internal/storage/dbproxy/proxy/endpoint.go | 67 ++-- internal/storage/dbproxy/proxy/server.go | 26 ++ .../proxy/server_epoch_internal_test.go | 67 ++++ internal/storage/dbproxy/proxy/server_test.go | 34 +- 11 files changed, 772 insertions(+), 22 deletions(-) create mode 100644 cmd/bd/proxied_local_listener_policy_linux_test.go create mode 100644 internal/storage/dbproxy/proxy/server_epoch_internal_test.go diff --git a/cmd/bd/db_proxy_child.go b/cmd/bd/db_proxy_child.go index ef4c3afa5..474981970 100644 --- a/cmd/bd/db_proxy_child.go +++ b/cmd/bd/db_proxy_child.go @@ -27,6 +27,7 @@ var ( dbProxyChildExternalPort int dbProxyChildExternalSocketPath string dbProxyChildExternalKeepAlive time.Duration + dbProxyChildStopEpoch string ) var dbProxyChildCmd = &cobra.Command{ @@ -64,6 +65,7 @@ not intended to be invoked directly by users.`, Port: dbProxyChildPort, IdleTimeout: dbProxyChildIdleTimeout, Server: srv, + StopEpoch: dbProxyChildStopEpoch, }) if err := p.ListenAndServe(cmd.Context()); err != nil { if errors.Is(err, proxy.ErrLockHeld) { @@ -101,6 +103,7 @@ func init() { dbProxyChildCmd.Flags().IntVar(&dbProxyChildExternalPort, "external-port", 0, "external backend: TCP port of the dolt sql-server") dbProxyChildCmd.Flags().StringVar(&dbProxyChildExternalSocketPath, "external-socket-path", "", "external backend: absolute path to a unix domain socket (overrides host/port)") dbProxyChildCmd.Flags().DurationVar(&dbProxyChildExternalKeepAlive, "external-keep-alive", 0, "external backend: TCP keepalive period (default 30s)") + dbProxyChildCmd.Flags().StringVar(&dbProxyChildStopEpoch, "stop-epoch", "", "stop epoch the parent observed at spawn; the proxy aborts before publishing if it advances") _ = dbProxyChildCmd.MarkFlagRequired("root") _ = dbProxyChildCmd.MarkFlagRequired("port") _ = dbProxyChildCmd.MarkFlagRequired("backend") diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 286fd136d..1433a9e72 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -800,6 +800,12 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // to ensure consistent path representation. beadsDir := beadsDirForInit + if initProxiedServer && externalConfig == nil { + if err := validateManagedProxiedServerConfigAtInit(beadsDir, serverConfigPath, serverRootPath); err != nil { + return fmt.Errorf("managed proxied-server config: %w", err) + } + } + // Prevent nested .beads directories // Check if current working directory is inside a .beads directory if strings.Contains(filepath.Clean(cwd), string(filepath.Separator)+".beads"+string(filepath.Separator)) || @@ -2048,7 +2054,7 @@ func init() { initCmd.Flags().Bool("external", false, "Server is externally managed (skip server startup); use with --shared-server or --server") initCmd.Flags().Bool("debug", false, "Run the managed Dolt sql-server with --loglevel=debug and CPU profiling (--prof cpu). Persisted to config.yaml as dolt.debug. No effect on externally-managed servers.") initCmd.Flags().Bool("proxied-server", false, "[EXPERIMENTAL] Use a per-workspace proxied dolt sql-server (proxy + child dolt) rooted at .beads/dolt") - initCmd.Flags().String("proxied-server-config-path", "", "[EXPERIMENTAL] Absolute path to an existing dolt sql-server YAML config (proxied-server mode only). When set, bd uses this file instead of auto-generating one. Relative paths are rejected.") + initCmd.Flags().String("proxied-server-config-path", "", "[EXPERIMENTAL] Absolute path to an existing dolt sql-server YAML config (proxied-server mode only). When set, bd uses this file instead of auto-generating one. Relative paths are rejected. Managed mode requires listener.host to be a numeric loopback IP (hostnames including localhost, non-loopback addresses, listener.socket, remotesapi, and cluster config are rejected); the same policy applies to BEADS_PROXIED_SERVER_CONFIG.") initCmd.Flags().String("proxied-server-log-path", "", "[EXPERIMENTAL] Absolute path to the proxied dolt sql-server log file (proxied-server mode only). Default: /dolt/server.log. Relative paths are rejected.") initCmd.Flags().String("proxied-server-root-path", "", "[EXPERIMENTAL] Absolute directory holding the proxied dolt sql-server's lockfiles, pidfiles, and child .dolt repository (proxied-server mode only). Default: /dolt. May not exist yet — bd will create it. Relative paths are rejected.") initCmd.Flags().Int("proxied-server-port", 0, "[EXPERIMENTAL] Fixed TCP port for the proxy's loopback listener (proxied-server mode only). Default 0 = an OS-assigned free port. Startup fails if the port is already in use.") diff --git a/cmd/bd/proxied_local_helpers_test.go b/cmd/bd/proxied_local_helpers_test.go index f6c5803a1..2b3a0cb9a 100644 --- a/cmd/bd/proxied_local_helpers_test.go +++ b/cmd/bd/proxied_local_helpers_test.go @@ -72,6 +72,9 @@ func bdManagedLocalInit(t *testing.T, bd, prefix string, idleTimeout time.Durati if info.IdleTimeout != idleTimeout { t.Fatalf("persisted managed-local idle timeout: got %s, want %s", info.IdleTimeout, idleTimeout) } + if info.Port != 0 { + t.Fatalf("default managed proxy port: got %d, want 0 so the child binds an OS-assigned port", info.Port) + } return p } diff --git a/cmd/bd/proxied_local_lifecycle_linux_test.go b/cmd/bd/proxied_local_lifecycle_linux_test.go index f94e4b5ab..1a3f02b2a 100644 --- a/cmd/bd/proxied_local_lifecycle_linux_test.go +++ b/cmd/bd/proxied_local_lifecycle_linux_test.go @@ -83,6 +83,9 @@ func TestManagedLocalProxiedLifecycleSmoke(t *testing.T) { if proxyPF == nil { t.Fatal("proxy.pid missing while a proxied connection is held open") } + if proxyPF.Port == 0 { + t.Fatal("proxy.pid published port 0 instead of the child's OS-assigned listener port") + } backendPF := readManagedBackendPidFile(t, p) if backendPF == nil { t.Fatal("proxy-child.pid missing while a proxied connection is held open") diff --git a/cmd/bd/proxied_local_listener_policy_linux_test.go b/cmd/bd/proxied_local_listener_policy_linux_test.go new file mode 100644 index 000000000..9c890d218 --- /dev/null +++ b/cmd/bd/proxied_local_listener_policy_linux_test.go @@ -0,0 +1,158 @@ +//go:build cgo && linux + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +func TestManagedLocalProxiedListenerPolicy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + + t.Run("non-loopback custom config refused at init", func(t *testing.T) { + dir := t.TempDir() + initGitRepoAt(t, dir) + configPath := writeListenerConfig( + t, + filepath.Join(t.TempDir(), "non-loopback.yaml"), + "0.0.0.0", + false, + false, + ) + + stdout, stderr, err := bdProxiedRunBuffersWithEnv( + t, + bd, + dir, + []string{"BEADS_PROXIED_SERVER_CONFIG=" + configPath}, + "init", + "--proxied-server", + "--quiet", + "--prefix", "mlpolicybad", + "--non-interactive", + "--skip-agents", + "--skip-hooks", + ) + if err == nil { + t.Fatalf("managed init unexpectedly accepted 0.0.0.0:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + assertManagedListenerPolicyOutput(t, stdout+stderr) + }) + + t.Run("non-loopback config under flagged root path refused at init", func(t *testing.T) { + dir := t.TempDir() + initGitRepoAt(t, dir) + rootDir := t.TempDir() + writeListenerConfig( + t, + filepath.Join(rootDir, proxiedServerConfigName), + "0.0.0.0", + false, + false, + ) + + stdout, stderr, err := bdProxiedRunBuffersWithEnv( + t, + bd, + dir, + nil, + "init", + "--proxied-server", + "--proxied-server-root-path", rootDir, + "--quiet", + "--prefix", "mlpolicyroot", + "--non-interactive", + "--skip-agents", + "--skip-hooks", + ) + if err == nil { + t.Fatalf("managed init unexpectedly accepted 0.0.0.0 under --proxied-server-root-path:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + assertManagedListenerPolicyOutput(t, stdout+stderr) + }) + + t.Run("loopback custom config completes lifecycle then hand edit is refused", func(t *testing.T) { + // PickFreePort here recreates the bind-close-launch race this PR + // removes from the proxy itself: the port is released before the + // managed dolt sql-server binds it. That is inherent to handing a + // pre-written custom config a concrete backend port, so the small + // EADDRINUSE flake window is accepted and documented rather than + // retried around. + backendPort, err := proxy.PickFreePort() + if err != nil { + t.Fatalf("pick custom backend port: %v", err) + } + configPath := writeListenerConfigWithPort( + t, + filepath.Join(t.TempDir(), "loopback.yaml"), + "127.0.0.1", + backendPort, + ) + + p := bdManagedLocalInit( + t, + bd, + "mlpolicyok", + 5*time.Minute, + "--proxied-server-config-path", configPath, + ) + held := openHeldManagedProxiedConn(t, bd, p) + issue := bdProxiedCreate(t, bd, p.dir, "loopback custom policy issue") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + var title string + err = held.Conn.QueryRowContext(ctx, "SELECT title FROM issues WHERE id = ?", issue.ID).Scan(&title) + cancel() + if err != nil { + t.Fatalf("SQL round-trip through loopback custom config: %v", err) + } + if title != issue.Title { + t.Fatalf("SQL round-trip title: got %q, want %q", title, issue.Title) + } + held.Release() + + if err := proxy.Shutdown(p.proxyRoot); err != nil { + t.Fatalf("stop managed proxy before runtime policy check: %v", err) + } + writeListenerConfigWithPort(t, configPath, "0.0.0.0", backendPort) + + out, err := bdProxiedRun(t, bd, p.dir, "list", "--json") + if err == nil { + t.Fatalf("runtime resolution unexpectedly accepted hand-edited 0.0.0.0 config:\n%s", out) + } + assertManagedListenerPolicyOutput(t, string(out)) + }) +} + +func writeListenerConfigWithPort(t *testing.T, path, host string, port int) string { + t.Helper() + body := []byte(fmt.Sprintf("listener:\n host: %q\n port: %d\n", host, port)) + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatalf("write listener config %s: %v", path, err) + } + return path +} + +func assertManagedListenerPolicyOutput(t *testing.T, output string) { + t.Helper() + for _, want := range []string{ + "numeric loopback IP literal", + "127.0.0.1", + "--proxied-server-external-host", + "--proxied-server-external-port", + "--proxied-server-external-socket-path", + } { + if !strings.Contains(output, want) { + t.Errorf("listener-policy output missing %q:\n%s", want, output) + } + } +} diff --git a/cmd/bd/proxied_server.go b/cmd/bd/proxied_server.go index dfec4b7dd..7d5621554 100644 --- a/cmd/bd/proxied_server.go +++ b/cmd/bd/proxied_server.go @@ -3,6 +3,7 @@ package main import ( "bytes" "fmt" + "net" "os" "os/exec" "path/filepath" @@ -26,6 +27,11 @@ import ( const archiveLevelSupportNoticeFormat = "Info: external dolt predates archive_level config support (need >= Dolt %s); " + "this managed server's background auto-GC may still produce zstd archives.\n" +const managedListenerHostRemediation = "set listener.host to 127.0.0.1 (or another 127.0.0.0/8 / ::1 " + + "loopback address); for non-local topology use " + + "--proxied-server-external-host with --proxied-server-external-port, " + + "--proxied-server-external-socket-path, or switch to non-proxied server mode" + // managedProxiedServerConfigMarker is written as a YAML comment header on // every config.yaml Beads itself generates (renderProxiedServerConfig). // YAML comments are ignored by Dolt's own parser, so this is invisible to @@ -261,7 +267,29 @@ func resolveProxiedServerLogPath(beadsDir string) (path string, isCustom bool, e // itself would accept it. NewYamlConfig on raw bytes is only safe for a // Beads-managed file (marker present), because Beads itself never writes // interpolation syntax into a file it generates. +// +// The managed listener policy (validateManagedServerConfigPolicy) is applied +// exactly once, on the resolved path, via the interpolation-aware +// validateProxiedServerConfig — never on raw pre-interpolation bytes. A +// marker config whose listener.host is a ${VAR} placeholder is therefore +// judged on what it expands to, matching the degrade-gracefully contract for +// hand-edited managed files below. func ensureProxiedServerConfig(beadsDir string, archiveLevelSupported bool) (string, error) { + path, err := resolveOrCreateProxiedServerConfig(beadsDir, archiveLevelSupported) + if err != nil { + return "", err + } + if err := validateProxiedServerConfig(path); err != nil { + return "", fmt.Errorf("ensureProxiedServerConfig: %w", err) + } + return path, nil +} + +// resolveOrCreateProxiedServerConfig is ensureProxiedServerConfig without the +// final policy gate: it only resolves, generates, reconciles, and warns. All +// callers go through the exported entry point so the policy check cannot be +// skipped or duplicated per branch. +func resolveOrCreateProxiedServerConfig(beadsDir string, archiveLevelSupported bool) (string, error) { path, isCustom, err := resolveProxiedServerConfigPath(beadsDir) if err != nil { return "", err @@ -403,12 +431,103 @@ func validateProxiedServerConfig(path string) error { if !info.Mode().IsRegular() { return fmt.Errorf("%s: not a regular file", path) } - if _, err := servercfg.YamlConfigFromFile(filesys.LocalFS, path); err != nil { + cfg, err := servercfg.YamlConfigFromFile(filesys.LocalFS, path) + if err != nil { return fmt.Errorf("%s: parse: %w", path, err) } + if err := validateManagedServerConfigPolicy(cfg); err != nil { + return fmt.Errorf("%s: %w", path, err) + } return nil } +// validateManagedServerConfigPolicy enforces the trust boundary for the Dolt +// child that Beads launches with root and an empty password: every listener +// the config can open must stay loopback-only, so the whole config is in +// scope, not just listener.host. External proxied topology never calls this +// validator because it launches no managed child. +func validateManagedServerConfigPolicy(cfg servercfg.ServerConfig) error { + // listener.host policy: an explicit numeric loopback IP, nothing else. + // An omitted host is rejected rather than trusted: servercfg resolves it + // to DefaultHost ("localhost"), and CheckForUnixSocket treats a + // "localhost" host as license to also open the shared-namespace default + // unix socket — strictly worse than the explicit hostname it looks like. + if yc, ok := cfg.(*servercfg.YAMLConfig); ok && yc.ListenerConfig.HostStr == nil { + return fmt.Errorf( + "managed proxied-server listener.host is omitted (Dolt would default to %q, "+ + "which can also open the default unix socket) and must be a numeric loopback IP literal: %s", + servercfg.DefaultHost, managedListenerHostRemediation, + ) + } + host := cfg.Host() + ip := net.ParseIP(host) + if ip == nil || !ip.IsLoopback() { + return fmt.Errorf( + "managed proxied-server listener.host %q must be a numeric loopback IP literal: %s", + host, managedListenerHostRemediation, + ) + } + // A unix socket bypasses the loopback TCP trust boundary entirely + // (shared filesystem namespace). Socket() maps an explicitly empty + // listener.socket to Dolt's default socket path, so it is rejected too. + if socket := cfg.Socket(); socket != "" { + return fmt.Errorf( + "managed proxied-server config sets listener.socket %q, which bypasses the loopback trust boundary: "+ + "remove listener.socket; %s", + socket, managedListenerHostRemediation, + ) + } + // remotesapi.port opens an HTTP+gRPC listener on all interfaces; there + // is no host knob for it, so its presence alone breaks the boundary. + if port := cfg.RemotesapiPort(); port != nil { + return fmt.Errorf( + "managed proxied-server config sets remotesapi.port %d, which listens on all interfaces: "+ + "remove the remotesapi block; %s", + *port, managedListenerHostRemediation, + ) + } + // cluster.remotesapi.port likewise listens on all interfaces. + if cfg.ClusterConfig() != nil { + return fmt.Errorf( + "managed proxied-server config sets a cluster block, whose remotesapi listens on all interfaces: "+ + "remove the cluster block; %s", + managedListenerHostRemediation, + ) + } + return nil +} + +// validateManagedProxiedServerConfigAtInit validates the effective existing +// managed-local config before init performs filesystem mutations. A missing +// Beads-default path is valid because init/runtime will generate the +// loopback-only config. Missing custom paths remain hard errors. +// +// explicitRootPath is the --proxied-server-root-path flag value: at this +// point the sidecar is not persisted yet, so resolveProxiedServerConfigPath +// cannot see the flag and it must be applied here. An explicit +// --proxied-server-config-path needs no handling here — init.go already +// validated it at flag-parse time, and when no config env override is set it +// IS the effective config, so there is nothing further to check. +func validateManagedProxiedServerConfigAtInit(beadsDir, explicitPath, explicitRootPath string) error { + if explicitPath != "" && os.Getenv("BEADS_PROXIED_SERVER_CONFIG") == "" { + return nil + } + path, isCustom, err := resolveProxiedServerConfigPath(beadsDir) + if err != nil { + return err + } + if !isCustom && explicitRootPath != "" && os.Getenv("BEADS_PROXIED_SERVER_ROOT_PATH") == "" { + path = filepath.Join(explicitRootPath, proxiedServerConfigName) + } + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) && !isCustom { + return nil + } + return fmt.Errorf("%s: %w", path, err) + } + return validateProxiedServerConfig(path) +} + func validateProxiedServerRootPath(path string) error { switch info, err := os.Stat(path); { case err == nil: diff --git a/cmd/bd/proxied_server_test.go b/cmd/bd/proxied_server_test.go index 487a6bde7..a04205626 100644 --- a/cmd/bd/proxied_server_test.go +++ b/cmd/bd/proxied_server_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -888,6 +889,309 @@ func TestValidateProxiedServerConfig(t *testing.T) { }) } +func TestValidateManagedServerConfigPolicy(t *testing.T) { + tests := []struct { + name string + mutate func(cfg *servercfg.YAMLConfig) + wantErr string // substring of the expected error; "" means accepted + }{ + {name: "IPv4 loopback"}, + { + name: "IPv4 loopback range", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("127.42.0.9") }, + }, + { + name: "IPv6 loopback", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("::1") }, + }, + { + name: "IPv4-mapped IPv6 loopback", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("::ffff:127.0.0.1") }, + }, + { + name: "omitted host rejected", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = nil }, + wantErr: "listener.host is omitted", + }, + { + name: "IPv4 wildcard", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("0.0.0.0") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "IPv6 wildcard", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("::") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "public IPv4", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("203.0.113.7") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "public IPv6", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("2001:db8::7") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "localhost hostname", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("localhost") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "other hostname", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("db.internal") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "garbage", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("not an address !") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "explicit empty binds wildcard", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.HostStr = stringPtr("") }, + wantErr: "numeric loopback IP literal", + }, + { + name: "unix socket rejected", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ListenerConfig.Socket = stringPtr("/tmp/mysql.sock") }, + wantErr: "listener.socket", + }, + { + name: "empty socket means default socket, rejected", + mutate: func(cfg *servercfg.YAMLConfig) { + cfg.ListenerConfig.Socket = stringPtr("") + }, + wantErr: "listener.socket", + }, + { + name: "remotesapi port rejected", + mutate: func(cfg *servercfg.YAMLConfig) { + port := 8080 + cfg.RemotesapiConfig.Port_ = &port + }, + wantErr: "remotesapi.port", + }, + { + name: "cluster block rejected", + mutate: func(cfg *servercfg.YAMLConfig) { cfg.ClusterCfg = &servercfg.ClusterYAMLConfig{} }, + wantErr: "cluster block", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &servercfg.YAMLConfig{ + ListenerConfig: servercfg.ListenerYAMLConfig{HostStr: stringPtr("127.0.0.1")}, + } + if tt.mutate != nil { + tt.mutate(cfg) + } + err := validateManagedServerConfigPolicy(cfg) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Contains(t, err.Error(), "127.0.0.1") + assert.Contains(t, err.Error(), "--proxied-server-external-host") + assert.Contains(t, err.Error(), "--proxied-server-external-port") + assert.Contains(t, err.Error(), "--proxied-server-external-socket-path") + assert.NotContains(t, err.Error(), "--server-") + }) + } +} + +func TestManagedListenerPolicyEveryConfigPathAtInitAndRuntime(t *testing.T) { + type setupFn func(t *testing.T, beadsDir string) string + tests := []struct { + name string + setup setupFn + }{ + { + name: "environment custom", + setup: func(t *testing.T, beadsDir string) string { + path := writeListenerConfig(t, filepath.Join(t.TempDir(), "env.yaml"), "0.0.0.0", false, false) + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", path) + return "" + }, + }, + { + name: "sidecar custom", + setup: func(t *testing.T, beadsDir string) string { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + path := writeListenerConfig(t, filepath.Join(t.TempDir(), "sidecar.yaml"), "0.0.0.0", false, false) + writeProxiedClientInfo(t, beadsDir, &configfile.ProxiedServerClientInfo{ConfigPath: path}) + return "" + }, + }, + { + name: "pre-marker default path", + setup: func(t *testing.T, beadsDir string) string { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + path := proxiedServerConfigPath(beadsDir) + writeListenerConfig(t, path, "0.0.0.0", false, false) + return "" + }, + }, + { + name: "marker file hand edit", + setup: func(t *testing.T, beadsDir string) string { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + path := proxiedServerConfigPath(beadsDir) + writeListenerConfig(t, path, "0.0.0.0", true, false) + return "" + }, + }, + { + name: "marker parse fallback with interpolation", + setup: func(t *testing.T, beadsDir string) string { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + t.Setenv("BEADS_TEST_MANAGED_LISTENER_PORT", "54321") + path := proxiedServerConfigPath(beadsDir) + writeListenerConfig(t, path, "0.0.0.0", true, true) + return "" + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + beadsDir := t.TempDir() + explicitPath := tt.setup(t, beadsDir) + + initErr := validateManagedProxiedServerConfigAtInit(beadsDir, explicitPath, "") + assertManagedListenerPolicyError(t, initErr) + + _, runtimeErr := ensureProxiedServerConfig(beadsDir, true) + assertManagedListenerPolicyError(t, runtimeErr) + }) + } +} + +func TestManagedListenerPolicyLoopbackCustomConfigPassesInitAndRuntime(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + beadsDir := t.TempDir() + path := writeListenerConfig(t, filepath.Join(t.TempDir(), "loopback.yaml"), "127.0.0.1", false, false) + writeProxiedClientInfo(t, beadsDir, &configfile.ProxiedServerClientInfo{ConfigPath: path}) + + require.NoError(t, validateManagedProxiedServerConfigAtInit(beadsDir, "", "")) + got, err := ensureProxiedServerConfig(beadsDir, true) + require.NoError(t, err) + assert.Equal(t, path, got) +} + +// TestManagedListenerPolicyOmittedHostRejected pins the policy decision that +// an omitted listener.host is NOT trusted: servercfg resolves it to +// "localhost", which CheckForUnixSocket also treats as license to open the +// shared-namespace default unix socket. +func TestManagedListenerPolicyOmittedHostRejected(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + beadsDir := t.TempDir() + path := filepath.Join(t.TempDir(), "omitted.yaml") + require.NoError(t, os.WriteFile(path, []byte("listener:\n port: 54321\n"), 0o600)) + writeProxiedClientInfo(t, beadsDir, &configfile.ProxiedServerClientInfo{ConfigPath: path}) + + initErr := validateManagedProxiedServerConfigAtInit(beadsDir, "", "") + require.Error(t, initErr) + assert.Contains(t, initErr.Error(), "listener.host is omitted") + + _, runtimeErr := ensureProxiedServerConfig(beadsDir, true) + require.Error(t, runtimeErr) + assert.Contains(t, runtimeErr.Error(), "listener.host is omitted") +} + +// TestManagedListenerPolicyMarkerInterpolatedHost pins the interpolation +// contract: a marker config whose listener.host is a ${VAR} placeholder is +// judged on what the placeholder expands to, not on the raw text. +func TestManagedListenerPolicyMarkerInterpolatedHost(t *testing.T) { + writeInterpolatedHostConfig := func(t *testing.T, beadsDir string) string { + t.Helper() + path := proxiedServerConfigPath(beadsDir) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + body := managedProxiedServerConfigMarker + + "listener:\n host: \"${BEADS_TEST_MANAGED_LISTENER_HOST}\"\n port: 54321\n" + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path + } + + t.Run("expands to loopback, accepted", func(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + t.Setenv("BEADS_TEST_MANAGED_LISTENER_HOST", "127.0.0.1") + beadsDir := t.TempDir() + want := writeInterpolatedHostConfig(t, beadsDir) + + got, err := ensureProxiedServerConfig(beadsDir, true) + require.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("expands to wildcard, rejected", func(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + t.Setenv("BEADS_TEST_MANAGED_LISTENER_HOST", "0.0.0.0") + beadsDir := t.TempDir() + writeInterpolatedHostConfig(t, beadsDir) + + _, err := ensureProxiedServerConfig(beadsDir, true) + assertManagedListenerPolicyError(t, err) + }) +} + +// TestManagedListenerPolicyRootPathFlagAtInit pins the F5c fix: at init time +// the sidecar is not persisted yet, so the --proxied-server-root-path flag +// value must be threaded into the validator explicitly for a pre-existing +// config.yaml under that root to be policy-checked. +func TestManagedListenerPolicyRootPathFlagAtInit(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + t.Setenv("BEADS_PROXIED_SERVER_ROOT_PATH", "") + beadsDir := t.TempDir() + rootDir := t.TempDir() + + t.Run("bad config under flagged root rejected", func(t *testing.T) { + writeListenerConfig(t, filepath.Join(rootDir, proxiedServerConfigName), "0.0.0.0", false, false) + err := validateManagedProxiedServerConfigAtInit(beadsDir, "", rootDir) + assertManagedListenerPolicyError(t, err) + }) + + t.Run("missing config under flagged root allowed", func(t *testing.T) { + emptyRoot := t.TempDir() + require.NoError(t, validateManagedProxiedServerConfigAtInit(beadsDir, "", emptyRoot)) + }) +} + +func stringPtr(value string) *string { + return &value +} + +func writeListenerConfig(t *testing.T, path, host string, marker, interpolatedPort bool) string { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + body := "" + if marker { + body = managedProxiedServerConfigMarker + } + port := "54321" + if interpolatedPort { + port = "${BEADS_TEST_MANAGED_LISTENER_PORT}" + } + body += fmt.Sprintf("listener:\n host: %q\n port: %s\n", host, port) + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +func assertManagedListenerPolicyError(t *testing.T, err error) { + t.Helper() + require.Error(t, err) + assert.Contains(t, err.Error(), "numeric loopback IP literal") + assert.Contains(t, err.Error(), "127.0.0.1") + assert.Contains(t, err.Error(), "--proxied-server-external-host") + assert.Contains(t, err.Error(), "--proxied-server-external-port") + assert.Contains(t, err.Error(), "--proxied-server-external-socket-path") + assert.NotContains(t, err.Error(), "--server-") +} + // TestCheckExistingBeadsDataAt_ProxiedServerNoData asserts that a proxied // workspace with metadata.json but no actual /proxieddb//.dolt // directory is treated as a fresh clone — init is allowed to proceed so the diff --git a/internal/storage/dbproxy/proxy/endpoint.go b/internal/storage/dbproxy/proxy/endpoint.go index fb57f4716..e780fe22c 100644 --- a/internal/storage/dbproxy/proxy/endpoint.go +++ b/internal/storage/dbproxy/proxy/endpoint.go @@ -167,6 +167,10 @@ type spawnMarker struct { var errStartInterrupted = errors.New("proxy startup interrupted by concurrent shutdown") func PickFreePort() (int, error) { + // The managed proxy no longer uses this bind-close allocator: its child + // binds port 0 and publishes the kernel-assigned port. The remaining + // production caller allocates the Dolt config port; that race requires + // the managed-config ownership/retry contract deferred to the PR-C RFC. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return 0, err @@ -326,16 +330,8 @@ func spawnAndHandoff( return Endpoint{}, err } - port := opts.Port - if port == 0 { - var err error - if port, err = PickFreePort(); err != nil { - return Endpoint{}, fmt.Errorf("pick port: %w", err) - } - } - handedOff = true - child, err := forkExecChild(rootDir, opts, port, stopEpoch, lock) + child, err := forkExecChild(rootDir, opts, opts.Port, stopEpoch, lock) if err != nil { return Endpoint{}, fmt.Errorf("fork child: %w", err) } @@ -363,32 +359,63 @@ func spawnAndHandoff( return Endpoint{}, fmt.Errorf("discover spawned proxy: %w", discovered.err) } select { - case <-child.done: + case childErr := <-child.done: if interrupted, ierr := stopEpochChanged(rootDir, stopEpoch); ierr != nil { return Endpoint{}, ierr } else if interrupted { return Endpoint{}, fmt.Errorf("%w for %s", errStartInterrupted, rootDir) } - return Endpoint{}, fmt.Errorf("proxy child on port %d exited before becoming ready (likely lost lock race)", port) + if childErr == nil { + childErr = errors.New("child exited without reporting an error") + } + // A LockHeldExitCode exit is a lost spawn race, not a listen + // failure; any other exit gets the child's log path so the real + // error (listen, backend start, ...) is findable. + var exitErr *exec.ExitError + if errors.As(childErr, &exitErr) && exitErr.ExitCode() == LockHeldExitCode { + return Endpoint{}, fmt.Errorf( + "proxy child lost the proxy.lock spawn race for %s: %w", + rootDir, childErr, + ) + } + if opts.Port != 0 { + return Endpoint{}, fmt.Errorf( + "proxy child exited before becoming ready on explicitly configured port %d (see %s): %w", + opts.Port, opts.LogFilePath, childErr, + ) + } + return Endpoint{}, fmt.Errorf( + "proxy child exited before publishing its OS-assigned port (see %s): %w", + opts.LogFilePath, childErr, + ) case <-hard.C: if err := killSpawnedChild(child); err != nil { - return Endpoint{}, fmt.Errorf("hard timeout waiting for proxy on port %d; safe child kill failed: %w", port, err) + return Endpoint{}, fmt.Errorf("hard timeout waiting for proxy on %s; safe child kill failed: %w", describeSpawnPort(opts.Port), err) } - return Endpoint{}, fmt.Errorf("hard timeout (%s) waiting for proxy on port %d", spawnReadyHardTimeout, port) + return Endpoint{}, fmt.Errorf("hard timeout (%s) waiting for proxy on %s", spawnReadyHardTimeout, describeSpawnPort(opts.Port)) case <-poll.C: } if time.Now().After(deadline) { if err := killSpawnedChild(child); err != nil { - return Endpoint{}, fmt.Errorf("timeout waiting for proxy on port %d; safe child kill failed: %w", port, err) + return Endpoint{}, fmt.Errorf("timeout waiting for proxy on %s; safe child kill failed: %w", describeSpawnPort(opts.Port), err) } - return Endpoint{}, fmt.Errorf("timeout waiting for proxy to become ready on port %d", port) + return Endpoint{}, fmt.Errorf("timeout waiting for proxy to become ready on %s", describeSpawnPort(opts.Port)) } } } +// describeSpawnPort renders a requested spawn port for wait/timeout +// messages: 0 is the default OS-assigned path, not a literal "port 0". +func describeSpawnPort(port int) string { + if port == 0 { + return "its OS-assigned port" + } + return fmt.Sprintf("port %d", port) +} + type spawnedProxyChild struct { cmd *exec.Cmd - done <-chan struct{} + done <-chan error handle *procid.Handle marker spawnMarker } @@ -417,6 +444,7 @@ func forkExecChild(rootDir string, opts OpenOpts, port int, stopEpoch string, lo "--port", strconv.Itoa(port), "--idle-timeout", idleTimeout.String(), "--backend", string(opts.Backend), + "--stop-epoch", stopEpoch, } if opts.ConfigFilePath != "" { args = append(args, "--config", opts.ConfigFilePath) @@ -488,11 +516,12 @@ func forkExecChild(rootDir string, opts OpenOpts, port int, stopEpoch string, lo return nil, fmt.Errorf("start proxy child: %w", err) } - done := make(chan struct{}) + done := make(chan error, 1) go func() { - defer close(done) - _ = cmd.Wait() + waitErr := cmd.Wait() _ = logFile.Close() + done <- waitErr + close(done) }() // Theoretical race: the birth token is captured after Start, so the child diff --git a/internal/storage/dbproxy/proxy/server.go b/internal/storage/dbproxy/proxy/server.go index 9e7fbc5c1..00103854c 100644 --- a/internal/storage/dbproxy/proxy/server.go +++ b/internal/storage/dbproxy/proxy/server.go @@ -33,6 +33,15 @@ type ProxyOpts struct { Port int IdleTimeout time.Duration Server server.DatabaseServer + // StopEpoch is the proxy stop epoch the spawning parent observed under + // proxy.lock before forking this proxy. ListenAndServe re-reads the + // epoch immediately before publishing proxy.pid and aborts if it has + // advanced, so a slow backend start cannot outlast a concurrent + // `bd dolt stop` and publish a running proxy after the stop returned. + // Empty means "no stop had ever run", which readStopEpoch also reports + // for a missing epoch file, so the zero value stays correct for direct + // (non-forked) callers such as tests. + StopEpoch string // Stats is optional. When non-nil, the proxy records per-event counters // against it; tests use Snapshot() to assert. Production code should // leave this nil. @@ -45,6 +54,7 @@ type proxyServer struct { idleTimeout time.Duration server server.DatabaseServer stats *Stats + stopEpoch string logger *log.Logger listener net.Listener @@ -88,6 +98,7 @@ func NewProxyServer(opts ProxyOpts) *proxyServer { idleTimeout: opts.IdleTimeout, server: opts.Server, stats: opts.Stats, + stopEpoch: opts.StopEpoch, } } @@ -204,6 +215,21 @@ func (p *proxyServer) ListenAndServe(parentCtx context.Context) error { identReply.ControlPort = control.Port() identMu.Unlock() + // Last fence before publishing: the spawn marker was cleared when this + // process took proxy.lock, so a `bd dolt stop` that began during a slow + // backend start has no record of this attempt. It did advance the stop + // epoch first, so re-check it here and abort instead of publishing a + // running proxy after that stop returned. + if changed, err := stopEpochChanged(p.rootDir, p.stopEpoch); err != nil { + p.stats.IncBackendStop() + _ = stopBackendBounded(p.server) + return fmt.Errorf("re-check proxy stop epoch before publish: %w", err) + } else if changed { + p.stats.IncBackendStop() + _ = stopBackendBounded(p.server) + return fmt.Errorf("%w for %s: stop epoch advanced during startup", errStartInterrupted, p.rootDir) + } + if err := pidfile.Write(p.rootDir, PIDFileName, pidfile.PidFile{ Pid: os.Getpid(), Port: dataPort.Port, diff --git a/internal/storage/dbproxy/proxy/server_epoch_internal_test.go b/internal/storage/dbproxy/proxy/server_epoch_internal_test.go new file mode 100644 index 000000000..0c77771d4 --- /dev/null +++ b/internal/storage/dbproxy/proxy/server_epoch_internal_test.go @@ -0,0 +1,67 @@ +package proxy + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// epochAdvancingServer simulates the F4 race deterministically: `bd dolt +// stop` advancing the stop epoch while the proxy child is inside its (slow) +// backend Start, after the spawn marker has already been cleared under +// proxy.lock. +type epochAdvancingServer struct { + *server.TestDatabaseServerImpl + rootDir string + t *testing.T +} + +func (s *epochAdvancingServer) Start(ctx context.Context) error { + if err := s.TestDatabaseServerImpl.Start(ctx); err != nil { + return err + } + require.NoError(s.t, advanceStopEpoch(s.rootDir)) + return nil +} + +// TestListenAndServe_StopEpochAdvanceDuringStartupAbortsPublish asserts that +// a stop which begins after the child took proxy.lock (and cleared the spawn +// marker) but before proxy.pid is written makes the child abort instead of +// publishing a running proxy after the stop returned. +func TestListenAndServe_StopEpochAdvanceDuringStartupAbortsPublish(t *testing.T) { + t.Parallel() + + root := t.TempDir() + baseline, err := readStopEpoch(root) + require.NoError(t, err) + + ts := server.New() + backend := &epochAdvancingServer{TestDatabaseServerImpl: ts, rootDir: root, t: t} + + p := NewProxyServer(ProxyOpts{ + RootDir: root, + Port: 0, + Server: backend, + StopEpoch: baseline, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + err = p.ListenAndServe(ctx) + require.Error(t, err) + assert.ErrorIs(t, err, errStartInterrupted) + + pf, readErr := pidfile.Read(root, PIDFileName) + require.NoError(t, readErr) + assert.Nil(t, pf, "proxy.pid must not be published after the stop epoch advanced") + + counters := ts.Snapshot() + assert.Equal(t, int64(1), counters.StartCalls) + assert.Equal(t, int64(1), counters.StopCalls, "aborting the publish must still stop the backend") +} diff --git a/internal/storage/dbproxy/proxy/server_test.go b/internal/storage/dbproxy/proxy/server_test.go index 37282bae2..369e2b662 100644 --- a/internal/storage/dbproxy/proxy/server_test.go +++ b/internal/storage/dbproxy/proxy/server_test.go @@ -177,6 +177,37 @@ func TestProxy_HappyPath_Echo(t *testing.T) { assertNoPidFile(t, root) } +func TestProxy_PortZeroPublishesActualDialablePort(t *testing.T) { + t.Parallel() + + root := t.TempDir() + ts := server.New() + h := runProxy(t, proxy.ProxyOpts{ + RootDir: root, + Port: 0, + Server: ts, + }) + waitListening(t, root, listenWait) + + pf, err := pidfile.Read(root, proxy.PIDFileName) + require.NoError(t, err) + require.NotNil(t, pf) + require.NotZero(t, pf.Port, "pidfile must publish the kernel-assigned port") + + conn := dialProxy(t, pf.Port) + _, err = conn.Write([]byte("port-zero")) + require.NoError(t, err) + got := make([]byte, len("port-zero")) + _, err = io.ReadFull(conn, got) + require.NoError(t, err) + assert.Equal(t, "port-zero", string(got)) + require.NoError(t, conn.Close()) + + h.Cancel() + require.NoError(t, h.waitErr(t, shutdownWait)) + assertNoPidFile(t, root) +} + func TestProxy_PidFile_WrittenAndRemoved(t *testing.T) { t.Parallel() @@ -243,7 +274,7 @@ func TestProxy_PublishesVerifiableIdentity(t *testing.T) { assert.Error(t, err) } -func TestProxy_ListenError_PortInUse(t *testing.T) { +func TestProxy_ExplicitPortInUseFailsWithoutRepick(t *testing.T) { t.Parallel() hold, err := net.Listen("tcp", "127.0.0.1:0") @@ -261,6 +292,7 @@ func TestProxy_ListenError_PortInUse(t *testing.T) { err = h.waitErr(t, shutdownWait) require.Error(t, err) assert.Contains(t, err.Error(), "listen on") + assert.Contains(t, err.Error(), strconv.Itoa(port)) s := stats.Snapshot() assert.Equal(t, int64(0), s.ListenAndServeCalls)