From d803ec9126028a76124c81ee90ca8068a1c744df Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 01:48:15 -0700 Subject: [PATCH] fix(dbproxy): retire the proxy when its Dolt backend exits cleanly (#6937) * fix(dbproxy): retire the proxy when its Dolt backend exits cleanly DoltServer supervised `dolt sql-server` with cmd.Wait inside an errgroup, and Running read the errgroup context. errgroup cancels that context only for a non-nil result, so a backend that shut down gracefully (dolt exits 0 on SIGTERM) was reported as running forever. The proxy's backend health watcher never fired: the proxy stayed up and adoptable in front of a dead store. With --proxied-server-idle-timeout 0 (the setting Gas City uses) it never went away; `bd dolt status` said "running" and every command, `bd ping` included, failed until `bd dolt stop`. The supervising goroutine now reports a clean exit as errBackendExited, so Running flips on any backend exit and Stop treats it as benign. This follows the existing proxied lifecycle contract (child death retires the owned proxy; the next bd invocation starts a fresh proxy/backend pair) rather than adding in-proxy restarts: spawn, adoption, stop epochs and identity already live in the on-demand start path, and a second restart loop inside the proxy would duplicate that and could mask a crash-looping backend. `bd dolt status` no longer headlines a proxy whose managed backend is gone as "running"; it says the proxy is up but not serving and points at `bd dolt stop` (for proxies from an older bd). The JSON payload is unchanged. `bd ping` already verifies by running SELECT 1 through the proxy, so after this fix a successful ping reflects a live pair. Tests: DoltServer Running goes false after SIGTERM (clean exit) and SIGKILL, and Stop succeeds; managed-local lane test that SIGTERMs the backend with idle timeout 0 and requires the proxy to retire and `bd ping --json` to succeed on a fresh pair; status render case. Co-Authored-By: Claude Opus 5.5 * fix(dolt status): report running=false for a proxy over a dead managed backend Review follow-up: the JSON running field now means "serving" (a live proxy and, on a managed topology, a live dolt behind it), so scripts that key off running alone are not fooled by a stranded proxy left by an older bd. proxy_pid/proxy_port and backend_* are unchanged. Adds the PR link to the CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 * docs(changelog): resolve merge markers left by the origin/main merge Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- .github/workflows/proxied-local-smoke.yml | 3 +- CHANGELOG.md | 13 ++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/dolt_proxied_lifecycle.go | 35 +++++++-- cmd/bd/dolt_proxied_lifecycle_test.go | 40 +++++++++++ cmd/bd/proxied_local_backend_exit_test.go | 71 +++++++++++++++++++ internal/storage/dbproxy/server/BUILD.bazel | 1 + internal/storage/dbproxy/server/doltserver.go | 14 +++- .../server/doltserver_exit_unix_test.go | 64 +++++++++++++++++ 9 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/proxied_local_backend_exit_test.go create mode 100644 internal/storage/dbproxy/server/doltserver_exit_unix_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 55967a0f2..bfa874fb1 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -133,7 +133,8 @@ jobs: TestManagedLocalProxiedBackupRoundTrip \ TestManagedLocalProxiedAutoBackup \ TestManagedLocalProxiedBackupRestoreRefusedWhileAttached \ - TestManagedLocalProxiedPurgeWispsPlaneRetention + TestManagedLocalProxiedPurgeWispsPlaneRetention \ + TestManagedLocalProxiedBackendCleanExitRetiresProxy do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index f176495a5..ea1413265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A proxied workspace's proxy retires when its Dolt backend exits + cleanly.** The proxy noticed its `dolt sql-server` child exiting only when + the exit status was non-zero. A backend that shut down gracefully (for + example on SIGTERM) left the proxy up and adoptable in front of a dead store: + with `--proxied-server-idle-timeout 0` it never went away, `bd dolt status` + called it running, and every command failed until `bd dolt stop`. The proxy + now retires on any backend exit, as it already did on a crash, so the next bd + command (including `bd ping`) starts a fresh proxy and backend. `bd dolt + status` no longer reports a proxy whose managed backend is gone as running: + the text says "not serving" and the JSON reports `running: false` (with + `proxy_pid` still set), including for a stranded proxy started by an older + bd. ([#6937](https://github.com/gastownhall/beads/pull/6937)) + - **`bd sql` no longer drops the rows of CTE queries or CALL result sets** ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7e7bddeaa..9a0066ff2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -929,6 +929,7 @@ go_test( "provenance_test.go", "proxied_direct_only_refusal_frontdoor_test.go", "proxied_integration_helpers_test.go", + "proxied_local_backend_exit_test.go", "proxied_local_dolt_lifecycle_test.go", "proxied_local_helpers_test.go", "proxied_local_identity_lifecycle_linux_test.go", diff --git a/cmd/bd/dolt_proxied_lifecycle.go b/cmd/bd/dolt_proxied_lifecycle.go index 31b7db6cb..28d06357f 100644 --- a/cmd/bd/dolt_proxied_lifecycle.go +++ b/cmd/bd/dolt_proxied_lifecycle.go @@ -43,9 +43,12 @@ func proxiedDoltStartRefusal() *ProxyCapabilityError { // proxiedDoltStatus is the `bd dolt status --json` payload for a proxied // workspace. It reports the two processes separately because they fail -// separately and an operator's next move differs: Running describes the proxy, -// which is the endpoint every bd command connects through, while the Backend* -// fields describe the dolt server behind it. +// separately and an operator's next move differs: Running says whether the +// workspace is serving (a live proxy, and on a managed topology a live dolt +// behind it), Proxy* describe the proxy every bd command connects through, and +// the Backend* fields describe the dolt server behind it. A proxy left in front +// of a dead managed backend (possible with proxies started by an older bd) +// therefore reports running=false with proxy_pid still set. type proxiedDoltStatus struct { Mode string `json:"mode"` Root string `json:"root"` @@ -98,9 +101,17 @@ func collectProxiedDoltStatus(beadsDir string) (proxiedDoltStatus, error) { status.BackendEndpoint = externalDoltEndpoint(*info.External) } } + status.Running = proxiedStatusServing(live.ProxyRunning, status.BackendManaged, status.BackendRunning) return status, nil } +// proxiedStatusServing reports whether a proxied workspace is serving: its +// proxy is live and, when bd manages the backend, that dolt server is live +// too. A proxy in front of a dead managed backend serves nothing. +func proxiedStatusServing(proxyRunning, backendManaged, backendRunning bool) bool { + return proxyRunning && (!backendManaged || backendRunning) +} + func externalDoltEndpoint(cfg configfile.ExternalDoltConfig) string { if cfg.Socket != "" { return "unix:" + cfg.Socket @@ -116,11 +127,20 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { return } - if status.Running { + // A live proxy in front of a managed backend that is gone serves nothing, + // so do not call it running. The proxy retires itself once it notices + // the backend exit; a proxy from an older bd may not. + backendDown := !status.Running && status.ProxyPID != 0 && status.BackendManaged && !status.BackendRunning + switch { + case backendDown: + fmt.Printf("Dolt server: not serving (%s): proxy is up but its dolt backend is not running\n", status.Mode) + fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) + fmt.Printf(" Proxy port: %d\n", status.ProxyPort) + case status.Running: fmt.Printf("Dolt server: running (%s)\n", status.Mode) fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) fmt.Printf(" Proxy port: %d\n", status.ProxyPort) - } else { + default: fmt.Printf("Dolt server: not running (%s)\n", status.Mode) } fmt.Printf(" Root: %s\n", status.Root) @@ -135,9 +155,12 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { if status.IdleTimeout != "" { fmt.Printf(" Idle timeout: %s\n", status.IdleTimeout) } - if !status.Running { + if !status.Running && !backendDown { fmt.Println(" The proxy starts on demand; the next bd command launches it.") } + if backendDown { + fmt.Println(" Run 'bd dolt stop' to retire the proxy; the next bd command starts a fresh proxy and backend.") + } if isDoltLocalOnly() { fmt.Println(" Remote sync: disabled (dolt.local-only=true)") } diff --git a/cmd/bd/dolt_proxied_lifecycle_test.go b/cmd/bd/dolt_proxied_lifecycle_test.go index d12f87d0b..aa31c22e6 100644 --- a/cmd/bd/dolt_proxied_lifecycle_test.go +++ b/cmd/bd/dolt_proxied_lifecycle_test.go @@ -151,6 +151,28 @@ func TestRenderProxiedDoltStatus(t *testing.T) { } }) + t.Run("text does not call a proxy over a dead managed backend running", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + stranded := running + stranded.Running = false + stranded.BackendRunning, stranded.BackendPID, stranded.BackendPort = false, 0, 0 + out := captureStdout(t, func() error { renderProxiedDoltStatus(stranded); return nil }) + if strings.Contains(out, "Dolt server: running") { + t.Errorf("proxy with a dead backend reported as running:\n%s", out) + } + for _, want := range []string{ + "Dolt server: not serving (proxied-server)", + "Proxy PID: 111", + "Backend: not running", + "bd dolt stop", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + t.Run("text marks an external backend as somebody else's", func(t *testing.T) { defer restoreJSONOutput(t)() jsonOutput = false @@ -180,6 +202,24 @@ func TestRenderProxiedDoltStatus(t *testing.T) { }) } +func TestProxiedStatusServing(t *testing.T) { + for _, tc := range []struct { + name string + proxyRunning, managed, backendRunning bool + want bool + }{ + {"managed pair live", true, true, true, true}, + {"proxy over dead managed backend", true, true, false, false}, + {"external backend", true, false, false, true}, + {"proxy down", false, true, true, false}, + {"nothing running", false, true, false, false}, + } { + if got := proxiedStatusServing(tc.proxyRunning, tc.managed, tc.backendRunning); got != tc.want { + t.Errorf("%s: proxiedStatusServing(%v, %v, %v) = %v, want %v", tc.name, tc.proxyRunning, tc.managed, tc.backendRunning, got, tc.want) + } + } +} + func restoreJSONOutput(t *testing.T) func() { t.Helper() orig := jsonOutput diff --git a/cmd/bd/proxied_local_backend_exit_test.go b/cmd/bd/proxied_local_backend_exit_test.go new file mode 100644 index 000000000..f4b5e336d --- /dev/null +++ b/cmd/bd/proxied_local_backend_exit_test.go @@ -0,0 +1,71 @@ +//go:build cgo && unix + +package main + +import ( + "encoding/json" + "strings" + "syscall" + "testing" + "time" +) + +// backendCleanExitRetireTimeout bounds how long the proxy may outlive a dolt +// backend that shut down gracefully. dolt's own SIGTERM shutdown dominates it; +// the proxy's health watcher polls every 100ms. +const backendCleanExitRetireTimeout = 30 * time.Second + +// TestManagedLocalProxiedBackendCleanExitRetiresProxy extends the child-death +// contract (see TestManagedLocalProxiedOutageReconnectContract, which kills +// the backend) to a backend that exits CLEANLY: SIGTERM makes dolt shut down +// gracefully with status 0. The proxy must retire just as it does for a +// crash, so the next bd command starts a fresh proxy/backend pair and `bd +// ping` answers from a live store. +// +// Before the fix the proxy only noticed non-zero exits. With idle timeout 0 +// (never idle out) it stayed up and adoptable in front of a dead backend +// indefinitely: `bd dolt status` said "running", and every command, `bd ping` +// included, failed until an operator ran `bd dolt stop`. +func TestManagedLocalProxiedBackendCleanExitRetiresProxy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdProxiedInit(t, bd, "clean_exit", "--proxied-server-idle-timeout", "0") + sentinel := bdProxiedCreate(t, bd, p.dir, "clean exit sentinel") + + proxyBefore := readManagedProxyPidFile(t, p) + if proxyBefore == nil || !processAlive(proxyBefore.Pid) { + t.Fatalf("managed-local proxy is not running: %+v", proxyBefore) + } + backendBefore := readManagedBackendPidFile(t, p) + if backendBefore == nil || !processAlive(backendBefore.Pid) { + t.Fatalf("managed-local backend is not running: %+v", backendBefore) + } + + if err := syscall.Kill(backendBefore.Pid, syscall.SIGTERM); err != nil { + t.Fatalf("SIGTERM dolt backend pid %d: %v", backendBefore.Pid, err) + } + waitForManagedProxiedShutdown(t, p, proxyBefore.Pid, backendBefore.Pid, backendCleanExitRetireTimeout) + + out, err := bdProxiedRun(t, bd, p.dir, "ping", "--json") + if err != nil { + t.Fatalf("bd ping after the backend exited: %v\n%s", err, out) + } + var ping struct { + Status string `json:"status"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ping); err != nil || ping.Status != "ok" { + t.Fatalf("bd ping --json: status=%q err=%v\n%s", ping.Status, err, out) + } + if got := bdProxiedShow(t, bd, p.dir, sentinel.ID); got.Title != "clean exit sentinel" { + t.Fatalf("sentinel after restart: %+v", got) + } + + proxyAfter := readManagedProxyPidFile(t, p) + backendAfter := readManagedBackendPidFile(t, p) + if proxyAfter == nil || proxyAfter.Pid == proxyBefore.Pid || !processAlive(proxyAfter.Pid) { + t.Fatalf("expected a fresh live proxy: before=%+v after=%+v", proxyBefore, proxyAfter) + } + if backendAfter == nil || backendAfter.Pid == backendBefore.Pid || !processAlive(backendAfter.Pid) { + t.Fatalf("expected a fresh live backend: before=%+v after=%+v", backendBefore, backendAfter) + } +} diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index d91c09a29..4b5c883ef 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -29,6 +29,7 @@ go_library( go_test( name = "server_test", srcs = [ + "doltserver_exit_unix_test.go", "doltserver_test.go", "external_dolt_server_test.go", "testmain_test.go", diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index 52ea71a23..0cd1a8a4b 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -36,6 +36,13 @@ const ( LockFileName = "proxy-child.lock" ) +// errBackendExited is the result of the supervising goroutine when the dolt +// sql-server exits on its own with status 0 (for example dolt's graceful +// shutdown on SIGTERM). errgroup cancels egCtx only for a non-nil result, and +// Running reads egCtx, so without it a cleanly exited backend would be +// reported as running forever. +var errBackendExited = errors.New("dolt sql-server exited") + const ( startReadyTimeout = 30 * time.Second startReadyPollInterval = 50 * time.Millisecond @@ -303,7 +310,10 @@ func (s *DoltServer) Start(ctx context.Context) error { eg.Go(func() error { defer lock.Unlock() - return cmd.Wait() + if err := cmd.Wait(); err != nil { + return err + } + return errBackendExited }) if err := s.waitReady(ctx); err != nil { @@ -358,7 +368,7 @@ func (s *DoltServer) Stop(ctx context.Context) error { if s.eg != nil { waitErr = s.eg.Wait() var exitErr *exec.ExitError - if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) { + if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) || errors.Is(waitErr, errBackendExited) { waitErr = nil } } diff --git a/internal/storage/dbproxy/server/doltserver_exit_unix_test.go b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go new file mode 100644 index 000000000..0dcc84fc9 --- /dev/null +++ b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go @@ -0,0 +1,64 @@ +//go:build unix + +package server_test + +import ( + "context" + "syscall" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// backendExitNoticeTimeout bounds how long Running may keep reporting a +// backend that has left the process table. dolt's own graceful shutdown on +// SIGTERM dominates this; the detection itself is immediate. +const ( + backendExitNoticeTimeout = 30 * time.Second + backendExitNoticePoll = 50 * time.Millisecond +) + +// TestDoltServer_RunningFalseAfterBackendExits verifies Running observes the +// dolt sql-server leaving on its own, whatever its exit status. A SIGTERM'd +// dolt shuts down gracefully and exits 0; before the fix only a non-zero exit +// flipped Running, so the proxy's backend health watcher never fired and the +// proxy stayed up (and adoptable) in front of a dead backend. +func TestDoltServer_RunningFalseAfterBackendExits(t *testing.T) { + for _, tc := range []struct { + name string + sig syscall.Signal + }{ + {name: "clean exit on SIGTERM", sig: syscall.SIGTERM}, + {name: "killed", sig: syscall.SIGKILL}, + } { + t.Run(tc.name, func(t *testing.T) { + s, rootDir := newDoltServer(t) + ctx := context.Background() + require.NoError(t, s.Start(ctx)) + require.True(t, s.Running(ctx)) + + pf, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + require.NotNil(t, pf) + require.NoError(t, syscall.Kill(pf.Pid, tc.sig)) + + deadline := time.Now().Add(backendExitNoticeTimeout) + for s.Running(ctx) && time.Now().Before(deadline) { + time.Sleep(backendExitNoticePoll) + } + require.False(t, s.Running(ctx), "Running still true %s after the dolt sql-server (pid %d) got %s", backendExitNoticeTimeout, pf.Pid, tc.sig) + + // A backend that exited on its own is not a Stop failure. + stopCtx, cancel := context.WithTimeout(ctx, stopTimeout) + defer cancel() + require.NoError(t, s.Stop(stopCtx)) + left, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + assert.Nil(t, left, "Stop must remove the backend pid record") + }) + } +}