From c0d8da42de5fd15c95adac85e342ba4a121da0fb Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 2 Sep 2026 21:15:27 -0700 Subject: [PATCH] fix(storage/uow): retry transient ping failures during openDB bootstrap (#6003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(uow): red — pingWithRetry does not exist yet for openDB's boot-time ping Adds TestPingWithRetryRecoversFromATransientBadConn (be-x49l6): a fake pinger exercising three cases — transient error retries then recovers, non-transient error fails without retrying, and an already-cancelled context fails promptly. Fails to compile until pingWithRetry/pinger are added to dolt_sql_provider.go. * feat: green — Fix: unretried ping in uow openDB turns a transient Dolt connection drop into a hard CI failure (refs be-x49l6) * refactor: drop what-comment on pingWithRetry (self-review, refs be-x49l6) Self-review flagged it as restating the function's own signature/behavior rather than explaining a non-obvious why; the pinger and isTransientPingError comments stay since they carry rationale the code doesn't otherwise convey. * test(uow): red — a hung boot ping is not bounded per attempt Adds the two behaviours review #6003 asked for, both failing: - a_hung_ping_is_capped_per_attempt_and_retried: blockingPinger stalls until its context is cancelled. backoff.Retry bounds only the gap between attempts, so with the caller's context handed straight to PingContext the call never returns at all. The subtest runs the call off-goroutine so this reports as "never returned" instead of a package timeout. - connection-level_net_error_retries_then_recovers: a *net.OpError (connection refused / reset / broken pipe) is currently classified non-transient and fails the boot ping outright. the_caller's_own_deadline_fails_without_retrying passes already and is kept as the regression guard for the trap #6000 fell into: making context.DeadlineExceeded retryable must not make the caller's own expiry retryable too. pingWithRetry takes attemptTimeout here but does not honour it yet, so the red is behavioural rather than a compile error. Refs be-g1i7n. * fix(uow): bound each boot ping attempt and widen the transient classifier Review #6003, point 1. backoff.Retry bounds the gap between attempts but cannot interrupt one in flight, and the DSN sets a dial Timeout (5s) with no ReadTimeout — so a server that accepts TCP and then stalls mid-handshake blocked for the caller's entire context, which is the exact scenario the PR summary claims to fix. Each attempt now runs under its own context.WithTimeout (pingAttemptTimeout, 10s — above the 5s dial timeout so a real dial timeout still surfaces as itself), and context.DeadlineExceeded is classified transient so the retry loop actually continues. pingWithRetry checks the caller's context *before* classifying. That is what keeps the retryable DeadlineExceeded from swallowing the caller's own expiry — the inverse of the regression #6000 introduced by capping each attempt and then treating the resulting DeadlineExceeded as permanent. Covered from both sides: a_hung_ping_is_capped_per_attempt_and_retried and the_caller's_own_deadline_fails_without_retrying. Review #6003, point 2. isTransientPingError stays separate from internal/storage/dolt's isRetryableError, and now says why in full: that one is a query-time classifier matching on error text, admitting migration-lock, read-only and MySQL-1105 shapes a boot ping cannot be in because nothing has executed on the connection yet. The coverage gap the review named is closed on this side instead — *net.OpError is transient, which is refused/reset/broken-pipe portably and by type rather than by substring, with permanent DNS failures excluded so an unresolvable host still fails fast instead of spending the whole budget. Refs be-g1i7n. --- internal/storage/uow/dolt_sql_provider.go | 94 +++++++- .../storage/uow/dolt_sql_provider_test.go | 216 ++++++++++++++++++ 2 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 internal/storage/uow/dolt_sql_provider_test.go diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index c1170031d..0aafb5bed 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -3,13 +3,16 @@ package uow import ( "context" "database/sql" + "database/sql/driver" "errors" "fmt" + "io" + "net" "sync/atomic" "time" "github.com/cenkalti/backoff/v4" - _ "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" @@ -378,12 +381,99 @@ func buildDSN(ep proxy.Endpoint, database, user, password, tlsConfigName string) }.String() } +// pinger is the subset of *sql.DB that pingWithRetry needs, narrowed so +// tests can script a sequence of transient failures without a real +// connection. +type pinger interface { + PingContext(ctx context.Context) error +} + +// pingAttemptTimeout bounds a single ping attempt. The DSN sets a dial +// Timeout (5s, internal/storage/dbproxy/util/dsn.go) but no ReadTimeout, so a +// server that accepts the TCP connection and then stalls mid-handshake is +// otherwise bounded only by the caller's context. Kept above the dial timeout +// so a genuine dial timeout still surfaces as itself instead of being masked +// by this cap. +const pingAttemptTimeout = 10 * time.Second + +// isTransientPingError reports whether err is a connection-level failure worth +// retrying — the shapes a Dolt server produces when it drops or stalls a +// connection mid-handshake or mid-ping — as opposed to a durable rejection +// (bad credentials, unknown database) that retrying cannot fix. +// +// Any *net.OpError counts. Before the handshake completes there is no +// application-level rejection that can reach us as one, so at boot an OpError +// is always the connection itself: refused while the server restarts, reset, +// or broken pipe. MySQL's own rejections arrive as *mysql.MySQLError and stay +// permanent. A name that does not resolve is the exception — it will not start +// resolving, so retrying only spends the whole budget before reporting the +// same misconfiguration. +// +// context.DeadlineExceeded counts only because pingWithRetry applies a +// per-attempt deadline and checks the caller's context *before* consulting +// this function, so a DeadlineExceeded that reaches here can only be that +// per-attempt cap. Do not call this without that guard: classifying the +// caller's own expiry as transient would retry a context that is already dead, +// which is the regression #6000 introduced from the other direction. +// +// Deliberately narrower than internal/storage/dolt's isRetryableError, which +// covers this same ground for query-time retries in server mode. That one +// matches on error text and additionally admits migration-lock, Dolt +// read-only, and MySQL-1105 shapes — server states that describe a *statement* +// being refused, which a boot ping cannot be in because nothing has executed on +// the connection yet. Matching sentinels and types with errors.Is/errors.As +// keeps this side typed, and keeps a bootstrap ping from inheriting retry +// semantics that only make sense mid-transaction. +func isTransientPingError(err error) bool { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && !dnsErr.IsTemporary && !dnsErr.IsTimeout { + return false + } + var opErr *net.OpError + return errors.Is(err, driver.ErrBadConn) || + errors.Is(err, mysql.ErrInvalidConn) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, context.DeadlineExceeded) || + errors.As(err, &opErr) +} + +// pingWithRetry pings until the server answers, the error proves durable, or +// bo's MaxElapsedTime is spent. Each attempt gets its own deadline: +// backoff.Retry bounds the gap between attempts but cannot interrupt one +// already in flight, so without this cap a server that accepts TCP and then +// stalls blocks for the caller's entire context. +func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff, attemptTimeout time.Duration) error { + return backoff.Retry(func() error { + attemptCtx, cancel := context.WithTimeout(ctx, attemptTimeout) + defer cancel() + + err := p.PingContext(attemptCtx) + if err == nil { + return nil + } + // The caller's context is spent, so nothing further can succeed and + // its Canceled/DeadlineExceeded must not be read as a transient + // per-attempt timeout. This check is what lets isTransientPingError + // treat DeadlineExceeded as retryable at all. + if ctx.Err() != nil { + return backoff.Permanent(err) + } + if isTransientPingError(err) { + return err + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) +} + func openDB(ctx context.Context, dsn string) (*sql.DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("uow: open db: %w", err) } - if err := conn.PingContext(ctx); err != nil { + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 30 * time.Second + if err := pingWithRetry(ctx, conn, bo, pingAttemptTimeout); err != nil { return nil, errors.Join(fmt.Errorf("uow: ping db: %w", err), conn.Close()) } return conn, nil diff --git a/internal/storage/uow/dolt_sql_provider_test.go b/internal/storage/uow/dolt_sql_provider_test.go new file mode 100644 index 000000000..4a6f84ed1 --- /dev/null +++ b/internal/storage/uow/dolt_sql_provider_test.go @@ -0,0 +1,216 @@ +package uow + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/go-sql-driver/mysql" +) + +// fakePinger is a pinger test double that returns a scripted sequence of +// responses and fails the test if called more times than scripted — that +// over-call case is exactly the retry-budget bug this test suite guards +// against. +type fakePinger struct { + t *testing.T + errs []error + calls int +} + +func (p *fakePinger) PingContext(context.Context) error { + p.t.Helper() + if p.calls >= len(p.errs) { + p.t.Fatalf("PingContext called more times than expected (call #%d, only %d responses scripted)", p.calls+1, len(p.errs)) + } + err := p.errs[p.calls] + p.calls++ + return err +} + +// blockingPinger never answers on its own: it stalls until the context it was +// handed is cancelled, then reports that context's error. This is the shape of +// a Dolt server that accepts the TCP connection and then stalls without ever +// completing the MySQL handshake — the DSN sets a dial Timeout but no +// ReadTimeout, so nothing below pingWithRetry bounds it. +type blockingPinger struct { + calls atomic.Int32 + release chan struct{} +} + +func newBlockingPinger(t *testing.T) *blockingPinger { + t.Helper() + p := &blockingPinger{release: make(chan struct{})} + // Unblock any attempt still parked when the test ends, so a failure here + // does not leak a goroutine into the rest of the package's tests. + t.Cleanup(func() { close(p.release) }) + return p +} + +func (p *blockingPinger) PingContext(ctx context.Context) error { + p.calls.Add(1) + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.release: + return errors.New("blockingPinger released at test cleanup") + } +} + +func testPingBackOff() *backoff.ExponentialBackOff { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = time.Second + return bo +} + +// testPingAttemptTimeout is generous: the subtests that use it script their +// ping responses and never actually stall, so the per-attempt cap must not be +// what ends them. +const testPingAttemptTimeout = time.Second + +func TestPingWithRetryRecoversFromATransientBadConn(t *testing.T) { + t.Run("transient error retries then recovers", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn, mysql.ErrInvalidConn, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil", err) + } + if p.calls != 3 { + t.Fatalf("PingContext called %d times, want exactly 3", p.calls) + } + }) + + t.Run("non-transient error fails without retrying", func(t *testing.T) { + wantErr := errors.New("Access denied") + p := &fakePinger{t: t, errs: []error{wantErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, wantErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, wantErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (non-transient errors must not retry)", p.calls) + } + }) + + t.Run("already-cancelled context fails promptly", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := pingWithRetry(ctx, p, testPingBackOff(), testPingAttemptTimeout) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil for an already-cancelled context") + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (cancelled context must not retry)", p.calls) + } + }) + + t.Run("connection-level net error retries then recovers", func(t *testing.T) { + // A boot ping cannot fail at the application level before the + // handshake completes, so any *net.OpError here is connection-level: + // connection refused (server restarting), reset, or broken pipe. + refused := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")} + p := &fakePinger{t: t, errs: []error{refused, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil (a connection-level net error is transient at boot)", err) + } + if p.calls != 2 { + t.Fatalf("PingContext called %d times, want exactly 2", p.calls) + } + }) + + t.Run("unresolvable host fails without retrying", func(t *testing.T) { + // The narrowing that pays for classifying *net.OpError broadly: a name + // that does not resolve will not start resolving, so retrying only + // spends the budget before reporting the same misconfiguration. + dnsErr := &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{ + Err: "no such host", Name: "no-such-dolt-host.invalid", IsNotFound: true, + }} + p := &fakePinger{t: t, errs: []error{dnsErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, dnsErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, dnsErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (a permanent DNS failure must not retry)", p.calls) + } + }) +} + +// TestPingWithRetryBoundsEachAttempt covers the half of the retry budget that +// MaxElapsedTime cannot reach: backoff.Retry bounds the gap *between* +// attempts, but it cannot interrupt an attempt already in flight. +func TestPingWithRetryBoundsEachAttempt(t *testing.T) { + t.Run("a hung ping is capped per attempt and retried", func(t *testing.T) { + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 200 * time.Millisecond + + // Run off-goroutine: before the per-attempt cap exists this call never + // returns at all, and the assertion we want is "it returned", not a + // 10-minute package timeout. + done := make(chan error, 1) + go func() { + done <- pingWithRetry(context.Background(), p, bo, 20*time.Millisecond) + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the retry budget is exhausted") + } + if got := p.calls.Load(); got < 2 { + t.Fatalf("PingContext called %d times, want >= 2 (each hung attempt must be capped and retried)", got) + } + case <-time.After(5 * time.Second): + t.Fatal("pingWithRetry() never returned: a hung ping is not bounded per attempt, so it blocks for the caller's whole context") + } + }) + + t.Run("the caller's own deadline fails without retrying", func(t *testing.T) { + // The regression guard for the trap #6000 fell into: once a + // per-attempt deadline exists, context.DeadlineExceeded becomes + // retryable — but only when it came from *that* deadline. The + // caller's own expiry must still stop immediately. + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 10 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + start := time.Now() + // Per-attempt cap far beyond the caller's deadline, so the only thing + // that can end this attempt is the caller's context. + err := pingWithRetry(ctx, p, bo, time.Hour) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the caller's context expires") + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (the caller's own deadline must not be retried as transient)", got) + } + if elapsed > 5*time.Second { + t.Fatalf("pingWithRetry() took %v after a 20ms caller deadline, want it to stop promptly", elapsed) + } + }) +}