diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index c1170031d..0aafb5bed 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -3,13 +3,16 @@ package uow import ( "context" "database/sql" + "database/sql/driver" "errors" "fmt" + "io" + "net" "sync/atomic" "time" "github.com/cenkalti/backoff/v4" - _ "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" @@ -378,12 +381,99 @@ func buildDSN(ep proxy.Endpoint, database, user, password, tlsConfigName string) }.String() } +// pinger is the subset of *sql.DB that pingWithRetry needs, narrowed so +// tests can script a sequence of transient failures without a real +// connection. +type pinger interface { + PingContext(ctx context.Context) error +} + +// pingAttemptTimeout bounds a single ping attempt. The DSN sets a dial +// Timeout (5s, internal/storage/dbproxy/util/dsn.go) but no ReadTimeout, so a +// server that accepts the TCP connection and then stalls mid-handshake is +// otherwise bounded only by the caller's context. Kept above the dial timeout +// so a genuine dial timeout still surfaces as itself instead of being masked +// by this cap. +const pingAttemptTimeout = 10 * time.Second + +// isTransientPingError reports whether err is a connection-level failure worth +// retrying — the shapes a Dolt server produces when it drops or stalls a +// connection mid-handshake or mid-ping — as opposed to a durable rejection +// (bad credentials, unknown database) that retrying cannot fix. +// +// Any *net.OpError counts. Before the handshake completes there is no +// application-level rejection that can reach us as one, so at boot an OpError +// is always the connection itself: refused while the server restarts, reset, +// or broken pipe. MySQL's own rejections arrive as *mysql.MySQLError and stay +// permanent. A name that does not resolve is the exception — it will not start +// resolving, so retrying only spends the whole budget before reporting the +// same misconfiguration. +// +// context.DeadlineExceeded counts only because pingWithRetry applies a +// per-attempt deadline and checks the caller's context *before* consulting +// this function, so a DeadlineExceeded that reaches here can only be that +// per-attempt cap. Do not call this without that guard: classifying the +// caller's own expiry as transient would retry a context that is already dead, +// which is the regression #6000 introduced from the other direction. +// +// Deliberately narrower than internal/storage/dolt's isRetryableError, which +// covers this same ground for query-time retries in server mode. That one +// matches on error text and additionally admits migration-lock, Dolt +// read-only, and MySQL-1105 shapes — server states that describe a *statement* +// being refused, which a boot ping cannot be in because nothing has executed on +// the connection yet. Matching sentinels and types with errors.Is/errors.As +// keeps this side typed, and keeps a bootstrap ping from inheriting retry +// semantics that only make sense mid-transaction. +func isTransientPingError(err error) bool { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && !dnsErr.IsTemporary && !dnsErr.IsTimeout { + return false + } + var opErr *net.OpError + return errors.Is(err, driver.ErrBadConn) || + errors.Is(err, mysql.ErrInvalidConn) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, context.DeadlineExceeded) || + errors.As(err, &opErr) +} + +// pingWithRetry pings until the server answers, the error proves durable, or +// bo's MaxElapsedTime is spent. Each attempt gets its own deadline: +// backoff.Retry bounds the gap between attempts but cannot interrupt one +// already in flight, so without this cap a server that accepts TCP and then +// stalls blocks for the caller's entire context. +func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff, attemptTimeout time.Duration) error { + return backoff.Retry(func() error { + attemptCtx, cancel := context.WithTimeout(ctx, attemptTimeout) + defer cancel() + + err := p.PingContext(attemptCtx) + if err == nil { + return nil + } + // The caller's context is spent, so nothing further can succeed and + // its Canceled/DeadlineExceeded must not be read as a transient + // per-attempt timeout. This check is what lets isTransientPingError + // treat DeadlineExceeded as retryable at all. + if ctx.Err() != nil { + return backoff.Permanent(err) + } + if isTransientPingError(err) { + return err + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) +} + func openDB(ctx context.Context, dsn string) (*sql.DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("uow: open db: %w", err) } - if err := conn.PingContext(ctx); err != nil { + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 30 * time.Second + if err := pingWithRetry(ctx, conn, bo, pingAttemptTimeout); err != nil { return nil, errors.Join(fmt.Errorf("uow: ping db: %w", err), conn.Close()) } return conn, nil diff --git a/internal/storage/uow/dolt_sql_provider_test.go b/internal/storage/uow/dolt_sql_provider_test.go new file mode 100644 index 000000000..4a6f84ed1 --- /dev/null +++ b/internal/storage/uow/dolt_sql_provider_test.go @@ -0,0 +1,216 @@ +package uow + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/go-sql-driver/mysql" +) + +// fakePinger is a pinger test double that returns a scripted sequence of +// responses and fails the test if called more times than scripted — that +// over-call case is exactly the retry-budget bug this test suite guards +// against. +type fakePinger struct { + t *testing.T + errs []error + calls int +} + +func (p *fakePinger) PingContext(context.Context) error { + p.t.Helper() + if p.calls >= len(p.errs) { + p.t.Fatalf("PingContext called more times than expected (call #%d, only %d responses scripted)", p.calls+1, len(p.errs)) + } + err := p.errs[p.calls] + p.calls++ + return err +} + +// blockingPinger never answers on its own: it stalls until the context it was +// handed is cancelled, then reports that context's error. This is the shape of +// a Dolt server that accepts the TCP connection and then stalls without ever +// completing the MySQL handshake — the DSN sets a dial Timeout but no +// ReadTimeout, so nothing below pingWithRetry bounds it. +type blockingPinger struct { + calls atomic.Int32 + release chan struct{} +} + +func newBlockingPinger(t *testing.T) *blockingPinger { + t.Helper() + p := &blockingPinger{release: make(chan struct{})} + // Unblock any attempt still parked when the test ends, so a failure here + // does not leak a goroutine into the rest of the package's tests. + t.Cleanup(func() { close(p.release) }) + return p +} + +func (p *blockingPinger) PingContext(ctx context.Context) error { + p.calls.Add(1) + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.release: + return errors.New("blockingPinger released at test cleanup") + } +} + +func testPingBackOff() *backoff.ExponentialBackOff { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = time.Second + return bo +} + +// testPingAttemptTimeout is generous: the subtests that use it script their +// ping responses and never actually stall, so the per-attempt cap must not be +// what ends them. +const testPingAttemptTimeout = time.Second + +func TestPingWithRetryRecoversFromATransientBadConn(t *testing.T) { + t.Run("transient error retries then recovers", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn, mysql.ErrInvalidConn, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil", err) + } + if p.calls != 3 { + t.Fatalf("PingContext called %d times, want exactly 3", p.calls) + } + }) + + t.Run("non-transient error fails without retrying", func(t *testing.T) { + wantErr := errors.New("Access denied") + p := &fakePinger{t: t, errs: []error{wantErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, wantErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, wantErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (non-transient errors must not retry)", p.calls) + } + }) + + t.Run("already-cancelled context fails promptly", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := pingWithRetry(ctx, p, testPingBackOff(), testPingAttemptTimeout) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil for an already-cancelled context") + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (cancelled context must not retry)", p.calls) + } + }) + + t.Run("connection-level net error retries then recovers", func(t *testing.T) { + // A boot ping cannot fail at the application level before the + // handshake completes, so any *net.OpError here is connection-level: + // connection refused (server restarting), reset, or broken pipe. + refused := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")} + p := &fakePinger{t: t, errs: []error{refused, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil (a connection-level net error is transient at boot)", err) + } + if p.calls != 2 { + t.Fatalf("PingContext called %d times, want exactly 2", p.calls) + } + }) + + t.Run("unresolvable host fails without retrying", func(t *testing.T) { + // The narrowing that pays for classifying *net.OpError broadly: a name + // that does not resolve will not start resolving, so retrying only + // spends the budget before reporting the same misconfiguration. + dnsErr := &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{ + Err: "no such host", Name: "no-such-dolt-host.invalid", IsNotFound: true, + }} + p := &fakePinger{t: t, errs: []error{dnsErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, dnsErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, dnsErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (a permanent DNS failure must not retry)", p.calls) + } + }) +} + +// TestPingWithRetryBoundsEachAttempt covers the half of the retry budget that +// MaxElapsedTime cannot reach: backoff.Retry bounds the gap *between* +// attempts, but it cannot interrupt an attempt already in flight. +func TestPingWithRetryBoundsEachAttempt(t *testing.T) { + t.Run("a hung ping is capped per attempt and retried", func(t *testing.T) { + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 200 * time.Millisecond + + // Run off-goroutine: before the per-attempt cap exists this call never + // returns at all, and the assertion we want is "it returned", not a + // 10-minute package timeout. + done := make(chan error, 1) + go func() { + done <- pingWithRetry(context.Background(), p, bo, 20*time.Millisecond) + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the retry budget is exhausted") + } + if got := p.calls.Load(); got < 2 { + t.Fatalf("PingContext called %d times, want >= 2 (each hung attempt must be capped and retried)", got) + } + case <-time.After(5 * time.Second): + t.Fatal("pingWithRetry() never returned: a hung ping is not bounded per attempt, so it blocks for the caller's whole context") + } + }) + + t.Run("the caller's own deadline fails without retrying", func(t *testing.T) { + // The regression guard for the trap #6000 fell into: once a + // per-attempt deadline exists, context.DeadlineExceeded becomes + // retryable — but only when it came from *that* deadline. The + // caller's own expiry must still stop immediately. + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 10 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + start := time.Now() + // Per-attempt cap far beyond the caller's deadline, so the only thing + // that can end this attempt is the caller's context. + err := pingWithRetry(ctx, p, bo, time.Hour) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the caller's context expires") + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (the caller's own deadline must not be retried as transient)", got) + } + if elapsed > 5*time.Second { + t.Fatalf("pingWithRetry() took %v after a 20ms caller deadline, want it to stop promptly", elapsed) + } + }) +}