From ee7f5c5053ab5888944ab59590f0057a3b9834f1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 00:28:39 +0000 Subject: [PATCH] fix(dolt): cap withRetryTx's wait so a losing writer keeps retrying withRetryTx replays a write that lost Dolt's commit-time merge on an exponential schedule: waits from 25ms, multiplier 1.5, jitter of half a wait either way, within 15s on a sql-server (5s embedded). The schedule set no MaxInterval, so it used the library default of 60s, and the waits grew to 2-5s each before the budget ran out. A writer that keeps losing conflicts got at most 16 attempts in the server budget (14-15 typically). That is routine when every mutation bumps one shared row, as it does with the events journal on and several concurrent writers. newWriteTxRetryBackOff now builds the schedule, and caps each wait at writeTxRetryMaxInterval (1s) before jitter. The budget and the first waits are unchanged; the later attempts keep arriving at about one a second. On the schedule alone, with attempts taking no time, the 15s budget now yields 19-29 attempts (median 23) instead of 14-16 (median 15), measured over 200,000 jittered runs; the worst case, every wait jittered to its longest, yields 18. The embedded budget goes from 11-13 to 11-16. Test: TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget drives newWriteTxRetryBackOff with a stepping clock (no sleeps), 50 jittered runs per mode. It checks that no wait exceeds the jittered cap (1.5s), that the schedule stays within its budget, and that a writer gets at least 18 attempts in 15s on a server and 11 in 5s embedded. With the schedule extracted but uncapped it fails ("wait 2.631086165s exceeds the jittered cap of 1.5s"); with only the cap check disabled, it reports "a writer that keeps losing gets 15 attempts in 15s, want at least 18". It passes with the cap. The package's Retry|Circuit|Serialization|Indeterminate| Concurrent|WriteTx tests pass against a dolt sql-server 2.2.0 test container. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/store.go | 27 +++++++-- .../storage/dolt/write_retry_backoff_test.go | 60 +++++++++++++++++++ 3 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 internal/storage/dolt/write_retry_backoff_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2be809cc9..4352d8adc 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -378,6 +378,7 @@ go_test( "wisp_validation_test.go", "wisps_cycle_test.go", "workspace_config_contract_test.go", + "write_retry_backoff_test.go", ], # Source-scan guards (journal/version scope completeness, helper sentinel # census) parse this package and ../issueops at test time; the PR4107 and @@ -417,6 +418,7 @@ go_test( "//internal/utils", "//internal/workapi", "//issueops", + "@com_github_cenkalti_backoff_v4//:backoff", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index e04352b54..656b8cc14 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1231,12 +1231,7 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f return ErrCircuitOpen } - bo := backoff.NewExponentialBackOff() - bo.InitialInterval = 25 * time.Millisecond - bo.MaxElapsedTime = 5 * time.Second - if s.serverMode { - bo.MaxElapsedTime = 15 * time.Second - } + bo := newWriteTxRetryBackOff(s.serverMode) var pending issueops.BlockedRecheck if err := backoff.Retry(func() error { var err error @@ -1290,6 +1285,26 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f return nil } +// writeTxRetryMaxInterval caps each wait between withRetryTx attempts (before +// jitter). Uncapped, the exponential waits stretch to several seconds each, so +// a writer that keeps losing commit-time conflicts gets at most 16 attempts in +// the server budget; capped, the attempts keep arriving through all of it. +const writeTxRetryMaxInterval = time.Second + +// newWriteTxRetryBackOff is withRetryTx's schedule: exponential waits from +// 25ms, capped at writeTxRetryMaxInterval and randomized by half in either +// direction, within a 15s budget on a sql-server (5s embedded). +func newWriteTxRetryBackOff(serverMode bool) *backoff.ExponentialBackOff { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 25 * time.Millisecond + bo.MaxInterval = writeTxRetryMaxInterval + bo.MaxElapsedTime = 5 * time.Second + if serverMode { + bo.MaxElapsedTime = 15 * time.Second + } + return bo +} + func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) error { pending, err := s.commitWriteTx(ctx, fn) if err != nil { diff --git a/internal/storage/dolt/write_retry_backoff_test.go b/internal/storage/dolt/write_retry_backoff_test.go new file mode 100644 index 000000000..ec667b3ff --- /dev/null +++ b/internal/storage/dolt/write_retry_backoff_test.go @@ -0,0 +1,60 @@ +package dolt + +import ( + "testing" + "time" + + "github.com/cenkalti/backoff/v4" +) + +// TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget pins the schedule +// withRetryTx replays a write on. A writer that keeps losing commit-time +// conflicts (routine when every mutation also bumps one shared counter row, as +// with the events journal on) must keep getting attempts through its whole +// budget. Uncapped, the exponential waits grow to several seconds each and the +// 15s server budget holds at most 16 attempts; capped at a second, it holds at +// least 18 even with every wait jittered to its longest. +func TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget(t *testing.T) { + for _, tc := range []struct { + name string + serverMode bool + budget time.Duration + minTries int + }{ + {name: "server", serverMode: true, budget: 15 * time.Second, minTries: 18}, + {name: "embedded", serverMode: false, budget: 5 * time.Second, minTries: 11}, + } { + t.Run(tc.name, func(t *testing.T) { + for run := 0; run < 50; run++ { + clock := &steppingClock{now: time.Unix(0, 0)} + bo := newWriteTxRetryBackOff(tc.serverMode) + bo.Clock = clock + bo.Reset() + tries := 1 + for { + next := bo.NextBackOff() + if next == backoff.Stop { + break + } + if longest := writeTxRetryMaxInterval * 3 / 2; next > longest { + t.Fatalf("wait %v exceeds the jittered cap of %v", next, longest) + } + clock.now = clock.now.Add(next) + tries++ + } + if tries < tc.minTries { + t.Fatalf("a writer that keeps losing gets %d attempts in %v, want at least %d", tries, tc.budget, tc.minTries) + } + if elapsed := clock.now.Sub(time.Unix(0, 0)); elapsed > tc.budget { + t.Fatalf("schedule waited %v, past its %v budget", elapsed, tc.budget) + } + } + }) + } +} + +// steppingClock is a backoff.Clock the test advances by each wait the +// schedule asks for, so the schedule runs without sleeping. +type steppingClock struct{ now time.Time } + +func (c *steppingClock) Now() time.Time { return c.now }