diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2be809cc9..4352d8adc 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -378,6 +378,7 @@ go_test( "wisp_validation_test.go", "wisps_cycle_test.go", "workspace_config_contract_test.go", + "write_retry_backoff_test.go", ], # Source-scan guards (journal/version scope completeness, helper sentinel # census) parse this package and ../issueops at test time; the PR4107 and @@ -417,6 +418,7 @@ go_test( "//internal/utils", "//internal/workapi", "//issueops", + "@com_github_cenkalti_backoff_v4//:backoff", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index e04352b54..656b8cc14 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1231,12 +1231,7 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f return ErrCircuitOpen } - bo := backoff.NewExponentialBackOff() - bo.InitialInterval = 25 * time.Millisecond - bo.MaxElapsedTime = 5 * time.Second - if s.serverMode { - bo.MaxElapsedTime = 15 * time.Second - } + bo := newWriteTxRetryBackOff(s.serverMode) var pending issueops.BlockedRecheck if err := backoff.Retry(func() error { var err error @@ -1290,6 +1285,26 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f return nil } +// writeTxRetryMaxInterval caps each wait between withRetryTx attempts (before +// jitter). Uncapped, the exponential waits stretch to several seconds each, so +// a writer that keeps losing commit-time conflicts gets at most 16 attempts in +// the server budget; capped, the attempts keep arriving through all of it. +const writeTxRetryMaxInterval = time.Second + +// newWriteTxRetryBackOff is withRetryTx's schedule: exponential waits from +// 25ms, capped at writeTxRetryMaxInterval and randomized by half in either +// direction, within a 15s budget on a sql-server (5s embedded). +func newWriteTxRetryBackOff(serverMode bool) *backoff.ExponentialBackOff { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 25 * time.Millisecond + bo.MaxInterval = writeTxRetryMaxInterval + bo.MaxElapsedTime = 5 * time.Second + if serverMode { + bo.MaxElapsedTime = 15 * time.Second + } + return bo +} + func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) error { pending, err := s.commitWriteTx(ctx, fn) if err != nil { diff --git a/internal/storage/dolt/write_retry_backoff_test.go b/internal/storage/dolt/write_retry_backoff_test.go new file mode 100644 index 000000000..ec667b3ff --- /dev/null +++ b/internal/storage/dolt/write_retry_backoff_test.go @@ -0,0 +1,60 @@ +package dolt + +import ( + "testing" + "time" + + "github.com/cenkalti/backoff/v4" +) + +// TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget pins the schedule +// withRetryTx replays a write on. A writer that keeps losing commit-time +// conflicts (routine when every mutation also bumps one shared counter row, as +// with the events journal on) must keep getting attempts through its whole +// budget. Uncapped, the exponential waits grow to several seconds each and the +// 15s server budget holds at most 16 attempts; capped at a second, it holds at +// least 18 even with every wait jittered to its longest. +func TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget(t *testing.T) { + for _, tc := range []struct { + name string + serverMode bool + budget time.Duration + minTries int + }{ + {name: "server", serverMode: true, budget: 15 * time.Second, minTries: 18}, + {name: "embedded", serverMode: false, budget: 5 * time.Second, minTries: 11}, + } { + t.Run(tc.name, func(t *testing.T) { + for run := 0; run < 50; run++ { + clock := &steppingClock{now: time.Unix(0, 0)} + bo := newWriteTxRetryBackOff(tc.serverMode) + bo.Clock = clock + bo.Reset() + tries := 1 + for { + next := bo.NextBackOff() + if next == backoff.Stop { + break + } + if longest := writeTxRetryMaxInterval * 3 / 2; next > longest { + t.Fatalf("wait %v exceeds the jittered cap of %v", next, longest) + } + clock.now = clock.now.Add(next) + tries++ + } + if tries < tc.minTries { + t.Fatalf("a writer that keeps losing gets %d attempts in %v, want at least %d", tries, tc.budget, tc.minTries) + } + if elapsed := clock.now.Sub(time.Unix(0, 0)); elapsed > tc.budget { + t.Fatalf("schedule waited %v, past its %v budget", elapsed, tc.budget) + } + } + }) + } +} + +// steppingClock is a backoff.Clock the test advances by each wait the +// schedule asks for, so the schedule runs without sleeping. +type steppingClock struct{ now time.Time } + +func (c *steppingClock) Now() time.Time { return c.now }