Compare commits

...
Sign in to create a new pull request.

1 commit

Author SHA1 Message Date
Julian Knutsen
ee7f5c5053 fix(dolt): cap withRetryTx's wait so a losing writer keeps retrying
withRetryTx replays a write that lost Dolt's commit-time merge on an
exponential schedule: waits from 25ms, multiplier 1.5, jitter of half a
wait either way, within 15s on a sql-server (5s embedded). The schedule set
no MaxInterval, so it used the library default of 60s, and the waits grew
to 2-5s each before the budget ran out. A writer that keeps losing
conflicts got at most 16 attempts in the server budget (14-15 typically).
That is routine when every mutation bumps one shared row, as it does with
the events journal on and several concurrent writers.

newWriteTxRetryBackOff now builds the schedule, and caps each wait at
writeTxRetryMaxInterval (1s) before jitter. The budget and the first waits
are unchanged; the later attempts keep arriving at about one a second. On
the schedule alone, with attempts taking no time, the 15s budget now
yields 19-29 attempts (median 23) instead of 14-16 (median 15), measured
over 200,000 jittered runs; the worst case, every wait jittered to its
longest, yields 18. The embedded budget goes from 11-13 to 11-16.

Test: TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget drives
newWriteTxRetryBackOff with a stepping clock (no sleeps), 50 jittered runs
per mode. It checks that no wait exceeds the jittered cap (1.5s), that the
schedule stays within its budget, and that a writer gets at least 18
attempts in 15s on a server and 11 in 5s embedded. With the schedule
extracted but uncapped it fails ("wait 2.631086165s exceeds the jittered
cap of 1.5s"); with only the cap check disabled, it reports "a writer that
keeps losing gets 15 attempts in 15s, want at least 18". It passes with
the cap. The package's Retry|Circuit|Serialization|Indeterminate|
Concurrent|WriteTx tests pass against a dolt sql-server 2.2.0 test
container.

Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-16 00:28:39 +00:00
3 changed files with 83 additions and 6 deletions

View file

@ -378,6 +378,7 @@ go_test(
"wisp_validation_test.go",
"wisps_cycle_test.go",
"workspace_config_contract_test.go",
"write_retry_backoff_test.go",
],
# Source-scan guards (journal/version scope completeness, helper sentinel
# census) parse this package and ../issueops at test time; the PR4107 and
@ -417,6 +418,7 @@ go_test(
"//internal/utils",
"//internal/workapi",
"//issueops",
"@com_github_cenkalti_backoff_v4//:backoff",
"@com_github_data_dog_go_sqlmock//:go-sqlmock",
"@com_github_go_sql_driver_mysql//:mysql",
"@com_github_google_uuid//:uuid",

View file

@ -1231,12 +1231,7 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f
return ErrCircuitOpen
}
bo := backoff.NewExponentialBackOff()
bo.InitialInterval = 25 * time.Millisecond
bo.MaxElapsedTime = 5 * time.Second
if s.serverMode {
bo.MaxElapsedTime = 15 * time.Second
}
bo := newWriteTxRetryBackOff(s.serverMode)
var pending issueops.BlockedRecheck
if err := backoff.Retry(func() error {
var err error
@ -1290,6 +1285,26 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f
return nil
}
// writeTxRetryMaxInterval caps each wait between withRetryTx attempts (before
// jitter). Uncapped, the exponential waits stretch to several seconds each, so
// a writer that keeps losing commit-time conflicts gets at most 16 attempts in
// the server budget; capped, the attempts keep arriving through all of it.
const writeTxRetryMaxInterval = time.Second
// newWriteTxRetryBackOff is withRetryTx's schedule: exponential waits from
// 25ms, capped at writeTxRetryMaxInterval and randomized by half in either
// direction, within a 15s budget on a sql-server (5s embedded).
func newWriteTxRetryBackOff(serverMode bool) *backoff.ExponentialBackOff {
bo := backoff.NewExponentialBackOff()
bo.InitialInterval = 25 * time.Millisecond
bo.MaxInterval = writeTxRetryMaxInterval
bo.MaxElapsedTime = 5 * time.Second
if serverMode {
bo.MaxElapsedTime = 15 * time.Second
}
return bo
}
func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
pending, err := s.commitWriteTx(ctx, fn)
if err != nil {

View file

@ -0,0 +1,60 @@
package dolt
import (
"testing"
"time"
"github.com/cenkalti/backoff/v4"
)
// TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget pins the schedule
// withRetryTx replays a write on. A writer that keeps losing commit-time
// conflicts (routine when every mutation also bumps one shared counter row, as
// with the events journal on) must keep getting attempts through its whole
// budget. Uncapped, the exponential waits grow to several seconds each and the
// 15s server budget holds at most 16 attempts; capped at a second, it holds at
// least 18 even with every wait jittered to its longest.
func TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget(t *testing.T) {
for _, tc := range []struct {
name string
serverMode bool
budget time.Duration
minTries int
}{
{name: "server", serverMode: true, budget: 15 * time.Second, minTries: 18},
{name: "embedded", serverMode: false, budget: 5 * time.Second, minTries: 11},
} {
t.Run(tc.name, func(t *testing.T) {
for run := 0; run < 50; run++ {
clock := &steppingClock{now: time.Unix(0, 0)}
bo := newWriteTxRetryBackOff(tc.serverMode)
bo.Clock = clock
bo.Reset()
tries := 1
for {
next := bo.NextBackOff()
if next == backoff.Stop {
break
}
if longest := writeTxRetryMaxInterval * 3 / 2; next > longest {
t.Fatalf("wait %v exceeds the jittered cap of %v", next, longest)
}
clock.now = clock.now.Add(next)
tries++
}
if tries < tc.minTries {
t.Fatalf("a writer that keeps losing gets %d attempts in %v, want at least %d", tries, tc.budget, tc.minTries)
}
if elapsed := clock.now.Sub(time.Unix(0, 0)); elapsed > tc.budget {
t.Fatalf("schedule waited %v, past its %v budget", elapsed, tc.budget)
}
}
})
}
}
// steppingClock is a backoff.Clock the test advances by each wait the
// schedule asks for, so the schedule runs without sleeping.
type steppingClock struct{ now time.Time }
func (c *steppingClock) Now() time.Time { return c.now }