Compare commits
1 commit
main
...
fix/write-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee7f5c5053 |
3 changed files with 83 additions and 6 deletions
|
|
@ -378,6 +378,7 @@ go_test(
|
|||
"wisp_validation_test.go",
|
||||
"wisps_cycle_test.go",
|
||||
"workspace_config_contract_test.go",
|
||||
"write_retry_backoff_test.go",
|
||||
],
|
||||
# Source-scan guards (journal/version scope completeness, helper sentinel
|
||||
# census) parse this package and ../issueops at test time; the PR4107 and
|
||||
|
|
@ -417,6 +418,7 @@ go_test(
|
|||
"//internal/utils",
|
||||
"//internal/workapi",
|
||||
"//issueops",
|
||||
"@com_github_cenkalti_backoff_v4//:backoff",
|
||||
"@com_github_data_dog_go_sqlmock//:go-sqlmock",
|
||||
"@com_github_go_sql_driver_mysql//:mysql",
|
||||
"@com_github_google_uuid//:uuid",
|
||||
|
|
|
|||
|
|
@ -1231,12 +1231,7 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f
|
|||
return ErrCircuitOpen
|
||||
}
|
||||
|
||||
bo := backoff.NewExponentialBackOff()
|
||||
bo.InitialInterval = 25 * time.Millisecond
|
||||
bo.MaxElapsedTime = 5 * time.Second
|
||||
if s.serverMode {
|
||||
bo.MaxElapsedTime = 15 * time.Second
|
||||
}
|
||||
bo := newWriteTxRetryBackOff(s.serverMode)
|
||||
var pending issueops.BlockedRecheck
|
||||
if err := backoff.Retry(func() error {
|
||||
var err error
|
||||
|
|
@ -1290,6 +1285,26 @@ func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn f
|
|||
return nil
|
||||
}
|
||||
|
||||
// writeTxRetryMaxInterval caps each wait between withRetryTx attempts (before
|
||||
// jitter). Uncapped, the exponential waits stretch to several seconds each, so
|
||||
// a writer that keeps losing commit-time conflicts gets at most 16 attempts in
|
||||
// the server budget; capped, the attempts keep arriving through all of it.
|
||||
const writeTxRetryMaxInterval = time.Second
|
||||
|
||||
// newWriteTxRetryBackOff is withRetryTx's schedule: exponential waits from
|
||||
// 25ms, capped at writeTxRetryMaxInterval and randomized by half in either
|
||||
// direction, within a 15s budget on a sql-server (5s embedded).
|
||||
func newWriteTxRetryBackOff(serverMode bool) *backoff.ExponentialBackOff {
|
||||
bo := backoff.NewExponentialBackOff()
|
||||
bo.InitialInterval = 25 * time.Millisecond
|
||||
bo.MaxInterval = writeTxRetryMaxInterval
|
||||
bo.MaxElapsedTime = 5 * time.Second
|
||||
if serverMode {
|
||||
bo.MaxElapsedTime = 15 * time.Second
|
||||
}
|
||||
return bo
|
||||
}
|
||||
|
||||
func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
|
||||
pending, err := s.commitWriteTx(ctx, fn)
|
||||
if err != nil {
|
||||
|
|
|
|||
60
internal/storage/dolt/write_retry_backoff_test.go
Normal file
60
internal/storage/dolt/write_retry_backoff_test.go
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
)
|
||||
|
||||
// TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget pins the schedule
|
||||
// withRetryTx replays a write on. A writer that keeps losing commit-time
|
||||
// conflicts (routine when every mutation also bumps one shared counter row, as
|
||||
// with the events journal on) must keep getting attempts through its whole
|
||||
// budget. Uncapped, the exponential waits grow to several seconds each and the
|
||||
// 15s server budget holds at most 16 attempts; capped at a second, it holds at
|
||||
// least 18 even with every wait jittered to its longest.
|
||||
func TestWriteTxRetryScheduleKeepsRetryingAcrossItsBudget(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
serverMode bool
|
||||
budget time.Duration
|
||||
minTries int
|
||||
}{
|
||||
{name: "server", serverMode: true, budget: 15 * time.Second, minTries: 18},
|
||||
{name: "embedded", serverMode: false, budget: 5 * time.Second, minTries: 11},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
for run := 0; run < 50; run++ {
|
||||
clock := &steppingClock{now: time.Unix(0, 0)}
|
||||
bo := newWriteTxRetryBackOff(tc.serverMode)
|
||||
bo.Clock = clock
|
||||
bo.Reset()
|
||||
tries := 1
|
||||
for {
|
||||
next := bo.NextBackOff()
|
||||
if next == backoff.Stop {
|
||||
break
|
||||
}
|
||||
if longest := writeTxRetryMaxInterval * 3 / 2; next > longest {
|
||||
t.Fatalf("wait %v exceeds the jittered cap of %v", next, longest)
|
||||
}
|
||||
clock.now = clock.now.Add(next)
|
||||
tries++
|
||||
}
|
||||
if tries < tc.minTries {
|
||||
t.Fatalf("a writer that keeps losing gets %d attempts in %v, want at least %d", tries, tc.budget, tc.minTries)
|
||||
}
|
||||
if elapsed := clock.now.Sub(time.Unix(0, 0)); elapsed > tc.budget {
|
||||
t.Fatalf("schedule waited %v, past its %v budget", elapsed, tc.budget)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// steppingClock is a backoff.Clock the test advances by each wait the
|
||||
// schedule asks for, so the schedule runs without sleeping.
|
||||
type steppingClock struct{ now time.Time }
|
||||
|
||||
func (c *steppingClock) Now() time.Time { return c.now }
|
||||
Loading…
Add table
Add a link
Reference in a new issue