fix(storage): a replay loop answers with its last attempt's own outcome
When the context ended during a replayed attempt, RunReplayableTransaction returned the earlier, rolled-back conflict wrapped with the context error, dropping what the last attempt returned. If that attempt's COMMIT was cut by the context (ErrCommitIndeterminate: the write may have landed), the caller was told the transaction lost a conflict and wrote nothing; a plain failure in the last attempt was replaced the same way. Once a replay has run, the last attempt's outcome is now returned unchanged unless it is itself a replayable conflict. The new test (a conflict, then an indeterminate or plain failure while the context is cancelled) fails without the fix. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c035e8d319
commit
0b9e19f232
2 changed files with 42 additions and 2 deletions
|
|
@ -108,7 +108,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin
|
|||
policy := backoff.WithContext(backoff.WithMaxRetries(bo, replayMaxAttempts-1), ctx)
|
||||
|
||||
attempts := 0
|
||||
var lastConflict error
|
||||
var lastConflict, lastAttempt error
|
||||
notify := func(err error, wait time.Duration) {
|
||||
txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed")))
|
||||
debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v\n", commitMsg, attempts, wait, err)
|
||||
|
|
@ -120,6 +120,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin
|
|||
err := backoff.RetryNotifyWithTimer(func() error {
|
||||
attempts++
|
||||
err := s.RunInTransaction(ctx, commitMsg, fn)
|
||||
lastAttempt = err
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -129,7 +130,12 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin
|
|||
}
|
||||
return backoff.Permanent(err)
|
||||
}, policy, notify, timer)
|
||||
if err == nil || lastConflict == nil {
|
||||
if err == nil || lastConflict == nil || !replayableConflict(lastAttempt) {
|
||||
// The last attempt's own outcome is the answer, unchanged: a callback
|
||||
// error, a non-conflict database error, or an indeterminate commit
|
||||
// that may have landed — even when the context ended during it.
|
||||
// Replacing it with an earlier, rolled-back conflict would tell the
|
||||
// caller nothing was written.
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, lastConflict) {
|
||||
|
|
|
|||
|
|
@ -277,3 +277,37 @@ func TestRunReplayableTransactionCountsReplays(t *testing.T) {
|
|||
t.Fatalf("exhausted count = %d, want 1 (only the loop that ran out of attempts)", got["exhausted"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome: once a replay
|
||||
// has run, the loop's answer is the LAST attempt's outcome. An attempt that
|
||||
// ends in an indeterminate commit (its COMMIT cut by the context, so the write
|
||||
// may have landed) or in a plain failure is returned unchanged even when the
|
||||
// context ended during it — never replaced by the earlier, rolled-back
|
||||
// conflict, which would tell the caller nothing was written.
|
||||
func TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome(t *testing.T) {
|
||||
for name, last := range map[string]error{
|
||||
"indeterminate commit": fmt.Errorf("sql commit (regular): %w: %w", context.Canceled, ErrCommitIndeterminate),
|
||||
"plain failure": errors.New("validation: dependency would create a cycle"),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
useInstantReplayTimer(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
tr := &scriptedTransactor{outcomes: []error{conflict(1), last}}
|
||||
calls := 0
|
||||
err := RunReplayableTransaction(ctx, tr, "bd: last outcome", func(Transaction) error {
|
||||
calls++
|
||||
if calls == 2 {
|
||||
cancel() // the context ends during the replayed attempt
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if !errors.Is(err, last) {
|
||||
t.Fatalf("err = %v, want the last attempt's own outcome %v", err, last)
|
||||
}
|
||||
if errors.Is(last, ErrCommitIndeterminate) && errors.Is(err, ErrTransactionConflict) {
|
||||
t.Fatalf("an indeterminate outcome was reported as a rolled-back conflict: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue