fix(storage): a replay loop answers with its last attempt's own outcome

When the context ended during a replayed attempt, RunReplayableTransaction
returned the earlier, rolled-back conflict wrapped with the context error,
dropping what the last attempt returned. If that attempt's COMMIT was cut by
the context (ErrCommitIndeterminate: the write may have landed), the caller
was told the transaction lost a conflict and wrote nothing; a plain failure
in the last attempt was replaced the same way.

Once a replay has run, the last attempt's outcome is now returned unchanged
unless it is itself a replayable conflict.

The new test (a conflict, then an indeterminate or plain failure while the
context is cancelled) fails without the fix.

Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Julian Knutsen 2026-10-16 19:00:00 +00:00
commit 0b9e19f232
2 changed files with 42 additions and 2 deletions

View file

@ -108,7 +108,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin
policy := backoff.WithContext(backoff.WithMaxRetries(bo, replayMaxAttempts-1), ctx)
attempts := 0
var lastConflict error
var lastConflict, lastAttempt error
notify := func(err error, wait time.Duration) {
txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed")))
debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v\n", commitMsg, attempts, wait, err)
@ -120,6 +120,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin
err := backoff.RetryNotifyWithTimer(func() error {
attempts++
err := s.RunInTransaction(ctx, commitMsg, fn)
lastAttempt = err
if err == nil {
return nil
}
@ -129,7 +130,12 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin
}
return backoff.Permanent(err)
}, policy, notify, timer)
if err == nil || lastConflict == nil {
if err == nil || lastConflict == nil || !replayableConflict(lastAttempt) {
// The last attempt's own outcome is the answer, unchanged: a callback
// error, a non-conflict database error, or an indeterminate commit
// that may have landed — even when the context ended during it.
// Replacing it with an earlier, rolled-back conflict would tell the
// caller nothing was written.
return err
}
if errors.Is(err, lastConflict) {

View file

@ -277,3 +277,37 @@ func TestRunReplayableTransactionCountsReplays(t *testing.T) {
t.Fatalf("exhausted count = %d, want 1 (only the loop that ran out of attempts)", got["exhausted"])
}
}
// TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome: once a replay
// has run, the loop's answer is the LAST attempt's outcome. An attempt that
// ends in an indeterminate commit (its COMMIT cut by the context, so the write
// may have landed) or in a plain failure is returned unchanged even when the
// context ended during it — never replaced by the earlier, rolled-back
// conflict, which would tell the caller nothing was written.
func TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome(t *testing.T) {
for name, last := range map[string]error{
"indeterminate commit": fmt.Errorf("sql commit (regular): %w: %w", context.Canceled, ErrCommitIndeterminate),
"plain failure": errors.New("validation: dependency would create a cycle"),
} {
t.Run(name, func(t *testing.T) {
useInstantReplayTimer(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
tr := &scriptedTransactor{outcomes: []error{conflict(1), last}}
calls := 0
err := RunReplayableTransaction(ctx, tr, "bd: last outcome", func(Transaction) error {
calls++
if calls == 2 {
cancel() // the context ends during the replayed attempt
}
return nil
})
if !errors.Is(err, last) {
t.Fatalf("err = %v, want the last attempt's own outcome %v", err, last)
}
if errors.Is(last, ErrCommitIndeterminate) && errors.Is(err, ErrTransactionConflict) {
t.Fatalf("an indeterminate outcome was reported as a rolled-back conflict: %v", err)
}
})
}
}