* fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fixup! fix(dolt): preserve transaction outcome boundary (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): retain indeterminate mixed claim updates (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): retain indeterminate guarded update errors (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): prevent lifecycle callback replay (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): classify publication commit ambiguity (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(embeddeddolt): mark commit response loss indeterminate (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): retain indeterminate claim outcomes (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): retry rollback-safe transaction setup (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): surface working-set staging failures (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): account direct commit publication failures (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(embeddeddolt): mark commit call response loss indeterminate (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): account indeterminate transaction commits (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): restore transaction circuit accounting (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(storage): preserve settlement commit boundaries (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): guard direct write circuit boundaries (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): close remaining circuit write gaps (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): account wisp dependency commits (ga-f7v2ft.79.1) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of CI Bot * fix(dolt): record circuit success only after claim verify; cut retry/patch complexity Maintainer review fixups for PR #5341 (three majors): - Correctness: verify-gated claim paths (ClaimReadyIssue, UnclaimIssue, UnclaimIssueIfAssignee, readyClaimer.ClaimNext, issueClaimer.Claim) now wrap verify+write in withCircuitWrite, so the circuit breaker records success only at the boundary after post-write verification returns nil — never on the bare SQL commit ahead of verification. Adds a regression test proving a committed-but-unverified claim does not reset the breaker. - Maintainability: replace hasNonCoordinationPatch's 25-branch || chain with a straight-line nonCoordinationPatchSignals []bool enumeration (cyclomatic 25 -> 3). Adds reflective exhaustiveness and count-guard tests so a new IssuePatch/LabelPatch/MetadataPatch field fails until it is classified. - Readability: split withRetryClassified (cognitive 33) into classifyManagedRetry and recordRetryFailure (cognitive 3/3), preserving retry, indeterminate-commit, and breaker-trip behavior for both callers. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(dolt): record circuit success after facade update verify; skip ignored tables in commit Maintainer review fixup for PR #5341 (attempt 2). Major: IssueLifecycle.Update ran its write on an unmanaged context, so withCircuitWrite recorded RecordSuccess when the SQL commit landed -- before verifiedClaimWrite's re-read could contradict it, laundering a phantom claim into breaker optimism. Wrap the verify path in withCircuitWrite and invoke write with the circuit-managed context so terminal success is recorded once at the boundary, only after verify returns nil (matching issueClaimer.Claim). The ordinary patch path keeps its original context and eager accounting. Minor: commitWorkingSet fed every dolt_status row into a fail-hard DOLT_ADD loop, so an ordinary Commit's success depended on Dolt's version-specific handling of DOLT_ADD on a dolt_ignore'd table. Filter ignored tables with the same anti-join HasCommittablePending uses. Nit: drop the dead nested-error branch in verifiedClaimWrite (err is already nil there). Regressions: facade claim and guarded-update verify failures must not reset the circuit breaker; a dirty ignored wisp is non-committable and Commit tolerates it. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(dolt/test): match commitWorkingSet dolt_ignore anti-join in sqlmock expectations commitWorkingSet selects dirty tables with the dolt_ignore anti-join (`SELECT s.table_name FROM dolt_status s WHERE NOT EXISTS (...)`) so ignored wisp/lease tables stay out of fail-hard staging, but the draincall and settlement sqlmock regression tests still expected the pre-anti-join `SELECT table_name FROM dolt_status`. Under QueryMatcherRegexp the status query no longer matched, so the commit path returned "could not match actual sql" before reaching the indeterminate-commit / staging-cause / circuit-trip assertions, failing the macOS Test job. Update the six status-query expectations to the anti-join form. Behavioral assertions are unchanged; this only realigns the mock's expected SQL with the production query the review validated. --------- Co-authored-by: CI Bot <ci@beads.test> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
328 lines
10 KiB
Go
328 lines
10 KiB
Go
package dolt
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"database/sql/driver"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
mysql "github.com/go-sql-driver/mysql"
|
|
)
|
|
|
|
// failureDriver provides deterministic pre-commit and commit-phase connection
|
|
// failures for the L6.4 retry-boundary tests.
|
|
type failureDriver struct {
|
|
begins atomic.Int32
|
|
prepares atomic.Int32
|
|
execs atomic.Int32
|
|
failBegin atomic.Int32
|
|
commitMu sync.Mutex
|
|
commitErrs []error
|
|
commitErr error
|
|
commitFunc func() error
|
|
}
|
|
|
|
var testConnectionLoss = errors.New("invalid connection")
|
|
|
|
func (d *failureDriver) Open(string) (driver.Conn, error) { return &failureConn{driver: d}, nil }
|
|
func (d *failureDriver) Connect(context.Context) (driver.Conn, error) {
|
|
return &failureConn{driver: d}, nil
|
|
}
|
|
func (d *failureDriver) Driver() driver.Driver { return d }
|
|
|
|
type failureConn struct{ driver *failureDriver }
|
|
|
|
func (c *failureConn) Prepare(string) (driver.Stmt, error) {
|
|
c.driver.prepares.Add(1)
|
|
return nil, errors.New("failure driver does not prepare statements")
|
|
}
|
|
func (c *failureConn) Close() error { return nil }
|
|
func (c *failureConn) ExecContext(context.Context, string, []driver.NamedValue) (driver.Result, error) {
|
|
c.driver.execs.Add(1)
|
|
return driver.RowsAffected(1), nil
|
|
}
|
|
func (c *failureConn) Begin() (driver.Tx, error) {
|
|
c.driver.begins.Add(1)
|
|
if c.driver.failBegin.Load() > 0 {
|
|
c.driver.failBegin.Add(-1)
|
|
return nil, testConnectionLoss
|
|
}
|
|
return &failureTx{driver: c.driver}, nil
|
|
}
|
|
|
|
type failureTx struct{ driver *failureDriver }
|
|
|
|
func (t *failureTx) Commit() error {
|
|
if t.driver.commitFunc != nil {
|
|
return t.driver.commitFunc()
|
|
}
|
|
return t.driver.nextCommitError()
|
|
}
|
|
func (t *failureTx) Rollback() error { return nil }
|
|
|
|
func (d *failureDriver) nextCommitError() error {
|
|
d.commitMu.Lock()
|
|
defer d.commitMu.Unlock()
|
|
if len(d.commitErrs) == 0 {
|
|
return d.commitErr
|
|
}
|
|
err := d.commitErrs[0]
|
|
d.commitErrs = d.commitErrs[1:]
|
|
return err
|
|
}
|
|
|
|
var _ driver.Connector = (*failureDriver)(nil)
|
|
var _ driver.ExecerContext = (*failureConn)(nil)
|
|
|
|
func newFailureStore(d *failureDriver) *DoltStore {
|
|
return &DoltStore{db: sql.OpenDB(d)}
|
|
}
|
|
|
|
// TestIndeterminateCommitIsSurfacedNotRetried pins L6.4: a connection loss
|
|
// during COMMIT is surfaced as indeterminate and the write is attempted once.
|
|
func TestIndeterminateCommitIsSurfacedNotRetried(t *testing.T) {
|
|
driver := &failureDriver{commitErrs: []error{testConnectionLoss}}
|
|
store := newFailureStore(driver)
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
err := store.withRetryTx(context.Background(), func(*sql.Tx) error { return nil })
|
|
if err == nil {
|
|
t.Fatal("lost commit returned nil; indeterminacy must be surfaced")
|
|
}
|
|
if !errors.Is(err, ErrCommitIndeterminate) {
|
|
t.Errorf("err = %v, want ErrCommitIndeterminate", err)
|
|
}
|
|
if !strings.Contains(err.Error(), "indeterminate") {
|
|
t.Errorf("err = %q, want indeterminate outcome", err)
|
|
}
|
|
if got := driver.begins.Load(); got != 1 {
|
|
t.Errorf("write attempts after lost commit = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestWithRetryTxIndeterminateCommitFailuresTripCircuitWithoutReplay(t *testing.T) {
|
|
t.Setenv("BEADS_TEST_MODE", "")
|
|
breaker := newTestCircuitBreaker(t)
|
|
driver := &failureDriver{commitErr: testConnectionLoss}
|
|
store := newFailureStore(driver)
|
|
store.breaker = breaker
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
ctx := t.Context()
|
|
var callbacks atomic.Int32
|
|
for attempt := 1; attempt <= circuitFailureThreshold; attempt++ {
|
|
err := store.withRetryTx(ctx, func(tx *sql.Tx) error {
|
|
callbacks.Add(1)
|
|
_, err := tx.ExecContext(ctx, "UPDATE issues SET title = ?", "changed")
|
|
return err
|
|
})
|
|
if !errors.Is(err, ErrCommitIndeterminate) {
|
|
t.Fatalf("withRetryTx() attempt %d error = %v, want ErrCommitIndeterminate", attempt, err)
|
|
}
|
|
if got := driver.begins.Load(); got != int32(attempt) {
|
|
t.Fatalf("transaction begins after attempt %d = %d, want %d", attempt, got, attempt)
|
|
}
|
|
if got := callbacks.Load(); got != int32(attempt) {
|
|
t.Fatalf("callback calls after attempt %d = %d, want %d", attempt, got, attempt)
|
|
}
|
|
if got := driver.execs.Load(); got != int32(attempt) {
|
|
t.Fatalf("SQL mutations after attempt %d = %d, want %d", attempt, got, attempt)
|
|
}
|
|
|
|
wantState := circuitClosed
|
|
if attempt == circuitFailureThreshold {
|
|
wantState = circuitOpen
|
|
}
|
|
if got := breaker.State(); got != wantState {
|
|
t.Fatalf("circuit state after attempt %d = %q, want %q", attempt, got, wantState)
|
|
}
|
|
}
|
|
|
|
beginCalls := driver.begins.Load()
|
|
callbackCalls := callbacks.Load()
|
|
mutationCalls := driver.execs.Load()
|
|
err := store.withRetryTx(ctx, func(tx *sql.Tx) error {
|
|
callbacks.Add(1)
|
|
_, execErr := tx.ExecContext(ctx, "UPDATE issues SET title = ?", "replayed")
|
|
return execErr
|
|
})
|
|
if !errors.Is(err, ErrCircuitOpen) {
|
|
t.Fatalf("withRetryTx() after circuit opened error = %v, want ErrCircuitOpen", err)
|
|
}
|
|
if got := driver.begins.Load(); got != beginCalls {
|
|
t.Fatalf("transaction begins after circuit opened = %d, want unchanged %d", got, beginCalls)
|
|
}
|
|
if got := callbacks.Load(); got != callbackCalls {
|
|
t.Fatalf("callback calls after circuit opened = %d, want unchanged %d", got, callbackCalls)
|
|
}
|
|
if got := driver.execs.Load(); got != mutationCalls {
|
|
t.Fatalf("SQL mutations after circuit opened = %d, want unchanged %d", got, mutationCalls)
|
|
}
|
|
}
|
|
|
|
func TestWithRetryTxPreCallbackConnectionFailureTripsCircuit(t *testing.T) {
|
|
t.Setenv("BEADS_TEST_MODE", "")
|
|
breaker := newTestCircuitBreaker(t)
|
|
for range circuitFailureThreshold - 1 {
|
|
breaker.RecordFailure()
|
|
}
|
|
driver := &failureDriver{}
|
|
driver.failBegin.Store(1)
|
|
store := newFailureStore(driver)
|
|
store.breaker = breaker
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
var callbacks atomic.Int32
|
|
err := store.withRetryTx(t.Context(), func(*sql.Tx) error {
|
|
callbacks.Add(1)
|
|
return nil
|
|
})
|
|
if !errors.Is(err, testConnectionLoss) {
|
|
t.Fatalf("withRetryTx() error = %v, want connection cause %v", err, testConnectionLoss)
|
|
}
|
|
if got := breaker.State(); got != circuitOpen {
|
|
t.Fatalf("circuit state = %q, want %q", got, circuitOpen)
|
|
}
|
|
if got := driver.begins.Load(); got != 1 {
|
|
t.Fatalf("transaction begins = %d, want 1", got)
|
|
}
|
|
if got := callbacks.Load(); got != 0 {
|
|
t.Fatalf("callback calls = %d, want 0 after pre-callback circuit trip", got)
|
|
}
|
|
}
|
|
|
|
func TestWithRetryTxSuccessResetsCircuitFailures(t *testing.T) {
|
|
t.Setenv("BEADS_TEST_MODE", "")
|
|
breaker := newTestCircuitBreaker(t)
|
|
for range circuitFailureThreshold - 1 {
|
|
breaker.RecordFailure()
|
|
}
|
|
driver := &failureDriver{}
|
|
store := newFailureStore(driver)
|
|
store.breaker = breaker
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
if err := store.withRetryTx(t.Context(), func(*sql.Tx) error { return nil }); err != nil {
|
|
t.Fatalf("withRetryTx() error = %v, want nil", err)
|
|
}
|
|
breaker.RecordFailure()
|
|
if got := breaker.State(); got != circuitClosed {
|
|
t.Fatalf("circuit state after success and one failure = %q, want %q", got, circuitClosed)
|
|
}
|
|
}
|
|
|
|
func TestDoltAutocommitRollbackCommitIsRetried(t *testing.T) {
|
|
rollback := &mysql.MySQLError{
|
|
Number: 1105,
|
|
Message: "Merge conflict detected, @autocommit transaction rolled back",
|
|
}
|
|
driver := &failureDriver{commitErrs: []error{rollback}}
|
|
store := newFailureStore(driver)
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
var callbacks atomic.Int32
|
|
if err := store.withRetryTx(context.Background(), func(*sql.Tx) error {
|
|
callbacks.Add(1)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("withRetryTx() error = %v, want nil", err)
|
|
}
|
|
if got := driver.begins.Load(); got != 2 {
|
|
t.Errorf("Begin calls = %d, want 2", got)
|
|
}
|
|
if got := callbacks.Load(); got != 2 {
|
|
t.Errorf("callback invocations = %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
func TestTyped1105CommitErrorIsDefiniteAndNotRetried(t *testing.T) {
|
|
cause := &mysql.MySQLError{Number: 1105, Message: "connection lost while validating commit"}
|
|
driver := &failureDriver{commitErrs: []error{cause}}
|
|
store := newFailureStore(driver)
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
var callbacks atomic.Int32
|
|
err := store.withRetryTx(context.Background(), func(*sql.Tx) error {
|
|
callbacks.Add(1)
|
|
return nil
|
|
})
|
|
if !errors.Is(err, cause) {
|
|
t.Fatalf("withRetryTx() error = %v, want %v", err, cause)
|
|
}
|
|
if errors.Is(err, ErrCommitIndeterminate) {
|
|
t.Fatalf("withRetryTx() error = %v must not be marked commit-indeterminate", err)
|
|
}
|
|
if got := driver.begins.Load(); got != 1 {
|
|
t.Errorf("Begin calls = %d, want 1", got)
|
|
}
|
|
if got := callbacks.Load(); got != 1 {
|
|
t.Errorf("callback invocations = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestWithWriteTxPacketSyncCommitIsIndeterminate(t *testing.T) {
|
|
driver := &failureDriver{commitErr: mysql.ErrPktSync}
|
|
store := newFailureStore(driver)
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
err := store.withRetryTx(context.Background(), func(*sql.Tx) error { return nil })
|
|
if !errors.Is(err, ErrCommitIndeterminate) {
|
|
t.Fatalf("withRetryTx() error = %v, want ErrCommitIndeterminate", err)
|
|
}
|
|
if !errors.Is(err, mysql.ErrPktSync) {
|
|
t.Fatalf("withRetryTx() error = %v, want cause %v", err, mysql.ErrPktSync)
|
|
}
|
|
if got := driver.begins.Load(); got != 1 {
|
|
t.Fatalf("write attempts = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
// TestPreCommitConnectionLossIsRetriedSilently is the safe-retry contrast: a
|
|
// connection failure before a transaction begins cannot have committed data.
|
|
func TestPreCommitConnectionLossIsRetriedSilently(t *testing.T) {
|
|
driver := &failureDriver{}
|
|
driver.failBegin.Store(2)
|
|
store := newFailureStore(driver)
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
if err := store.withRetryTx(context.Background(), func(*sql.Tx) error { return nil }); err != nil {
|
|
t.Fatalf("pre-commit connection loss surfaced: %v", err)
|
|
}
|
|
if got := driver.begins.Load(); got != 3 {
|
|
t.Errorf("Begin calls = %d, want 3", got)
|
|
}
|
|
}
|
|
|
|
func TestExecContextCommitResponseLossIsIndeterminateAndMutationRunsOnce(t *testing.T) {
|
|
var commitCalls atomic.Int32
|
|
driver := &failureDriver{}
|
|
driver.commitFunc = func() error {
|
|
if commitCalls.Add(1) == 1 {
|
|
return testConnectionLoss
|
|
}
|
|
return nil
|
|
}
|
|
store := newFailureStore(driver)
|
|
defer func() { _ = store.db.Close() }()
|
|
|
|
_, err := store.execContext(context.Background(), "UPDATE issues SET title = ?", "changed")
|
|
if !errors.Is(err, ErrCommitIndeterminate) {
|
|
t.Fatalf("execContext() error = %v, want ErrCommitIndeterminate", err)
|
|
}
|
|
if !errors.Is(err, testConnectionLoss) {
|
|
t.Fatalf("execContext() error = %v, want cause %v", err, testConnectionLoss)
|
|
}
|
|
if got := driver.execs.Load(); got != 1 {
|
|
t.Fatalf("SQL mutation calls = %d, want 1", got)
|
|
}
|
|
if got := driver.begins.Load(); got != 1 {
|
|
t.Fatalf("transaction begins = %d, want 1", got)
|
|
}
|
|
if got := commitCalls.Load(); got != 1 {
|
|
t.Fatalf("commit calls = %d, want 1", got)
|
|
}
|
|
}
|