diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 273110621..3e06abe50 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1231,6 +1231,7 @@ go_test( "todo_embedded_test.go", "tracker_frontdoor_proxy_test.go", "tracker_proxy_uow_conformance_integration_test.go", + "transaction_replay_cgo_test.go", "types_builtin_test.go", "types_create_agreement_embedded_test.go", "types_create_agreement_test.go", diff --git a/cmd/bd/batch.go b/cmd/bd/batch.go index 99e8b76e1..9ef95c6c3 100644 --- a/cmd/bd/batch.go +++ b/cmd/bd/batch.go @@ -174,17 +174,7 @@ normal 'bd' subcommands for interactive/read operations.`, if proxied { results, err = runBatchProxiedServer(ctx, ops, commitMsg) } else { - results = make([]batchOpResult, 0, len(ops)) - err = transact(ctx, store, commitMsg, func(tx storage.Transaction) error { - for _, op := range ops { - res, rerr := runBatchOp(ctx, tx, op) - if rerr != nil { - return fmt.Errorf("line %d (%s): %w", op.line, op.raw, rerr) - } - results = append(results, res) - } - return nil - }) + results, err = runBatchTransaction(ctx, store, commitMsg, ops) } if err != nil { if jsonOutput { @@ -222,6 +212,25 @@ func init() { rootCmd.AddCommand(batchCmd) } +// runBatchTransaction runs ops in one transaction and reports each op's +// result. transact replays the transaction after a lost concurrent-write +// conflict, so each attempt starts with no results. +func runBatchTransaction(ctx context.Context, s storage.DoltStorage, commitMsg string, ops []batchOp) ([]batchOpResult, error) { + var results []batchOpResult + err := transact(ctx, s, commitMsg, func(tx storage.Transaction) error { + results = make([]batchOpResult, 0, len(ops)) + for _, op := range ops { + res, err := runBatchOp(ctx, tx, op) + if err != nil { + return fmt.Errorf("line %d (%s): %w", op.line, op.raw, err) + } + results = append(results, res) + } + return nil + }) + return results, err +} + // batchOp is one parsed command line. type batchOp struct { line int // 1-based source line number diff --git a/cmd/bd/batch_test.go b/cmd/bd/batch_test.go index 834bd49f9..6d3d00d09 100644 --- a/cmd/bd/batch_test.go +++ b/cmd/bd/batch_test.go @@ -1,11 +1,13 @@ package main import ( + "context" "errors" "reflect" "strings" "testing" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" publicops "github.com/steveyegge/beads/issueops" ) @@ -372,3 +374,53 @@ func TestBatchCmd_Registered(t *testing.T) { t.Error("batch missing --message flag") } } + +// batchReplayStore runs each transaction's callback against batchReplayTx and +// makes the first transaction lose a write conflict at commit, as it does when +// a concurrent writer wins. +type batchReplayStore struct { + storage.DoltStorage + transactions int +} + +func (s *batchReplayStore) RunInTransaction(_ context.Context, _ string, fn func(tx storage.Transaction) error) error { + s.transactions++ + if err := fn(batchReplayTx{}); err != nil { + return err + } + if s.transactions == 1 { + return storage.MarkTransactionConflict(errors.New("sql commit (regular): serialization failure")) + } + return nil +} + +type batchReplayTx struct{ storage.Transaction } + +func (batchReplayTx) CloseIssue(context.Context, string, string, string, string) error { return nil } + +func (batchReplayTx) UpdateIssue(context.Context, string, map[string]interface{}, string) error { + return nil +} + +// TestRunBatchTransactionReplayReportsEachOpOnce: a batch whose transaction +// loses a write conflict is replayed whole, and the report must describe the +// transaction that committed, not the rolled-back attempt's ops as well. +func TestRunBatchTransactionReplayReportsEachOpOnce(t *testing.T) { + origExplicit := commandDidExplicitDoltCommit + t.Cleanup(func() { commandDidExplicitDoltCommit = origExplicit }) + ops, err := parseBatchScript(strings.NewReader("close bd-1 done\nupdate bd-2 priority=1\n")) + if err != nil { + t.Fatalf("parseBatchScript: %v", err) + } + st := &batchReplayStore{} + results, err := runBatchTransaction(context.Background(), st, "bd: batch", ops) + if err != nil { + t.Fatalf("runBatchTransaction was not replayed to success: %v", err) + } + if st.transactions != 2 { + t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.transactions) + } + if len(results) != len(ops) { + t.Fatalf("results = %+v, want one per op (%d): the rolled-back attempt's ops were reported too", results, len(ops)) + } +} diff --git a/cmd/bd/create_atomic.go b/cmd/bd/create_atomic.go index f2f425ed9..bebd1090e 100644 --- a/cmd/bd/create_atomic.go +++ b/cmd/bd/create_atomic.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -62,7 +63,13 @@ func createIssueWithDeps(ctx context.Context, st storage.DoltStorage, issue *typ commitMsg = "bd: create " + issue.ID } + // tx.CreateIssue writes what it mints back into issue, and the transaction + // is replayed after a lost concurrent-write conflict: each attempt + // restarts from the caller's request, or a replay would take the + // rolled-back attempt's ID as the caller's own. + restore := issueops.SnapshotCreateInputs([]*types.Issue{issue}) return transactHonoringAutoCommit(ctx, st, commitMsg, func(tx storage.Transaction) error { + restore() if err := tx.CreateIssue(ctx, issue, actor); err != nil { return err } diff --git a/cmd/bd/create_atomic_test.go b/cmd/bd/create_atomic_test.go index 3121a326a..0cea725f2 100644 --- a/cmd/bd/create_atomic_test.go +++ b/cmd/bd/create_atomic_test.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "strings" "testing" @@ -17,11 +18,12 @@ type createAtomicFakeTx struct { created []*types.Issue deps []*types.Dependency failDep string // DependsOnID whose AddDependency fails + mintID string // the ID an auto-minted create receives in this transaction } func (t *createAtomicFakeTx) CreateIssue(_ context.Context, issue *types.Issue, _ string) error { if issue.ID == "" { - issue.ID = "fake-1" + issue.ID = t.mintID } cp := *issue t.created = append(t.created, &cp) @@ -46,6 +48,10 @@ type createAtomicFakeStore struct { committedIssues []*types.Issue committedDeps []*types.Dependency failDep string + // lostConflicts makes the first transactions lose a write conflict at + // commit: the callback runs, nothing it wrote lands, and the failure is + // marked as one a replay may retry. + lostConflicts int } func (s *createAtomicFakeStore) IsInfraTypeCtx(context.Context, types.IssueType) bool { return false } @@ -90,11 +96,16 @@ func (s *createAtomicFakeStore) CreateIssue(_ context.Context, issue *types.Issu func (s *createAtomicFakeStore) RunInTransaction(_ context.Context, _ string, fn func(tx storage.Transaction) error) error { s.txCount++ - tx := &createAtomicFakeTx{failDep: s.failDep} + // Each transaction mints the next number, as counter mode does when a + // concurrent writer has taken the previous one. + tx := &createAtomicFakeTx{failDep: s.failDep, mintID: fmt.Sprintf("fake-%d", s.txCount)} if err := fn(tx); err != nil { // Rollback: discard everything recorded inside the transaction. return err } + if s.txCount <= s.lostConflicts { + return storage.MarkTransactionConflict(errors.New("sql commit (regular): serialization failure")) + } s.committedIssues = append(s.committedIssues, tx.created...) s.committedDeps = append(s.committedDeps, tx.deps...) return nil @@ -186,6 +197,36 @@ func TestCreateIssueWithDepsRollsBackOnDepFailure(t *testing.T) { } } +// TestCreateIssueWithDepsReplaysFromTheCallersRequest: a create that loses a +// write conflict is replayed, and the replay must mint its own ID. Re-using +// the rolled-back attempt's ID as if the caller had asked for it would take +// the explicit-ID path and could overwrite the writer that won the conflict. +func TestCreateIssueWithDepsReplaysFromTheCallersRequest(t *testing.T) { + saveCreateAtomicGlobals(t) + st := &createAtomicFakeStore{lostConflicts: 1} + issue := &types.Issue{Title: "replayed create", Status: types.StatusOpen, IssueType: types.TypeTask} + + if err := createIssueWithDeps(context.Background(), st, issue, "tester", createDepEdges{parentID: "fake-parent"}); err != nil { + t.Fatalf("createIssueWithDeps was not replayed to success: %v", err) + } + if st.txCount != 2 { + t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.txCount) + } + if len(st.committedIssues) != 1 || st.committedIssues[0].ID != "fake-2" { + var ids []string + for _, committed := range st.committedIssues { + ids = append(ids, committed.ID) + } + t.Fatalf("committed issues = %v, want the one the replay minted (fake-2), not the rolled-back attempt's fake-1", ids) + } + if len(st.committedDeps) != 1 || st.committedDeps[0].IssueID != "fake-2" { + t.Fatalf("committed deps = %+v, want the parent edge from fake-2", st.committedDeps) + } + if issue.ID != "fake-2" { + t.Fatalf("caller's issue reports %q, want the committed fake-2", issue.ID) + } +} + func TestCreateIssueWithDepsEmptyEdgesUsesStoreCreate(t *testing.T) { saveCreateAtomicGlobals(t) st := &createAtomicFakeStore{} diff --git a/cmd/bd/dolt_autocommit.go b/cmd/bd/dolt_autocommit.go index a32205e85..0a5739232 100644 --- a/cmd/bd/dolt_autocommit.go +++ b/cmd/bd/dolt_autocommit.go @@ -14,8 +14,13 @@ import ( // DOLT_COMMIT occurred, preventing the redundant maybeAutoCommit in // PersistentPostRun. Use this instead of calling store.RunInTransaction // directly from command handlers. +// +// A transaction that loses a write conflict with a concurrent writer is +// replayed whole (storage.RunReplayableTransaction), so fn must be safe to run +// again from the top: it touches the database only through tx and re-derives +// every value it hands back on each run. func transact(ctx context.Context, s storage.DoltStorage, commitMsg string, fn func(tx storage.Transaction) error) error { - err := s.RunInTransaction(ctx, commitMsg, fn) + err := storage.RunReplayableTransaction(ctx, s, commitMsg, fn) if err == nil { commandDidExplicitDoltCommit = true } @@ -26,7 +31,9 @@ func transact(ctx context.Context, s storage.DoltStorage, commitMsg string, fn f // part of command auto-commit policy. In batch/off modes the SQL transaction // still commits, but no Dolt version commit is created — the blank message // makes StageAndCommit a no-op, in embedded and SQL-server mode alike -// (bd-4wamg: batch used to be silently inert in server mode). +// (bd-4wamg: batch used to be silently inert in server mode). Like transact, +// it replays a transaction that lost a concurrent-write conflict, so fn must +// be safe to run again from the top. func transactHonoringAutoCommit(ctx context.Context, s storage.DoltStorage, commitMsg string, fn func(tx storage.Transaction) error) error { msg := commitMsg committedExplicitly := strings.TrimSpace(msg) != "" @@ -39,7 +46,7 @@ func transactHonoringAutoCommit(ctx context.Context, s storage.DoltStorage, comm committedExplicitly = false } - err = s.RunInTransaction(ctx, msg, fn) + err = storage.RunReplayableTransaction(ctx, s, msg, fn) if err == nil && committedExplicitly { commandDidExplicitDoltCommit = true } diff --git a/cmd/bd/transaction_replay_cgo_test.go b/cmd/bd/transaction_replay_cgo_test.go new file mode 100644 index 000000000..8cc91020b --- /dev/null +++ b/cmd/bd/transaction_replay_cgo_test.go @@ -0,0 +1,183 @@ +//go:build cgo + +package main + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/types" +) + +// openJournalStores opens n stores (n connection pools, as n concurrent bd +// processes would have) on one fresh database of the test server, with the +// events journal on: every mutation also bumps the single bd_events_seq row +// inside its transaction, so writers whose transactions overlap collide at +// commit. +func openJournalStores(t *testing.T, n int) []*dolt.DoltStore { + t.Helper() + ensureTestMode(t) + if testDoltServerPort == 0 { + t.Skip("Dolt test server not available, skipping") + } + if testutil.DoltContainerCrashed() { + t.Skipf("Dolt test server crashed: %v", testutil.DoltContainerCrashError()) + } + dbName := uniqueTestDBName(t) + stores := make([]*dolt.DoltStore, 0, n) + t.Cleanup(func() { + for _, s := range stores { + _ = s.Close() + } + dropTestDatabase(dbName, testDoltServerPort) + }) + for i := 0; i < n; i++ { + dbPath := filepath.Join(t.TempDir(), ".beads", "beads.db") + writeTestMetadata(t, dbPath, dbName) + ctx, cancel := context.WithTimeout(context.Background(), testStoreOpenTimeout) + doltNewMutex.Lock() + s, err := dolt.New(ctx, &dolt.Config{ + Path: dbPath, + ServerHost: "127.0.0.1", + ServerPort: testDoltServerPort, + Database: dbName, + CreateIfMissing: i == 0, + MaxOpenConns: 4, + }) + doltNewMutex.Unlock() + if err == nil && i == 0 { + err = s.SetConfig(ctx, "issue_prefix", "rp") + } + cancel() + if err != nil { + t.Fatalf("open store %d on %s: %v", i, dbName, err) + } + s.SetEventsJournalEnabled(true) + stores = append(stores, s) + } + return stores +} + +// replayBarrierStore holds its first transaction open after the callback has +// written, until released, so a test can order two writers' commits. +type replayBarrierStore struct { + *dolt.DoltStore + attempts atomic.Int32 + prepared chan struct{} + release chan struct{} +} + +func newReplayBarrierStore(s *dolt.DoltStore) *replayBarrierStore { + return &replayBarrierStore{DoltStore: s, prepared: make(chan struct{}), release: make(chan struct{})} +} + +func (b *replayBarrierStore) RunInTransaction(ctx context.Context, msg string, fn func(tx storage.Transaction) error) error { + return b.DoltStore.RunInTransaction(ctx, msg, func(tx storage.Transaction) error { + if err := fn(tx); err != nil { + return err + } + if b.attempts.Add(1) == 1 { + close(b.prepared) + select { + case <-b.release: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) +} + +// threeStepFormula is a cooked formula: a root and three chained steps, the +// shape `bd mol wisp` clones. +func threeStepFormula() *TemplateSubgraph { + root := &types.Issue{ID: "formula", Title: "three-step formula", IssueType: types.TypeEpic, Status: types.StatusOpen, Priority: 2} + sg := &TemplateSubgraph{Root: root, Issues: []*types.Issue{root}, IssueMap: map[string]*types.Issue{root.ID: root}} + var previous *types.Issue + for _, name := range []string{"first", "second", "third"} { + step := &types.Issue{ID: "formula." + name, Title: name + " step", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2} + sg.Issues = append(sg.Issues, step) + sg.IssueMap[step.ID] = step + sg.Dependencies = append(sg.Dependencies, &types.Dependency{IssueID: step.ID, DependsOnID: root.ID, Type: types.DepParentChild}) + if previous != nil { + sg.Dependencies = append(sg.Dependencies, &types.Dependency{IssueID: step.ID, DependsOnID: previous.ID, Type: types.DepBlocks}) + } + previous = step + } + return sg +} + +func countWisps(t *testing.T, ctx context.Context, db *sql.DB, ids map[string]string) int { + t.Helper() + total := 0 + for _, id := range ids { + var n int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM wisps WHERE id = ?", id).Scan(&n); err != nil { + t.Fatalf("count wisp %s: %v", id, err) + } + if n != 1 { + t.Fatalf("wisp %s has %d rows, want exactly 1", id, n) + } + total += n + } + return total +} + +// TestMolWispReplaysLostJournalConflict: two `bd mol wisp` runs instantiate a +// formula at the same moment with the events journal on, and the second one +// to commit loses with "sql commit (regular): Error 1213 ... try restarting +// transaction". The clone must replay its whole transaction and succeed, with +// every step created exactly once. +func TestMolWispReplaysLostJournalConflict(t *testing.T) { + stores := openJournalStores(t, 2) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + winner, loser := newReplayBarrierStore(stores[0]), newReplayBarrierStore(stores[1]) + type outcome struct { + res *InstantiateResult + err error + } + done := [2]chan outcome{make(chan outcome, 1), make(chan outcome, 1)} + for i, s := range []*replayBarrierStore{winner, loser} { + go func() { + res, err := cloneSubgraph(ctx, s, threeStepFormula(), CloneOptions{ + Actor: fmt.Sprintf("writer-%d", i), Ephemeral: true, Prefix: types.IDPrefixWisp, + }) + done[i] <- outcome{res, err} + }() + } + for _, s := range []*replayBarrierStore{winner, loser} { + select { + case <-s.prepared: + case <-ctx.Done(): + t.Fatalf("a clone never wrote its wisps: %v", ctx.Err()) + } + } + close(winner.release) + won := <-done[0] + close(loser.release) + lost := <-done[1] + + if won.err != nil { + t.Fatalf("winner: %v", won.err) + } + if lost.err != nil { + t.Fatalf("loser failed instead of replaying the conflict: %v", lost.err) + } + if got := loser.attempts.Load(); got != 2 { + t.Fatalf("loser transaction attempts = %d, want 2 (one lost to the conflict, one replay)", got) + } + db := stores[0].DB() + if n := countWisps(t, ctx, db, won.res.IDMapping) + countWisps(t, ctx, db, lost.res.IDMapping); n != 8 { + t.Fatalf("wisp rows = %d, want 8 (two clones of a root and three steps)", n) + } +} diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index acb3b8e08..f8824b9ae 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "remote_ref.go", "storage.go", "sync.go", + "transaction_replay.go", "version_control.go", "versioned.go", "wake_defers.go", @@ -69,11 +70,16 @@ go_library( importpath = "github.com/steveyegge/beads/internal/storage", visibility = ["//:__subpackages__"], deps = [ + "//internal/debug", "//internal/hooks", "//internal/types", "//issueops", "//journalops", "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_metric//:metric", ], ) @@ -102,6 +108,7 @@ go_test( "hook_role_census_test.go", "journal_facade_test.go", "leg_contract_wiring_test.go", + "merge_slot_test.go", "metadata_cas_test.go", "metadata_edit_value_test.go", "metadata_jsonpath_test.go", @@ -109,6 +116,7 @@ go_test( "metadata_test.go", "remote_ref_test.go", "role_accessor_decorator_test.go", + "transaction_replay_test.go", ], data = [ ":tree_go_srcs", @@ -123,6 +131,10 @@ go_test( "//issueops", "//journalops", "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_sdk_metric//:metric", + "@io_opentelemetry_go_otel_sdk_metric//metricdata", ], ) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2be809cc9..2449c4d13 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -181,6 +181,7 @@ go_test( "create_batch_equivalence_test.go", "create_guard_test.go", "create_issue_commit_test.go", + "create_replay_test.go", "create_rollback_test.go", "credentials_test.go", "cross_project_test.go", @@ -289,6 +290,7 @@ go_test( "molecule_contract_test.go", "multistore_multiprocess_test.go", "open_test.go", + "peer_store_test.go", "pool_timeouts_test.go", "port_provenance_test.go", "portsource_propagation_test.go", @@ -342,6 +344,7 @@ go_test( "smart_gate_data_behind_test.go", "smart_remote_migrate_gate_test.go", "socket_fallback_test.go", + "sql_wire_proxy_test.go", "stats_reporter_contract_test.go", "store_branch_pin_test.go", "store_unit_test.go", @@ -352,8 +355,10 @@ go_test( "testmain_helper_sentinel_test.go", "testmain_test.go", "transaction_borrow_test.go", + "transaction_conflict_test.go", "transaction_ignored_checkout_test.go", "transaction_phase_test.go", + "transaction_replay_test.go", "transaction_test.go", "transaction_wisp_lifecycle_persistence_test.go", "tree_walker_contract_test.go", diff --git a/internal/storage/dolt/create_replay_test.go b/internal/storage/dolt/create_replay_test.go new file mode 100644 index 000000000..b01a9cc47 --- /dev/null +++ b/internal/storage/dolt/create_replay_test.go @@ -0,0 +1,297 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/rowid" + "github.com/steveyegge/beads/internal/types" +) + +// A create that loses Dolt's commit-time merge to a concurrent writer is +// rolled back by the server and replayed by withRetryTx. These tests prove the +// replay starts from the caller's request: it mints its own ID and derives its +// inline edges and comments from that ID, rather than re-using what the +// rolled-back attempt wrote back into the caller's issue. + +// TestConcurrentCounterModeCreatesNeverShareAnID: in counter mode two writers +// that overlap mint the same number, so one of them loses at commit and is +// replayed. A replay that re-used the rolled-back attempt's ID took the +// explicit-ID upsert path and overwrote the winner's issue, and both callers +// were told the same ID. +func TestConcurrentCounterModeCreatesNeverShareAnID(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if err := base.SetConfig(ctx, "issue_id_mode", "counter"); err != nil { + t.Fatalf("set counter mode: %v", err) + } + const writers, rounds = 4, 3 + stores := []*DoltStore{base} + for len(stores) < writers { + stores = append(stores, openPeerStore(t, ctx, base)) + } + + var mu sync.Mutex + titleByID := map[string]string{} + errs := make(chan error, writers*rounds*3) + record := func(issue *types.Issue) { + mu.Lock() + defer mu.Unlock() + if prev, dup := titleByID[issue.ID]; dup { + errs <- fmt.Errorf("id %s handed to both %q and %q", issue.ID, prev, issue.Title) + return + } + titleByID[issue.ID] = issue.Title + } + newIssue := func(title string) *types.Issue { + return &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + } + var wg sync.WaitGroup + start := make(chan struct{}) + for w, s := range stores { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for r := range rounds { + single := newIssue(fmt.Sprintf("single %d-%d", w, r)) + if err := s.CreateIssue(ctx, single, "writer"); err != nil { + errs <- fmt.Errorf("create %q: %w", single.Title, err) + } else { + record(single) + } + batch := []*types.Issue{newIssue(fmt.Sprintf("batch %d-%d-a", w, r)), newIssue(fmt.Sprintf("batch %d-%d-b", w, r))} + if err := s.CreateIssues(ctx, batch, "writer"); err != nil { + errs <- fmt.Errorf("create batch %d-%d: %w", w, r, err) + } else { + record(batch[0]) + record(batch[1]) + } + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + for id, title := range titleByID { + got, err := base.GetIssue(ctx, id) + if err != nil { + t.Errorf("read %s: %v", id, err) + continue + } + if got.Title != title { + t.Errorf("issue %s holds %q, want %q: a replayed create overwrote it", id, got.Title, title) + } + } + if want := writers * rounds * 3; len(titleByID) != want && !t.Failed() { + t.Fatalf("distinct issues = %d, want %d", len(titleByID), want) + } +} + +// replayIDModes are the two ID modes a replayed create must restart in: in +// counter mode the competitor mints the very number the loser's rolled-back +// attempt had; in hash mode the replay mints a fresh ID, and the events journal +// makes the two overlapping writers collide. +var replayIDModes = []struct { + name string + counterMode bool +}{ + {name: "counter mode", counterMode: true}, + {name: "hash mode", counterMode: false}, +} + +func setUpReplayIDMode(t *testing.T, ctx context.Context, base *DoltStore, counterMode bool) { + t.Helper() + if counterMode { + if err := base.SetConfig(ctx, "issue_id_mode", "counter"); err != nil { + t.Fatalf("set counter mode: %v", err) + } + return + } + base.SetEventsJournalEnabled(true) +} + +// TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue holds a batch create +// open (through the skipped-edge callback it runs inside its transaction) until +// a competitor has committed, so the batch loses at commit and is replayed. Its +// inline edge named no source, so the create filled the source in from the +// minted ID: the replay must fill it in again from its own ID. +func TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue(t *testing.T) { + for _, mode := range replayIDModes { + t.Run(mode.name, func(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + setUpReplayIDMode(t, ctx, base, mode.counterMode) + peer := openPeerStore(t, ctx, base) + peer.SetEventsJournalEnabled(!mode.counterMode) + if err := base.CreateIssue(ctx, &types.Issue{ID: "test-anchor", Title: "anchor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed anchor: %v", err) + } + + prepared, release := make(chan struct{}), make(chan struct{}) + var attempts atomic.Int32 + opts := storage.BatchCreateOptions{ + // Runs inside the create transaction, after the anchor edge is + // written: hold the first attempt open until the competitor + // has committed. + OnSkippedDependency: func(_, _, _ string) { + if attempts.Add(1) == 1 { + close(prepared) + <-release + } + }, + } + loser := &types.Issue{ + Title: "loser", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: "test-anchor", Type: types.DepBlocks}, + {DependsOnID: "test-missing", Type: types.DepRelated}, + }, + } + done := make(chan error, 1) + go func() { done <- base.CreateIssuesWithFullOptions(ctx, []*types.Issue{loser}, "loser", opts) }() + waitForTransactionPrepared(t, ctx, prepared, "loser") + competitor := &types.Issue{Title: "competitor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := peer.CreateIssue(ctx, competitor, "competitor"); err != nil { + close(release) + t.Fatalf("competitor create: %v", err) + } + close(release) + if err := <-done; err != nil { + t.Fatalf("loser create was not replayed to success: %v", err) + } + if n := attempts.Load(); n < 2 { + t.Fatalf("loser create ran %d attempt(s), want a replay: the competitor's commit did not conflict", n) + } + assertCompetitorIntact(t, ctx, base, loser, competitor) + if got := edgeSourcesInto(t, ctx, base, "test-anchor"); len(got) != 1 || got[0] != loser.ID { + t.Fatalf("edges into test-anchor come from %v, want exactly [%s] (the loser's own issue); competitor is %s", got, loser.ID, competitor.ID) + } + }) + } +} + +// TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue holds a single +// create's COMMIT at a proxy until a competitor has committed, so the create +// loses and is replayed. Its inline comment carried no ID, so the create +// derived one from the minted issue ID: the replay must derive it again from +// its own. +func TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue(t *testing.T) { + for _, mode := range replayIDModes { + t.Run(mode.name, func(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + setUpReplayIDMode(t, ctx, base, mode.counterMode) + proxy := startSQLWireProxy(t) + via := openStoreOnPort(t, ctx, base, proxy.port()) + via.SetEventsJournalEnabled(!mode.counterMode) + + loser := &types.Issue{ + Title: "loser", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + Comments: []*types.Comment{{Author: "loser", Text: "filed with the issue"}}, + } + mark := proxy.mark() + held, release := proxy.holdNext(isCommitStatement) + done := make(chan error, 1) + go func() { done <- via.CreateIssue(ctx, loser, "loser") }() + select { + case <-held: + case err := <-done: + t.Fatalf("loser create finished before its COMMIT was held: %v", err) + case <-ctx.Done(): + t.Fatalf("loser create never reached COMMIT: %v", ctx.Err()) + } + competitor := &types.Issue{Title: "competitor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := base.CreateIssue(ctx, competitor, "competitor"); err != nil { + release() + t.Fatalf("competitor create: %v", err) + } + release() + if err := <-done; err != nil { + t.Fatalf("loser create was not replayed to success: %v", err) + } + insertsComment := func(q string) bool { return strings.HasPrefix(q, "INSERT INTO comments") } + if n := proxy.statementsSince(mark, insertsComment); n != 2 { + t.Fatalf("loser create wrote its comment in %d attempt(s), want 2 (one lost to the competitor, one replay)", n) + } + assertCompetitorIntact(t, ctx, base, loser, competitor) + assertCommentDerivedFromOwnIssue(t, ctx, base, loser.ID) + }) + } +} + +func assertCompetitorIntact(t *testing.T, ctx context.Context, base *DoltStore, loser, competitor *types.Issue) { + t.Helper() + if loser.ID == competitor.ID { + t.Fatalf("loser and competitor were both handed %s", loser.ID) + } + got, err := base.GetIssue(ctx, competitor.ID) + if err != nil { + t.Fatalf("read competitor %s: %v", competitor.ID, err) + } + if got.Title != "competitor" { + t.Fatalf("competitor %s holds %q: the replayed create overwrote it", competitor.ID, got.Title) + } +} + +func edgeSourcesInto(t *testing.T, ctx context.Context, s *DoltStore, target string) []string { + t.Helper() + rows, err := s.db.QueryContext(ctx, "SELECT issue_id FROM dependencies WHERE depends_on_issue_id = ?", target) + if err != nil { + t.Fatalf("query edges into %s: %v", target, err) + } + defer rows.Close() + var sources []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan edge source: %v", err) + } + sources = append(sources, id) + } + if err := rows.Err(); err != nil { + t.Fatalf("read edges into %s: %v", target, err) + } + sort.Strings(sources) + return sources +} + +// assertCommentDerivedFromOwnIssue checks issueID's one comment carries the ID +// derived from issueID itself, not one left over from a rolled-back attempt +// that derived it from that attempt's issue ID. +func assertCommentDerivedFromOwnIssue(t *testing.T, ctx context.Context, s *DoltStore, issueID string) { + t.Helper() + comments, err := s.GetIssueComments(ctx, issueID) + if err != nil { + t.Fatalf("read comments of %s: %v", issueID, err) + } + if len(comments) != 1 { + t.Fatalf("%s has %d comments, want 1", issueID, len(comments)) + } + c := comments[0] + want := rowid.New("comments", 0, rowid.Digest([]sql.NullString{ + {String: issueID, Valid: true}, {String: c.Author, Valid: true}, + {String: c.Text, Valid: true}, {String: issueops.FormatAuxTime(c.CreatedAt), Valid: true}, + })) + if c.ID != want { + t.Fatalf("comment on %s has id %s, want %s (derived from its own issue): the replay re-used the rolled-back attempt's comment id", issueID, c.ID, want) + } +} diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index b6a8362b6..cb5a3b690 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -40,7 +40,14 @@ func (s *DoltStore) createIssue(ctx context.Context, issue *types.Issue, actor s } var result issueops.CreateIssueResult + // withRetryTx replays the body after a lost commit-time conflict, and the + // create writes its minted ID (and what derives from it) back into the + // caller's issue: each attempt restarts from the caller's request, or the + // replay would take the rolled-back ID as the caller's own and upsert over + // the winner's row. + restore := issueops.SnapshotCreateInputs([]*types.Issue{issue}) if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + restore() // SkipPrefixValidation matches legacy behavior: single-issue path does // not validate prefixes for explicit IDs. bc, err := issueops.NewBatchContext(ctx, tx, storage.BatchCreateOptions{ @@ -113,14 +120,19 @@ func (s *DoltStore) createIssuesWithFullOptions(ctx context.Context, issues []*t issue.Ephemeral = true } } + restore := issueops.SnapshotCreateInputs(issues) return s.withRetryTx(ctx, func(tx *sql.Tx) error { + restore() _, err := issueops.CreateIssuesInTxWithResult(ctx, tx, issues, actor, opts) return err }) } + // Each attempt restarts from the caller's request; see createIssue. + restore := issueops.SnapshotCreateInputs(issues) var result issueops.CreateIssuesResult if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + restore() var err error result, err = issueops.CreateIssuesInTxWithResult(ctx, tx, issues, actor, opts) return err diff --git a/internal/storage/dolt/peer_store_test.go b/internal/storage/dolt/peer_store_test.go new file mode 100644 index 000000000..9a6d6653e --- /dev/null +++ b/internal/storage/dolt/peer_store_test.go @@ -0,0 +1,33 @@ +package dolt + +import ( + "context" + "testing" +) + +// openPeerStore opens a second store on store's database, with a connection +// pool of its own as a second bd process would have. +func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore { + t.Helper() + return openStoreOnPort(t, ctx, store, testServerPort) +} + +// openStoreOnPort opens a store on store's database through whatever listens +// on port: the test server itself, or a test proxy in front of it. +func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore { + t.Helper() + peer, err := New(ctx, &Config{ + Path: t.TempDir(), + CommitterName: "test", + CommitterEmail: "test@example.com", + ServerHost: "127.0.0.1", + ServerPort: port, + Database: store.database, + MaxOpenConns: 2, + }) + if err != nil { + t.Fatalf("open store on %s through port %d: %v", store.database, port, err) + } + t.Cleanup(func() { _ = peer.Close() }) + return peer +} diff --git a/internal/storage/dolt/sql_wire_proxy_test.go b/internal/storage/dolt/sql_wire_proxy_test.go new file mode 100644 index 000000000..98e40b518 --- /dev/null +++ b/internal/storage/dolt/sql_wire_proxy_test.go @@ -0,0 +1,205 @@ +package dolt + +import ( + "fmt" + "io" + "net" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// comQuery is the MySQL command byte of a text-protocol statement. Every +// statement a store sends travels as one: its DSN interpolates parameters. +const comQuery = 0x03 + +// sqlWireProxy forwards MySQL wire traffic between stores under test and the +// test Dolt server. It records every statement it forwards and can trap the +// next one that matches a predicate, either holding it until the test releases +// it (to order two writers' commits deterministically) or forwarding it and +// cutting the client off before the server's answer arrives (a response lost +// after the server acted on the statement). Production code needs no test hook. +type sqlWireProxy struct { + ln net.Listener + target string + + mu sync.Mutex + trap *sqlWireTrap + held []*sqlWireTrap + queries []string + conns []net.Conn +} + +type sqlWireTrap struct { + match func(query string) bool + drop bool + sprung chan struct{} + release chan struct{} + released sync.Once +} + +func (trap *sqlWireTrap) open() { trap.released.Do(func() { close(trap.release) }) } + +func startSQLWireProxy(t *testing.T) *sqlWireProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + p := &sqlWireProxy{ln: ln, target: fmt.Sprintf("127.0.0.1:%d", testServerPort)} + go func() { + for { + client, err := ln.Accept() + if err != nil { + return + } + go p.handle(client) + } + }() + t.Cleanup(p.close) + return p +} + +func (p *sqlWireProxy) port() int { return p.ln.Addr().(*net.TCPAddr).Port } + +// holdNext traps the next statement matching match and keeps it from the +// server until release is called. sprung is closed once it is caught. +func (p *sqlWireProxy) holdNext(match func(query string) bool) (sprung <-chan struct{}, release func()) { + trap := p.arm(match, false) + return trap.sprung, trap.open +} + +// dropResponseToNext traps the next statement matching match: the server runs +// it, and the client's connection is cut instead of receiving the answer. +func (p *sqlWireProxy) dropResponseToNext(match func(query string) bool) (sprung <-chan struct{}) { + return p.arm(match, true).sprung +} + +func (p *sqlWireProxy) arm(match func(query string) bool, drop bool) *sqlWireTrap { + trap := &sqlWireTrap{match: match, drop: drop, sprung: make(chan struct{}), release: make(chan struct{})} + p.mu.Lock() + defer p.mu.Unlock() + if p.trap != nil { + p.trap.open() + } + p.trap = trap + return trap +} + +// mark returns a position in the statement log for statementsSince. +func (p *sqlWireProxy) mark() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.queries) +} + +// statementsSince counts the statements forwarded after mark that match. +func (p *sqlWireProxy) statementsSince(mark int, match func(query string) bool) int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, q := range p.queries[mark:] { + if match(q) { + n++ + } + } + return n +} + +// intercept logs query and hands back the armed trap when it catches query. +func (p *sqlWireProxy) intercept(query string) *sqlWireTrap { + p.mu.Lock() + defer p.mu.Unlock() + p.queries = append(p.queries, query) + trap := p.trap + if trap == nil || !trap.match(query) { + return nil + } + p.trap = nil + if !trap.drop { + p.held = append(p.held, trap) + } + close(trap.sprung) + return trap +} + +func (p *sqlWireProxy) close() { + _ = p.ln.Close() + p.mu.Lock() + defer p.mu.Unlock() + if p.trap != nil { + p.trap.open() + } + for _, trap := range p.held { + trap.open() + } + for _, c := range p.conns { + _ = c.Close() + } +} + +func (p *sqlWireProxy) handle(client net.Conn) { + server, err := net.Dial("tcp", p.target) + if err != nil { + _ = client.Close() + return + } + p.mu.Lock() + p.conns = append(p.conns, client, server) + p.mu.Unlock() + var once sync.Once + closeBoth := func() { once.Do(func() { _ = client.Close(); _ = server.Close() }) } + var dropping atomic.Bool + go func() { + buf := make([]byte, 64*1024) + for { + n, err := server.Read(buf) + if n > 0 { + if dropping.Load() { + closeBoth() + return + } + if _, werr := client.Write(buf[:n]); werr != nil { + closeBoth() + return + } + } + if err != nil { + closeBoth() + return + } + } + }() + hdr := make([]byte, 4) + for { + if _, err := io.ReadFull(client, hdr); err != nil { + closeBoth() + return + } + n := int(hdr[0]) | int(hdr[1])<<8 | int(hdr[2])<<16 + packet := make([]byte, 4+n) + copy(packet, hdr) + if _, err := io.ReadFull(client, packet[4:]); err != nil { + closeBoth() + return + } + // A command starts a new exchange (sequence id 0); the handshake's + // packets do not. + if hdr[3] == 0 && n > 1 && packet[4] == comQuery { + if trap := p.intercept(strings.TrimSpace(string(packet[5:]))); trap != nil { + if trap.drop { + dropping.Store(true) + } else { + <-trap.release + } + } + } + if _, err := server.Write(packet); err != nil { + closeBoth() + return + } + } +} + +func isCommitStatement(query string) bool { return strings.EqualFold(query, "COMMIT") } diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index ff2be619b..522a2717d 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -67,10 +67,12 @@ func (t *doltTransaction) CreateIssueImport(ctx context.Context, issue *types.Is // RunInTransaction executes a function within a database transaction. Its // callback is invoked at most once per call; callers retry explicitly after a -// callback has started when their operation is safe to repeat. The commitMsg is -// used for the DOLT_COMMIT that makes regular writes visible in Dolt history. -// Wisp routing is handled by individual transaction methods based on -// ID/Ephemeral. +// callback has started when their operation is safe to repeat. A failure that +// proves the server rolled the whole transaction back after losing a write +// conflict is marked storage.ErrTransactionConflict so such a caller can tell +// (storage.RunReplayableTransaction). The commitMsg is used for the +// DOLT_COMMIT that makes regular writes visible in Dolt history. Wisp routing +// is handled by individual transaction methods based on ID/Ephemeral. func (s *DoltStore) RunInTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { return s.runInTransaction(ctx, commitMsg, fn, s.runDoltTransaction) } @@ -98,12 +100,37 @@ func (s *DoltStore) runInTransaction( if callbackErr == nil && errors.Is(err, ErrCommitIndeterminate) { return err } - return backoff.Permanent(err) + return backoff.Permanent(markTransactionConflict(err)) } return err }) } +// markTransactionConflict tags a failure that proves the server rolled the +// whole transaction back after losing a write conflict as +// storage.ErrTransactionConflict, leaving its message unchanged. Nothing such a +// transaction wrote is durable, so a caller whose callback is safe to repeat +// may run it again. +// +// - 1213 (serialization failure or deadlock) is what Dolt's commit-time merge +// raises when a concurrent writer committed a conflicting change first; +// the remedy is exactly a replay. +// - 1205 (lock-wait timeout) is tagged for engines that raise it. The engine +// may roll back only the statement, but the failed transaction is always +// rolled back whole here before any replay. +// +// Dolt's 1105 "@autocommit transaction rolled back" is not tagged: it reports +// merge conflicts already in the working set, which a replay meets again. A +// failure after the SQL commit landed carries ErrCommitIndeterminate and is +// never tagged: its rows may be durable, and replaying them could apply them +// twice. +func markTransactionConflict(err error) error { + if err == nil || errors.Is(err, ErrCommitIndeterminate) || !isSerializationError(err) { + return err + } + return storage.MarkTransactionConflict(err) +} + // RunInIssueLifecycleTransaction runs a lifecycle transition and its durable // side effects through one SQL transaction and one Dolt commit attempt. func (s *DoltStore) RunInIssueLifecycleTransaction(ctx context.Context, commitMsg string, fn func(tx storage.IssueLifecycleTransaction) error) error { @@ -140,7 +167,7 @@ func (s *DoltStore) runInIssueLifecycleTransaction( if callbackErr == nil && errors.Is(err, ErrCommitIndeterminate) { return err } - return backoff.Permanent(err) + return backoff.Permanent(markTransactionConflict(err)) } return err }) diff --git a/internal/storage/dolt/transaction_conflict_test.go b/internal/storage/dolt/transaction_conflict_test.go new file mode 100644 index 000000000..4ba0b2e0d --- /dev/null +++ b/internal/storage/dolt/transaction_conflict_test.go @@ -0,0 +1,119 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage" +) + +// transactionFailureCases are failures a public transaction can surface after +// its callback ran, and whether each proves the server rolled the whole +// transaction back after losing a write conflict, so that a caller may replay +// it. +func transactionFailureCases() []struct { + name string + err error + conflict bool +} { + serialization := &mysql.MySQLError{Number: 1213, SQLState: [5]byte{'4', '0', '0', '0', '1'}, Message: "serialization failure: this transaction conflicts with a committed transaction from another client, try restarting transaction"} + return []struct { + name string + err error + conflict bool + }{ + {name: "1213 serialization failure at the regular commit", err: wrapSQLCommitError("sql commit (regular)", serialization), conflict: true}, + {name: "1205 lock wait timeout", err: fmt.Errorf("callback write: %w", &mysql.MySQLError{Number: 1205, Message: "Lock wait timeout exceeded; try restarting transaction"}), conflict: true}, + // Dolt's autocommit rollback reports merge conflicts already in the + // working set; a replay meets them again, so it is not a conflict a + // replay can win. + {name: "Dolt autocommit rollback", err: &mysql.MySQLError{Number: 1105, Message: "Merge conflict detected, @autocommit transaction rolled back"}, conflict: false}, + {name: "other 1105", err: &mysql.MySQLError{Number: 1105, Message: "command denied to user 'op'@'%'"}, conflict: false}, + {name: "duplicate key", err: &mysql.MySQLError{Number: 1062, Message: "Duplicate entry"}, conflict: false}, + // A serialization failure on the Dolt commit that follows a + // successful SQL commit does not roll the transaction back: its rows + // landed. + {name: "1213 after the SQL commit landed", err: fmt.Errorf("stage and commit after regular SQL commit: %w: %w", serialization, ErrCommitIndeterminate), conflict: false}, + {name: "lost commit response", err: wrapSQLCommitError("sql commit (regular)", mysql.ErrInvalidConn), conflict: false}, + // Classification is by the decoded server error, never by text. + {name: "untyped text naming 1213", err: errors.New("Error 1213 (40001): serialization failure"), conflict: false}, + } +} + +func TestRunInTransactionMarksOnlyRollbackGuaranteedConflicts(t *testing.T) { + for _, tc := range transactionFailureCases() { + t.Run(tc.name, func(t *testing.T) { + store := &DoltStore{} + callbacks := 0 + err := store.runInTransaction(context.Background(), "test: conflict classification", func(storage.Transaction) error { + callbacks++ + return nil + }, func(_ context.Context, _ string, fn func(storage.Transaction) error) error { + if err := fn(nil); err != nil { + return err + } + return tc.err + }) + assertTransactionConflictClassification(t, err, tc.err, tc.conflict) + if callbacks != 1 { + t.Fatalf("callback calls = %d, want 1: RunInTransaction itself never replays a callback", callbacks) + } + }) + } +} + +func TestRunInIssueLifecycleTransactionMarksOnlyRollbackGuaranteedConflicts(t *testing.T) { + for _, tc := range transactionFailureCases() { + t.Run(tc.name, func(t *testing.T) { + store := &DoltStore{} + callbacks := 0 + err := store.runInIssueLifecycleTransaction(context.Background(), "test: conflict classification", func(storage.IssueLifecycleTransaction) error { + callbacks++ + return nil + }, func(_ context.Context, fn func(*sql.Tx) error) error { + if err := fn(nil); err != nil { + return err + } + return tc.err + }) + assertTransactionConflictClassification(t, err, tc.err, tc.conflict) + if callbacks != 1 { + t.Fatalf("callback calls = %d, want 1", callbacks) + } + }) + } +} + +func assertTransactionConflictClassification(t *testing.T, err, cause error, conflict bool) { + t.Helper() + if !errors.Is(err, cause) { + t.Fatalf("err = %v, want the original failure %v in the chain", err, cause) + } + if got := errors.Is(err, storage.ErrTransactionConflict); got != conflict { + t.Fatalf("errors.Is(err, ErrTransactionConflict) = %v, want %v (err = %v)", got, conflict, err) + } + if err.Error() != cause.Error() { + t.Fatalf("message = %q, want the unchanged %q", err.Error(), cause.Error()) + } +} + +// A callback's own failure that wraps a rollback-guaranteed conflict (a write +// statement inside the transaction that lost) is a conflict too: the callback +// error rolls the transaction back. +func TestRunInTransactionMarksConflictRaisedInsideCallback(t *testing.T) { + store := &DoltStore{} + inner := &mysql.MySQLError{Number: 1213, Message: "serialization failure"} + err := store.runInTransaction(context.Background(), "test: conflict in callback", func(storage.Transaction) error { + return fmt.Errorf("adding dependency: %w", inner) + }, func(_ context.Context, _ string, fn func(storage.Transaction) error) error { + return fn(nil) + }) + if !errors.Is(err, storage.ErrTransactionConflict) || !errors.Is(err, inner) { + t.Fatalf("err = %v, want a marked conflict wrapping the statement's 1213", err) + } +} diff --git a/internal/storage/dolt/transaction_replay_test.go b/internal/storage/dolt/transaction_replay_test.go new file mode 100644 index 000000000..060fad443 --- /dev/null +++ b/internal/storage/dolt/transaction_replay_test.go @@ -0,0 +1,329 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// With the events journal on, every mutation also bumps the single +// bd_events_seq row inside its own transaction, so two writers whose +// transactions overlap at all collide when the second one commits: Dolt rolls +// it back with MySQL 1213 ("serialization failure ... try restarting +// transaction"). These tests order two such transactions deterministically: +// both write, the winner commits, then the loser commits and loses. + +// overlapCreates runs two transactions that each create one issue, holds both +// open until both have written, then lets the winner commit before the loser. +// run decides how each transaction is driven; the hold applies only to each +// transaction's first attempt. +func overlapCreates(t *testing.T, ctx context.Context, winner, loser *DoltStore, ephemeral bool, + run func(s *DoltStore, fn func(tx storage.Transaction) error) error, +) (winnerErr, loserErr error, loserCallbacks int32, ids [2]string) { + t.Helper() + prepared := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + release := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + done := [2]chan error{make(chan error, 1), make(chan error, 1)} + var calls [2]atomic.Int32 + for i, s := range []*DoltStore{winner, loser} { + go func() { + done[i] <- run(s, func(tx storage.Transaction) error { + attempt := calls[i].Add(1) + issue := &types.Issue{ + Title: []string{"winner", "loser"}[i], + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: ephemeral, + } + if err := tx.CreateIssue(ctx, issue, "tester"); err != nil { + return err + } + ids[i] = issue.ID + if attempt == 1 { + close(prepared[i]) + return waitForTransactionRelease(ctx, release[i]) + } + return nil + }) + }() + } + waitForTransactionPrepared(t, ctx, prepared[0], "winner") + waitForTransactionPrepared(t, ctx, prepared[1], "loser") + close(release[0]) + winnerErr = <-done[0] + close(release[1]) + loserErr = <-done[1] + return winnerErr, loserErr, calls[1].Load(), ids +} + +func countRows(t *testing.T, ctx context.Context, db *sql.DB, table, column, value string) int { + t.Helper() + var n int + //nolint:gosec // G202: table and column are fixed literals chosen by the test + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table+" WHERE "+column+" = ?", value).Scan(&n); err != nil { + t.Fatalf("count %s rows with %s = %s: %v", table, column, value, err) + } + return n +} + +func journalRows(t *testing.T, ctx context.Context, db *sql.DB) int { + t.Helper() + var n int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal").Scan(&n); err != nil { + t.Fatalf("count journal rows: %v", err) + } + return n +} + +// TestRunInTransactionMarksLostJournalConflict: RunInTransaction keeps its +// at-most-once contract and does not replay the loser, but marks its failure +// as a rolled-back conflict so a caller whose callback is safe to repeat can. +func TestRunInTransactionMarksLostJournalConflict(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + store.SetEventsJournalEnabled(true) + peer.SetEventsJournalEnabled(true) + + winnerErr, loserErr, loserCalls, _ := overlapCreates(t, ctx, store, peer, true, + func(s *DoltStore, fn func(tx storage.Transaction) error) error { + return s.RunInTransaction(ctx, "bd: clone template subgraph", fn) + }) + if winnerErr != nil { + t.Fatalf("winner: %v", winnerErr) + } + if loserErr == nil { + t.Fatal("loser committed although both transactions bumped the journal counter; the overlap did not conflict") + } + if !errors.Is(loserErr, storage.ErrTransactionConflict) { + t.Fatalf("loser error = %v, want it marked storage.ErrTransactionConflict", loserErr) + } + if errors.Is(loserErr, ErrCommitIndeterminate) { + t.Fatalf("loser error = %v must not be indeterminate: the server rolled it back", loserErr) + } + if loserCalls != 1 { + t.Fatalf("loser callback ran %d times, want 1 (RunInTransaction never replays)", loserCalls) + } +} + +// TestRunReplayableTransactionReplaysLostJournalConflict drives the same +// overlap through storage.RunReplayableTransaction: the loser's WHOLE +// transaction is replayed once, both creates land exactly once, and the +// journal records exactly one create for each. +func TestRunReplayableTransactionReplaysLostJournalConflict(t *testing.T) { + for _, tc := range []struct { + name string + ephemeral bool + table string + }{ + {name: "wisp", ephemeral: true, table: "wisps"}, + {name: "durable", ephemeral: false, table: "issues"}, + } { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + store.SetEventsJournalEnabled(true) + peer.SetEventsJournalEnabled(true) + journalBefore := journalRows(t, ctx, store.db) + + winnerErr, loserErr, loserCalls, ids := overlapCreates(t, ctx, store, peer, tc.ephemeral, + func(s *DoltStore, fn func(tx storage.Transaction) error) error { + return storage.RunReplayableTransaction(ctx, s, "bd: clone template subgraph", fn) + }) + if winnerErr != nil { + t.Fatalf("winner: %v", winnerErr) + } + if loserErr != nil { + t.Fatalf("loser was not replayed to success: %v", loserErr) + } + if loserCalls != 2 { + t.Fatalf("loser callback ran %d times, want 2 (one lost attempt, one replay)", loserCalls) + } + for i, id := range ids { + if id == "" { + t.Fatalf("transaction %d reported no issue id", i) + } + if n := countRows(t, ctx, store.db, tc.table, "id", id); n != 1 { + t.Fatalf("%s rows for %s = %d, want exactly 1", tc.table, id, n) + } + } + perCreate := countRows(t, ctx, store.db, "bd_events_journal", "issue_id", ids[0]) + if grew := journalRows(t, ctx, store.db) - journalBefore; grew != 2*perCreate { + t.Fatalf("journal grew by %d rows, want %d: the replay must record its create exactly as the winner did", grew, 2*perCreate) + } + }) + } +} + +// TestRunReplayableTransactionNeverReplaysLostCommitResponse: a COMMIT whose +// response is lost may have landed, so it surfaces as indeterminate and is +// never replayed, with the journal on (one transaction) or off (a regular and +// an ignored transaction). +func TestRunReplayableTransactionNeverReplaysLostCommitResponse(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + proxy := startSQLWireProxy(t) + via := openStoreOnPort(t, ctx, base, proxy.port()) + + for _, journal := range []bool{false, true} { + t.Run(fmt.Sprintf("journal=%v", journal), func(t *testing.T) { + via.SetEventsJournalEnabled(journal) + title := fmt.Sprintf("lost-commit-journal-%v", journal) + var calls atomic.Int32 + sprung := proxy.dropResponseToNext(isCommitStatement) + err := storage.RunReplayableTransaction(ctx, via, "bd: lost commit", func(tx storage.Transaction) error { + calls.Add(1) + return tx.CreateIssue(ctx, &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "tester") + }) + select { + case <-sprung: + default: + t.Fatalf("the transaction never reached COMMIT (err = %v)", err) + } + if err == nil { + t.Fatal("a lost commit response reported success") + } + if !errors.Is(err, ErrCommitIndeterminate) || errors.Is(err, storage.ErrTransactionConflict) { + t.Fatalf("err = %v, want indeterminate and not a replayable conflict", err) + } + if n := calls.Load(); n != 1 { + t.Fatalf("callback ran %d times: a lost commit response must never be replayed", n) + } + if n := countRows(t, ctx, base.db, "issues", "title", title); n != 1 { + t.Fatalf("rows titled %q = %d, want 1: the server committed it once", title, n) + } + }) + } +} + +// splitTxOverlap runs a winner on peer and a loser on store, both with the +// journal off, so each runs the default split shape: a regular transaction +// for Dolt-tracked tables and an ignored one for the wisp tables. Both write, +// the winner commits, then the loser commits. +func splitTxOverlap(t *testing.T, ctx context.Context, store, peer *DoltStore, + winnerWrite, loserWrite func(tx storage.Transaction) error, +) (loserErr error, loserCalls int32) { + t.Helper() + prepared := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + release := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + done := [2]chan error{make(chan error, 1), make(chan error, 1)} + var calls atomic.Int32 + go func() { + done[0] <- peer.RunInTransaction(ctx, "winner", func(tx storage.Transaction) error { + if err := winnerWrite(tx); err != nil { + return err + } + close(prepared[0]) + return waitForTransactionRelease(ctx, release[0]) + }) + }() + go func() { + done[1] <- storage.RunReplayableTransaction(ctx, store, "loser", func(tx storage.Transaction) error { + attempt := calls.Add(1) + if err := loserWrite(tx); err != nil { + return err + } + if attempt == 1 { + close(prepared[1]) + return waitForTransactionRelease(ctx, release[1]) + } + return nil + }) + }() + waitForTransactionPrepared(t, ctx, prepared[0], "winner") + waitForTransactionPrepared(t, ctx, prepared[1], "loser") + close(release[0]) + if err := <-done[0]; err != nil { + t.Fatalf("winner: %v", err) + } + close(release[1]) + return <-done[1], calls.Load() +} + +// TestSplitTransactionRegularConflictRollsBackBothPlanes: the loser's REGULAR +// commit loses, so both of its transactions roll back and the replay writes +// each plane exactly once; no wisp from the lost attempt survives. +func TestSplitTransactionRegularConflictRollsBackBothPlanes(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + createPerm(t, ctx, store, "test-shared") + + loserErr, loserCalls := splitTxOverlap(t, ctx, store, peer, + func(tx storage.Transaction) error { + return tx.UpdateIssue(ctx, "test-shared", map[string]interface{}{"title": "winner"}, "w") + }, + func(tx storage.Transaction) error { + if err := tx.UpdateIssue(ctx, "test-shared", map[string]interface{}{"title": "loser"}, "l"); err != nil { + return err + } + return tx.CreateIssue(ctx, &types.Issue{Title: "split-wisp", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Ephemeral: true}, "l") + }) + if loserErr != nil { + t.Fatalf("loser: %v", loserErr) + } + if loserCalls != 2 { + t.Fatalf("loser attempts = %d, want 2", loserCalls) + } + if n := countRows(t, ctx, store.db, "wisps", "title", "split-wisp"); n != 1 { + t.Fatalf("wisp rows = %d, want 1: the lost attempt's ignored-plane write leaked", n) + } + got, err := store.GetIssue(ctx, "test-shared") + if err != nil { + t.Fatalf("read test-shared: %v", err) + } + if got.Title != "loser" { + t.Fatalf("title = %q, want the replayed loser's", got.Title) + } +} + +// TestSplitTransactionIgnoredConflictAfterRegularCommitIsNotReplayed: the +// loser's IGNORED commit loses after its regular commit landed. That outcome +// is partial, so it is indeterminate and never a replayable conflict. +func TestSplitTransactionIgnoredConflictAfterRegularCommitIsNotReplayed(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + createWisp(t, ctx, store, "test-wisp-shared") + + loserErr, loserCalls := splitTxOverlap(t, ctx, store, peer, + func(tx storage.Transaction) error { + return tx.UpdateIssue(ctx, "test-wisp-shared", map[string]interface{}{"title": "winner"}, "w") + }, + func(tx storage.Transaction) error { + if err := tx.CreateIssue(ctx, &types.Issue{Title: "split-durable", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "l"); err != nil { + return err + } + return tx.UpdateIssue(ctx, "test-wisp-shared", map[string]interface{}{"title": "loser"}, "l") + }) + if loserErr == nil { + t.Fatal("loser succeeded; the ignored-plane overlap did not conflict") + } + if errors.Is(loserErr, storage.ErrTransactionConflict) || !errors.Is(loserErr, ErrCommitIndeterminate) { + t.Fatalf("err = %v, want indeterminate, not a replayable conflict", loserErr) + } + if loserCalls != 1 { + t.Fatalf("loser attempts = %d, want 1: no replay after the regular commit landed", loserCalls) + } + if n := countRows(t, ctx, store.db, "issues", "title", "split-durable"); n != 1 { + t.Fatalf("durable rows = %d, want 1", n) + } +} diff --git a/internal/storage/dolt/transaction_test.go b/internal/storage/dolt/transaction_test.go index fc7e138d4..45c7cf891 100644 --- a/internal/storage/dolt/transaction_test.go +++ b/internal/storage/dolt/transaction_test.go @@ -243,6 +243,9 @@ func TestRunInTransactionSerializationConflictInvokesCallbacksOnce(t *testing.T) if errors.Is(err, ErrCommitIndeterminate) { t.Fatalf("stale transaction error = %v unexpectedly marks an indeterminate commit", err) } + if !errors.Is(err, storage.ErrTransactionConflict) { + t.Fatalf("stale transaction error = %v, want it marked storage.ErrTransactionConflict so a caller may replay it", err) + } if got := callsA.Load(); got != 1 { t.Errorf("winner callback calls = %d, want 1", got) } diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 3d7ab0898..168967ffa 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -36,6 +36,7 @@ go_library( "create_dep_batch.go", "create_fastpath_testhook.go", "create_only_guard.go", + "create_replay.go", "cycle_report.go", "cycles.go", "delete.go", @@ -145,6 +146,7 @@ go_test( "create_batch_cache_test.go", "create_only_errors_test.go", "create_only_guard_test.go", + "create_replay_test.go", "create_test.go", "custom_types_parity_test.go", "cycle_report_test.go", diff --git a/internal/storage/issueops/create_replay.go b/internal/storage/issueops/create_replay.go new file mode 100644 index 000000000..fb8cf567a --- /dev/null +++ b/internal/storage/issueops/create_replay.go @@ -0,0 +1,67 @@ +package issueops + +import "github.com/steveyegge/beads/internal/types" + +// SnapshotCreateInputs records what the create path writes back into the +// caller's issues and returns a func that puts it back, so a create that a +// transaction retry replays starts every attempt from the caller's request +// instead of from what a rolled-back attempt minted. +// +// The create path writes back three things: +// +// - the Issue struct itself: the minted ID (assignCreateIssueIDInTx) and the +// defaulted timestamps and content hash (PrepareIssueForInsert); +// - through each inline dependency, the owning issue's ID as the edge's +// source when the edge named none (PersistDependenciesWithOptionsResult); +// - through each inline comment, the content-derived ID when the comment +// carried none (PersistComments). +// +// Restoring only the struct would leave the last two pointing at the +// rolled-back attempt's ID. The dependency and comment objects stay the +// caller's: only the fields the create path writes are reset. A new write into +// caller-owned state on the create path needs its restore here. +func SnapshotCreateInputs(issues []*types.Issue) (restore func()) { + type depSource struct { + dep *types.Dependency + issueID string + } + type commentID struct { + comment *types.Comment + id string + } + type issueState struct { + issue *types.Issue + fields types.Issue + deps []depSource + comments []commentID + } + saved := make([]issueState, 0, len(issues)) + for _, issue := range issues { + if issue == nil { + continue + } + state := issueState{issue: issue, fields: *issue} + for _, dep := range issue.Dependencies { + if dep != nil { + state.deps = append(state.deps, depSource{dep: dep, issueID: dep.IssueID}) + } + } + for _, comment := range issue.Comments { + if comment != nil { + state.comments = append(state.comments, commentID{comment: comment, id: comment.ID}) + } + } + saved = append(saved, state) + } + return func() { + for _, state := range saved { + *state.issue = state.fields + for _, d := range state.deps { + d.dep.IssueID = d.issueID + } + for _, c := range state.comments { + c.comment.ID = c.id + } + } + } +} diff --git a/internal/storage/issueops/create_replay_test.go b/internal/storage/issueops/create_replay_test.go new file mode 100644 index 000000000..fa1ce3bfe --- /dev/null +++ b/internal/storage/issueops/create_replay_test.go @@ -0,0 +1,70 @@ +package issueops + +import ( + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack covers every +// write the create path makes into a caller's issues, including the two it +// makes through pointers the Issue struct only holds: the minted ID copied into +// each inline dependency that named no source, and the derived ID stamped onto +// each inline comment that had none. A struct copy restores neither. +func TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack(t *testing.T) { + presetEdge := &types.Dependency{IssueID: "test-explicit", DependsOnID: "test-other", Type: types.DepRelated} + inlineEdge := &types.Dependency{DependsOnID: "test-anchor", Type: types.DepBlocks} + inlineComment := &types.Comment{Author: "a", Text: "first"} + importedComment := &types.Comment{ID: "c-imported", Author: "b", Text: "second"} + issue := &types.Issue{ + Title: "loser", + Labels: []string{"area:storage"}, + Dependencies: []*types.Dependency{inlineEdge, nil, presetEdge}, + Comments: []*types.Comment{inlineComment, importedComment, nil}, + } + second := &types.Issue{Title: "second"} + restore := SnapshotCreateInputs([]*types.Issue{issue, nil, second}) + + // What one create attempt leaves behind. + issue.ID = "test-1" + issue.CreatedAt = time.Now() + issue.UpdatedAt = issue.CreatedAt + issue.ContentHash = "attempt-1-hash" + inlineEdge.IssueID = issue.ID + inlineComment.ID = "c-attempt-1" + second.ID = "test-2" + + restore() + + if issue.ID != "" || !issue.CreatedAt.IsZero() || !issue.UpdatedAt.IsZero() || issue.ContentHash != "" { + t.Fatalf("issue after restore = id %q, created %v, updated %v, hash %q; want the caller's empty values", + issue.ID, issue.CreatedAt, issue.UpdatedAt, issue.ContentHash) + } + if second.ID != "" { + t.Fatalf("second issue id after restore = %q, want empty", second.ID) + } + if inlineEdge.IssueID != "" { + t.Fatalf("inline edge source after restore = %q, want empty so the replay derives it from the new id", inlineEdge.IssueID) + } + if presetEdge.IssueID != "test-explicit" { + t.Fatalf("explicit edge source after restore = %q, want the caller's %q", presetEdge.IssueID, "test-explicit") + } + if inlineComment.ID != "" { + t.Fatalf("inline comment id after restore = %q, want empty so the replay derives it again", inlineComment.ID) + } + if importedComment.ID != "c-imported" { + t.Fatalf("imported comment id after restore = %q, want the caller's %q", importedComment.ID, "c-imported") + } + if issue.Dependencies[0] != inlineEdge || issue.Comments[0] != inlineComment { + t.Fatal("restore replaced the caller's dependency or comment objects instead of resetting them") + } + + // Every replay starts from the same request. + issue.ID = "test-3" + inlineEdge.IssueID = issue.ID + restore() + if issue.ID != "" || inlineEdge.IssueID != "" { + t.Fatalf("second restore left id %q and edge source %q", issue.ID, inlineEdge.IssueID) + } +} diff --git a/internal/storage/merge_slot.go b/internal/storage/merge_slot.go index 3ae4c7718..af21eb1fb 100644 --- a/internal/storage/merge_slot.go +++ b/internal/storage/merge_slot.go @@ -78,8 +78,10 @@ func MergeSlotCheckImpl(ctx context.Context, s Storage) (*MergeSlotStatus, error } // MergeSlotAcquireImpl is the shared implementation of Storage.MergeSlotAcquire. -// It uses RunInTransaction to ensure atomic check-and-set, preventing two -// agents from simultaneously acquiring the slot. +// It runs the check-and-set in one transaction, preventing two agents from +// simultaneously acquiring the slot. Two agents acquiring at once is exactly +// the race the slot arbitrates: the loser's transaction is replayed whole, so +// it re-reads the slot and queues behind the winner instead of failing. func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, wait bool) (*MergeSlotResult, error) { if holder == "" { return nil, fmt.Errorf("merge-slot acquire: holder must not be empty") @@ -87,11 +89,11 @@ func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, slotID := MergeSlotID(ctx, s) var result MergeSlotResult - result.SlotID = slotID - err := s.RunInTransaction(ctx, + err := RunReplayableTransaction(ctx, s, fmt.Sprintf("bd: acquire merge slot %s for %s", slotID, holder), func(tx Transaction) error { + result = MergeSlotResult{SlotID: slotID} slot, err := tx.GetIssue(ctx, slotID) if err != nil || slot == nil { return fmt.Errorf("merge slot not found: %s (run 'bd merge-slot create' first)", slotID) @@ -153,7 +155,7 @@ func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, func MergeSlotReleaseImpl(ctx context.Context, s Storage, holder, actor string) error { slotID := MergeSlotID(ctx, s) - return s.RunInTransaction(ctx, + return RunReplayableTransaction(ctx, s, fmt.Sprintf("bd: release merge slot %s", slotID), func(tx Transaction) error { slot, err := tx.GetIssue(ctx, slotID) diff --git a/internal/storage/merge_slot_test.go b/internal/storage/merge_slot_test.go new file mode 100644 index 000000000..32712f3d5 --- /dev/null +++ b/internal/storage/merge_slot_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// mergeSlotRaceStore holds one merge slot row and runs each transaction's +// callback against a copy of it, keeping the callback's writes only when the +// transaction commits. Its first transaction loses a write conflict to a +// competitor whose acquire commits first. +type mergeSlotRaceStore struct { + Storage + slot types.Issue + competitor string + transactions int +} + +func (s *mergeSlotRaceStore) GetConfig(context.Context, string) (string, error) { return "test", nil } + +func (s *mergeSlotRaceStore) RunInTransaction(_ context.Context, _ string, fn func(tx Transaction) error) error { + s.transactions++ + tx := &mergeSlotRaceTx{slot: s.slot} + if err := fn(tx); err != nil { + return err + } + if s.transactions == 1 { + s.slot.Status = types.StatusInProgress + s.slot.Metadata = json.RawMessage(`{"holder":"` + s.competitor + `"}`) + return MarkTransactionConflict(errors.New("sql commit (regular): serialization failure")) + } + s.slot = tx.slot + return nil +} + +type mergeSlotRaceTx struct { + Transaction + slot types.Issue +} + +func (tx *mergeSlotRaceTx) GetIssue(_ context.Context, id string) (*types.Issue, error) { + if id != tx.slot.ID { + return nil, ErrNotFound + } + slot := tx.slot + return &slot, nil +} + +func (tx *mergeSlotRaceTx) UpdateIssue(_ context.Context, _ string, updates map[string]interface{}, _ string) error { + if status, ok := updates["status"].(types.Status); ok { + tx.slot.Status = status + } + if meta, ok := updates["metadata"].(string); ok { + tx.slot.Metadata = json.RawMessage(meta) + } + return nil +} + +// TestMergeSlotAcquireReplayReportsTheReplaysOutcome: two agents acquiring at +// once is the race the slot arbitrates. The loser's transaction is replayed, +// finds the slot held, and must report exactly that, not the acquisition its +// rolled-back attempt made. +func TestMergeSlotAcquireReplayReportsTheReplaysOutcome(t *testing.T) { + useInstantReplayTimer(t) + st := &mergeSlotRaceStore{slot: types.Issue{ID: "test-merge-slot", Status: types.StatusOpen}, competitor: "other"} + res, err := MergeSlotAcquireImpl(context.Background(), st, "me", "me", false) + if err != nil { + t.Fatalf("acquire was not replayed to an answer: %v", err) + } + if st.transactions != 2 { + t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.transactions) + } + if res.Acquired || res.Holder != "other" || res.SlotID != "test-merge-slot" { + t.Fatalf("result = %+v, want the slot reported held by other: the rolled-back attempt's acquisition leaked into the replay's result", res) + } + if st.slot.Status != types.StatusInProgress || string(st.slot.Metadata) != `{"holder":"other"}` { + t.Fatalf("slot = %s %s, want the competitor's acquisition intact", st.slot.Status, st.slot.Metadata) + } +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index eaff2090c..5f7a9703c 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -636,7 +636,10 @@ type Storage interface { // starts, it is invoked at most once for this public call: callers retry // explicitly when repeating their work is safe. An error wrapping // ErrCommitIndeterminate means the durable outcome may be unknown and must - // not be blindly replayed. + // not be blindly replayed. An error wrapping ErrTransactionConflict means + // the transaction lost a concurrent-write conflict and was rolled back + // whole; RunReplayableTransaction replays such a transaction for a caller + // whose callback is safe to repeat. RunInTransaction(ctx context.Context, commitMsg string, fn func(tx Transaction) error) error // MergeSlot — serialized conflict resolution primitive. diff --git a/internal/storage/transaction_replay.go b/internal/storage/transaction_replay.go new file mode 100644 index 000000000..b8488d26b --- /dev/null +++ b/internal/storage/transaction_replay.go @@ -0,0 +1,160 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/cenkalti/backoff/v4" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/steveyegge/beads/internal/debug" +) + +// ErrTransactionConflict marks a transaction the database rolled back whole +// because it lost a write conflict with a concurrent transaction: a +// serialization failure or deadlock (MySQL 1213, what Dolt's commit-time merge +// raises) or a lock-wait timeout (1205). Nothing the transaction wrote is +// durable, and the server's own remedy is to run the WHOLE transaction again. +// Backends attach it with MarkTransactionConflict; it never accompanies +// ErrCommitIndeterminate. +var ErrTransactionConflict = errors.New("transaction lost a write conflict with a concurrent transaction and was rolled back") + +// transactionConflictError tags a backend error as ErrTransactionConflict +// without changing the message the user sees. +type transactionConflictError struct{ err error } + +func (e *transactionConflictError) Error() string { return e.err.Error() } + +func (e *transactionConflictError) Unwrap() []error { + return []error{e.err, ErrTransactionConflict} +} + +// MarkTransactionConflict returns err tagged so that +// errors.Is(err, ErrTransactionConflict) holds, keeping its message and its +// cause chain intact. Only a backend that knows the failure rolled the whole +// transaction back may call it. +func MarkTransactionConflict(err error) error { + if err == nil || errors.Is(err, ErrTransactionConflict) { + return err + } + return &transactionConflictError{err: err} +} + +// Transactor is the one store capability RunReplayableTransaction needs. +type Transactor interface { + RunInTransaction(ctx context.Context, commitMsg string, fn func(tx Transaction) error) error +} + +// The replay policy. Each attempt is a complete transaction, so the attempt +// bound is what caps the work; the elapsed bound only stops a run whose +// attempts are themselves slow. The waits grow exponentially, are capped so +// the later attempts keep arriving, and are randomized by half their length in +// either direction: writers that collide tend to start together, and a fixed +// schedule would have every loser collide again in lockstep. +const ( + replayMaxAttempts = 20 + replayInitialInterval = 25 * time.Millisecond + replayMaxInterval = time.Second + replayRandomizationFactor = 0.5 + replayMaxElapsed = 30 * time.Second +) + +// replayTimer supplies the timer for one replay loop; nil uses the backoff +// library's real timer. Tests substitute one that fires immediately. +var replayTimer func() backoff.Timer + +// txReplays counts whole-transaction replays, labeled outcome=replayed for each +// replay and outcome=exhausted when a loop runs out of attempts on conflicts: +// the RunReplayableTransaction counterpart of withRetryTx's write-retry +// counter. It is registered against the global delegating provider, so it +// forwards to the real provider once telemetry is initialized. +var txReplays = newTxReplaysCounter(otel.Meter("github.com/steveyegge/beads/storage")) + +func newTxReplaysCounter(meter metric.Meter) metric.Int64Counter { + counter, _ := meter.Int64Counter("bd.tx_replays_total", + metric.WithDescription("Whole-transaction replays after a lost write conflict (label: outcome=replayed|exhausted)"), + metric.WithUnit("{replay}"), + ) + return counter +} + +// RunReplayableTransaction runs fn through s.RunInTransaction and, while an +// attempt fails with ErrTransactionConflict, waits a jittered backoff and runs +// the WHOLE transaction again: a fresh BEGIN, the callback from the top, and a +// fresh COMMIT. RunInTransaction invokes its callback at most once and leaves +// the decision to replay to the caller, because only the caller knows whether +// its callback is safe to repeat. +// +// Use it only for a callback that touches the database exclusively through tx, +// re-derives every value it hands back on each invocation (plain assignment or +// an explicit reset at its top, never an append into captured state), and has +// no other side effects. Any other failure — a callback error, a non-conflict +// database error, or an ErrCommitIndeterminate outcome that may already have +// landed — is returned after the one attempt that produced it. +// +// The loop makes at most replayMaxAttempts attempts within replayMaxElapsed. +// When it gives up it returns the last conflict, which still matches +// ErrTransactionConflict, annotated with the number of attempts made. +func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg string, fn func(tx Transaction) error) error { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = replayInitialInterval + bo.MaxInterval = replayMaxInterval + bo.RandomizationFactor = replayRandomizationFactor + bo.MaxElapsedTime = replayMaxElapsed + policy := backoff.WithContext(backoff.WithMaxRetries(bo, replayMaxAttempts-1), ctx) + + attempts := 0 + var lastConflict, lastAttempt error + notify := func(err error, wait time.Duration) { + txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed"))) + debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v\n", commitMsg, attempts, wait, err) + } + var timer backoff.Timer + if replayTimer != nil { + timer = replayTimer() + } + err := backoff.RetryNotifyWithTimer(func() error { + attempts++ + err := s.RunInTransaction(ctx, commitMsg, fn) + lastAttempt = err + if err == nil { + return nil + } + if replayableConflict(err) { + lastConflict = err + return err + } + return backoff.Permanent(err) + }, policy, notify, timer) + if err == nil || lastConflict == nil || !replayableConflict(lastAttempt) { + // The last attempt's own outcome is the answer, unchanged: a callback + // error, a non-conflict database error, or an indeterminate commit + // that may have landed — even when the context ended during it. + // Replacing it with an earlier, rolled-back conflict would tell the + // caller nothing was written. + return err + } + if errors.Is(err, lastConflict) { + // The loop ended on the conflict it was replaying: its budget ran out. + txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "exhausted"))) + return fmt.Errorf("%w (gave up after %d attempts: the transaction kept losing write conflicts with concurrent writers)", err, attempts) + } + if ctxErr := ctx.Err(); ctxErr != nil { + // The context ended while waiting to replay: report both why the loop + // stopped and the conflict it was retrying. That is not exhaustion. + return fmt.Errorf("%w (stopped retrying after %d attempt(s): %w)", lastConflict, attempts, ctxErr) + } + return err +} + +// replayableConflict reports whether err proves the attempt rolled back +// because of a concurrent-write conflict. Indeterminacy always wins: a write +// whose commit response was lost may have landed, so it is never replayed even +// if something also marked it as a conflict. +func replayableConflict(err error) bool { + return errors.Is(err, ErrTransactionConflict) && !errors.Is(err, ErrCommitIndeterminate) +} diff --git a/internal/storage/transaction_replay_test.go b/internal/storage/transaction_replay_test.go new file mode 100644 index 000000000..a6e507eca --- /dev/null +++ b/internal/storage/transaction_replay_test.go @@ -0,0 +1,313 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// scriptedTransactor answers each RunInTransaction call with the next scripted +// outcome, invoking the callback first as a real store would. +type scriptedTransactor struct { + outcomes []error + calls int +} + +func (s *scriptedTransactor) RunInTransaction(_ context.Context, _ string, fn func(tx Transaction) error) error { + s.calls++ + if err := fn(nil); err != nil { + return err + } + if len(s.outcomes) == 0 { + return nil + } + out := s.outcomes[0] + s.outcomes = s.outcomes[1:] + return out +} + +// instantTimer fires immediately and records every wait the policy asked for, +// so the replay loop runs without sleeping. +type instantTimer struct { + c chan time.Time + delays []time.Duration +} + +func (t *instantTimer) Start(d time.Duration) { + t.delays = append(t.delays, d) + t.c <- time.Now() +} +func (t *instantTimer) Stop() {} +func (t *instantTimer) C() <-chan time.Time { return t.c } + +func useInstantReplayTimer(t *testing.T) *instantTimer { + t.Helper() + timer := &instantTimer{c: make(chan time.Time, 1)} + prev := replayTimer + replayTimer = func() backoff.Timer { return timer } + t.Cleanup(func() { replayTimer = prev }) + return timer +} + +// conflict is what a backend returns when the server rolled the whole +// transaction back after losing a concurrent-write race. +func conflict(n int) error { + return MarkTransactionConflict(fmt.Errorf("sql commit (regular): serialization failure #%d", n)) +} + +func conflicts(n int) []error { + out := make([]error, n) + for i := range out { + out[i] = conflict(i) + } + return out +} + +func TestMarkTransactionConflictKeepsMessageAndCause(t *testing.T) { + cause := errors.New("sql commit (regular): Error 1213 (40001): serialization failure") + marked := MarkTransactionConflict(cause) + if !errors.Is(marked, ErrTransactionConflict) { + t.Fatalf("marked error %v does not match ErrTransactionConflict", marked) + } + if !errors.Is(marked, cause) { + t.Fatalf("marked error %v lost its cause", marked) + } + if marked.Error() != cause.Error() { + t.Fatalf("marked message = %q, want the unchanged %q", marked.Error(), cause.Error()) + } + if MarkTransactionConflict(nil) != nil { + t.Fatal("MarkTransactionConflict(nil) must stay nil") + } + if again := MarkTransactionConflict(marked); again != marked { + t.Fatal("marking an already-marked error must not wrap it twice") + } +} + +func TestRunReplayableTransactionReplaysWholeTransactionOnConflict(t *testing.T) { + timer := useInstantReplayTimer(t) + tr := &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}} + runs := 0 + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { + runs++ + return nil + }) + if err != nil { + t.Fatalf("RunReplayableTransaction = %v, want success after two lost conflicts", err) + } + if tr.calls != 3 || runs != 3 { + t.Fatalf("transactions=%d callbacks=%d, want the WHOLE transaction, callback included, run 3 times", tr.calls, runs) + } + if len(timer.delays) != 2 { + t.Fatalf("backoff waits = %v, want one wait before each replay", timer.delays) + } +} + +func TestRunReplayableTransactionDoesNotReplayOtherFailures(t *testing.T) { + useInstantReplayTimer(t) + for name, failure := range map[string]error{ + "plain error": errors.New("dependency would create a cycle"), + "indeterminate commit": fmt.Errorf("stage and commit after regular SQL commit: %w", ErrCommitIndeterminate), + // A conflict marker never overrides indeterminacy: the write may + // already have landed, so replaying it could apply it twice. + "marked but indeterminate": MarkTransactionConflict(fmt.Errorf("lost response: %w", ErrCommitIndeterminate)), + } { + t.Run(name, func(t *testing.T) { + tr := &scriptedTransactor{outcomes: []error{failure, nil}} + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }) + if !errors.Is(err, failure) { + t.Fatalf("err = %v, want the original failure", err) + } + if tr.calls != 1 { + t.Fatalf("transactions = %d, want exactly one attempt for a failure that is not a conflict", tr.calls) + } + }) + } +} + +func TestRunReplayableTransactionCallbackErrorIsNotReplayed(t *testing.T) { + useInstantReplayTimer(t) + tr := &scriptedTransactor{} + want := errors.New("issue not found") + runs := 0 + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { + runs++ + return want + }) + if !errors.Is(err, want) || runs != 1 { + t.Fatalf("err=%v runs=%d, want the callback's own error after one run", err, runs) + } +} + +func TestRunReplayableTransactionBoundsAttempts(t *testing.T) { + timer := useInstantReplayTimer(t) + tr := &scriptedTransactor{outcomes: conflicts(replayMaxAttempts + 5)} + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }) + if err == nil { + t.Fatal("RunReplayableTransaction succeeded while every attempt lost a conflict") + } + if tr.calls != replayMaxAttempts { + t.Fatalf("transactions = %d, want exactly replayMaxAttempts (%d)", tr.calls, replayMaxAttempts) + } + if !errors.Is(err, ErrTransactionConflict) { + t.Fatalf("exhausted error %v must still classify as a transaction conflict", err) + } + if !strings.Contains(err.Error(), fmt.Sprintf("serialization failure #%d", replayMaxAttempts-1)) { + t.Fatalf("exhausted error %q must carry the LAST attempt's failure", err) + } + if !strings.Contains(err.Error(), fmt.Sprintf("%d attempts", replayMaxAttempts)) { + t.Fatalf("exhausted error %q must say how many attempts were made", err) + } + longest := time.Duration(float64(replayMaxInterval) * (1 + replayRandomizationFactor)) + for i, d := range timer.delays { + if d <= 0 || d > longest { + t.Fatalf("wait %d = %v, want a positive wait no longer than the jittered cap %v", i, d, longest) + } + } +} + +func TestRunReplayableTransactionBackoffIsJittered(t *testing.T) { + // Writers that collide tend to retry together; a fixed schedule would + // have every loser collide again at the same instant. + distinct := map[time.Duration]bool{} + for run := 0; run < 8; run++ { + timer := useInstantReplayTimer(t) + tr := &scriptedTransactor{outcomes: []error{conflict(1), nil}} + if err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }); err != nil { + t.Fatal(err) + } + if len(timer.delays) != 1 { + t.Fatalf("waits = %v, want one", timer.delays) + } + distinct[timer.delays[0]] = true + } + if len(distinct) < 2 { + t.Fatalf("first waits were identical across runs (%v); the backoff is not jittered", distinct) + } +} + +func TestRunReplayableTransactionStopsWhenContextEnds(t *testing.T) { + useInstantReplayTimer(t) + ctx, cancel := context.WithCancel(context.Background()) + tr := &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}} + runs := 0 + err := RunReplayableTransaction(ctx, tr, "bd: test", func(Transaction) error { + runs++ + cancel() + return nil + }) + if err == nil { + t.Fatal("RunReplayableTransaction kept going after its context was canceled") + } + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrTransactionConflict) { + t.Fatalf("err = %v, want both the cancellation and the conflict that was being retried", err) + } + if runs != 1 { + t.Fatalf("callback runs = %d, want 1 (no replay after cancellation)", runs) + } +} + +// TestRunReplayableTransactionCountsReplays makes replays observable the way +// withRetryTx's retries are: each replay and each loop that gives up is +// counted on bd.tx_replays_total, labeled by outcome. A loop that stops on a +// different failure after a replay did not give up on a conflict. +func TestRunReplayableTransactionCountsReplays(t *testing.T) { + useInstantReplayTimer(t) + // The counter is swapped for one built by the same constructor on a + // test-local provider, so the global meter provider is never touched. + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + prev := txReplays + txReplays = newTxReplaysCounter(provider.Meter("test")) + t.Cleanup(func() { + txReplays = prev + _ = provider.Shutdown(context.Background()) + }) + + noop := func(Transaction) error { return nil } + if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}}, "bd: replayed", noop); err != nil { + t.Fatal(err) + } + if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: conflicts(replayMaxAttempts)}, "bd: exhausted", noop); err == nil { + t.Fatal("the exhausted loop succeeded") + } + if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), errors.New("constraint violated")}}, "bd: failed", noop); err == nil { + t.Fatal("the loop that ended on a plain failure succeeded") + } + // The context ends while the loop waits to replay: not exhausted. + ctx, cancel := context.WithCancel(context.Background()) + if err := RunReplayableTransaction(ctx, &scriptedTransactor{outcomes: []error{conflict(1), nil}}, "bd: canceled", func(Transaction) error { + cancel() + return nil + }); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want the cancellation", err) + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatal(err) + } + got := map[string]int64{} + for _, scope := range rm.ScopeMetrics { + for _, m := range scope.Metrics { + if m.Name != "bd.tx_replays_total" { + continue + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + t.Fatalf("bd.tx_replays_total is %T, want an int64 sum", m.Data) + } + for _, dp := range sum.DataPoints { + outcome, _ := dp.Attributes.Value(attribute.Key("outcome")) + got[outcome.AsString()] += dp.Value + } + } + } + if want := int64(2 + replayMaxAttempts - 1 + 1); got["replayed"] != want { + t.Fatalf("replayed count = %d, want %d (two, then every replay of the exhausted loop, then one)", got["replayed"], want) + } + if got["exhausted"] != 1 { + t.Fatalf("exhausted count = %d, want 1 (only the loop that ran out of attempts)", got["exhausted"]) + } +} + +// TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome: once a replay +// has run, the loop's answer is the LAST attempt's outcome. An attempt that +// ends in an indeterminate commit (its COMMIT cut by the context, so the write +// may have landed) or in a plain failure is returned unchanged even when the +// context ended during it — never replaced by the earlier, rolled-back +// conflict, which would tell the caller nothing was written. +func TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome(t *testing.T) { + for name, last := range map[string]error{ + "indeterminate commit": fmt.Errorf("sql commit (regular): %w: %w", context.Canceled, ErrCommitIndeterminate), + "plain failure": errors.New("validation: dependency would create a cycle"), + } { + t.Run(name, func(t *testing.T) { + useInstantReplayTimer(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tr := &scriptedTransactor{outcomes: []error{conflict(1), last}} + calls := 0 + err := RunReplayableTransaction(ctx, tr, "bd: last outcome", func(Transaction) error { + calls++ + if calls == 2 { + cancel() // the context ends during the replayed attempt + } + return nil + }) + if !errors.Is(err, last) { + t.Fatalf("err = %v, want the last attempt's own outcome %v", err, last) + } + if errors.Is(last, ErrCommitIndeterminate) && errors.Is(err, ErrTransactionConflict) { + t.Fatalf("an indeterminate outcome was reported as a rolled-back conflict: %v", err) + } + }) + } +}