From e77510f8d75c902b0ac004a6baa293c6b03af45d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 00:29:59 +0000 Subject: [PATCH] fix(dolt): wisp writes replay a transaction that loses a write conflict Every wisp write in the Dolt store ran a single-attempt raw transaction: BeginTx, the issueops mutation, then Commit. Wisp tables are dolt_ignored, but they are not free of contention. With the events journal on, each wisp write also bumps the shared bd_events_seq row inside its transaction, so two writers whose transactions overlap collide at commit. Dolt then rolls the later one back with MySQL 1213 ("serialization failure: this transaction conflicts with a committed transaction from another client, try restarting transaction"). The permanent-issue paths run through withRetryTx and replay that rollback. The wisp paths surfaced it to the caller. These now run their body through withRetryTx: - wisps.go: updateWisp, updateWispChecked, closeWisp, closeWispChecked, deleteWisp, deleteWispBatchTx, claimWisp, addWispDependency; - slots.go: mergeMetadataWisp, clearMetadataWisp; - dependencies.go: the wisp arm of RemoveDependencyWithOptions. withRetryTx's transaction scopes the journal and the blocked-state recheck as these bodies did, and runs the post-commit recheck commitSQLTxAndRecheck used to run. That helper now has no callers and is removed. It also scopes versioned history, which is inert on the wisp plane (RecordVersionInTx returns early on IsWisp), so the version-scope guard's eight wisp-plane exemptions go too. The checked paths keep their compare-and-swap. A precondition refusal is a permanent error, so it is not retried. A concurrent write that wins the race makes the transaction lose at commit, and the replay re-reads the row and refuses with ErrVersionMismatch, ErrAssigneeMismatch or ErrStatusMismatch. A commit whose response is lost is still ErrCommitIndeterminate and never replayed. Test: TestJournaledWispWritesSurviveConcurrentWriters runs four stores (four connection pools) on one database with the journal on. Each runs three rounds of update, checked update, claim, link, metadata merge, metadata clear, unlink, close or checked close, delete, and batch delete against its own wisps. It then checks every write landed. On main 67 of the 120 writes fail with Error 1213, and every converted path fails at least once ("commit update wisp: Error 1213 (40001): serialization failure ..."). With this change none do. The package's Wisp|Slot|Metadata|ClaimWrapper|CheckedWrapper|TypeConflict|BlockedRecheck| ScopeIsScoped|ScopeVersioned|DependencyEvents|RemoveDependency|Molecule| CloseIssueChecked|CommitBoundary tests pass, against a dolt sql-server 2.2.0 test container. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/blocked_recheck_raw.go | 15 - .../storage/dolt/blocked_recheck_raw_test.go | 4 +- .../dolt/checked_wrapper_smoke_test.go | 4 +- .../storage/dolt/claim_wrapper_smoke_test.go | 6 +- internal/storage/dolt/dependencies.go | 24 +- .../dolt/dependencies_type_conflict_test.go | 8 +- internal/storage/dolt/peer_store_test.go | 33 ++ internal/storage/dolt/slots.go | 41 +- .../dolt/version_scope_completeness_test.go | 28 +- .../dolt/wisp_write_contention_test.go | 128 ++++++ internal/storage/dolt/wisps.go | 385 +++++++----------- 12 files changed, 344 insertions(+), 334 deletions(-) create mode 100644 internal/storage/dolt/peer_store_test.go create mode 100644 internal/storage/dolt/wisp_write_contention_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2be809cc9..7ee4f3bac 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -289,6 +289,7 @@ go_test( "molecule_contract_test.go", "multistore_multiprocess_test.go", "open_test.go", + "peer_store_test.go", "pool_timeouts_test.go", "port_provenance_test.go", "portsource_propagation_test.go", @@ -376,6 +377,7 @@ go_test( "wisp_set_routing_test.go", "wisp_set_scoped_test.go", "wisp_validation_test.go", + "wisp_write_contention_test.go", "wisps_cycle_test.go", "workspace_config_contract_test.go", ], diff --git a/internal/storage/dolt/blocked_recheck_raw.go b/internal/storage/dolt/blocked_recheck_raw.go index cd9130ba9..f4eeea3e4 100644 --- a/internal/storage/dolt/blocked_recheck_raw.go +++ b/internal/storage/dolt/blocked_recheck_raw.go @@ -2,7 +2,6 @@ package dolt import ( "context" - "database/sql" "github.com/steveyegge/beads/internal/storage/issueops" ) @@ -14,17 +13,3 @@ import ( func (s *DoltStore) settleBlockedRecheck(ctx context.Context, pending issueops.BlockedRecheck) { logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) } - -// commitSQLTxAndRecheck is commitSQLTx for a raw write transaction that was -// scoped with issueops.ScopeBlockedRecheckTransaction: once tx has committed -// it settles the dependents tx's unblocking writes recorded. The wisp writers -// and the legacy dependency removal run on raw transactions outside -// withWriteTx, so without this their writes would record into nothing and -// keep the #6716 race. -func (s *DoltStore) commitSQLTxAndRecheck(ctx context.Context, op string, tx *sql.Tx) error { - if err := s.commitSQLTx(ctx, op, tx); err != nil { - return err - } - s.settleBlockedRecheck(ctx, issueops.TakeBlockedRecheck(tx)) - return nil -} diff --git a/internal/storage/dolt/blocked_recheck_raw_test.go b/internal/storage/dolt/blocked_recheck_raw_test.go index a7609d68f..6abd04a62 100644 --- a/internal/storage/dolt/blocked_recheck_raw_test.go +++ b/internal/storage/dolt/blocked_recheck_raw_test.go @@ -111,8 +111,8 @@ func TestBlockedRecheck_RunInTransactionCommittingLast(t *testing.T) { } } -// TestBlockedRecheck_ConcurrentWispCloses races the raw-transaction wisp -// close path (closeWisp) against itself. It cannot pin the interleaving the +// TestBlockedRecheck_ConcurrentWispCloses races the wisp close path +// (closeWisp) against itself. It cannot pin the interleaving the // way the tests above do, so it only proves the settled outcome: whichever // close commits last, the dependent ends up ready. func TestBlockedRecheck_ConcurrentWispCloses(t *testing.T) { diff --git a/internal/storage/dolt/checked_wrapper_smoke_test.go b/internal/storage/dolt/checked_wrapper_smoke_test.go index 0190956bc..99bc6d35b 100644 --- a/internal/storage/dolt/checked_wrapper_smoke_test.go +++ b/internal/storage/dolt/checked_wrapper_smoke_test.go @@ -95,8 +95,8 @@ func TestCheckedWrappersRouteTheirPreconditions(t *testing.T) { }) // The wisp branch is a THIRD composition — updateWispChecked runs the same - // guards over a bare BeginTx with a deferred Rollback rather than - // withRetryTx — so it is routed to separately here. + // guards in a transaction of its own, with no Dolt commit — so it is + // routed to separately here. t.Run("UpdateRoutesAWispToTheWispChecker", func(t *testing.T) { createWisp(t, ctx, store, "wrapsmoke-wisp") before := get(t, "wrapsmoke-wisp").Title diff --git a/internal/storage/dolt/claim_wrapper_smoke_test.go b/internal/storage/dolt/claim_wrapper_smoke_test.go index 5bef8722e..8a15a4982 100644 --- a/internal/storage/dolt/claim_wrapper_smoke_test.go +++ b/internal/storage/dolt/claim_wrapper_smoke_test.go @@ -106,9 +106,9 @@ func TestClaimIssueWrapperRoutesItsOwnBranches(t *testing.T) { // would be a test that cannot fail. // // C-C. And the wisp branch's refusal, which is a different question from - // C-A: the branch runs its CAS over a bare BeginTx with a deferred - // Rollback, so a refusal has to both surface the sentinel AND leave the - // ephemeral row alone. The role cannot ask this at all. + // C-A: the branch runs its CAS in a transaction of its own, so a refusal + // has to both surface the sentinel AND leave the ephemeral row alone. The + // role cannot ask this at all. t.Run("RefusesAWispHeldByAnotherActorAndLeavesItAlone", func(t *testing.T) { createWisp(t, ctx, store, "clwrap-wisp-held") if err := store.ClaimIssue(ctx, "clwrap-wisp-held", "worker1"); err != nil { diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index cad4758ce..bdb625687 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -97,28 +97,10 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de return s.withCircuitWrite(ctx, func(ctx context.Context) error { // Wisps live in dolt_ignored tables — skip Dolt versioning entirely. if s.isActiveWisp(ctx, issueID) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - // Scoped even though RecordVersionInTx returns early on IsWisp and - // can mint nothing here: the two arms below share one mutator, and - // a scope that is present on one arm only is the asymmetry this - // method's own history is made of. Uniform beats clever. - clearVersionScope := s.scopeVersionedHistoryTransaction(tx) - defer clearVersionScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() - if _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent); err != nil { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent) return err - } - if err := s.commitSQLTxAndRecheck(ctx, "commit remove wisp dependency", tx); err != nil { - return err - } - return nil + }) } tx, err := s.db.BeginTx(ctx, nil) diff --git a/internal/storage/dolt/dependencies_type_conflict_test.go b/internal/storage/dolt/dependencies_type_conflict_test.go index 26b20e356..f791706be 100644 --- a/internal/storage/dolt/dependencies_type_conflict_test.go +++ b/internal/storage/dolt/dependencies_type_conflict_test.go @@ -26,9 +26,11 @@ import ( // // Both DoltStore.AddDependency write seams funnel through the single typed-conflict // return at issueops/dependencies.go, but wrap it differently, so both are covered: -// - issues source: commits via withRetryTx (backoff-permanent, no retry loop). -// - wisp source: addWispDependency (dolt/wisps.go) uses its own BeginTx/Commit, -// returning the conflict before it reaches Commit — a distinct wrapping point. +// - issues source: commits via withRetryTx and a Dolt commit (backoff-permanent, +// no retry loop). +// - wisp source: addWispDependency (dolt/wisps.go) runs its own withRetryTx with +// no Dolt commit, returning the conflict before it reaches Commit — a distinct +// wrapping point. // // The external/cross-prefix route shares the same issueops/dependencies.go return // and is not separately exercised here. diff --git a/internal/storage/dolt/peer_store_test.go b/internal/storage/dolt/peer_store_test.go new file mode 100644 index 000000000..9a6d6653e --- /dev/null +++ b/internal/storage/dolt/peer_store_test.go @@ -0,0 +1,33 @@ +package dolt + +import ( + "context" + "testing" +) + +// openPeerStore opens a second store on store's database, with a connection +// pool of its own as a second bd process would have. +func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore { + t.Helper() + return openStoreOnPort(t, ctx, store, testServerPort) +} + +// openStoreOnPort opens a store on store's database through whatever listens +// on port: the test server itself, or a test proxy in front of it. +func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore { + t.Helper() + peer, err := New(ctx, &Config{ + Path: t.TempDir(), + CommitterName: "test", + CommitterEmail: "test@example.com", + ServerHost: "127.0.0.1", + ServerPort: port, + Database: store.database, + MaxOpenConns: 2, + }) + if err != nil { + t.Fatalf("open store on %s through port %d: %v", store.database, port, err) + } + t.Cleanup(func() { _ = peer.Close() }) + return peer +} diff --git a/internal/storage/dolt/slots.go b/internal/storage/dolt/slots.go index e4c251c57..9ce80d493 100644 --- a/internal/storage/dolt/slots.go +++ b/internal/storage/dolt/slots.go @@ -49,24 +49,12 @@ func (s *DoltStore) MergeMetadata(ctx context.Context, issueID, key string, valu // mergeMetadataWisp merges a metadata key on a wisp. Mirrors closeWisp: no Dolt // versioning since wisps live in dolt_ignored tables. The read and write still -// share the one transaction, so the atomic-merge property holds. +// share the one transaction, which withRetryTx replays whole, so the +// atomic-merge property holds. func (s *DoltStore) mergeMetadataWisp(ctx context.Context, issueID, key string, value json.RawMessage, actor string) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - - if err := issueops.MergeMetadataInTx(ctx, tx, issueID, key, value, actor); err != nil { - return err - } - if err := s.commitSQLTx(ctx, "commit merge metadata wisp", tx); err != nil { - return err - } - return nil + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + return issueops.MergeMetadataInTx(ctx, tx, issueID, key, value, actor) + }) } // SlotSet sets a key-value pair in the issue's metadata JSON. @@ -152,20 +140,7 @@ func (s *DoltStore) SlotClear(ctx context.Context, issueID, key, actor string) e // clearMetadataWisp clears a metadata key on a wisp. Mirrors mergeMetadataWisp / // closeWisp: no Dolt versioning since wisps live in dolt_ignored tables. func (s *DoltStore) clearMetadataWisp(ctx context.Context, issueID, key, actor string) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - - if err := issueops.DeleteMetadataInTx(ctx, tx, issueID, key, actor); err != nil { - return err - } - if err := s.commitSQLTx(ctx, "commit clear metadata wisp", tx); err != nil { - return err - } - return nil + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + return issueops.DeleteMetadataInTx(ctx, tx, issueID, key, actor) + }) } diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 56ef883f0..9ade91254 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -19,28 +19,11 @@ var versionScopeCalls = map[string]bool{ // reason. The staleness check below fails if one stops being flagged, so an // exemption cannot rot. // -// Every entry is a WISP-PLANE mutation. They reach the shared issueops -// mutators, which is why the scan flags them, but RecordVersionInTx returns -// early on IsWisp(issue): issue_versions is the permanent-issues history and -// wisps are the ephemeral plane, whose own current_revision column is shape -// parity only (migration 0067's "SEMANTIC ASYMMETRY, DELIBERATE" note). -// Binding activation to these transactions could not mint a row, so it would -// be inert code that reads as coverage. -// // The journal guard exempts a different set — the is_blocked recompute passes // — which deliberately does NOT appear here: those recomputes never reach a // versioning mutator at all, so this scan does not flag them, and listing them // would rot immediately under the staleness check below. -var rawTxVersionScopeExemptions = map[string]string{ - "DoltStore.claimWisp": "wisp-plane claim: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", - "DoltStore.closeWisp": "wisp-plane close: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", - "DoltStore.closeWispChecked": "wisp-plane checked close: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", - "DoltStore.updateWisp": "wisp-plane update: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", - "DoltStore.updateWispChecked": "wisp-plane checked update: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", - "DoltStore.addWispDependency": "wisp-plane dependency edit: wisp edges live in wisp_dependencies and RecordVersionInTx returns early on IsWisp", - "DoltStore.mergeMetadataWisp": "wisp slot metadata merge: mutates the wisp plane only, and RecordVersionInTx returns early on IsWisp", - "DoltStore.clearMetadataWisp": "wisp slot metadata clear: mutates the wisp plane only, and RecordVersionInTx returns early on IsWisp", -} +var rawTxVersionScopeExemptions = map[string]string{} // TestEveryRawTxVersionScopeIsScopedOrExempt is the versioned-history twin of // TestEveryRawTxJournalScopeIsScopedOrExempt, and exists for the same reason: @@ -57,12 +40,9 @@ var rawTxVersionScopeExemptions = map[string]string{ // — a lie that reads as well-formed history. // // KNOWN LIMIT, inherited from the journal guard's shape: the scan is per -// FUNCTION, not per transaction. RemoveDependencyWithOptions begins two -// transactions in one body (wisp arm, non-wisp arm) and satisfies this guard -// once EITHER arm scopes — measured, by deleting each arm's scope in turn: -// only deleting both turns it red. That is the reason the fix scoped the -// inert wisp arm too rather than the issues-plane arm alone; a half-scoped -// two-arm function is a shape this guard cannot see. +// FUNCTION, not per transaction. A function that begins two transactions in +// one body satisfies this guard once EITHER of them scopes; a half-scoped +// two-transaction function is a shape this guard cannot see. func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { issueFns, err := journalscan.ParsePackage("../issueops") if err != nil { diff --git a/internal/storage/dolt/wisp_write_contention_test.go b/internal/storage/dolt/wisp_write_contention_test.go new file mode 100644 index 000000000..c4e48850e --- /dev/null +++ b/internal/storage/dolt/wisp_write_contention_test.go @@ -0,0 +1,128 @@ +package dolt + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// TestJournaledWispWritesSurviveConcurrentWriters drives every wisp write path +// from several stores at once with the events journal on. Each of those writes +// bumps the one bd_events_seq row inside its own transaction, so two writers +// whose transactions overlap collide at commit and the later one is rolled back +// with MySQL 1213 ("serialization failure ... try restarting transaction"). +// Every path must replay its transaction instead of surfacing that conflict. +func TestJournaledWispWritesSurviveConcurrentWriters(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + const writers, rounds = 4, 3 + stores := []*DoltStore{base} + for len(stores) < writers { + stores = append(stores, openPeerStore(t, ctx, base)) + } + + type target struct{ wisp, anchor, doomed, swept string } + targets := make([][]target, writers) + for w := range writers { + for r := range rounds { + tg := target{ + wisp: fmt.Sprintf("test-wisp-%d-%d", w, r), + anchor: fmt.Sprintf("test-anchor-%d-%d", w, r), + doomed: fmt.Sprintf("test-doomed-%d-%d", w, r), + swept: fmt.Sprintf("test-swept-%d-%d", w, r), + } + createWisp(t, ctx, base, tg.wisp) + createPerm(t, ctx, base, tg.anchor) + createWisp(t, ctx, base, tg.doomed) + createWisp(t, ctx, base, tg.swept) + targets[w] = append(targets[w], tg) + } + } + for _, s := range stores { + s.SetEventsJournalEnabled(true) + } + + var wg sync.WaitGroup + errs := make(chan error, writers*rounds*11) + start := make(chan struct{}) + for w, s := range stores { + wg.Add(1) + go func() { + defer wg.Done() + <-start + actor := fmt.Sprintf("writer-%d", w) + try := func(op, id string, err error) { + if err != nil { + errs <- fmt.Errorf("%s %s %s: %w", actor, op, id, err) + } + } + for r, tg := range targets[w] { + try("update", tg.wisp, s.UpdateIssue(ctx, tg.wisp, map[string]interface{}{"notes": "noted by " + actor}, actor)) + try("checked update", tg.wisp, s.UpdateIssueChecked(ctx, tg.wisp, map[string]interface{}{"priority": 1}, actor, + storage.UpdateIssueOptions{ExpectedStatus: strPtr(string(types.StatusOpen))})) + try("claim", tg.wisp, s.ClaimIssue(ctx, tg.wisp, actor)) + try("link", tg.wisp, s.AddDependency(ctx, &types.Dependency{IssueID: tg.wisp, DependsOnID: tg.anchor, Type: types.DepRelated}, actor)) + try("merge metadata", tg.wisp, s.MergeMetadata(ctx, tg.wisp, "owner", json.RawMessage(`"`+actor+`"`), actor)) + try("clear metadata", tg.wisp, s.SlotClear(ctx, tg.wisp, "owner", actor)) + try("unlink", tg.wisp, s.RemoveDependency(ctx, tg.wisp, tg.anchor, actor)) + if r%2 == 0 { + try("close", tg.wisp, s.CloseIssue(ctx, tg.wisp, "done", actor, "")) + } else { + _, err := s.CloseIssueChecked(ctx, tg.wisp, actor, storage.CloseIssueOptions{Reason: "done"}) + try("checked close", tg.wisp, err) + } + try("delete", tg.doomed, s.DeleteIssue(ctx, tg.doomed)) + _, err := s.DeleteIssues(ctx, []string{tg.swept}, false, true, false) + try("batch delete", tg.swept, err) + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + if t.Failed() { + return + } + + for w := range writers { + actor := fmt.Sprintf("writer-%d", w) + for _, tg := range targets[w] { + got, err := base.GetIssue(ctx, tg.wisp) + if err != nil { + t.Fatalf("read %s: %v", tg.wisp, err) + } + if got.Status != types.StatusClosed || got.Assignee != actor || got.Notes != "noted by "+actor || got.Priority != 1 { + t.Errorf("%s = status %s, assignee %q, notes %q, priority %d; want every write of %s applied", + tg.wisp, got.Status, got.Assignee, got.Notes, got.Priority, actor) + } + if len(got.Metadata) > 0 && string(got.Metadata) != "{}" { + t.Errorf("%s metadata = %s, want the merged key cleared", tg.wisp, got.Metadata) + } + deps, err := base.GetDependencyRecords(ctx, tg.wisp) + if err != nil { + t.Fatalf("read edges of %s: %v", tg.wisp, err) + } + if len(deps) != 0 { + t.Errorf("%s keeps %d edge(s), want the link removed", tg.wisp, len(deps)) + } + for _, gone := range []string{tg.doomed, tg.swept} { + if _, err := base.GetIssue(ctx, gone); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("read %s after its delete = %v, want ErrNotFound", gone, err) + } + } + } + } +} diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index e8452a674..be0dcfa10 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -185,23 +185,17 @@ func (s *DoltStore) getWispLabels(ctx context.Context, issueID string) ([]string // updateWisp updates fields on a wisp in the wisps table. // Delegates SQL work to issueops.UpdateIssueInTx; no Dolt versioning needed // since wisps live in dolt_ignored tables. +// +// Like the permanent paths, every wisp write runs through withRetryTx. Wisp +// tables are dolt_ignored but not free of contention: with the events journal +// on, each wisp write also bumps the shared bd_events_seq row inside its +// transaction, so a writer whose transaction overlaps another's loses at +// commit (MySQL 1213, rolled back whole) and is replayed. func (s *DoltStore) updateWisp(ctx context.Context, id string, updates map[string]interface{}, actor string) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() - - if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor) return err - } - - return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) + }) } // updateWispChecked updates a wisp with the optional atomic preconditions of @@ -210,57 +204,34 @@ func (s *DoltStore) updateWisp(ctx context.Context, id string, updates map[strin // storage.ErrVersionMismatch) and the opts.ExpectedAssignee/ExpectedStatus // field guards (issueops.CheckExpectedFieldsInTx → ErrAssigneeMismatch/ // ErrStatusMismatch), so a stale precondition refuses before any write and the -// deferred Rollback discards the transaction (a true compare-and-swap). Like -// updateWisp it uses a bare BeginTx/Commit with no withRetryTx (consistent with -// the rest of the wisp write path — do not add one here); wisps live in -// dolt_ignored tables, so there is no DOLT_COMMIT. +// transaction rolls back (a true compare-and-swap). Like updateWisp it runs +// through withRetryTx: a concurrent wisp write that commits during this +// transaction makes it lose at commit, and the replay re-reads the +// preconditions and refuses. Wisps live in dolt_ignored tables, so there is no +// DOLT_COMMIT. func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates map[string]interface{}, actor string, opts storage.UpdateIssueOptions) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() - - if opts.ExpectedVersion != nil { - if err := issueops.CheckVersionInTx(ctx, tx, id, *opts.ExpectedVersion); err != nil { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + if opts.ExpectedVersion != nil { + if err := issueops.CheckVersionInTx(ctx, tx, id, *opts.ExpectedVersion); err != nil { + return err + } + } + if err := issueops.CheckExpectedFieldsInTx(ctx, tx, id, opts.ExpectedAssignee, opts.ExpectedStatus); err != nil { return err } - } - if err := issueops.CheckExpectedFieldsInTx(ctx, tx, id, opts.ExpectedAssignee, opts.ExpectedStatus); err != nil { + _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor) return err - } - if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { - return err - } - - return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) + }) } // closeWisp closes a wisp in the wisps table. // Delegates SQL work to issueops.CloseIssueInTx; no Dolt versioning needed // since wisps live in dolt_ignored tables. func (s *DoltStore) closeWisp(ctx context.Context, id string, reason string, actor string, session string) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() - - if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session) return err - } - - return s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx) + }) } // closeWispChecked closes a wisp with the is_blocked guard, mirroring closeWisp @@ -269,97 +240,78 @@ func (s *DoltStore) closeWisp(ctx context.Context, id string, reason string, act // storage.ErrVersionMismatch when the row's RowVersion no longer matches (an // orthogonal CAS that Force does not bypass). The checks and the close share the // same transaction; wisps live in dolt_ignored tables, so there is no -// DOLT_COMMIT. On any rejection the deferred Rollback discards the transaction — -// no close or event is written. +// DOLT_COMMIT. On any rejection the transaction rolls back — no close or event +// is written. // -// Unlike the permanent path, the wisp close uses a bare BeginTx/Commit with no -// withRetryTx (consistent with the rest of the wisp write path — do not add one -// here). So the CAS's read-side limb still returns ErrVersionMismatch for a -// writer that committed before this tx began, but a CONCURRENT wisp mutation -// that loses the race at commit surfaces as a transaction/serialization error -// rather than ErrVersionMismatch. Either way atomicity holds: no lost update and -// no stale close. +// Like the permanent path, the wisp close runs through withRetryTx: a +// CONCURRENT wisp mutation that wins the race makes this transaction lose at +// commit, and the replay re-reads the row, so the CAS refuses with +// ErrVersionMismatch (or finds the wisp already closed) instead of surfacing a +// serialization error. Either way atomicity holds: no lost update and no stale +// close. func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return storage.CloseIssueResult{}, fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() - - res, err := issueops.CloseIssueCheckedInTx(ctx, tx, id, opts.Reason, actor, opts.Session, opts.Force, opts.ExpectedVersion) - if err != nil { + var result storage.CloseIssueResult + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + res, err := issueops.CloseIssueCheckedInTx(ctx, tx, id, opts.Reason, actor, opts.Session, opts.Force, opts.ExpectedVersion) + if err != nil { + return err + } + result = storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren} + return nil + }); err != nil { return storage.CloseIssueResult{}, err } - - if err := s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx); err != nil { - return storage.CloseIssueResult{}, err - } - return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil + return result, nil } // deleteWisp permanently removes a wisp and its related data. func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, []string{id}) + if aerr != nil { + return fmt.Errorf("affected by wisp delete for %s: %w", id, aerr) + } - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() + // Edges are journaled before the row goes, while their source snapshots can + // still be read. An active wisp is a bead like any other, so its delete is + // a journalled mutation, not silent cleanup. + // The wisp delete surface carries no actor, so these record none. + if err := issueops.RecordDependencyRemovalsForIssuesInTx(ctx, tx, []string{id}, ""); err != nil { + return fmt.Errorf("journal dependency removals for wisp %s: %w", id, err) + } - affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, []string{id}) - if aerr != nil { - return fmt.Errorf("affected by wisp delete for %s: %w", id, aerr) - } + result, err := tx.ExecContext(ctx, "DELETE FROM wisps WHERE id = ?", id) + if err != nil { + return fmt.Errorf("failed to delete wisp: %w", err) + } - // Edges are journaled before the row goes, while their source snapshots can - // still be read. An active wisp is a bead like any other, so its delete is - // a journalled mutation, not silent cleanup. - // The wisp delete surface carries no actor, so these record none. - if err := issueops.RecordDependencyRemovalsForIssuesInTx(ctx, tx, []string{id}, ""); err != nil { - return fmt.Errorf("journal dependency removals for wisp %s: %w", id, err) - } + rows, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } + if rows == 0 { + return fmt.Errorf("wisp not found: %s", id) + } + // The rows==0 return above keeps this actually-deleted-only. The wisp + // delete surface carries no actor, so the row records none. + if err := issueops.RecordDeleteInTx(ctx, tx, id, ""); err != nil { + return err + } - result, err := tx.ExecContext(ctx, "DELETE FROM wisps WHERE id = ?", id) - if err != nil { - return fmt.Errorf("failed to delete wisp: %w", err) - } + if err := issueops.DeleteWispFromDependenciesInTx(ctx, tx, id); err != nil { + return err + } - rows, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get rows affected: %w", err) - } - if rows == 0 { - return fmt.Errorf("wisp not found: %s", id) - } - // The rows==0 return above keeps this actually-deleted-only. The wisp - // delete surface carries no actor, so the row records none. - if err := issueops.RecordDeleteInTx(ctx, tx, id, ""); err != nil { - return err - } + if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { + return fmt.Errorf("delete wisp aux rows for %s: %w", id, err) + } - if err := issueops.DeleteWispFromDependenciesInTx(ctx, tx, id); err != nil { - return err - } - - if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { - return fmt.Errorf("delete wisp aux rows for %s: %w", id, err) - } - - if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { - return fmt.Errorf("recompute is_blocked after wisp delete for %s: %w", id, err) - } - issueops.NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) - - return s.commitSQLTxAndRecheck(ctx, "commit delete wisp", tx) + if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { + return fmt.Errorf("recompute is_blocked after wisp delete for %s: %w", id, err) + } + issueops.NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) + return nil + }) } // deleteWispBatch permanently removes multiple wisps using one transaction per @@ -400,94 +352,75 @@ func (s *DoltStore) deleteWispBatch(ctx context.Context, ids []string) (int, err // Keeping each transaction to ≤200 wisps (8 DELETE statements) ensures it // completes well within Dolt's 10 s write timeout. func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, error) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return 0, fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) - defer clearRecheckScope() - - affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, ids) - if aerr != nil { - return 0, fmt.Errorf("affected by batched wisp delete: %w", aerr) - } - - // Resolve WHICH wisps this batch actually removes before the DELETE runs: - // afterwards they are gone, and RowsAffected reports a count, not a set. - // GC hands this path ids it scanned earlier, so an already-collected wisp is - // a routine case, and a phantom delete record would tell a consumer to drop - // a bead this transaction never touched. - deletedIDs, err := issueops.ExistingIssueIDsInTableInTx(ctx, tx, "wisps", ids) - if err != nil { - return 0, fmt.Errorf("resolve existing wisps for batch delete: %w", err) - } - // Edges are journaled before the rows go, while their source snapshots can - // still be read. - // The batched wisp delete surface carries no actor, so these record none. - if err := issueops.RecordDependencyRemovalsForIssuesInTx(ctx, tx, deletedIDs, ""); err != nil { - return 0, fmt.Errorf("journal dependency removals for batched wisp delete: %w", err) - } - - inClause, args := doltBuildSQLInClause(ids) - - //nolint:gosec // G201: inClause contains only ? markers - result, err := tx.ExecContext(ctx, - fmt.Sprintf("DELETE FROM wisps WHERE id IN (%s)", inClause), - args...) - if err != nil { - return 0, fmt.Errorf("failed to batch delete wisps: %w", err) - } - rowsAffected, _ := result.RowsAffected() - - // The batched wisp delete surface carries no actor, so the rows record none. - for _, id := range deletedIDs { - if err := issueops.RecordDeleteInTx(ctx, tx, id, ""); err != nil { - return 0, err + var deleted int + if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, ids) + if aerr != nil { + return fmt.Errorf("affected by batched wisp delete: %w", aerr) } - } - if err := issueops.DeleteWispsFromDependenciesInTx(ctx, tx, ids); err != nil { + // Resolve WHICH wisps this batch actually removes before the DELETE runs: + // afterwards they are gone, and RowsAffected reports a count, not a set. + // GC hands this path ids it scanned earlier, so an already-collected wisp is + // a routine case, and a phantom delete record would tell a consumer to drop + // a bead this transaction never touched. + deletedIDs, err := issueops.ExistingIssueIDsInTableInTx(ctx, tx, "wisps", ids) + if err != nil { + return fmt.Errorf("resolve existing wisps for batch delete: %w", err) + } + // Edges are journaled before the rows go, while their source snapshots can + // still be read. + // The batched wisp delete surface carries no actor, so these record none. + if err := issueops.RecordDependencyRemovalsForIssuesInTx(ctx, tx, deletedIDs, ""); err != nil { + return fmt.Errorf("journal dependency removals for batched wisp delete: %w", err) + } + + inClause, args := doltBuildSQLInClause(ids) + + //nolint:gosec // G201: inClause contains only ? markers + result, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM wisps WHERE id IN (%s)", inClause), + args...) + if err != nil { + return fmt.Errorf("failed to batch delete wisps: %w", err) + } + rowsAffected, _ := result.RowsAffected() + + // The batched wisp delete surface carries no actor, so the rows record none. + for _, id := range deletedIDs { + if err := issueops.RecordDeleteInTx(ctx, tx, id, ""); err != nil { + return err + } + } + + if err := issueops.DeleteWispsFromDependenciesInTx(ctx, tx, ids); err != nil { + return err + } + + if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, ids); err != nil { + return fmt.Errorf("delete wisp aux rows: %w", err) + } + + if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { + return fmt.Errorf("recompute is_blocked after batched wisp delete: %w", err) + } + issueops.NoteDeleteBlockedRecheck(tx, ids, "", affectedIssues, affectedWisps) + deleted = int(rowsAffected) + return nil + }); err != nil { return 0, err } - - if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, ids); err != nil { - return 0, fmt.Errorf("delete wisp aux rows: %w", err) - } - - if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { - return 0, fmt.Errorf("recompute is_blocked after batched wisp delete: %w", err) - } - issueops.NoteDeleteBlockedRecheck(tx, ids, "", affectedIssues, affectedWisps) - - if err := s.commitSQLTxAndRecheck(ctx, "commit batch wisp delete", tx); err != nil { - return 0, err - } - - return int(rowsAffected), nil + return deleted, nil } // claimWisp atomically claims a wisp. // Delegates SQL work to issueops.ClaimIssueInTx; no Dolt versioning needed // since wisps live in dolt_ignored tables. func (s *DoltStore) claimWisp(ctx context.Context, id string, actor string) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - - if _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor); err != nil { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor) return err - } - - return s.commitSQLTx(ctx, "commit claim wisp", tx) + }) } // ListWisps returns ephemeral issues matching the filter. @@ -663,29 +596,19 @@ func (s *DoltStore) getWispsByIDs(ctx context.Context, ids []string) ([]*types.I // addWispDependency adds a dependency to the wisp_dependencies table. func (s *DoltStore) addWispDependency(ctx context.Context, dep *types.Dependency, actor string, isCrossPrefix, emitEvent bool) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - clearJournalScope := s.scopeEventsJournalTransaction(tx) - defer clearJournalScope() - - kind := issueops.ClassifyDepTarget(ctx, tx, dep, isCrossPrefix) - // Wisp source/event tables are dolt_ignored (committed with the SQL tx, not - // via selective doltAddAndCommit), so the event-written flag is not needed here. - if _, err := issueops.AddDependencyInTx(ctx, tx, dep, actor, issueops.AddDependencyOpts{ - SourceTable: "wisps", - WriteTable: "wisp_dependencies", - IsCrossPrefix: isCrossPrefix, - TargetKind: &kind, - EmitEvent: emitEvent, - }); err != nil { + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + kind := issueops.ClassifyDepTarget(ctx, tx, dep, isCrossPrefix) + // Wisp source/event tables are dolt_ignored (committed with the SQL tx, not + // via selective doltAddAndCommit), so the event-written flag is not needed here. + _, err := issueops.AddDependencyInTx(ctx, tx, dep, actor, issueops.AddDependencyOpts{ + SourceTable: "wisps", + WriteTable: "wisp_dependencies", + IsCrossPrefix: isCrossPrefix, + TargetKind: &kind, + EmitEvent: emitEvent, + }) return err - } - - return s.commitSQLTx(ctx, "commit add wisp dependency", tx) + }) } // getWispDependencies retrieves issues that a wisp depends on.