From 30c3fbc13773bd733df00f6afcc104def8331681 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 00:04:24 +0000 Subject: [PATCH 1/4] fix(dolt): replayed creates restart from the caller's request withRetryTx replays a create whose transaction lost Dolt's commit-time merge to a concurrent writer, but the create writes what it minted back into the caller's issue, and the replay started from that instead of from the request. This is the mechanism #7289 describes. The rolled-back attempt's ID then looked caller-supplied, so the replay took the explicit-ID upsert path. In counter mode, where the competing writer minted the same number, that overwrote the competitor's issue and handed one ID to both callers. The create path writes back through three routes, and a replay has to undo all of them: - the Issue struct: the minted ID, the defaulted timestamps and the content hash; - each inline dependency that named no source gets the minted ID as its source (PersistDependenciesWithOptionsResult); - each inline comment without an ID gets one derived from the minted ID (PersistComments). issueops.SnapshotCreateInputs records those fields before the first attempt and returns a restore func. DoltStore.createIssue and both createIssuesWithFullOptions paths call it at the top of every withRetryTx attempt. A struct-only restore is not enough: the replay then mints a new ID while the edge and the comment still point at the old one, and in hash mode the inline edge fails its foreign key (Error 1452). An audit of the create path found no other write into caller-owned state. Tests: - TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack (unit). It fails against a struct-only restore ("inline edge source after restore = test-1"). - TestConcurrentCounterModeCreatesNeverShareAnID (real Dolt: 4 stores x 3 rounds of single and batch creates, counter mode). On main it fails: "id test-1 handed to both "single 3-0" and "single 2-0"", and "issue test-12 holds "batch 2-2-a" ...: a replayed create overwrote it". - TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue holds a batch create open in its transaction until a competitor commits. It runs in counter and hash mode. Counter mode fails on main ("loser and competitor were both handed test-1"). With a struct-only restore both modes fail; hash mode fails with "Error 1452 ... Foreign key violation on fk: fk_dep_issue". - TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue holds a single create's COMMIT at a TCP proxy until a competitor commits. It also runs in counter and hash mode. Counter mode fails on main as above. With a struct-only restore both modes fail ("comment on test-2 has id ..., want ... (derived from its own issue)"). All pass with this change, against a dolt sql-server 2.2.0 test container. The replay tests use only test-side seams: an OnSkippedDependency callback and a MySQL-wire proxy in the test package. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/storage/dolt/BUILD.bazel | 3 + internal/storage/dolt/create_replay_test.go | 297 ++++++++++++++++++ internal/storage/dolt/issues.go | 12 + internal/storage/dolt/peer_store_test.go | 33 ++ internal/storage/dolt/sql_wire_proxy_test.go | 205 ++++++++++++ internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/create_replay.go | 67 ++++ .../storage/issueops/create_replay_test.go | 70 +++++ 8 files changed, 689 insertions(+) create mode 100644 internal/storage/dolt/create_replay_test.go create mode 100644 internal/storage/dolt/peer_store_test.go create mode 100644 internal/storage/dolt/sql_wire_proxy_test.go create mode 100644 internal/storage/issueops/create_replay.go create mode 100644 internal/storage/issueops/create_replay_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2be809cc9..f69d3cfcb 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -181,6 +181,7 @@ go_test( "create_batch_equivalence_test.go", "create_guard_test.go", "create_issue_commit_test.go", + "create_replay_test.go", "create_rollback_test.go", "credentials_test.go", "cross_project_test.go", @@ -289,6 +290,7 @@ go_test( "molecule_contract_test.go", "multistore_multiprocess_test.go", "open_test.go", + "peer_store_test.go", "pool_timeouts_test.go", "port_provenance_test.go", "portsource_propagation_test.go", @@ -342,6 +344,7 @@ go_test( "smart_gate_data_behind_test.go", "smart_remote_migrate_gate_test.go", "socket_fallback_test.go", + "sql_wire_proxy_test.go", "stats_reporter_contract_test.go", "store_branch_pin_test.go", "store_unit_test.go", diff --git a/internal/storage/dolt/create_replay_test.go b/internal/storage/dolt/create_replay_test.go new file mode 100644 index 000000000..b01a9cc47 --- /dev/null +++ b/internal/storage/dolt/create_replay_test.go @@ -0,0 +1,297 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/rowid" + "github.com/steveyegge/beads/internal/types" +) + +// A create that loses Dolt's commit-time merge to a concurrent writer is +// rolled back by the server and replayed by withRetryTx. These tests prove the +// replay starts from the caller's request: it mints its own ID and derives its +// inline edges and comments from that ID, rather than re-using what the +// rolled-back attempt wrote back into the caller's issue. + +// TestConcurrentCounterModeCreatesNeverShareAnID: in counter mode two writers +// that overlap mint the same number, so one of them loses at commit and is +// replayed. A replay that re-used the rolled-back attempt's ID took the +// explicit-ID upsert path and overwrote the winner's issue, and both callers +// were told the same ID. +func TestConcurrentCounterModeCreatesNeverShareAnID(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if err := base.SetConfig(ctx, "issue_id_mode", "counter"); err != nil { + t.Fatalf("set counter mode: %v", err) + } + const writers, rounds = 4, 3 + stores := []*DoltStore{base} + for len(stores) < writers { + stores = append(stores, openPeerStore(t, ctx, base)) + } + + var mu sync.Mutex + titleByID := map[string]string{} + errs := make(chan error, writers*rounds*3) + record := func(issue *types.Issue) { + mu.Lock() + defer mu.Unlock() + if prev, dup := titleByID[issue.ID]; dup { + errs <- fmt.Errorf("id %s handed to both %q and %q", issue.ID, prev, issue.Title) + return + } + titleByID[issue.ID] = issue.Title + } + newIssue := func(title string) *types.Issue { + return &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + } + var wg sync.WaitGroup + start := make(chan struct{}) + for w, s := range stores { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for r := range rounds { + single := newIssue(fmt.Sprintf("single %d-%d", w, r)) + if err := s.CreateIssue(ctx, single, "writer"); err != nil { + errs <- fmt.Errorf("create %q: %w", single.Title, err) + } else { + record(single) + } + batch := []*types.Issue{newIssue(fmt.Sprintf("batch %d-%d-a", w, r)), newIssue(fmt.Sprintf("batch %d-%d-b", w, r))} + if err := s.CreateIssues(ctx, batch, "writer"); err != nil { + errs <- fmt.Errorf("create batch %d-%d: %w", w, r, err) + } else { + record(batch[0]) + record(batch[1]) + } + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + for id, title := range titleByID { + got, err := base.GetIssue(ctx, id) + if err != nil { + t.Errorf("read %s: %v", id, err) + continue + } + if got.Title != title { + t.Errorf("issue %s holds %q, want %q: a replayed create overwrote it", id, got.Title, title) + } + } + if want := writers * rounds * 3; len(titleByID) != want && !t.Failed() { + t.Fatalf("distinct issues = %d, want %d", len(titleByID), want) + } +} + +// replayIDModes are the two ID modes a replayed create must restart in: in +// counter mode the competitor mints the very number the loser's rolled-back +// attempt had; in hash mode the replay mints a fresh ID, and the events journal +// makes the two overlapping writers collide. +var replayIDModes = []struct { + name string + counterMode bool +}{ + {name: "counter mode", counterMode: true}, + {name: "hash mode", counterMode: false}, +} + +func setUpReplayIDMode(t *testing.T, ctx context.Context, base *DoltStore, counterMode bool) { + t.Helper() + if counterMode { + if err := base.SetConfig(ctx, "issue_id_mode", "counter"); err != nil { + t.Fatalf("set counter mode: %v", err) + } + return + } + base.SetEventsJournalEnabled(true) +} + +// TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue holds a batch create +// open (through the skipped-edge callback it runs inside its transaction) until +// a competitor has committed, so the batch loses at commit and is replayed. Its +// inline edge named no source, so the create filled the source in from the +// minted ID: the replay must fill it in again from its own ID. +func TestReplayedBatchCreateKeepsInlineEdgeOnItsOwnIssue(t *testing.T) { + for _, mode := range replayIDModes { + t.Run(mode.name, func(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + setUpReplayIDMode(t, ctx, base, mode.counterMode) + peer := openPeerStore(t, ctx, base) + peer.SetEventsJournalEnabled(!mode.counterMode) + if err := base.CreateIssue(ctx, &types.Issue{ID: "test-anchor", Title: "anchor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed anchor: %v", err) + } + + prepared, release := make(chan struct{}), make(chan struct{}) + var attempts atomic.Int32 + opts := storage.BatchCreateOptions{ + // Runs inside the create transaction, after the anchor edge is + // written: hold the first attempt open until the competitor + // has committed. + OnSkippedDependency: func(_, _, _ string) { + if attempts.Add(1) == 1 { + close(prepared) + <-release + } + }, + } + loser := &types.Issue{ + Title: "loser", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + Dependencies: []*types.Dependency{ + {DependsOnID: "test-anchor", Type: types.DepBlocks}, + {DependsOnID: "test-missing", Type: types.DepRelated}, + }, + } + done := make(chan error, 1) + go func() { done <- base.CreateIssuesWithFullOptions(ctx, []*types.Issue{loser}, "loser", opts) }() + waitForTransactionPrepared(t, ctx, prepared, "loser") + competitor := &types.Issue{Title: "competitor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := peer.CreateIssue(ctx, competitor, "competitor"); err != nil { + close(release) + t.Fatalf("competitor create: %v", err) + } + close(release) + if err := <-done; err != nil { + t.Fatalf("loser create was not replayed to success: %v", err) + } + if n := attempts.Load(); n < 2 { + t.Fatalf("loser create ran %d attempt(s), want a replay: the competitor's commit did not conflict", n) + } + assertCompetitorIntact(t, ctx, base, loser, competitor) + if got := edgeSourcesInto(t, ctx, base, "test-anchor"); len(got) != 1 || got[0] != loser.ID { + t.Fatalf("edges into test-anchor come from %v, want exactly [%s] (the loser's own issue); competitor is %s", got, loser.ID, competitor.ID) + } + }) + } +} + +// TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue holds a single +// create's COMMIT at a proxy until a competitor has committed, so the create +// loses and is replayed. Its inline comment carried no ID, so the create +// derived one from the minted issue ID: the replay must derive it again from +// its own. +func TestReplayedCreateDerivesInlineCommentIDFromItsOwnIssue(t *testing.T) { + for _, mode := range replayIDModes { + t.Run(mode.name, func(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + setUpReplayIDMode(t, ctx, base, mode.counterMode) + proxy := startSQLWireProxy(t) + via := openStoreOnPort(t, ctx, base, proxy.port()) + via.SetEventsJournalEnabled(!mode.counterMode) + + loser := &types.Issue{ + Title: "loser", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + Comments: []*types.Comment{{Author: "loser", Text: "filed with the issue"}}, + } + mark := proxy.mark() + held, release := proxy.holdNext(isCommitStatement) + done := make(chan error, 1) + go func() { done <- via.CreateIssue(ctx, loser, "loser") }() + select { + case <-held: + case err := <-done: + t.Fatalf("loser create finished before its COMMIT was held: %v", err) + case <-ctx.Done(): + t.Fatalf("loser create never reached COMMIT: %v", ctx.Err()) + } + competitor := &types.Issue{Title: "competitor", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := base.CreateIssue(ctx, competitor, "competitor"); err != nil { + release() + t.Fatalf("competitor create: %v", err) + } + release() + if err := <-done; err != nil { + t.Fatalf("loser create was not replayed to success: %v", err) + } + insertsComment := func(q string) bool { return strings.HasPrefix(q, "INSERT INTO comments") } + if n := proxy.statementsSince(mark, insertsComment); n != 2 { + t.Fatalf("loser create wrote its comment in %d attempt(s), want 2 (one lost to the competitor, one replay)", n) + } + assertCompetitorIntact(t, ctx, base, loser, competitor) + assertCommentDerivedFromOwnIssue(t, ctx, base, loser.ID) + }) + } +} + +func assertCompetitorIntact(t *testing.T, ctx context.Context, base *DoltStore, loser, competitor *types.Issue) { + t.Helper() + if loser.ID == competitor.ID { + t.Fatalf("loser and competitor were both handed %s", loser.ID) + } + got, err := base.GetIssue(ctx, competitor.ID) + if err != nil { + t.Fatalf("read competitor %s: %v", competitor.ID, err) + } + if got.Title != "competitor" { + t.Fatalf("competitor %s holds %q: the replayed create overwrote it", competitor.ID, got.Title) + } +} + +func edgeSourcesInto(t *testing.T, ctx context.Context, s *DoltStore, target string) []string { + t.Helper() + rows, err := s.db.QueryContext(ctx, "SELECT issue_id FROM dependencies WHERE depends_on_issue_id = ?", target) + if err != nil { + t.Fatalf("query edges into %s: %v", target, err) + } + defer rows.Close() + var sources []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan edge source: %v", err) + } + sources = append(sources, id) + } + if err := rows.Err(); err != nil { + t.Fatalf("read edges into %s: %v", target, err) + } + sort.Strings(sources) + return sources +} + +// assertCommentDerivedFromOwnIssue checks issueID's one comment carries the ID +// derived from issueID itself, not one left over from a rolled-back attempt +// that derived it from that attempt's issue ID. +func assertCommentDerivedFromOwnIssue(t *testing.T, ctx context.Context, s *DoltStore, issueID string) { + t.Helper() + comments, err := s.GetIssueComments(ctx, issueID) + if err != nil { + t.Fatalf("read comments of %s: %v", issueID, err) + } + if len(comments) != 1 { + t.Fatalf("%s has %d comments, want 1", issueID, len(comments)) + } + c := comments[0] + want := rowid.New("comments", 0, rowid.Digest([]sql.NullString{ + {String: issueID, Valid: true}, {String: c.Author, Valid: true}, + {String: c.Text, Valid: true}, {String: issueops.FormatAuxTime(c.CreatedAt), Valid: true}, + })) + if c.ID != want { + t.Fatalf("comment on %s has id %s, want %s (derived from its own issue): the replay re-used the rolled-back attempt's comment id", issueID, c.ID, want) + } +} diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index b6a8362b6..cb5a3b690 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -40,7 +40,14 @@ func (s *DoltStore) createIssue(ctx context.Context, issue *types.Issue, actor s } var result issueops.CreateIssueResult + // withRetryTx replays the body after a lost commit-time conflict, and the + // create writes its minted ID (and what derives from it) back into the + // caller's issue: each attempt restarts from the caller's request, or the + // replay would take the rolled-back ID as the caller's own and upsert over + // the winner's row. + restore := issueops.SnapshotCreateInputs([]*types.Issue{issue}) if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + restore() // SkipPrefixValidation matches legacy behavior: single-issue path does // not validate prefixes for explicit IDs. bc, err := issueops.NewBatchContext(ctx, tx, storage.BatchCreateOptions{ @@ -113,14 +120,19 @@ func (s *DoltStore) createIssuesWithFullOptions(ctx context.Context, issues []*t issue.Ephemeral = true } } + restore := issueops.SnapshotCreateInputs(issues) return s.withRetryTx(ctx, func(tx *sql.Tx) error { + restore() _, err := issueops.CreateIssuesInTxWithResult(ctx, tx, issues, actor, opts) return err }) } + // Each attempt restarts from the caller's request; see createIssue. + restore := issueops.SnapshotCreateInputs(issues) var result issueops.CreateIssuesResult if err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + restore() var err error result, err = issueops.CreateIssuesInTxWithResult(ctx, tx, issues, actor, opts) return err diff --git a/internal/storage/dolt/peer_store_test.go b/internal/storage/dolt/peer_store_test.go new file mode 100644 index 000000000..9a6d6653e --- /dev/null +++ b/internal/storage/dolt/peer_store_test.go @@ -0,0 +1,33 @@ +package dolt + +import ( + "context" + "testing" +) + +// openPeerStore opens a second store on store's database, with a connection +// pool of its own as a second bd process would have. +func openPeerStore(t *testing.T, ctx context.Context, store *DoltStore) *DoltStore { + t.Helper() + return openStoreOnPort(t, ctx, store, testServerPort) +} + +// openStoreOnPort opens a store on store's database through whatever listens +// on port: the test server itself, or a test proxy in front of it. +func openStoreOnPort(t *testing.T, ctx context.Context, store *DoltStore, port int) *DoltStore { + t.Helper() + peer, err := New(ctx, &Config{ + Path: t.TempDir(), + CommitterName: "test", + CommitterEmail: "test@example.com", + ServerHost: "127.0.0.1", + ServerPort: port, + Database: store.database, + MaxOpenConns: 2, + }) + if err != nil { + t.Fatalf("open store on %s through port %d: %v", store.database, port, err) + } + t.Cleanup(func() { _ = peer.Close() }) + return peer +} diff --git a/internal/storage/dolt/sql_wire_proxy_test.go b/internal/storage/dolt/sql_wire_proxy_test.go new file mode 100644 index 000000000..98e40b518 --- /dev/null +++ b/internal/storage/dolt/sql_wire_proxy_test.go @@ -0,0 +1,205 @@ +package dolt + +import ( + "fmt" + "io" + "net" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// comQuery is the MySQL command byte of a text-protocol statement. Every +// statement a store sends travels as one: its DSN interpolates parameters. +const comQuery = 0x03 + +// sqlWireProxy forwards MySQL wire traffic between stores under test and the +// test Dolt server. It records every statement it forwards and can trap the +// next one that matches a predicate, either holding it until the test releases +// it (to order two writers' commits deterministically) or forwarding it and +// cutting the client off before the server's answer arrives (a response lost +// after the server acted on the statement). Production code needs no test hook. +type sqlWireProxy struct { + ln net.Listener + target string + + mu sync.Mutex + trap *sqlWireTrap + held []*sqlWireTrap + queries []string + conns []net.Conn +} + +type sqlWireTrap struct { + match func(query string) bool + drop bool + sprung chan struct{} + release chan struct{} + released sync.Once +} + +func (trap *sqlWireTrap) open() { trap.released.Do(func() { close(trap.release) }) } + +func startSQLWireProxy(t *testing.T) *sqlWireProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + p := &sqlWireProxy{ln: ln, target: fmt.Sprintf("127.0.0.1:%d", testServerPort)} + go func() { + for { + client, err := ln.Accept() + if err != nil { + return + } + go p.handle(client) + } + }() + t.Cleanup(p.close) + return p +} + +func (p *sqlWireProxy) port() int { return p.ln.Addr().(*net.TCPAddr).Port } + +// holdNext traps the next statement matching match and keeps it from the +// server until release is called. sprung is closed once it is caught. +func (p *sqlWireProxy) holdNext(match func(query string) bool) (sprung <-chan struct{}, release func()) { + trap := p.arm(match, false) + return trap.sprung, trap.open +} + +// dropResponseToNext traps the next statement matching match: the server runs +// it, and the client's connection is cut instead of receiving the answer. +func (p *sqlWireProxy) dropResponseToNext(match func(query string) bool) (sprung <-chan struct{}) { + return p.arm(match, true).sprung +} + +func (p *sqlWireProxy) arm(match func(query string) bool, drop bool) *sqlWireTrap { + trap := &sqlWireTrap{match: match, drop: drop, sprung: make(chan struct{}), release: make(chan struct{})} + p.mu.Lock() + defer p.mu.Unlock() + if p.trap != nil { + p.trap.open() + } + p.trap = trap + return trap +} + +// mark returns a position in the statement log for statementsSince. +func (p *sqlWireProxy) mark() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.queries) +} + +// statementsSince counts the statements forwarded after mark that match. +func (p *sqlWireProxy) statementsSince(mark int, match func(query string) bool) int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, q := range p.queries[mark:] { + if match(q) { + n++ + } + } + return n +} + +// intercept logs query and hands back the armed trap when it catches query. +func (p *sqlWireProxy) intercept(query string) *sqlWireTrap { + p.mu.Lock() + defer p.mu.Unlock() + p.queries = append(p.queries, query) + trap := p.trap + if trap == nil || !trap.match(query) { + return nil + } + p.trap = nil + if !trap.drop { + p.held = append(p.held, trap) + } + close(trap.sprung) + return trap +} + +func (p *sqlWireProxy) close() { + _ = p.ln.Close() + p.mu.Lock() + defer p.mu.Unlock() + if p.trap != nil { + p.trap.open() + } + for _, trap := range p.held { + trap.open() + } + for _, c := range p.conns { + _ = c.Close() + } +} + +func (p *sqlWireProxy) handle(client net.Conn) { + server, err := net.Dial("tcp", p.target) + if err != nil { + _ = client.Close() + return + } + p.mu.Lock() + p.conns = append(p.conns, client, server) + p.mu.Unlock() + var once sync.Once + closeBoth := func() { once.Do(func() { _ = client.Close(); _ = server.Close() }) } + var dropping atomic.Bool + go func() { + buf := make([]byte, 64*1024) + for { + n, err := server.Read(buf) + if n > 0 { + if dropping.Load() { + closeBoth() + return + } + if _, werr := client.Write(buf[:n]); werr != nil { + closeBoth() + return + } + } + if err != nil { + closeBoth() + return + } + } + }() + hdr := make([]byte, 4) + for { + if _, err := io.ReadFull(client, hdr); err != nil { + closeBoth() + return + } + n := int(hdr[0]) | int(hdr[1])<<8 | int(hdr[2])<<16 + packet := make([]byte, 4+n) + copy(packet, hdr) + if _, err := io.ReadFull(client, packet[4:]); err != nil { + closeBoth() + return + } + // A command starts a new exchange (sequence id 0); the handshake's + // packets do not. + if hdr[3] == 0 && n > 1 && packet[4] == comQuery { + if trap := p.intercept(strings.TrimSpace(string(packet[5:]))); trap != nil { + if trap.drop { + dropping.Store(true) + } else { + <-trap.release + } + } + } + if _, err := server.Write(packet); err != nil { + closeBoth() + return + } + } +} + +func isCommitStatement(query string) bool { return strings.EqualFold(query, "COMMIT") } diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 3d7ab0898..168967ffa 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -36,6 +36,7 @@ go_library( "create_dep_batch.go", "create_fastpath_testhook.go", "create_only_guard.go", + "create_replay.go", "cycle_report.go", "cycles.go", "delete.go", @@ -145,6 +146,7 @@ go_test( "create_batch_cache_test.go", "create_only_errors_test.go", "create_only_guard_test.go", + "create_replay_test.go", "create_test.go", "custom_types_parity_test.go", "cycle_report_test.go", diff --git a/internal/storage/issueops/create_replay.go b/internal/storage/issueops/create_replay.go new file mode 100644 index 000000000..fb8cf567a --- /dev/null +++ b/internal/storage/issueops/create_replay.go @@ -0,0 +1,67 @@ +package issueops + +import "github.com/steveyegge/beads/internal/types" + +// SnapshotCreateInputs records what the create path writes back into the +// caller's issues and returns a func that puts it back, so a create that a +// transaction retry replays starts every attempt from the caller's request +// instead of from what a rolled-back attempt minted. +// +// The create path writes back three things: +// +// - the Issue struct itself: the minted ID (assignCreateIssueIDInTx) and the +// defaulted timestamps and content hash (PrepareIssueForInsert); +// - through each inline dependency, the owning issue's ID as the edge's +// source when the edge named none (PersistDependenciesWithOptionsResult); +// - through each inline comment, the content-derived ID when the comment +// carried none (PersistComments). +// +// Restoring only the struct would leave the last two pointing at the +// rolled-back attempt's ID. The dependency and comment objects stay the +// caller's: only the fields the create path writes are reset. A new write into +// caller-owned state on the create path needs its restore here. +func SnapshotCreateInputs(issues []*types.Issue) (restore func()) { + type depSource struct { + dep *types.Dependency + issueID string + } + type commentID struct { + comment *types.Comment + id string + } + type issueState struct { + issue *types.Issue + fields types.Issue + deps []depSource + comments []commentID + } + saved := make([]issueState, 0, len(issues)) + for _, issue := range issues { + if issue == nil { + continue + } + state := issueState{issue: issue, fields: *issue} + for _, dep := range issue.Dependencies { + if dep != nil { + state.deps = append(state.deps, depSource{dep: dep, issueID: dep.IssueID}) + } + } + for _, comment := range issue.Comments { + if comment != nil { + state.comments = append(state.comments, commentID{comment: comment, id: comment.ID}) + } + } + saved = append(saved, state) + } + return func() { + for _, state := range saved { + *state.issue = state.fields + for _, d := range state.deps { + d.dep.IssueID = d.issueID + } + for _, c := range state.comments { + c.comment.ID = c.id + } + } + } +} diff --git a/internal/storage/issueops/create_replay_test.go b/internal/storage/issueops/create_replay_test.go new file mode 100644 index 000000000..fa1ce3bfe --- /dev/null +++ b/internal/storage/issueops/create_replay_test.go @@ -0,0 +1,70 @@ +package issueops + +import ( + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack covers every +// write the create path makes into a caller's issues, including the two it +// makes through pointers the Issue struct only holds: the minted ID copied into +// each inline dependency that named no source, and the derived ID stamped onto +// each inline comment that had none. A struct copy restores neither. +func TestSnapshotCreateInputsRestoresWhatTheCreatePathWritesBack(t *testing.T) { + presetEdge := &types.Dependency{IssueID: "test-explicit", DependsOnID: "test-other", Type: types.DepRelated} + inlineEdge := &types.Dependency{DependsOnID: "test-anchor", Type: types.DepBlocks} + inlineComment := &types.Comment{Author: "a", Text: "first"} + importedComment := &types.Comment{ID: "c-imported", Author: "b", Text: "second"} + issue := &types.Issue{ + Title: "loser", + Labels: []string{"area:storage"}, + Dependencies: []*types.Dependency{inlineEdge, nil, presetEdge}, + Comments: []*types.Comment{inlineComment, importedComment, nil}, + } + second := &types.Issue{Title: "second"} + restore := SnapshotCreateInputs([]*types.Issue{issue, nil, second}) + + // What one create attempt leaves behind. + issue.ID = "test-1" + issue.CreatedAt = time.Now() + issue.UpdatedAt = issue.CreatedAt + issue.ContentHash = "attempt-1-hash" + inlineEdge.IssueID = issue.ID + inlineComment.ID = "c-attempt-1" + second.ID = "test-2" + + restore() + + if issue.ID != "" || !issue.CreatedAt.IsZero() || !issue.UpdatedAt.IsZero() || issue.ContentHash != "" { + t.Fatalf("issue after restore = id %q, created %v, updated %v, hash %q; want the caller's empty values", + issue.ID, issue.CreatedAt, issue.UpdatedAt, issue.ContentHash) + } + if second.ID != "" { + t.Fatalf("second issue id after restore = %q, want empty", second.ID) + } + if inlineEdge.IssueID != "" { + t.Fatalf("inline edge source after restore = %q, want empty so the replay derives it from the new id", inlineEdge.IssueID) + } + if presetEdge.IssueID != "test-explicit" { + t.Fatalf("explicit edge source after restore = %q, want the caller's %q", presetEdge.IssueID, "test-explicit") + } + if inlineComment.ID != "" { + t.Fatalf("inline comment id after restore = %q, want empty so the replay derives it again", inlineComment.ID) + } + if importedComment.ID != "c-imported" { + t.Fatalf("imported comment id after restore = %q, want the caller's %q", importedComment.ID, "c-imported") + } + if issue.Dependencies[0] != inlineEdge || issue.Comments[0] != inlineComment { + t.Fatal("restore replaced the caller's dependency or comment objects instead of resetting them") + } + + // Every replay starts from the same request. + issue.ID = "test-3" + inlineEdge.IssueID = issue.ID + restore() + if issue.ID != "" || inlineEdge.IssueID != "" { + t.Fatalf("second restore left id %q and edge source %q", issue.ID, inlineEdge.IssueID) + } +} From 2d063e575fe94739090b56f639c3c72f0c6194a7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 00:39:00 +0000 Subject: [PATCH 2/4] fix(storage): replay transactions that lose a commit-time write conflict Several writers share one Dolt sql-server. With the events journal enabled, every mutation also bumps the single bd_events_seq row inside its own transaction, so writers whose transactions overlap collide at commit. Dolt rolls the later one back with MySQL 1213 ("serialization failure ... try restarting transaction"). RunInTransaction invokes its callback at most once, by contract. Every command that writes through it, such as bd mol wisp, bd mol pour/bond/squash/burn, bd cook --persist, bd batch and bd label propagate, failed outright with "sql commit (regular): Error 1213 ...", although nothing had been written and the server's remedy is to run the transaction again. RunInTransaction keeps its at-most-once contract. A caller whose callback is safe to repeat now replays explicitly: - storage.ErrTransactionConflict marks a transaction the database rolled back whole after losing a write conflict, and storage.MarkTransactionConflict attaches it without changing the error's message or cause chain. DoltStore.RunInTransaction and RunInIssueLifecycleTransaction attach it to 1213 and 1205 failures only (isSerializationError). They never attach it alongside ErrCommitIndeterminate: a failure after the SQL commit landed may be durable. Dolt's 1105 "@autocommit transaction rolled back" is not marked either, because it reports merge conflicts already in the working set, which a replay meets again. Classification uses the decoded server error, never message text. - storage.RunReplayableTransaction runs the WHOLE transaction again on that marker: a fresh BEGIN, the callback from the top, and a fresh COMMIT. It makes at most 20 attempts within 30s, with exponential waits from 25ms capped at 1s and jittered by half either way. Every other failure is returned after the attempt that produced it. It logs each replay (BD_DEBUG) and counts bd.tx_replays_total{outcome=replayed|exhausted}. - cmd/bd's transact and transactHonoringAutoCommit, and merge-slot acquire and release, use it. Each callback was audited for replay safety: - bd batch appended results into captured state. runBatchTransaction now starts each attempt with no results. - createIssueWithDeps (bd create-form, bd q) re-used a rolled-back attempt's minted ID. It now restores the caller's request with issueops.SnapshotCreateInputs from the commit below. - Merge-slot acquire kept a rolled-back attempt's result fields. It now resets them per attempt. - The rest are replay-safe as written: label propagate, migrate issues, mol squash, cloneSubgraph (wisp/pour/bond), bond protos and molecules, burn wisps, cook --persist and the proto delete. Each builds fresh values per run and assigns its result; cook passes explicit IDs and no inline edges or comments. bd create --graph already replays through BatchApplier's withRetryTx, and bd close --continue claims through the Claimer/Lifecycle roles, which do the same. Tests: - internal/storage unit tests cover the marker (message and cause kept, idempotent) and the replay policy: whole-transaction replay, attempt bound, jitter, cancellation, no replay of plain, callback or indeterminate failures (even when marked), and replay counting. Each test fails against the mutation it guards. - TestMergeSlotAcquireReplayReportsTheReplaysOutcome fails on main ("acquire was not replayed to an answer: sql commit (regular): serialization failure"). Replayed without the reset, it reports the leak: "result = &{... Acquired:true ... Holder:other ...}". - The DoltStore classifier table covers 1213 at commit, 1205, callback-raised 1213, Dolt's 1105, another 1105, a duplicate key, a 1213 after the SQL commit landed, a lost commit response, and untyped text naming 1213, for both transaction runners. The positive cases fail on main. - Real Dolt (dolt sql-server 2.2.0 testcontainer). Two journaled transactions are ordered so the winner commits first: RunInTransaction marks the loser and runs its callback once, and RunReplayableTransaction replays it exactly once, with every row and journal record written once (wisp and durable). On main the loser fails with "sql commit (regular): Error 1213 (40001): serialization failure: ...". A COMMIT whose response is dropped by a test proxy is indeterminate and never replayed, with the journal on and off. With the journal off, a REGULAR-commit loss rolls both planes back and replays as one, and an IGNORED-commit loss after the regular commit is indeterminate and not replayed. - cmd/bd: TestMolWispReplaysLostJournalConflict runs two cloneSubgraph calls (bd mol wisp) on a journaled database. On main the loser fails with the 1213 above; now it replays once and all eight wisps land once. TestRunBatchTransactionReplayReportsEachOpOnce and TestCreateIssueWithDepsReplaysFromTheCallersRequest fail without transact's replay, and again without the per-attempt reset/restore ("results = [...4 entries...]", "committed issues = [fake-1]"). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/bd/BUILD.bazel | 1 + cmd/bd/batch.go | 31 +- cmd/bd/batch_test.go | 52 +++ cmd/bd/create_atomic.go | 7 + cmd/bd/create_atomic_test.go | 45 ++- cmd/bd/dolt_autocommit.go | 13 +- cmd/bd/transaction_replay_cgo_test.go | 183 ++++++++++ internal/storage/BUILD.bazel | 13 + internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/transaction.go | 39 ++- .../storage/dolt/transaction_conflict_test.go | 119 +++++++ .../storage/dolt/transaction_replay_test.go | 329 ++++++++++++++++++ internal/storage/dolt/transaction_test.go | 3 + internal/storage/merge_slot.go | 12 +- internal/storage/merge_slot_test.go | 83 +++++ internal/storage/storage.go | 5 +- internal/storage/transaction_replay.go | 151 ++++++++ internal/storage/transaction_replay_test.go | 264 ++++++++++++++ 18 files changed, 1324 insertions(+), 28 deletions(-) create mode 100644 cmd/bd/transaction_replay_cgo_test.go create mode 100644 internal/storage/dolt/transaction_conflict_test.go create mode 100644 internal/storage/dolt/transaction_replay_test.go create mode 100644 internal/storage/merge_slot_test.go create mode 100644 internal/storage/transaction_replay.go create mode 100644 internal/storage/transaction_replay_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 273110621..3e06abe50 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1231,6 +1231,7 @@ go_test( "todo_embedded_test.go", "tracker_frontdoor_proxy_test.go", "tracker_proxy_uow_conformance_integration_test.go", + "transaction_replay_cgo_test.go", "types_builtin_test.go", "types_create_agreement_embedded_test.go", "types_create_agreement_test.go", diff --git a/cmd/bd/batch.go b/cmd/bd/batch.go index 99e8b76e1..9ef95c6c3 100644 --- a/cmd/bd/batch.go +++ b/cmd/bd/batch.go @@ -174,17 +174,7 @@ normal 'bd' subcommands for interactive/read operations.`, if proxied { results, err = runBatchProxiedServer(ctx, ops, commitMsg) } else { - results = make([]batchOpResult, 0, len(ops)) - err = transact(ctx, store, commitMsg, func(tx storage.Transaction) error { - for _, op := range ops { - res, rerr := runBatchOp(ctx, tx, op) - if rerr != nil { - return fmt.Errorf("line %d (%s): %w", op.line, op.raw, rerr) - } - results = append(results, res) - } - return nil - }) + results, err = runBatchTransaction(ctx, store, commitMsg, ops) } if err != nil { if jsonOutput { @@ -222,6 +212,25 @@ func init() { rootCmd.AddCommand(batchCmd) } +// runBatchTransaction runs ops in one transaction and reports each op's +// result. transact replays the transaction after a lost concurrent-write +// conflict, so each attempt starts with no results. +func runBatchTransaction(ctx context.Context, s storage.DoltStorage, commitMsg string, ops []batchOp) ([]batchOpResult, error) { + var results []batchOpResult + err := transact(ctx, s, commitMsg, func(tx storage.Transaction) error { + results = make([]batchOpResult, 0, len(ops)) + for _, op := range ops { + res, err := runBatchOp(ctx, tx, op) + if err != nil { + return fmt.Errorf("line %d (%s): %w", op.line, op.raw, err) + } + results = append(results, res) + } + return nil + }) + return results, err +} + // batchOp is one parsed command line. type batchOp struct { line int // 1-based source line number diff --git a/cmd/bd/batch_test.go b/cmd/bd/batch_test.go index 834bd49f9..6d3d00d09 100644 --- a/cmd/bd/batch_test.go +++ b/cmd/bd/batch_test.go @@ -1,11 +1,13 @@ package main import ( + "context" "errors" "reflect" "strings" "testing" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" publicops "github.com/steveyegge/beads/issueops" ) @@ -372,3 +374,53 @@ func TestBatchCmd_Registered(t *testing.T) { t.Error("batch missing --message flag") } } + +// batchReplayStore runs each transaction's callback against batchReplayTx and +// makes the first transaction lose a write conflict at commit, as it does when +// a concurrent writer wins. +type batchReplayStore struct { + storage.DoltStorage + transactions int +} + +func (s *batchReplayStore) RunInTransaction(_ context.Context, _ string, fn func(tx storage.Transaction) error) error { + s.transactions++ + if err := fn(batchReplayTx{}); err != nil { + return err + } + if s.transactions == 1 { + return storage.MarkTransactionConflict(errors.New("sql commit (regular): serialization failure")) + } + return nil +} + +type batchReplayTx struct{ storage.Transaction } + +func (batchReplayTx) CloseIssue(context.Context, string, string, string, string) error { return nil } + +func (batchReplayTx) UpdateIssue(context.Context, string, map[string]interface{}, string) error { + return nil +} + +// TestRunBatchTransactionReplayReportsEachOpOnce: a batch whose transaction +// loses a write conflict is replayed whole, and the report must describe the +// transaction that committed, not the rolled-back attempt's ops as well. +func TestRunBatchTransactionReplayReportsEachOpOnce(t *testing.T) { + origExplicit := commandDidExplicitDoltCommit + t.Cleanup(func() { commandDidExplicitDoltCommit = origExplicit }) + ops, err := parseBatchScript(strings.NewReader("close bd-1 done\nupdate bd-2 priority=1\n")) + if err != nil { + t.Fatalf("parseBatchScript: %v", err) + } + st := &batchReplayStore{} + results, err := runBatchTransaction(context.Background(), st, "bd: batch", ops) + if err != nil { + t.Fatalf("runBatchTransaction was not replayed to success: %v", err) + } + if st.transactions != 2 { + t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.transactions) + } + if len(results) != len(ops) { + t.Fatalf("results = %+v, want one per op (%d): the rolled-back attempt's ops were reported too", results, len(ops)) + } +} diff --git a/cmd/bd/create_atomic.go b/cmd/bd/create_atomic.go index f2f425ed9..bebd1090e 100644 --- a/cmd/bd/create_atomic.go +++ b/cmd/bd/create_atomic.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -62,7 +63,13 @@ func createIssueWithDeps(ctx context.Context, st storage.DoltStorage, issue *typ commitMsg = "bd: create " + issue.ID } + // tx.CreateIssue writes what it mints back into issue, and the transaction + // is replayed after a lost concurrent-write conflict: each attempt + // restarts from the caller's request, or a replay would take the + // rolled-back attempt's ID as the caller's own. + restore := issueops.SnapshotCreateInputs([]*types.Issue{issue}) return transactHonoringAutoCommit(ctx, st, commitMsg, func(tx storage.Transaction) error { + restore() if err := tx.CreateIssue(ctx, issue, actor); err != nil { return err } diff --git a/cmd/bd/create_atomic_test.go b/cmd/bd/create_atomic_test.go index 3121a326a..0cea725f2 100644 --- a/cmd/bd/create_atomic_test.go +++ b/cmd/bd/create_atomic_test.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "strings" "testing" @@ -17,11 +18,12 @@ type createAtomicFakeTx struct { created []*types.Issue deps []*types.Dependency failDep string // DependsOnID whose AddDependency fails + mintID string // the ID an auto-minted create receives in this transaction } func (t *createAtomicFakeTx) CreateIssue(_ context.Context, issue *types.Issue, _ string) error { if issue.ID == "" { - issue.ID = "fake-1" + issue.ID = t.mintID } cp := *issue t.created = append(t.created, &cp) @@ -46,6 +48,10 @@ type createAtomicFakeStore struct { committedIssues []*types.Issue committedDeps []*types.Dependency failDep string + // lostConflicts makes the first transactions lose a write conflict at + // commit: the callback runs, nothing it wrote lands, and the failure is + // marked as one a replay may retry. + lostConflicts int } func (s *createAtomicFakeStore) IsInfraTypeCtx(context.Context, types.IssueType) bool { return false } @@ -90,11 +96,16 @@ func (s *createAtomicFakeStore) CreateIssue(_ context.Context, issue *types.Issu func (s *createAtomicFakeStore) RunInTransaction(_ context.Context, _ string, fn func(tx storage.Transaction) error) error { s.txCount++ - tx := &createAtomicFakeTx{failDep: s.failDep} + // Each transaction mints the next number, as counter mode does when a + // concurrent writer has taken the previous one. + tx := &createAtomicFakeTx{failDep: s.failDep, mintID: fmt.Sprintf("fake-%d", s.txCount)} if err := fn(tx); err != nil { // Rollback: discard everything recorded inside the transaction. return err } + if s.txCount <= s.lostConflicts { + return storage.MarkTransactionConflict(errors.New("sql commit (regular): serialization failure")) + } s.committedIssues = append(s.committedIssues, tx.created...) s.committedDeps = append(s.committedDeps, tx.deps...) return nil @@ -186,6 +197,36 @@ func TestCreateIssueWithDepsRollsBackOnDepFailure(t *testing.T) { } } +// TestCreateIssueWithDepsReplaysFromTheCallersRequest: a create that loses a +// write conflict is replayed, and the replay must mint its own ID. Re-using +// the rolled-back attempt's ID as if the caller had asked for it would take +// the explicit-ID path and could overwrite the writer that won the conflict. +func TestCreateIssueWithDepsReplaysFromTheCallersRequest(t *testing.T) { + saveCreateAtomicGlobals(t) + st := &createAtomicFakeStore{lostConflicts: 1} + issue := &types.Issue{Title: "replayed create", Status: types.StatusOpen, IssueType: types.TypeTask} + + if err := createIssueWithDeps(context.Background(), st, issue, "tester", createDepEdges{parentID: "fake-parent"}); err != nil { + t.Fatalf("createIssueWithDeps was not replayed to success: %v", err) + } + if st.txCount != 2 { + t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.txCount) + } + if len(st.committedIssues) != 1 || st.committedIssues[0].ID != "fake-2" { + var ids []string + for _, committed := range st.committedIssues { + ids = append(ids, committed.ID) + } + t.Fatalf("committed issues = %v, want the one the replay minted (fake-2), not the rolled-back attempt's fake-1", ids) + } + if len(st.committedDeps) != 1 || st.committedDeps[0].IssueID != "fake-2" { + t.Fatalf("committed deps = %+v, want the parent edge from fake-2", st.committedDeps) + } + if issue.ID != "fake-2" { + t.Fatalf("caller's issue reports %q, want the committed fake-2", issue.ID) + } +} + func TestCreateIssueWithDepsEmptyEdgesUsesStoreCreate(t *testing.T) { saveCreateAtomicGlobals(t) st := &createAtomicFakeStore{} diff --git a/cmd/bd/dolt_autocommit.go b/cmd/bd/dolt_autocommit.go index a32205e85..0a5739232 100644 --- a/cmd/bd/dolt_autocommit.go +++ b/cmd/bd/dolt_autocommit.go @@ -14,8 +14,13 @@ import ( // DOLT_COMMIT occurred, preventing the redundant maybeAutoCommit in // PersistentPostRun. Use this instead of calling store.RunInTransaction // directly from command handlers. +// +// A transaction that loses a write conflict with a concurrent writer is +// replayed whole (storage.RunReplayableTransaction), so fn must be safe to run +// again from the top: it touches the database only through tx and re-derives +// every value it hands back on each run. func transact(ctx context.Context, s storage.DoltStorage, commitMsg string, fn func(tx storage.Transaction) error) error { - err := s.RunInTransaction(ctx, commitMsg, fn) + err := storage.RunReplayableTransaction(ctx, s, commitMsg, fn) if err == nil { commandDidExplicitDoltCommit = true } @@ -26,7 +31,9 @@ func transact(ctx context.Context, s storage.DoltStorage, commitMsg string, fn f // part of command auto-commit policy. In batch/off modes the SQL transaction // still commits, but no Dolt version commit is created — the blank message // makes StageAndCommit a no-op, in embedded and SQL-server mode alike -// (bd-4wamg: batch used to be silently inert in server mode). +// (bd-4wamg: batch used to be silently inert in server mode). Like transact, +// it replays a transaction that lost a concurrent-write conflict, so fn must +// be safe to run again from the top. func transactHonoringAutoCommit(ctx context.Context, s storage.DoltStorage, commitMsg string, fn func(tx storage.Transaction) error) error { msg := commitMsg committedExplicitly := strings.TrimSpace(msg) != "" @@ -39,7 +46,7 @@ func transactHonoringAutoCommit(ctx context.Context, s storage.DoltStorage, comm committedExplicitly = false } - err = s.RunInTransaction(ctx, msg, fn) + err = storage.RunReplayableTransaction(ctx, s, msg, fn) if err == nil && committedExplicitly { commandDidExplicitDoltCommit = true } diff --git a/cmd/bd/transaction_replay_cgo_test.go b/cmd/bd/transaction_replay_cgo_test.go new file mode 100644 index 000000000..8cc91020b --- /dev/null +++ b/cmd/bd/transaction_replay_cgo_test.go @@ -0,0 +1,183 @@ +//go:build cgo + +package main + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/types" +) + +// openJournalStores opens n stores (n connection pools, as n concurrent bd +// processes would have) on one fresh database of the test server, with the +// events journal on: every mutation also bumps the single bd_events_seq row +// inside its transaction, so writers whose transactions overlap collide at +// commit. +func openJournalStores(t *testing.T, n int) []*dolt.DoltStore { + t.Helper() + ensureTestMode(t) + if testDoltServerPort == 0 { + t.Skip("Dolt test server not available, skipping") + } + if testutil.DoltContainerCrashed() { + t.Skipf("Dolt test server crashed: %v", testutil.DoltContainerCrashError()) + } + dbName := uniqueTestDBName(t) + stores := make([]*dolt.DoltStore, 0, n) + t.Cleanup(func() { + for _, s := range stores { + _ = s.Close() + } + dropTestDatabase(dbName, testDoltServerPort) + }) + for i := 0; i < n; i++ { + dbPath := filepath.Join(t.TempDir(), ".beads", "beads.db") + writeTestMetadata(t, dbPath, dbName) + ctx, cancel := context.WithTimeout(context.Background(), testStoreOpenTimeout) + doltNewMutex.Lock() + s, err := dolt.New(ctx, &dolt.Config{ + Path: dbPath, + ServerHost: "127.0.0.1", + ServerPort: testDoltServerPort, + Database: dbName, + CreateIfMissing: i == 0, + MaxOpenConns: 4, + }) + doltNewMutex.Unlock() + if err == nil && i == 0 { + err = s.SetConfig(ctx, "issue_prefix", "rp") + } + cancel() + if err != nil { + t.Fatalf("open store %d on %s: %v", i, dbName, err) + } + s.SetEventsJournalEnabled(true) + stores = append(stores, s) + } + return stores +} + +// replayBarrierStore holds its first transaction open after the callback has +// written, until released, so a test can order two writers' commits. +type replayBarrierStore struct { + *dolt.DoltStore + attempts atomic.Int32 + prepared chan struct{} + release chan struct{} +} + +func newReplayBarrierStore(s *dolt.DoltStore) *replayBarrierStore { + return &replayBarrierStore{DoltStore: s, prepared: make(chan struct{}), release: make(chan struct{})} +} + +func (b *replayBarrierStore) RunInTransaction(ctx context.Context, msg string, fn func(tx storage.Transaction) error) error { + return b.DoltStore.RunInTransaction(ctx, msg, func(tx storage.Transaction) error { + if err := fn(tx); err != nil { + return err + } + if b.attempts.Add(1) == 1 { + close(b.prepared) + select { + case <-b.release: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) +} + +// threeStepFormula is a cooked formula: a root and three chained steps, the +// shape `bd mol wisp` clones. +func threeStepFormula() *TemplateSubgraph { + root := &types.Issue{ID: "formula", Title: "three-step formula", IssueType: types.TypeEpic, Status: types.StatusOpen, Priority: 2} + sg := &TemplateSubgraph{Root: root, Issues: []*types.Issue{root}, IssueMap: map[string]*types.Issue{root.ID: root}} + var previous *types.Issue + for _, name := range []string{"first", "second", "third"} { + step := &types.Issue{ID: "formula." + name, Title: name + " step", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2} + sg.Issues = append(sg.Issues, step) + sg.IssueMap[step.ID] = step + sg.Dependencies = append(sg.Dependencies, &types.Dependency{IssueID: step.ID, DependsOnID: root.ID, Type: types.DepParentChild}) + if previous != nil { + sg.Dependencies = append(sg.Dependencies, &types.Dependency{IssueID: step.ID, DependsOnID: previous.ID, Type: types.DepBlocks}) + } + previous = step + } + return sg +} + +func countWisps(t *testing.T, ctx context.Context, db *sql.DB, ids map[string]string) int { + t.Helper() + total := 0 + for _, id := range ids { + var n int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM wisps WHERE id = ?", id).Scan(&n); err != nil { + t.Fatalf("count wisp %s: %v", id, err) + } + if n != 1 { + t.Fatalf("wisp %s has %d rows, want exactly 1", id, n) + } + total += n + } + return total +} + +// TestMolWispReplaysLostJournalConflict: two `bd mol wisp` runs instantiate a +// formula at the same moment with the events journal on, and the second one +// to commit loses with "sql commit (regular): Error 1213 ... try restarting +// transaction". The clone must replay its whole transaction and succeed, with +// every step created exactly once. +func TestMolWispReplaysLostJournalConflict(t *testing.T) { + stores := openJournalStores(t, 2) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + winner, loser := newReplayBarrierStore(stores[0]), newReplayBarrierStore(stores[1]) + type outcome struct { + res *InstantiateResult + err error + } + done := [2]chan outcome{make(chan outcome, 1), make(chan outcome, 1)} + for i, s := range []*replayBarrierStore{winner, loser} { + go func() { + res, err := cloneSubgraph(ctx, s, threeStepFormula(), CloneOptions{ + Actor: fmt.Sprintf("writer-%d", i), Ephemeral: true, Prefix: types.IDPrefixWisp, + }) + done[i] <- outcome{res, err} + }() + } + for _, s := range []*replayBarrierStore{winner, loser} { + select { + case <-s.prepared: + case <-ctx.Done(): + t.Fatalf("a clone never wrote its wisps: %v", ctx.Err()) + } + } + close(winner.release) + won := <-done[0] + close(loser.release) + lost := <-done[1] + + if won.err != nil { + t.Fatalf("winner: %v", won.err) + } + if lost.err != nil { + t.Fatalf("loser failed instead of replaying the conflict: %v", lost.err) + } + if got := loser.attempts.Load(); got != 2 { + t.Fatalf("loser transaction attempts = %d, want 2 (one lost to the conflict, one replay)", got) + } + db := stores[0].DB() + if n := countWisps(t, ctx, db, won.res.IDMapping) + countWisps(t, ctx, db, lost.res.IDMapping); n != 8 { + t.Fatalf("wisp rows = %d, want 8 (two clones of a root and three steps)", n) + } +} diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index acb3b8e08..e25dfc463 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "remote_ref.go", "storage.go", "sync.go", + "transaction_replay.go", "version_control.go", "versioned.go", "wake_defers.go", @@ -69,11 +70,16 @@ go_library( importpath = "github.com/steveyegge/beads/internal/storage", visibility = ["//:__subpackages__"], deps = [ + "//internal/debug", "//internal/hooks", "//internal/types", "//issueops", "//journalops", "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_metric//:metric", ], ) @@ -102,6 +108,7 @@ go_test( "hook_role_census_test.go", "journal_facade_test.go", "leg_contract_wiring_test.go", + "merge_slot_test.go", "metadata_cas_test.go", "metadata_edit_value_test.go", "metadata_jsonpath_test.go", @@ -109,6 +116,7 @@ go_test( "metadata_test.go", "remote_ref_test.go", "role_accessor_decorator_test.go", + "transaction_replay_test.go", ], data = [ ":tree_go_srcs", @@ -123,6 +131,11 @@ go_test( "//issueops", "//journalops", "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_sdk_metric//:metric", + "@io_opentelemetry_go_otel_sdk_metric//metricdata", ], ) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index f69d3cfcb..2449c4d13 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -355,8 +355,10 @@ go_test( "testmain_helper_sentinel_test.go", "testmain_test.go", "transaction_borrow_test.go", + "transaction_conflict_test.go", "transaction_ignored_checkout_test.go", "transaction_phase_test.go", + "transaction_replay_test.go", "transaction_test.go", "transaction_wisp_lifecycle_persistence_test.go", "tree_walker_contract_test.go", diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index ff2be619b..522a2717d 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -67,10 +67,12 @@ func (t *doltTransaction) CreateIssueImport(ctx context.Context, issue *types.Is // RunInTransaction executes a function within a database transaction. Its // callback is invoked at most once per call; callers retry explicitly after a -// callback has started when their operation is safe to repeat. The commitMsg is -// used for the DOLT_COMMIT that makes regular writes visible in Dolt history. -// Wisp routing is handled by individual transaction methods based on -// ID/Ephemeral. +// callback has started when their operation is safe to repeat. A failure that +// proves the server rolled the whole transaction back after losing a write +// conflict is marked storage.ErrTransactionConflict so such a caller can tell +// (storage.RunReplayableTransaction). The commitMsg is used for the +// DOLT_COMMIT that makes regular writes visible in Dolt history. Wisp routing +// is handled by individual transaction methods based on ID/Ephemeral. func (s *DoltStore) RunInTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { return s.runInTransaction(ctx, commitMsg, fn, s.runDoltTransaction) } @@ -98,12 +100,37 @@ func (s *DoltStore) runInTransaction( if callbackErr == nil && errors.Is(err, ErrCommitIndeterminate) { return err } - return backoff.Permanent(err) + return backoff.Permanent(markTransactionConflict(err)) } return err }) } +// markTransactionConflict tags a failure that proves the server rolled the +// whole transaction back after losing a write conflict as +// storage.ErrTransactionConflict, leaving its message unchanged. Nothing such a +// transaction wrote is durable, so a caller whose callback is safe to repeat +// may run it again. +// +// - 1213 (serialization failure or deadlock) is what Dolt's commit-time merge +// raises when a concurrent writer committed a conflicting change first; +// the remedy is exactly a replay. +// - 1205 (lock-wait timeout) is tagged for engines that raise it. The engine +// may roll back only the statement, but the failed transaction is always +// rolled back whole here before any replay. +// +// Dolt's 1105 "@autocommit transaction rolled back" is not tagged: it reports +// merge conflicts already in the working set, which a replay meets again. A +// failure after the SQL commit landed carries ErrCommitIndeterminate and is +// never tagged: its rows may be durable, and replaying them could apply them +// twice. +func markTransactionConflict(err error) error { + if err == nil || errors.Is(err, ErrCommitIndeterminate) || !isSerializationError(err) { + return err + } + return storage.MarkTransactionConflict(err) +} + // RunInIssueLifecycleTransaction runs a lifecycle transition and its durable // side effects through one SQL transaction and one Dolt commit attempt. func (s *DoltStore) RunInIssueLifecycleTransaction(ctx context.Context, commitMsg string, fn func(tx storage.IssueLifecycleTransaction) error) error { @@ -140,7 +167,7 @@ func (s *DoltStore) runInIssueLifecycleTransaction( if callbackErr == nil && errors.Is(err, ErrCommitIndeterminate) { return err } - return backoff.Permanent(err) + return backoff.Permanent(markTransactionConflict(err)) } return err }) diff --git a/internal/storage/dolt/transaction_conflict_test.go b/internal/storage/dolt/transaction_conflict_test.go new file mode 100644 index 000000000..4ba0b2e0d --- /dev/null +++ b/internal/storage/dolt/transaction_conflict_test.go @@ -0,0 +1,119 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage" +) + +// transactionFailureCases are failures a public transaction can surface after +// its callback ran, and whether each proves the server rolled the whole +// transaction back after losing a write conflict, so that a caller may replay +// it. +func transactionFailureCases() []struct { + name string + err error + conflict bool +} { + serialization := &mysql.MySQLError{Number: 1213, SQLState: [5]byte{'4', '0', '0', '0', '1'}, Message: "serialization failure: this transaction conflicts with a committed transaction from another client, try restarting transaction"} + return []struct { + name string + err error + conflict bool + }{ + {name: "1213 serialization failure at the regular commit", err: wrapSQLCommitError("sql commit (regular)", serialization), conflict: true}, + {name: "1205 lock wait timeout", err: fmt.Errorf("callback write: %w", &mysql.MySQLError{Number: 1205, Message: "Lock wait timeout exceeded; try restarting transaction"}), conflict: true}, + // Dolt's autocommit rollback reports merge conflicts already in the + // working set; a replay meets them again, so it is not a conflict a + // replay can win. + {name: "Dolt autocommit rollback", err: &mysql.MySQLError{Number: 1105, Message: "Merge conflict detected, @autocommit transaction rolled back"}, conflict: false}, + {name: "other 1105", err: &mysql.MySQLError{Number: 1105, Message: "command denied to user 'op'@'%'"}, conflict: false}, + {name: "duplicate key", err: &mysql.MySQLError{Number: 1062, Message: "Duplicate entry"}, conflict: false}, + // A serialization failure on the Dolt commit that follows a + // successful SQL commit does not roll the transaction back: its rows + // landed. + {name: "1213 after the SQL commit landed", err: fmt.Errorf("stage and commit after regular SQL commit: %w: %w", serialization, ErrCommitIndeterminate), conflict: false}, + {name: "lost commit response", err: wrapSQLCommitError("sql commit (regular)", mysql.ErrInvalidConn), conflict: false}, + // Classification is by the decoded server error, never by text. + {name: "untyped text naming 1213", err: errors.New("Error 1213 (40001): serialization failure"), conflict: false}, + } +} + +func TestRunInTransactionMarksOnlyRollbackGuaranteedConflicts(t *testing.T) { + for _, tc := range transactionFailureCases() { + t.Run(tc.name, func(t *testing.T) { + store := &DoltStore{} + callbacks := 0 + err := store.runInTransaction(context.Background(), "test: conflict classification", func(storage.Transaction) error { + callbacks++ + return nil + }, func(_ context.Context, _ string, fn func(storage.Transaction) error) error { + if err := fn(nil); err != nil { + return err + } + return tc.err + }) + assertTransactionConflictClassification(t, err, tc.err, tc.conflict) + if callbacks != 1 { + t.Fatalf("callback calls = %d, want 1: RunInTransaction itself never replays a callback", callbacks) + } + }) + } +} + +func TestRunInIssueLifecycleTransactionMarksOnlyRollbackGuaranteedConflicts(t *testing.T) { + for _, tc := range transactionFailureCases() { + t.Run(tc.name, func(t *testing.T) { + store := &DoltStore{} + callbacks := 0 + err := store.runInIssueLifecycleTransaction(context.Background(), "test: conflict classification", func(storage.IssueLifecycleTransaction) error { + callbacks++ + return nil + }, func(_ context.Context, fn func(*sql.Tx) error) error { + if err := fn(nil); err != nil { + return err + } + return tc.err + }) + assertTransactionConflictClassification(t, err, tc.err, tc.conflict) + if callbacks != 1 { + t.Fatalf("callback calls = %d, want 1", callbacks) + } + }) + } +} + +func assertTransactionConflictClassification(t *testing.T, err, cause error, conflict bool) { + t.Helper() + if !errors.Is(err, cause) { + t.Fatalf("err = %v, want the original failure %v in the chain", err, cause) + } + if got := errors.Is(err, storage.ErrTransactionConflict); got != conflict { + t.Fatalf("errors.Is(err, ErrTransactionConflict) = %v, want %v (err = %v)", got, conflict, err) + } + if err.Error() != cause.Error() { + t.Fatalf("message = %q, want the unchanged %q", err.Error(), cause.Error()) + } +} + +// A callback's own failure that wraps a rollback-guaranteed conflict (a write +// statement inside the transaction that lost) is a conflict too: the callback +// error rolls the transaction back. +func TestRunInTransactionMarksConflictRaisedInsideCallback(t *testing.T) { + store := &DoltStore{} + inner := &mysql.MySQLError{Number: 1213, Message: "serialization failure"} + err := store.runInTransaction(context.Background(), "test: conflict in callback", func(storage.Transaction) error { + return fmt.Errorf("adding dependency: %w", inner) + }, func(_ context.Context, _ string, fn func(storage.Transaction) error) error { + return fn(nil) + }) + if !errors.Is(err, storage.ErrTransactionConflict) || !errors.Is(err, inner) { + t.Fatalf("err = %v, want a marked conflict wrapping the statement's 1213", err) + } +} diff --git a/internal/storage/dolt/transaction_replay_test.go b/internal/storage/dolt/transaction_replay_test.go new file mode 100644 index 000000000..060fad443 --- /dev/null +++ b/internal/storage/dolt/transaction_replay_test.go @@ -0,0 +1,329 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// With the events journal on, every mutation also bumps the single +// bd_events_seq row inside its own transaction, so two writers whose +// transactions overlap at all collide when the second one commits: Dolt rolls +// it back with MySQL 1213 ("serialization failure ... try restarting +// transaction"). These tests order two such transactions deterministically: +// both write, the winner commits, then the loser commits and loses. + +// overlapCreates runs two transactions that each create one issue, holds both +// open until both have written, then lets the winner commit before the loser. +// run decides how each transaction is driven; the hold applies only to each +// transaction's first attempt. +func overlapCreates(t *testing.T, ctx context.Context, winner, loser *DoltStore, ephemeral bool, + run func(s *DoltStore, fn func(tx storage.Transaction) error) error, +) (winnerErr, loserErr error, loserCallbacks int32, ids [2]string) { + t.Helper() + prepared := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + release := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + done := [2]chan error{make(chan error, 1), make(chan error, 1)} + var calls [2]atomic.Int32 + for i, s := range []*DoltStore{winner, loser} { + go func() { + done[i] <- run(s, func(tx storage.Transaction) error { + attempt := calls[i].Add(1) + issue := &types.Issue{ + Title: []string{"winner", "loser"}[i], + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: ephemeral, + } + if err := tx.CreateIssue(ctx, issue, "tester"); err != nil { + return err + } + ids[i] = issue.ID + if attempt == 1 { + close(prepared[i]) + return waitForTransactionRelease(ctx, release[i]) + } + return nil + }) + }() + } + waitForTransactionPrepared(t, ctx, prepared[0], "winner") + waitForTransactionPrepared(t, ctx, prepared[1], "loser") + close(release[0]) + winnerErr = <-done[0] + close(release[1]) + loserErr = <-done[1] + return winnerErr, loserErr, calls[1].Load(), ids +} + +func countRows(t *testing.T, ctx context.Context, db *sql.DB, table, column, value string) int { + t.Helper() + var n int + //nolint:gosec // G202: table and column are fixed literals chosen by the test + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table+" WHERE "+column+" = ?", value).Scan(&n); err != nil { + t.Fatalf("count %s rows with %s = %s: %v", table, column, value, err) + } + return n +} + +func journalRows(t *testing.T, ctx context.Context, db *sql.DB) int { + t.Helper() + var n int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal").Scan(&n); err != nil { + t.Fatalf("count journal rows: %v", err) + } + return n +} + +// TestRunInTransactionMarksLostJournalConflict: RunInTransaction keeps its +// at-most-once contract and does not replay the loser, but marks its failure +// as a rolled-back conflict so a caller whose callback is safe to repeat can. +func TestRunInTransactionMarksLostJournalConflict(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + store.SetEventsJournalEnabled(true) + peer.SetEventsJournalEnabled(true) + + winnerErr, loserErr, loserCalls, _ := overlapCreates(t, ctx, store, peer, true, + func(s *DoltStore, fn func(tx storage.Transaction) error) error { + return s.RunInTransaction(ctx, "bd: clone template subgraph", fn) + }) + if winnerErr != nil { + t.Fatalf("winner: %v", winnerErr) + } + if loserErr == nil { + t.Fatal("loser committed although both transactions bumped the journal counter; the overlap did not conflict") + } + if !errors.Is(loserErr, storage.ErrTransactionConflict) { + t.Fatalf("loser error = %v, want it marked storage.ErrTransactionConflict", loserErr) + } + if errors.Is(loserErr, ErrCommitIndeterminate) { + t.Fatalf("loser error = %v must not be indeterminate: the server rolled it back", loserErr) + } + if loserCalls != 1 { + t.Fatalf("loser callback ran %d times, want 1 (RunInTransaction never replays)", loserCalls) + } +} + +// TestRunReplayableTransactionReplaysLostJournalConflict drives the same +// overlap through storage.RunReplayableTransaction: the loser's WHOLE +// transaction is replayed once, both creates land exactly once, and the +// journal records exactly one create for each. +func TestRunReplayableTransactionReplaysLostJournalConflict(t *testing.T) { + for _, tc := range []struct { + name string + ephemeral bool + table string + }{ + {name: "wisp", ephemeral: true, table: "wisps"}, + {name: "durable", ephemeral: false, table: "issues"}, + } { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + store.SetEventsJournalEnabled(true) + peer.SetEventsJournalEnabled(true) + journalBefore := journalRows(t, ctx, store.db) + + winnerErr, loserErr, loserCalls, ids := overlapCreates(t, ctx, store, peer, tc.ephemeral, + func(s *DoltStore, fn func(tx storage.Transaction) error) error { + return storage.RunReplayableTransaction(ctx, s, "bd: clone template subgraph", fn) + }) + if winnerErr != nil { + t.Fatalf("winner: %v", winnerErr) + } + if loserErr != nil { + t.Fatalf("loser was not replayed to success: %v", loserErr) + } + if loserCalls != 2 { + t.Fatalf("loser callback ran %d times, want 2 (one lost attempt, one replay)", loserCalls) + } + for i, id := range ids { + if id == "" { + t.Fatalf("transaction %d reported no issue id", i) + } + if n := countRows(t, ctx, store.db, tc.table, "id", id); n != 1 { + t.Fatalf("%s rows for %s = %d, want exactly 1", tc.table, id, n) + } + } + perCreate := countRows(t, ctx, store.db, "bd_events_journal", "issue_id", ids[0]) + if grew := journalRows(t, ctx, store.db) - journalBefore; grew != 2*perCreate { + t.Fatalf("journal grew by %d rows, want %d: the replay must record its create exactly as the winner did", grew, 2*perCreate) + } + }) + } +} + +// TestRunReplayableTransactionNeverReplaysLostCommitResponse: a COMMIT whose +// response is lost may have landed, so it surfaces as indeterminate and is +// never replayed, with the journal on (one transaction) or off (a regular and +// an ignored transaction). +func TestRunReplayableTransactionNeverReplaysLostCommitResponse(t *testing.T) { + base, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + proxy := startSQLWireProxy(t) + via := openStoreOnPort(t, ctx, base, proxy.port()) + + for _, journal := range []bool{false, true} { + t.Run(fmt.Sprintf("journal=%v", journal), func(t *testing.T) { + via.SetEventsJournalEnabled(journal) + title := fmt.Sprintf("lost-commit-journal-%v", journal) + var calls atomic.Int32 + sprung := proxy.dropResponseToNext(isCommitStatement) + err := storage.RunReplayableTransaction(ctx, via, "bd: lost commit", func(tx storage.Transaction) error { + calls.Add(1) + return tx.CreateIssue(ctx, &types.Issue{Title: title, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "tester") + }) + select { + case <-sprung: + default: + t.Fatalf("the transaction never reached COMMIT (err = %v)", err) + } + if err == nil { + t.Fatal("a lost commit response reported success") + } + if !errors.Is(err, ErrCommitIndeterminate) || errors.Is(err, storage.ErrTransactionConflict) { + t.Fatalf("err = %v, want indeterminate and not a replayable conflict", err) + } + if n := calls.Load(); n != 1 { + t.Fatalf("callback ran %d times: a lost commit response must never be replayed", n) + } + if n := countRows(t, ctx, base.db, "issues", "title", title); n != 1 { + t.Fatalf("rows titled %q = %d, want 1: the server committed it once", title, n) + } + }) + } +} + +// splitTxOverlap runs a winner on peer and a loser on store, both with the +// journal off, so each runs the default split shape: a regular transaction +// for Dolt-tracked tables and an ignored one for the wisp tables. Both write, +// the winner commits, then the loser commits. +func splitTxOverlap(t *testing.T, ctx context.Context, store, peer *DoltStore, + winnerWrite, loserWrite func(tx storage.Transaction) error, +) (loserErr error, loserCalls int32) { + t.Helper() + prepared := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + release := [2]chan struct{}{make(chan struct{}), make(chan struct{})} + done := [2]chan error{make(chan error, 1), make(chan error, 1)} + var calls atomic.Int32 + go func() { + done[0] <- peer.RunInTransaction(ctx, "winner", func(tx storage.Transaction) error { + if err := winnerWrite(tx); err != nil { + return err + } + close(prepared[0]) + return waitForTransactionRelease(ctx, release[0]) + }) + }() + go func() { + done[1] <- storage.RunReplayableTransaction(ctx, store, "loser", func(tx storage.Transaction) error { + attempt := calls.Add(1) + if err := loserWrite(tx); err != nil { + return err + } + if attempt == 1 { + close(prepared[1]) + return waitForTransactionRelease(ctx, release[1]) + } + return nil + }) + }() + waitForTransactionPrepared(t, ctx, prepared[0], "winner") + waitForTransactionPrepared(t, ctx, prepared[1], "loser") + close(release[0]) + if err := <-done[0]; err != nil { + t.Fatalf("winner: %v", err) + } + close(release[1]) + return <-done[1], calls.Load() +} + +// TestSplitTransactionRegularConflictRollsBackBothPlanes: the loser's REGULAR +// commit loses, so both of its transactions roll back and the replay writes +// each plane exactly once; no wisp from the lost attempt survives. +func TestSplitTransactionRegularConflictRollsBackBothPlanes(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + createPerm(t, ctx, store, "test-shared") + + loserErr, loserCalls := splitTxOverlap(t, ctx, store, peer, + func(tx storage.Transaction) error { + return tx.UpdateIssue(ctx, "test-shared", map[string]interface{}{"title": "winner"}, "w") + }, + func(tx storage.Transaction) error { + if err := tx.UpdateIssue(ctx, "test-shared", map[string]interface{}{"title": "loser"}, "l"); err != nil { + return err + } + return tx.CreateIssue(ctx, &types.Issue{Title: "split-wisp", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Ephemeral: true}, "l") + }) + if loserErr != nil { + t.Fatalf("loser: %v", loserErr) + } + if loserCalls != 2 { + t.Fatalf("loser attempts = %d, want 2", loserCalls) + } + if n := countRows(t, ctx, store.db, "wisps", "title", "split-wisp"); n != 1 { + t.Fatalf("wisp rows = %d, want 1: the lost attempt's ignored-plane write leaked", n) + } + got, err := store.GetIssue(ctx, "test-shared") + if err != nil { + t.Fatalf("read test-shared: %v", err) + } + if got.Title != "loser" { + t.Fatalf("title = %q, want the replayed loser's", got.Title) + } +} + +// TestSplitTransactionIgnoredConflictAfterRegularCommitIsNotReplayed: the +// loser's IGNORED commit loses after its regular commit landed. That outcome +// is partial, so it is indeterminate and never a replayable conflict. +func TestSplitTransactionIgnoredConflictAfterRegularCommitIsNotReplayed(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := concurrentTestContext(t) + defer cancel() + peer := openPeerStore(t, ctx, store) + createWisp(t, ctx, store, "test-wisp-shared") + + loserErr, loserCalls := splitTxOverlap(t, ctx, store, peer, + func(tx storage.Transaction) error { + return tx.UpdateIssue(ctx, "test-wisp-shared", map[string]interface{}{"title": "winner"}, "w") + }, + func(tx storage.Transaction) error { + if err := tx.CreateIssue(ctx, &types.Issue{Title: "split-durable", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "l"); err != nil { + return err + } + return tx.UpdateIssue(ctx, "test-wisp-shared", map[string]interface{}{"title": "loser"}, "l") + }) + if loserErr == nil { + t.Fatal("loser succeeded; the ignored-plane overlap did not conflict") + } + if errors.Is(loserErr, storage.ErrTransactionConflict) || !errors.Is(loserErr, ErrCommitIndeterminate) { + t.Fatalf("err = %v, want indeterminate, not a replayable conflict", loserErr) + } + if loserCalls != 1 { + t.Fatalf("loser attempts = %d, want 1: no replay after the regular commit landed", loserCalls) + } + if n := countRows(t, ctx, store.db, "issues", "title", "split-durable"); n != 1 { + t.Fatalf("durable rows = %d, want 1", n) + } +} diff --git a/internal/storage/dolt/transaction_test.go b/internal/storage/dolt/transaction_test.go index fc7e138d4..45c7cf891 100644 --- a/internal/storage/dolt/transaction_test.go +++ b/internal/storage/dolt/transaction_test.go @@ -243,6 +243,9 @@ func TestRunInTransactionSerializationConflictInvokesCallbacksOnce(t *testing.T) if errors.Is(err, ErrCommitIndeterminate) { t.Fatalf("stale transaction error = %v unexpectedly marks an indeterminate commit", err) } + if !errors.Is(err, storage.ErrTransactionConflict) { + t.Fatalf("stale transaction error = %v, want it marked storage.ErrTransactionConflict so a caller may replay it", err) + } if got := callsA.Load(); got != 1 { t.Errorf("winner callback calls = %d, want 1", got) } diff --git a/internal/storage/merge_slot.go b/internal/storage/merge_slot.go index 3ae4c7718..af21eb1fb 100644 --- a/internal/storage/merge_slot.go +++ b/internal/storage/merge_slot.go @@ -78,8 +78,10 @@ func MergeSlotCheckImpl(ctx context.Context, s Storage) (*MergeSlotStatus, error } // MergeSlotAcquireImpl is the shared implementation of Storage.MergeSlotAcquire. -// It uses RunInTransaction to ensure atomic check-and-set, preventing two -// agents from simultaneously acquiring the slot. +// It runs the check-and-set in one transaction, preventing two agents from +// simultaneously acquiring the slot. Two agents acquiring at once is exactly +// the race the slot arbitrates: the loser's transaction is replayed whole, so +// it re-reads the slot and queues behind the winner instead of failing. func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, wait bool) (*MergeSlotResult, error) { if holder == "" { return nil, fmt.Errorf("merge-slot acquire: holder must not be empty") @@ -87,11 +89,11 @@ func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, slotID := MergeSlotID(ctx, s) var result MergeSlotResult - result.SlotID = slotID - err := s.RunInTransaction(ctx, + err := RunReplayableTransaction(ctx, s, fmt.Sprintf("bd: acquire merge slot %s for %s", slotID, holder), func(tx Transaction) error { + result = MergeSlotResult{SlotID: slotID} slot, err := tx.GetIssue(ctx, slotID) if err != nil || slot == nil { return fmt.Errorf("merge slot not found: %s (run 'bd merge-slot create' first)", slotID) @@ -153,7 +155,7 @@ func MergeSlotAcquireImpl(ctx context.Context, s Storage, holder, actor string, func MergeSlotReleaseImpl(ctx context.Context, s Storage, holder, actor string) error { slotID := MergeSlotID(ctx, s) - return s.RunInTransaction(ctx, + return RunReplayableTransaction(ctx, s, fmt.Sprintf("bd: release merge slot %s", slotID), func(tx Transaction) error { slot, err := tx.GetIssue(ctx, slotID) diff --git a/internal/storage/merge_slot_test.go b/internal/storage/merge_slot_test.go new file mode 100644 index 000000000..32712f3d5 --- /dev/null +++ b/internal/storage/merge_slot_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// mergeSlotRaceStore holds one merge slot row and runs each transaction's +// callback against a copy of it, keeping the callback's writes only when the +// transaction commits. Its first transaction loses a write conflict to a +// competitor whose acquire commits first. +type mergeSlotRaceStore struct { + Storage + slot types.Issue + competitor string + transactions int +} + +func (s *mergeSlotRaceStore) GetConfig(context.Context, string) (string, error) { return "test", nil } + +func (s *mergeSlotRaceStore) RunInTransaction(_ context.Context, _ string, fn func(tx Transaction) error) error { + s.transactions++ + tx := &mergeSlotRaceTx{slot: s.slot} + if err := fn(tx); err != nil { + return err + } + if s.transactions == 1 { + s.slot.Status = types.StatusInProgress + s.slot.Metadata = json.RawMessage(`{"holder":"` + s.competitor + `"}`) + return MarkTransactionConflict(errors.New("sql commit (regular): serialization failure")) + } + s.slot = tx.slot + return nil +} + +type mergeSlotRaceTx struct { + Transaction + slot types.Issue +} + +func (tx *mergeSlotRaceTx) GetIssue(_ context.Context, id string) (*types.Issue, error) { + if id != tx.slot.ID { + return nil, ErrNotFound + } + slot := tx.slot + return &slot, nil +} + +func (tx *mergeSlotRaceTx) UpdateIssue(_ context.Context, _ string, updates map[string]interface{}, _ string) error { + if status, ok := updates["status"].(types.Status); ok { + tx.slot.Status = status + } + if meta, ok := updates["metadata"].(string); ok { + tx.slot.Metadata = json.RawMessage(meta) + } + return nil +} + +// TestMergeSlotAcquireReplayReportsTheReplaysOutcome: two agents acquiring at +// once is the race the slot arbitrates. The loser's transaction is replayed, +// finds the slot held, and must report exactly that, not the acquisition its +// rolled-back attempt made. +func TestMergeSlotAcquireReplayReportsTheReplaysOutcome(t *testing.T) { + useInstantReplayTimer(t) + st := &mergeSlotRaceStore{slot: types.Issue{ID: "test-merge-slot", Status: types.StatusOpen}, competitor: "other"} + res, err := MergeSlotAcquireImpl(context.Background(), st, "me", "me", false) + if err != nil { + t.Fatalf("acquire was not replayed to an answer: %v", err) + } + if st.transactions != 2 { + t.Fatalf("transactions = %d, want 2 (one lost conflict, one replay)", st.transactions) + } + if res.Acquired || res.Holder != "other" || res.SlotID != "test-merge-slot" { + t.Fatalf("result = %+v, want the slot reported held by other: the rolled-back attempt's acquisition leaked into the replay's result", res) + } + if st.slot.Status != types.StatusInProgress || string(st.slot.Metadata) != `{"holder":"other"}` { + t.Fatalf("slot = %s %s, want the competitor's acquisition intact", st.slot.Status, st.slot.Metadata) + } +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index eaff2090c..5f7a9703c 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -636,7 +636,10 @@ type Storage interface { // starts, it is invoked at most once for this public call: callers retry // explicitly when repeating their work is safe. An error wrapping // ErrCommitIndeterminate means the durable outcome may be unknown and must - // not be blindly replayed. + // not be blindly replayed. An error wrapping ErrTransactionConflict means + // the transaction lost a concurrent-write conflict and was rolled back + // whole; RunReplayableTransaction replays such a transaction for a caller + // whose callback is safe to repeat. RunInTransaction(ctx context.Context, commitMsg string, fn func(tx Transaction) error) error // MergeSlot — serialized conflict resolution primitive. diff --git a/internal/storage/transaction_replay.go b/internal/storage/transaction_replay.go new file mode 100644 index 000000000..1e6dea495 --- /dev/null +++ b/internal/storage/transaction_replay.go @@ -0,0 +1,151 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/cenkalti/backoff/v4" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/steveyegge/beads/internal/debug" +) + +// ErrTransactionConflict marks a transaction the database rolled back whole +// because it lost a write conflict with a concurrent transaction: a +// serialization failure or deadlock (MySQL 1213, what Dolt's commit-time merge +// raises) or a lock-wait timeout (1205). Nothing the transaction wrote is +// durable, and the server's own remedy is to run the WHOLE transaction again. +// Backends attach it with MarkTransactionConflict; it never accompanies +// ErrCommitIndeterminate. +var ErrTransactionConflict = errors.New("transaction lost a write conflict with a concurrent transaction and was rolled back") + +// transactionConflictError tags a backend error as ErrTransactionConflict +// without changing the message the user sees. +type transactionConflictError struct{ err error } + +func (e *transactionConflictError) Error() string { return e.err.Error() } + +func (e *transactionConflictError) Unwrap() []error { + return []error{e.err, ErrTransactionConflict} +} + +// MarkTransactionConflict returns err tagged so that +// errors.Is(err, ErrTransactionConflict) holds, keeping its message and its +// cause chain intact. Only a backend that knows the failure rolled the whole +// transaction back may call it. +func MarkTransactionConflict(err error) error { + if err == nil || errors.Is(err, ErrTransactionConflict) { + return err + } + return &transactionConflictError{err: err} +} + +// Transactor is the one store capability RunReplayableTransaction needs. +type Transactor interface { + RunInTransaction(ctx context.Context, commitMsg string, fn func(tx Transaction) error) error +} + +// The replay policy. Each attempt is a complete transaction, so the attempt +// bound is what caps the work; the elapsed bound only stops a run whose +// attempts are themselves slow. The waits grow exponentially, are capped so +// the later attempts keep arriving, and are randomized by half their length in +// either direction: writers that collide tend to start together, and a fixed +// schedule would have every loser collide again in lockstep. +const ( + replayMaxAttempts = 20 + replayInitialInterval = 25 * time.Millisecond + replayMaxInterval = time.Second + replayRandomizationFactor = 0.5 + replayMaxElapsed = 30 * time.Second +) + +// replayTimer supplies the timer for one replay loop; nil uses the backoff +// library's real timer. Tests substitute one that fires immediately. +var replayTimer func() backoff.Timer + +// txReplays counts whole-transaction replays, labeled outcome=replayed for each +// replay and outcome=exhausted when a loop gives up: the +// RunReplayableTransaction counterpart of withRetryTx's write-retry counter. It +// is registered against the global delegating provider, so it forwards to the +// real provider once telemetry is initialized. +var txReplays metric.Int64Counter + +func init() { + txReplays, _ = otel.Meter("github.com/steveyegge/beads/storage").Int64Counter("bd.tx_replays_total", + metric.WithDescription("Whole-transaction replays after a lost write conflict (label: outcome=replayed|exhausted)"), + metric.WithUnit("{replay}"), + ) +} + +// RunReplayableTransaction runs fn through s.RunInTransaction and, while an +// attempt fails with ErrTransactionConflict, waits a jittered backoff and runs +// the WHOLE transaction again: a fresh BEGIN, the callback from the top, and a +// fresh COMMIT. RunInTransaction invokes its callback at most once and leaves +// the decision to replay to the caller, because only the caller knows whether +// its callback is safe to repeat. +// +// Use it only for a callback that touches the database exclusively through tx, +// re-derives every value it hands back on each invocation (plain assignment or +// an explicit reset at its top, never an append into captured state), and has +// no other side effects. Any other failure — a callback error, a non-conflict +// database error, or an ErrCommitIndeterminate outcome that may already have +// landed — is returned after the one attempt that produced it. +// +// The loop makes at most replayMaxAttempts attempts within replayMaxElapsed. +// When it gives up it returns the last conflict, which still matches +// ErrTransactionConflict, annotated with the number of attempts made. +func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg string, fn func(tx Transaction) error) error { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = replayInitialInterval + bo.MaxInterval = replayMaxInterval + bo.RandomizationFactor = replayRandomizationFactor + bo.MaxElapsedTime = replayMaxElapsed + policy := backoff.WithContext(backoff.WithMaxRetries(bo, replayMaxAttempts-1), ctx) + + attempts := 0 + var lastConflict error + notify := func(err error, wait time.Duration) { + txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed"))) + debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v", commitMsg, attempts, wait, err) + } + var timer backoff.Timer + if replayTimer != nil { + timer = replayTimer() + } + err := backoff.RetryNotifyWithTimer(func() error { + attempts++ + err := s.RunInTransaction(ctx, commitMsg, fn) + if err == nil { + return nil + } + if replayableConflict(err) { + lastConflict = err + return err + } + return backoff.Permanent(err) + }, policy, notify, timer) + if err == nil || lastConflict == nil { + return err + } + if ctxErr := ctx.Err(); ctxErr != nil && !errors.Is(err, lastConflict) { + txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "exhausted"))) + return fmt.Errorf("%w (stopped retrying after %d attempt(s): %w)", lastConflict, attempts, ctxErr) + } + if errors.Is(err, lastConflict) { + txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "exhausted"))) + return fmt.Errorf("%w (gave up after %d attempts: the transaction kept losing write conflicts with concurrent writers)", err, attempts) + } + return err +} + +// replayableConflict reports whether err proves the attempt rolled back +// because of a concurrent-write conflict. Indeterminacy always wins: a write +// whose commit response was lost may have landed, so it is never replayed even +// if something also marked it as a conflict. +func replayableConflict(err error) bool { + return errors.Is(err, ErrTransactionConflict) && !errors.Is(err, ErrCommitIndeterminate) +} diff --git a/internal/storage/transaction_replay_test.go b/internal/storage/transaction_replay_test.go new file mode 100644 index 000000000..9685a2089 --- /dev/null +++ b/internal/storage/transaction_replay_test.go @@ -0,0 +1,264 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// scriptedTransactor answers each RunInTransaction call with the next scripted +// outcome, invoking the callback first as a real store would. +type scriptedTransactor struct { + outcomes []error + calls int +} + +func (s *scriptedTransactor) RunInTransaction(_ context.Context, _ string, fn func(tx Transaction) error) error { + s.calls++ + if err := fn(nil); err != nil { + return err + } + if len(s.outcomes) == 0 { + return nil + } + out := s.outcomes[0] + s.outcomes = s.outcomes[1:] + return out +} + +// instantTimer fires immediately and records every wait the policy asked for, +// so the replay loop runs without sleeping. +type instantTimer struct { + c chan time.Time + delays []time.Duration +} + +func (t *instantTimer) Start(d time.Duration) { + t.delays = append(t.delays, d) + t.c <- time.Now() +} +func (t *instantTimer) Stop() {} +func (t *instantTimer) C() <-chan time.Time { return t.c } + +func useInstantReplayTimer(t *testing.T) *instantTimer { + t.Helper() + timer := &instantTimer{c: make(chan time.Time, 1)} + prev := replayTimer + replayTimer = func() backoff.Timer { return timer } + t.Cleanup(func() { replayTimer = prev }) + return timer +} + +// conflict is what a backend returns when the server rolled the whole +// transaction back after losing a concurrent-write race. +func conflict(n int) error { + return MarkTransactionConflict(fmt.Errorf("sql commit (regular): serialization failure #%d", n)) +} + +func conflicts(n int) []error { + out := make([]error, n) + for i := range out { + out[i] = conflict(i) + } + return out +} + +func TestMarkTransactionConflictKeepsMessageAndCause(t *testing.T) { + cause := errors.New("sql commit (regular): Error 1213 (40001): serialization failure") + marked := MarkTransactionConflict(cause) + if !errors.Is(marked, ErrTransactionConflict) { + t.Fatalf("marked error %v does not match ErrTransactionConflict", marked) + } + if !errors.Is(marked, cause) { + t.Fatalf("marked error %v lost its cause", marked) + } + if marked.Error() != cause.Error() { + t.Fatalf("marked message = %q, want the unchanged %q", marked.Error(), cause.Error()) + } + if MarkTransactionConflict(nil) != nil { + t.Fatal("MarkTransactionConflict(nil) must stay nil") + } + if again := MarkTransactionConflict(marked); again != marked { + t.Fatal("marking an already-marked error must not wrap it twice") + } +} + +func TestRunReplayableTransactionReplaysWholeTransactionOnConflict(t *testing.T) { + timer := useInstantReplayTimer(t) + tr := &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}} + runs := 0 + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { + runs++ + return nil + }) + if err != nil { + t.Fatalf("RunReplayableTransaction = %v, want success after two lost conflicts", err) + } + if tr.calls != 3 || runs != 3 { + t.Fatalf("transactions=%d callbacks=%d, want the WHOLE transaction, callback included, run 3 times", tr.calls, runs) + } + if len(timer.delays) != 2 { + t.Fatalf("backoff waits = %v, want one wait before each replay", timer.delays) + } +} + +func TestRunReplayableTransactionDoesNotReplayOtherFailures(t *testing.T) { + useInstantReplayTimer(t) + for name, failure := range map[string]error{ + "plain error": errors.New("dependency would create a cycle"), + "indeterminate commit": fmt.Errorf("stage and commit after regular SQL commit: %w", ErrCommitIndeterminate), + // A conflict marker never overrides indeterminacy: the write may + // already have landed, so replaying it could apply it twice. + "marked but indeterminate": MarkTransactionConflict(fmt.Errorf("lost response: %w", ErrCommitIndeterminate)), + } { + t.Run(name, func(t *testing.T) { + tr := &scriptedTransactor{outcomes: []error{failure, nil}} + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }) + if !errors.Is(err, failure) { + t.Fatalf("err = %v, want the original failure", err) + } + if tr.calls != 1 { + t.Fatalf("transactions = %d, want exactly one attempt for a failure that is not a conflict", tr.calls) + } + }) + } +} + +func TestRunReplayableTransactionCallbackErrorIsNotReplayed(t *testing.T) { + useInstantReplayTimer(t) + tr := &scriptedTransactor{} + want := errors.New("issue not found") + runs := 0 + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { + runs++ + return want + }) + if !errors.Is(err, want) || runs != 1 { + t.Fatalf("err=%v runs=%d, want the callback's own error after one run", err, runs) + } +} + +func TestRunReplayableTransactionBoundsAttempts(t *testing.T) { + timer := useInstantReplayTimer(t) + tr := &scriptedTransactor{outcomes: conflicts(replayMaxAttempts + 5)} + err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }) + if err == nil { + t.Fatal("RunReplayableTransaction succeeded while every attempt lost a conflict") + } + if tr.calls != replayMaxAttempts { + t.Fatalf("transactions = %d, want exactly replayMaxAttempts (%d)", tr.calls, replayMaxAttempts) + } + if !errors.Is(err, ErrTransactionConflict) { + t.Fatalf("exhausted error %v must still classify as a transaction conflict", err) + } + if !strings.Contains(err.Error(), fmt.Sprintf("serialization failure #%d", replayMaxAttempts-1)) { + t.Fatalf("exhausted error %q must carry the LAST attempt's failure", err) + } + if !strings.Contains(err.Error(), fmt.Sprintf("%d attempts", replayMaxAttempts)) { + t.Fatalf("exhausted error %q must say how many attempts were made", err) + } + longest := time.Duration(float64(replayMaxInterval) * (1 + replayRandomizationFactor)) + for i, d := range timer.delays { + if d <= 0 || d > longest { + t.Fatalf("wait %d = %v, want a positive wait no longer than the jittered cap %v", i, d, longest) + } + } +} + +func TestRunReplayableTransactionBackoffIsJittered(t *testing.T) { + // Writers that collide tend to retry together; a fixed schedule would + // have every loser collide again at the same instant. + distinct := map[time.Duration]bool{} + for run := 0; run < 8; run++ { + timer := useInstantReplayTimer(t) + tr := &scriptedTransactor{outcomes: []error{conflict(1), nil}} + if err := RunReplayableTransaction(context.Background(), tr, "bd: test", func(Transaction) error { return nil }); err != nil { + t.Fatal(err) + } + if len(timer.delays) != 1 { + t.Fatalf("waits = %v, want one", timer.delays) + } + distinct[timer.delays[0]] = true + } + if len(distinct) < 2 { + t.Fatalf("first waits were identical across runs (%v); the backoff is not jittered", distinct) + } +} + +func TestRunReplayableTransactionStopsWhenContextEnds(t *testing.T) { + useInstantReplayTimer(t) + ctx, cancel := context.WithCancel(context.Background()) + tr := &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}} + runs := 0 + err := RunReplayableTransaction(ctx, tr, "bd: test", func(Transaction) error { + runs++ + cancel() + return nil + }) + if err == nil { + t.Fatal("RunReplayableTransaction kept going after its context was canceled") + } + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrTransactionConflict) { + t.Fatalf("err = %v, want both the cancellation and the conflict that was being retried", err) + } + if runs != 1 { + t.Fatalf("callback runs = %d, want 1 (no replay after cancellation)", runs) + } +} + +// TestRunReplayableTransactionCountsReplays makes replays observable the way +// withRetryTx's retries are: each replay and each loop that gives up is +// counted on bd.tx_replays_total, labeled by outcome. A loop that stops on a +// different failure after a replay did not give up on a conflict. +func TestRunReplayableTransactionCountsReplays(t *testing.T) { + useInstantReplayTimer(t) + reader := sdkmetric.NewManualReader() + otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) + + noop := func(Transaction) error { return nil } + if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}}, "bd: replayed", noop); err != nil { + t.Fatal(err) + } + if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: conflicts(replayMaxAttempts)}, "bd: exhausted", noop); err == nil { + t.Fatal("the exhausted loop succeeded") + } + if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), errors.New("constraint violated")}}, "bd: failed", noop); err == nil { + t.Fatal("the loop that ended on a plain failure succeeded") + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatal(err) + } + got := map[string]int64{} + for _, scope := range rm.ScopeMetrics { + for _, m := range scope.Metrics { + if m.Name != "bd.tx_replays_total" { + continue + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + t.Fatalf("bd.tx_replays_total is %T, want an int64 sum", m.Data) + } + for _, dp := range sum.DataPoints { + outcome, _ := dp.Attributes.Value(attribute.Key("outcome")) + got[outcome.AsString()] += dp.Value + } + } + } + if want := int64(2 + replayMaxAttempts - 1 + 1); got["replayed"] != want { + t.Fatalf("replayed count = %d, want %d (two, then every replay of the exhausted loop, then one)", got["replayed"], want) + } + if got["exhausted"] != 1 { + t.Fatalf("exhausted count = %d, want 1", got["exhausted"]) + } +} From c035e8d319e1f22cd5a41916903ae2286d528e46 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 06:04:01 +0000 Subject: [PATCH 3/4] fix(storage): count a replay exhaustion only when the attempts ran out Review follow-up: - bd.tx_replays_total{outcome=exhausted} also counted a loop whose context ended while it waited to replay. It now counts only a loop that ran out of attempts on conflicts; a cancellation still reports both why the loop stopped and the conflict it was retrying. - The replay debug log line now ends with a newline. - The metric test no longer replaces the global meter provider: it builds the counter with the same constructor (newTxReplaysCounter) on a test-local provider, swaps the package counter, and restores it. Counting a cancellation as exhausted again fails the metric test (exhausted 2, want 1). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/storage/BUILD.bazel | 1 - internal/storage/transaction_replay.go | 27 ++++++++++++--------- internal/storage/transaction_replay_test.go | 21 +++++++++++++--- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index e25dfc463..f8824b9ae 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -132,7 +132,6 @@ go_test( "//journalops", "//memoryops", "@com_github_cenkalti_backoff_v4//:backoff", - "@io_opentelemetry_go_otel//:otel", "@io_opentelemetry_go_otel//attribute", "@io_opentelemetry_go_otel_sdk_metric//:metric", "@io_opentelemetry_go_otel_sdk_metric//metricdata", diff --git a/internal/storage/transaction_replay.go b/internal/storage/transaction_replay.go index 1e6dea495..d032a5509 100644 --- a/internal/storage/transaction_replay.go +++ b/internal/storage/transaction_replay.go @@ -68,17 +68,18 @@ const ( var replayTimer func() backoff.Timer // txReplays counts whole-transaction replays, labeled outcome=replayed for each -// replay and outcome=exhausted when a loop gives up: the -// RunReplayableTransaction counterpart of withRetryTx's write-retry counter. It -// is registered against the global delegating provider, so it forwards to the -// real provider once telemetry is initialized. -var txReplays metric.Int64Counter +// replay and outcome=exhausted when a loop runs out of attempts on conflicts: +// the RunReplayableTransaction counterpart of withRetryTx's write-retry +// counter. It is registered against the global delegating provider, so it +// forwards to the real provider once telemetry is initialized. +var txReplays = newTxReplaysCounter(otel.Meter("github.com/steveyegge/beads/storage")) -func init() { - txReplays, _ = otel.Meter("github.com/steveyegge/beads/storage").Int64Counter("bd.tx_replays_total", +func newTxReplaysCounter(meter metric.Meter) metric.Int64Counter { + counter, _ := meter.Int64Counter("bd.tx_replays_total", metric.WithDescription("Whole-transaction replays after a lost write conflict (label: outcome=replayed|exhausted)"), metric.WithUnit("{replay}"), ) + return counter } // RunReplayableTransaction runs fn through s.RunInTransaction and, while an @@ -110,7 +111,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin var lastConflict error notify := func(err error, wait time.Duration) { txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed"))) - debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v", commitMsg, attempts, wait, err) + debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v\n", commitMsg, attempts, wait, err) } var timer backoff.Timer if replayTimer != nil { @@ -131,14 +132,16 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin if err == nil || lastConflict == nil { return err } - if ctxErr := ctx.Err(); ctxErr != nil && !errors.Is(err, lastConflict) { - txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "exhausted"))) - return fmt.Errorf("%w (stopped retrying after %d attempt(s): %w)", lastConflict, attempts, ctxErr) - } if errors.Is(err, lastConflict) { + // The loop ended on the conflict it was replaying: its budget ran out. txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "exhausted"))) return fmt.Errorf("%w (gave up after %d attempts: the transaction kept losing write conflicts with concurrent writers)", err, attempts) } + if ctxErr := ctx.Err(); ctxErr != nil { + // The context ended while waiting to replay: report both why the loop + // stopped and the conflict it was retrying. That is not exhaustion. + return fmt.Errorf("%w (stopped retrying after %d attempt(s): %w)", lastConflict, attempts, ctxErr) + } return err } diff --git a/internal/storage/transaction_replay_test.go b/internal/storage/transaction_replay_test.go index 9685a2089..bf348f437 100644 --- a/internal/storage/transaction_replay_test.go +++ b/internal/storage/transaction_replay_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/cenkalti/backoff/v4" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" @@ -221,8 +220,16 @@ func TestRunReplayableTransactionStopsWhenContextEnds(t *testing.T) { // different failure after a replay did not give up on a conflict. func TestRunReplayableTransactionCountsReplays(t *testing.T) { useInstantReplayTimer(t) + // The counter is swapped for one built by the same constructor on a + // test-local provider, so the global meter provider is never touched. reader := sdkmetric.NewManualReader() - otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + prev := txReplays + txReplays = newTxReplaysCounter(provider.Meter("test")) + t.Cleanup(func() { + txReplays = prev + _ = provider.Shutdown(context.Background()) + }) noop := func(Transaction) error { return nil } if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), conflict(2), nil}}, "bd: replayed", noop); err != nil { @@ -234,6 +241,14 @@ func TestRunReplayableTransactionCountsReplays(t *testing.T) { if err := RunReplayableTransaction(context.Background(), &scriptedTransactor{outcomes: []error{conflict(1), errors.New("constraint violated")}}, "bd: failed", noop); err == nil { t.Fatal("the loop that ended on a plain failure succeeded") } + // The context ends while the loop waits to replay: not exhausted. + ctx, cancel := context.WithCancel(context.Background()) + if err := RunReplayableTransaction(ctx, &scriptedTransactor{outcomes: []error{conflict(1), nil}}, "bd: canceled", func(Transaction) error { + cancel() + return nil + }); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want the cancellation", err) + } var rm metricdata.ResourceMetrics if err := reader.Collect(context.Background(), &rm); err != nil { @@ -259,6 +274,6 @@ func TestRunReplayableTransactionCountsReplays(t *testing.T) { t.Fatalf("replayed count = %d, want %d (two, then every replay of the exhausted loop, then one)", got["replayed"], want) } if got["exhausted"] != 1 { - t.Fatalf("exhausted count = %d, want 1", got["exhausted"]) + t.Fatalf("exhausted count = %d, want 1 (only the loop that ran out of attempts)", got["exhausted"]) } } From 0b9e19f23289665244ebf1917ecf920f1d18ca4c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 16 Oct 2026 19:00:00 +0000 Subject: [PATCH 4/4] fix(storage): a replay loop answers with its last attempt's own outcome When the context ended during a replayed attempt, RunReplayableTransaction returned the earlier, rolled-back conflict wrapped with the context error, dropping what the last attempt returned. If that attempt's COMMIT was cut by the context (ErrCommitIndeterminate: the write may have landed), the caller was told the transaction lost a conflict and wrote nothing; a plain failure in the last attempt was replaced the same way. Once a replay has run, the last attempt's outcome is now returned unchanged unless it is itself a replayable conflict. The new test (a conflict, then an indeterminate or plain failure while the context is cancelled) fails without the fix. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/storage/transaction_replay.go | 10 ++++-- internal/storage/transaction_replay_test.go | 34 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/internal/storage/transaction_replay.go b/internal/storage/transaction_replay.go index d032a5509..b8488d26b 100644 --- a/internal/storage/transaction_replay.go +++ b/internal/storage/transaction_replay.go @@ -108,7 +108,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin policy := backoff.WithContext(backoff.WithMaxRetries(bo, replayMaxAttempts-1), ctx) attempts := 0 - var lastConflict error + var lastConflict, lastAttempt error notify := func(err error, wait time.Duration) { txReplays.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", "replayed"))) debug.Logf("transaction %q lost a write conflict on attempt %d; replaying in %v: %v\n", commitMsg, attempts, wait, err) @@ -120,6 +120,7 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin err := backoff.RetryNotifyWithTimer(func() error { attempts++ err := s.RunInTransaction(ctx, commitMsg, fn) + lastAttempt = err if err == nil { return nil } @@ -129,7 +130,12 @@ func RunReplayableTransaction(ctx context.Context, s Transactor, commitMsg strin } return backoff.Permanent(err) }, policy, notify, timer) - if err == nil || lastConflict == nil { + if err == nil || lastConflict == nil || !replayableConflict(lastAttempt) { + // The last attempt's own outcome is the answer, unchanged: a callback + // error, a non-conflict database error, or an indeterminate commit + // that may have landed — even when the context ended during it. + // Replacing it with an earlier, rolled-back conflict would tell the + // caller nothing was written. return err } if errors.Is(err, lastConflict) { diff --git a/internal/storage/transaction_replay_test.go b/internal/storage/transaction_replay_test.go index bf348f437..a6e507eca 100644 --- a/internal/storage/transaction_replay_test.go +++ b/internal/storage/transaction_replay_test.go @@ -277,3 +277,37 @@ func TestRunReplayableTransactionCountsReplays(t *testing.T) { t.Fatalf("exhausted count = %d, want 1 (only the loop that ran out of attempts)", got["exhausted"]) } } + +// TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome: once a replay +// has run, the loop's answer is the LAST attempt's outcome. An attempt that +// ends in an indeterminate commit (its COMMIT cut by the context, so the write +// may have landed) or in a plain failure is returned unchanged even when the +// context ended during it — never replaced by the earlier, rolled-back +// conflict, which would tell the caller nothing was written. +func TestRunReplayableTransactionKeepsTheLastAttemptsOwnOutcome(t *testing.T) { + for name, last := range map[string]error{ + "indeterminate commit": fmt.Errorf("sql commit (regular): %w: %w", context.Canceled, ErrCommitIndeterminate), + "plain failure": errors.New("validation: dependency would create a cycle"), + } { + t.Run(name, func(t *testing.T) { + useInstantReplayTimer(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tr := &scriptedTransactor{outcomes: []error{conflict(1), last}} + calls := 0 + err := RunReplayableTransaction(ctx, tr, "bd: last outcome", func(Transaction) error { + calls++ + if calls == 2 { + cancel() // the context ends during the replayed attempt + } + return nil + }) + if !errors.Is(err, last) { + t.Fatalf("err = %v, want the last attempt's own outcome %v", err, last) + } + if errors.Is(last, ErrCommitIndeterminate) && errors.Is(err, ErrTransactionConflict) { + t.Fatalf("an indeterminate outcome was reported as a rolled-back conflict: %v", err) + } + }) + } +}