diff --git a/cmd/bd/mutations.go b/cmd/bd/mutations.go index aedd77d4e..746c21382 100644 --- a/cmd/bd/mutations.go +++ b/cmd/bd/mutations.go @@ -37,7 +37,18 @@ var mutationsCmd = &cobra.Command{ The journal records every committed issue mutation as an ordered, replayable row. Enable it with 'bd config set mutations-journal true' (or -BD_MUTATIONS_JOURNAL=1). Records are emitted only while it is enabled.`, +BD_MUTATIONS_JOURNAL=1). Records are emitted only while it is enabled. + +Coverage and scope: + - Every mutation through bd's normal write paths (create, update, close, + reopen, delete, claim, dependency add/remove, label add/remove, comment) is + journaled in the same transaction as the change. Raw DML run through + 'bd sql' bypasses those paths and is NOT journaled — a known non-coverage. + - The journal is per-branch working-set state (dolt_ignored): it records the + mutations committed on the writer's active branch. Rows arrive by direct + write, not by merge, so a consumer must read the journal on the same branch + the writer commits to; a branch checkout or merge does not carry journal + rows across branches.`, } var mutationsTailCmd = &cobra.Command{ @@ -50,8 +61,9 @@ Each line is a JSON record: "issue_id":"...","issue":{...|null},"dep":{"kind":..,"target":..}} Record contract (stable for external consumers): - seq int64 engine-assigned, strictly increasing, never reused or reset - ts string UTC timestamp the row was committed + seq int64 counter-assigned inside the mutation's transaction; gapless, + strictly increasing in commit order, never reused or reset + ts string UTC insert time, stamped inside the committing transaction op string one of the six ops above issue_id string the mutated issue's id issue object full issue state AFTER the mutation; null on delete @@ -91,16 +103,20 @@ var mutationsPruneCmd = &cobra.Command{ Use after a consumer has durably processed everything up to that seq. The journal is clone-local operational state, so pruning never affects issue data. -Two retention floors protect recent history and can only reduce what a prune -removes (a lagging consumer is never cut off by an over-eager --before): +Two retention floors compose onto --before and can only reduce what a prune +removes: mutations-journal-retain-days keep every row younger than N days mutations-journal-retain-rows always keep the newest N rows +Note the floors are time-based and count-based — they are NOT a consumer +watermark. They protect only the recent window; a consumer that has fallen +further behind than both floors allow can still be pruned past and lose records. Consumers are responsible for tracking their own watermark (the highest seq they -have durably processed). Pruned history cannot be recovered from the workspace — -the journal is the only local copy. On a Dolt backend, pair a prune with -'dolt gc' to reclaim the space, since the table is working-set (dolt_ignored) -state that ordinary Dolt commits never garbage-collect.`, +have durably processed) and for pruning no further than they have consumed. +Pruned history cannot be recovered from the workspace — the journal is the only +local copy. On a Dolt backend, pair a prune with 'dolt gc' to reclaim the space, +since the table is working-set (dolt_ignored) state that ordinary Dolt commits +never garbage-collect.`, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, _ []string) error { @@ -271,29 +287,30 @@ func readJournalProxied(ctx context.Context, query string, args ...any) ([]mutat } // pruneJournalProxied applies the retain floors through the proxied-server unit -// of work, reusing the shared floor helpers. +// of work. The floor orchestration lives in issueops.ComputeMutationsPruneWhere; +// only the ceil read and the DELETE are expressed in raw SQL here. func pruneJournalProxied(ctx context.Context, before int64, retainDays, retainRows int) (int64, error) { if uowProvider == nil { return 0, fmt.Errorf("proxied-server UOW provider not initialized") } - var ( - rowsCeil int64 - rowsCeilOK bool - ) - if retainRows > 0 { + where, args, skip, err := issueops.ComputeMutationsPruneWhere(before, retainDays, retainRows, time.Now().UTC(), func() (int64, bool, error) { res, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, error) { return uw.RawSQLUseCase().Query(ctx, issueops.MutationsPruneRowsCeilQuery(), retainRows) }) if err != nil { - return 0, err + return 0, false, err } if res == nil || len(res.Rows) == 0 || len(res.Rows[0]) == 0 { - // The whole journal is inside the retained window: nothing to prune. - return 0, nil + return 0, false, nil } - rowsCeil, rowsCeilOK = toInt64(res.Rows[0][0]), true + return toInt64(res.Rows[0][0]), true, nil + }) + if err != nil { + return 0, err + } + if skip { + return 0, nil } - where, args := issueops.BuildMutationsPruneWhere(before, retainDays, time.Now().UTC(), rowsCeil, rowsCeilOK) return uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (int64, string, error) { n, err := uw.RawSQLUseCase().Exec(ctx, "DELETE FROM bd_mutations_journal WHERE "+where, args...) if err != nil { diff --git a/internal/storage/dolt/journal_test.go b/internal/storage/dolt/journal_test.go index 6c9dabdf3..eec072641 100644 --- a/internal/storage/dolt/journal_test.go +++ b/internal/storage/dolt/journal_test.go @@ -10,7 +10,7 @@ import ( // TestMutationsJournal_EmbeddedPlumbing drives mutations through the DoltStore // (the embedded/DoltStorage write plumbing, which bottoms out in the issueops // *InTx functions) against real Dolt and asserts the journal at the issueops -// seam records each op with an engine-assigned monotonic seq. This is the second +// seam records each op with a counter-assigned monotonic seq. This is the second // of the two plumbings; the first is exercised in domain/db. func TestMutationsJournal_EmbeddedPlumbing(t *testing.T) { store, cleanup := setupTestStore(t) @@ -75,3 +75,55 @@ func TestMutationsJournal_EmbeddedPlumbing(t *testing.T) { prev = got[i].seq } } + +// TestMutationsJournal_NoPhantomDeletes asserts the bulk delete (DeleteIssuesInTx) +// journals a delete only for ids that actually removed a row — never a phantom +// for an id that matched nothing. +func TestMutationsJournal_NoPhantomDeletes(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx := context.Background() + + enableJournalForTest(t) + + mk := func(id string) *types.Issue { + return &types.Issue{ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen} + } + if err := store.CreateIssue(ctx, mk("bd-pd-1"), "actor"); err != nil { + t.Fatalf("create 1: %v", err) + } + if err := store.CreateIssue(ctx, mk("bd-pd-2"), "actor"); err != nil { + t.Fatalf("create 2: %v", err) + } + if _, err := store.db.ExecContext(ctx, "DELETE FROM bd_mutations_journal"); err != nil { + t.Fatalf("clear journal: %v", err) + } + + // Delete a mix of present and absent ids; force avoids the dependent gate. + if _, err := store.DeleteIssues(ctx, []string{"bd-pd-1", "bd-pd-missing-a", "bd-pd-2", "bd-pd-missing-b"}, false, true, false); err != nil { + t.Fatalf("delete: %v", err) + } + + rows, err := store.db.QueryContext(ctx, "SELECT op, issue_id FROM bd_mutations_journal ORDER BY seq ASC") + if err != nil { + t.Fatalf("query journal: %v", err) + } + defer rows.Close() + deleted := map[string]bool{} + for rows.Next() { + var op, id string + if err := rows.Scan(&op, &id); err != nil { + t.Fatalf("scan: %v", err) + } + if op != "delete" { + t.Fatalf("unexpected op %q for %s", op, id) + } + deleted[id] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("rows err: %v", err) + } + if len(deleted) != 2 || !deleted["bd-pd-1"] || !deleted["bd-pd-2"] { + t.Fatalf("journal must record deletes only for present ids, got %v", deleted) + } +} diff --git a/internal/storage/domain/db/comment.go b/internal/storage/domain/db/comment.go index 7dbd150e9..9bad9a5ea 100644 --- a/internal/storage/domain/db/comment.go +++ b/internal/storage/domain/db/comment.go @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -140,6 +141,12 @@ func (r *commentSQLRepositoryImpl) Insert(ctx context.Context, issueID, author, return nil, fmt.Errorf("db: CommentSQLRepository.Insert: %w", err) } + // Journal the comment as an update to the issue in the same transaction — a + // comment mutates the issue's bead state, recorded like a label change. + if err := issueops.RecordMutationInTx(ctx, r.runner, issueops.MutationUpdate, issueID); err != nil { + return nil, err + } + return &types.Comment{ ID: id, IssueID: issueID, diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 945b1f4f6..a6213fe35 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -795,9 +795,19 @@ func (r *issueSQLRepositoryImpl) DeleteByIDs(ctx context.Context, ids []string, placeholders[i] = "?" args[i] = id } + inClause := strings.Join(placeholders, ",") + + // Which ids in this batch actually exist? In this transaction nothing else + // removes them, so the present set is exactly the set the DELETE removes — + // journal a delete only for those, never a phantom for an absent id. + present, err := r.presentIDs(ctx, table, inClause, args) + if err != nil { + return total, err + } + //nolint:gosec // G201: table is a hardcoded constant; placeholders are ?. res, err := r.runner.ExecContext(ctx, - fmt.Sprintf("DELETE FROM %s WHERE id IN (%s)", table, strings.Join(placeholders, ",")), + fmt.Sprintf("DELETE FROM %s WHERE id IN (%s)", table, inClause), args...) if err != nil { return total, fmt.Errorf("db: IssueSQLRepository.DeleteByIDs from %s: %w", table, err) @@ -807,9 +817,9 @@ func (r *issueSQLRepositoryImpl) DeleteByIDs(ctx context.Context, ids []string, return total, fmt.Errorf("db: IssueSQLRepository.DeleteByIDs rows affected: %w", err) } total += int(n) - // Journal each deleted id in the same transaction. The delete use case - // passes the cascade-expanded set, so this records cascade deletes too. - for _, id := range batch { + // Journal each actually-deleted id in the same transaction. The delete use + // case passes the cascade-expanded set, so this records cascade deletes too. + for _, id := range present { if err := issueops.RecordDeleteInTx(ctx, r.runner, id); err != nil { return total, err } @@ -827,6 +837,29 @@ func (r *issueSQLRepositoryImpl) DeleteByIDs(ctx context.Context, ids []string, return total, nil } +// presentIDs returns the subset of a batch (rendered as inClause + args) whose +// ids exist in table, so DeleteByIDs journals a delete only for rows actually +// removed. table is a hardcoded issue/wisp constant. +// +//nolint:gosec // G201: table is a hardcoded constant; inClause is ? placeholders. +func (r *issueSQLRepositoryImpl) presentIDs(ctx context.Context, table, inClause string, args []any) ([]string, error) { + rows, err := r.runner.QueryContext(ctx, + fmt.Sprintf("SELECT id FROM %s WHERE id IN (%s)", table, inClause), args...) + if err != nil { + return nil, fmt.Errorf("db: IssueSQLRepository.DeleteByIDs check existing from %s: %w", table, err) + } + defer rows.Close() + var present []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("db: IssueSQLRepository.DeleteByIDs scan existing: %w", err) + } + present = append(present, id) + } + return present, rows.Err() +} + func (r *issueSQLRepositoryImpl) PartitionWispIDs(ctx context.Context, ids []string) ([]string, []string, error) { return issueops.PartitionWispIDsInTx(ctx, r.runner, ids) } diff --git a/internal/storage/domain/db/journal_behavior_test.go b/internal/storage/domain/db/journal_behavior_test.go index 67b3a1ed8..08625cd86 100644 --- a/internal/storage/domain/db/journal_behavior_test.go +++ b/internal/storage/domain/db/journal_behavior_test.go @@ -140,6 +140,34 @@ func (s *testSuite) TestMutationsJournal_CascadeDelete() { s.True(deleted["bd-cd-child"], "cascade-deleted child must be journaled, got %v", deleted) } +// TestMutationsJournal_NoPhantomDeletes asserts DeleteByIDs journals a delete +// only for ids that actually removed a row — never a phantom delete for an id +// that matched nothing (the batched DELETE reports only a per-batch total). +func (s *testSuite) TestMutationsJournal_NoPhantomDeletes() { + s.enableJournalForTest() + ctx := s.Ctx() + ir := s.issueRepo() + + s.Require().NoError(ir.Insert(ctx, newTestIssue("bd-pd-1", "t"), "actor", domain.InsertIssueOpts{})) + s.Require().NoError(ir.Insert(ctx, newTestIssue("bd-pd-2", "t"), "actor", domain.InsertIssueOpts{})) + // clear setup rows so we assert only on the delete. + _, err := s.Runner().ExecContext(ctx, "DELETE FROM bd_mutations_journal") + s.Require().NoError(err) + + // Delete a mix: two real ids and two that do not exist. + n, err := ir.DeleteByIDs(ctx, []string{"bd-pd-1", "bd-pd-missing-a", "bd-pd-2", "bd-pd-missing-b"}, domain.IssueTableOpts{}) + s.Require().NoError(err) + s.Equal(2, n, "only the two present ids are deleted") + + deleted := map[string]bool{} + for _, r := range s.readJournal() { + s.Equal("delete", r.Op, "only delete rows expected: %+v", r) + deleted[r.IssueID] = true + } + s.Equal(map[string]bool{"bd-pd-1": true, "bd-pd-2": true}, deleted, + "journal must record a delete only for ids that removed a row, no phantoms") +} + // TestMutationsJournal_DisabledWritesNothing asserts the default-off knob writes // no rows. func (s *testSuite) TestMutationsJournal_DisabledWritesNothing() { diff --git a/internal/storage/domain/db/journal_completeness_test.go b/internal/storage/domain/db/journal_completeness_test.go index fcbe38269..664c09fb1 100644 --- a/internal/storage/domain/db/journal_completeness_test.go +++ b/internal/storage/domain/db/journal_completeness_test.go @@ -1,48 +1,45 @@ package db import ( - "go/ast" - "go/parser" - "go/token" - "io/fs" - "regexp" - "strings" "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" ) // TestEveryRepositoryMutatorJournals is the completeness guard for the // unit-of-work write plumbing. Unlike the DoltStorage plumbing, several of this // package's repository mutators reimplement their own SQL instead of routing -// through issueops, so the issueops seam alone does NOT cover them. This test -// enumerates every mutator method on the issue/dependency/label repositories and -// asserts each one journals — either by calling an issueops.Record*InTx emit -// helper directly, or by delegating to an issueops function that emits -// (CloseIssueInTx / ReopenIssueInTx / ClaimReadyIssueInTx / -// ReclaimExpiredLeasesInTx), or by delegating to a sibling repository method -// that does. +// through issueops, so the issueops seam alone does NOT cover them. // -// A new reimplemented mutator that forgets to journal fails this test. +// The guard is STRUCTURAL, not name-based: it does not trust a verb regex on the +// method name or a broad "Insert" coverage token (either of which could let a +// mutator named off-pattern, or a false delegation, ship un-journaled). Instead +// it detects a mutator by BEHAVIOR — any method on a work-bead repository whose +// body (directly, or through a free function it calls) executes an INSERT / +// UPDATE / DELETE against a work-bead table — and asserts each one journals. +// +// A new reimplemented mutator that writes a bead table and forgets to journal +// fails this test even if it is named nothing like a mutator. func TestEveryRepositoryMutatorJournals(t *testing.T) { - // Repository receiver types whose mutator methods must journal. - mutatorReceivers := map[string]bool{ + // Repository receiver types that own work-bead state. issue/dependency/label + // reimplement issueops SQL; comment writes the comments bead table. + beadReceivers := map[string]bool{ "issueSQLRepositoryImpl": true, "dependencySQLRepositoryImpl": true, "labelSQLRepositoryImpl": true, + "commentSQLRepositoryImpl": true, } - // A method name is a mutator when it starts with one of these verbs. - mutatorVerb := regexp.MustCompile(`^(Insert|Update|Delete|Claim|Close|Reopen|Unclaim|Reclaim|Promote|Reparent|Rename|Add|Remove|Set)`) - - // Methods that are legitimately not journalled: bulk cascade cleanups run - // while their parent issue delete (already journalled) removes the rows. - allowNoJournal := map[string]bool{ - "DeleteAllForIDs": true, + // Bead-mutating methods that legitimately do NOT journal, each with a reason. + // The staleness check below fails if any of these stops being a bead mutator, + // so a rename or refactor cannot silently strand an exemption. + exempt := map[string]string{ + "DeleteAllForIDs": "bulk edge/label cleanup runs under a parent issue delete that already journals each removed bead", + "markDirectBlockedSource": "maintains the derived is_blocked column as a side effect of a journaled dependency insert", } - // Calls that count as journalling coverage: the direct emit helpers, the - // issueops functions that emit internally, and delegation to a sibling repo - // method (matched loosely by the "Insert" method name for InsertBatch). - coverageCalls := map[string]bool{ + // Direct emit helpers and the issueops functions that journal internally. + emitCalls := map[string]bool{ "RecordMutationInTx": true, "RecordDeleteInTx": true, "RecordDepMutationInTx": true, @@ -50,66 +47,53 @@ func TestEveryRepositoryMutatorJournals(t *testing.T) { "ReopenIssueInTx": true, "ClaimReadyIssueInTx": true, "ReclaimExpiredLeasesInTx": true, - "Insert": true, // InsertBatch delegates to Insert } - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { - return !strings.HasSuffix(fi.Name(), "_test.go") - }, 0) + fns, err := journalscan.ParsePackage(".") if err != nil { t.Fatalf("parse domain/db package: %v", err) } - var checked int - for _, pkg := range pkgs { - for _, file := range pkg.Files { - for _, decl := range file.Decls { - fn, ok := decl.(*ast.FuncDecl) - if !ok || fn.Recv == nil || len(fn.Recv.List) == 0 { - continue - } - recv := receiverTypeName(fn.Recv.List[0].Type) - if !mutatorReceivers[recv] { - continue - } - name := fn.Name.Name - if !mutatorVerb.MatchString(name) || allowNoJournal[name] { - continue - } + // beadDML: fn writes a bead table directly, or calls a free function that + // transitively does (e.g. Insert -> insertIssueRow). Only free-function + // (bare-identifier) calls propagate DML, so an events/leases write reached + // through a selector (r.events.Record) does not count as a bead mutation. + beadDML := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.OwnBeadDML }, + func(f *journalscan.FuncInfo) []string { return f.IdentCalls }) - covered := false - ast.Inspect(fn, func(n ast.Node) bool { - call, ok := n.(*ast.CallExpr) - if !ok { - return true - } - if sel, ok := call.Fun.(*ast.SelectorExpr); ok && coverageCalls[sel.Sel.Name] { - covered = true - } - return true - }) - checked++ - if !covered { - t.Errorf("%s.%s is a mutator that does not journal: it neither calls an issueops.Record*InTx emit helper, nor delegates to an emitting issueops function, nor to a sibling mutator that journals", recv, name) - } + // emits: fn calls an emit helper directly, or calls any package function + // (free or method-by-name) that emits. + emits := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.CallsAnyOf(emitCalls) }, + func(f *journalscan.FuncInfo) []string { return f.AllCallNames() }) + + seenExempt := map[string]bool{} + var checked int + for name, f := range fns { + if !beadReceivers[f.Recv] || !beadDML[name] { + continue + } + if reason, ok := exempt[f.Name]; ok { + if reason == "" { + t.Errorf("%s has an empty exemption reason", f.Name) } + seenExempt[f.Name] = true + continue + } + checked++ + if !emits[name] { + t.Errorf("%s.%s writes a work-bead table but does not journal: it neither calls a Record*InTx emit helper, nor delegates to an emitting issueops function, nor to a sibling mutator that journals", f.Recv, f.Name) } } if checked == 0 { - t.Fatal("guard found no mutator methods — receiver names or parsing changed; the completeness guard is not actually running") + t.Fatal("guard found no bead-mutating repository methods — receiver names, DML detection, or parsing changed; the completeness guard is not actually running") + } + // Staleness: every exemption must still name a real bead mutator. + for m := range exempt { + if !seenExempt[m] { + t.Errorf("exemption %q no longer matches a bead-mutating repository method — remove it", m) + } } } - -// receiverTypeName returns the bare type name of a method receiver -// (e.g. *issueSQLRepositoryImpl -> issueSQLRepositoryImpl). -func receiverTypeName(expr ast.Expr) string { - if star, ok := expr.(*ast.StarExpr); ok { - expr = star.X - } - if ident, ok := expr.(*ast.Ident); ok { - return ident.Name - } - return "" -} diff --git a/internal/storage/domain/db/journal_prune_test.go b/internal/storage/domain/db/journal_prune_test.go index 3d3cc034b..78ab705f3 100644 --- a/internal/storage/domain/db/journal_prune_test.go +++ b/internal/storage/domain/db/journal_prune_test.go @@ -9,22 +9,27 @@ import ( // TestMutationsJournal_PruneFloors exercises the real prune predicate // (issueops.BuildMutationsPruneWhere + MutationsPruneRowsCeilQuery) against Dolt: // retain-rows keeps the newest N rows, retain-days keeps rows within the window, -// both floors compose, and the AUTO_INCREMENT seq keeps climbing after a prune -// (never resets). Rows are inserted directly with controlled ts so age-based +// both floors compose, and the seq keeps climbing after a prune (never resets, +// because it is drawn from the durable bd_mutations_seq counter, which prune +// never touches). Rows are inserted directly with controlled ts so age-based // pruning is deterministic. func (s *testSuite) TestMutationsJournal_PruneFloors() { ctx := s.Ctx() _, err := s.Runner().ExecContext(ctx, "DELETE FROM bd_mutations_journal") s.Require().NoError(err) + // insert allocates seq from the durable counter (mirroring + // issueops.nextMutationSeq) so seq never resets across the DELETEs below. insert := func(id string, ts time.Time) int64 { - _, err := s.Runner().ExecContext(ctx, - `INSERT INTO bd_mutations_journal (ts, op, issue_id, issue_json, dep_json) VALUES (?, 'create', ?, NULL, NULL)`, - ts.UTC(), id) + _, err := s.Runner().ExecContext(ctx, "UPDATE bd_mutations_seq SET next_seq = next_seq + 1 WHERE id = 0") s.Require().NoError(err) var seq int64 s.Require().NoError(s.Runner().QueryRowContext(ctx, - "SELECT seq FROM bd_mutations_journal WHERE issue_id = ?", id).Scan(&seq)) + "SELECT next_seq FROM bd_mutations_seq WHERE id = 0").Scan(&seq)) + _, err = s.Runner().ExecContext(ctx, + `INSERT INTO bd_mutations_journal (seq, ts, op, issue_id, issue_json, dep_json) VALUES (?, ?, 'create', ?, NULL, NULL)`, + seq, ts.UTC(), id) + s.Require().NoError(err) return seq } remaining := func() []string { @@ -85,5 +90,5 @@ func (s *testSuite) TestMutationsJournal_PruneFloors() { // Seq never resets: a fresh insert after pruning gets a seq above everything. after := insert("d6", now) - s.Greater(after, dLast, "AUTO_INCREMENT seq must keep climbing after a prune (never reset)") + s.Greater(after, dLast, "counter-drawn seq must keep climbing after a prune (never reset)") } diff --git a/internal/storage/domain/db/journal_seq_test.go b/internal/storage/domain/db/journal_seq_test.go index 8ea5863b0..023536258 100644 --- a/internal/storage/domain/db/journal_seq_test.go +++ b/internal/storage/domain/db/journal_seq_test.go @@ -2,53 +2,171 @@ package db import ( "database/sql" + "errors" + "sync" + + gomysql "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/testutil" ) -// TestMutationsJournal_SeqMonotonicAcrossHandles proves the journal seq is a -// single shared, strictly-increasing sequence across independent connections to -// the same database — the guarantee that holds across processes and restarts, -// since the AUTO_INCREMENT counter lives in the durable (dolt_ignored) table, -// not in any one handle. A second sql.DB stands in for a second process. -func (s *testSuite) TestMutationsJournal_SeqMonotonicAcrossHandles() { +// isSerializationFailure reports whether err is a Dolt/MySQL serialization +// failure (1213 deadlock / 1205 lock-wait-timeout). Dolt has no blocking row +// locks in server mode; concurrent writers to the same cell lose the +// commit-time merge with one of these codes and the loser is rolled back. +func isSerializationFailure(err error) bool { + var myErr *gomysql.MySQLError + if errors.As(err, &myErr) { + return myErr.Number == 1213 || myErr.Number == 1205 + } + return false +} + +// TestMutationsJournal_CommitOrderedGaplessSeq is the FINDING 1 proof: under two +// OVERLAPPING transactions, the journal seq order equals commit-visibility order +// with no gaps, on bd's actual (deferred-commit, optimistic-merge) Dolt server +// model. +// +// The seq is drawn from the single-row bd_mutations_seq counter inside each +// mutation's own transaction. The two inserts touch DIFFERENT issue rows and +// DIFFERENT event rows — the ONLY cell they share is the counter row — so any +// serialization here is caused solely by the counter. That is exactly the point: +// with the former AUTO_INCREMENT seq the two commits would NOT conflict (distinct +// PKs assigned at insert), so commit order could invert seq order and a tailing +// consumer would skip the lower seq. The counter converts that silent skip into +// a detected conflict + retry that preserves commit order. +func (s *testSuite) TestMutationsJournal_CommitOrderedGaplessSeq() { ctx := s.Ctx() + issueops.SetJournalEnabled(true) + s.T().Cleanup(func() { issueops.SetJournalEnabled(false) }) _, err := s.Runner().ExecContext(ctx, "DELETE FROM bd_mutations_journal") s.Require().NoError(err) port := testutil.DoltContainerPortInt() s.Require().NotZero(port) dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: port, User: "root", Database: s.dbName}.String() - db2, err := sql.Open("mysql", dsn) + dbB, err := sql.Open("mysql", dsn) s.Require().NoError(err) - defer db2.Close() - s.Require().NoError(db2.PingContext(ctx)) + defer dbB.Close() - insert := func(runner *sql.DB, id string) { - _, err := runner.ExecContext(ctx, - `INSERT INTO bd_mutations_journal (ts, op, issue_id, issue_json, dep_json) VALUES (UTC_TIMESTAMP(), 'create', ?, NULL, NULL)`, id) - s.Require().NoError(err) + insertOnTx := func(tx *sql.Tx, id string) error { + ir := NewIssueSQLRepository(tx) + return ir.Insert(ctx, newTestIssue(id, "t"), "actor", domain.InsertIssueOpts{}) } - // Interleave writes across the two handles. - insert(s.db, "h-a1") - insert(db2, "h-b1") - insert(s.db, "h-a2") - insert(db2, "h-b2") - - rows, err := s.Runner().QueryContext(ctx, "SELECT seq FROM bd_mutations_journal ORDER BY seq ASC") + // Two overlapping transactions on independent connections. Both allocate a + // seq from the counter BEFORE either commits. + txA, err := s.db.BeginTx(ctx, nil) s.Require().NoError(err) - defer rows.Close() - var seqs []int64 - for rows.Next() { - var seq int64 - s.Require().NoError(rows.Scan(&seq)) - seqs = append(seqs, seq) + txB, err := dbB.BeginTx(ctx, nil) + s.Require().NoError(err) + + s.Require().NoError(insertOnTx(txA, "bd-seq-a"), "insert A") + s.Require().NoError(insertOnTx(txB, "bd-seq-b"), "insert B") + + // Commit A first; it wins. + s.Require().NoError(txA.Commit(), "commit A") + // B committed second and both bumped the same counter row from the same base, + // so B must lose the commit-time merge (rolled back server-side) rather than + // silently committing an out-of-order/duplicate seq. + errB := txB.Commit() + s.Require().Error(errB, "B must not commit an inverted/duplicate seq") + s.Require().Truef(isSerializationFailure(errB), + "B must fail with a serialization error (guaranteed rollback), got: %v", errB) + + // Retry B in a fresh transaction (this is what the real write paths — + // withRetryTx / uow.RunTx — do). It re-reads the committed counter and gets a + // strictly greater seq. + txB2, err := dbB.BeginTx(ctx, nil) + s.Require().NoError(err) + s.Require().NoError(insertOnTx(txB2, "bd-seq-b"), "retry insert B") + s.Require().NoError(txB2.Commit(), "commit retried B") + + got := s.readJournal() + s.Require().Len(got, 2, "exactly two journal rows survive: %+v", got) + // seq order == commit order: A (committed first) < B (committed second). + s.Equal("bd-seq-a", got[0].IssueID, "first seq is the first committer") + s.Equal("bd-seq-b", got[1].IssueID, "second seq is the second committer") + // Gapless: the two surviving seqs are consecutive — B's rolled-back attempt + // burned no seq. + s.Equalf(got[0].Seq+1, got[1].Seq, "seqs must be gapless: %d then %d", got[0].Seq, got[1].Seq) +} + +// TestMutationsJournal_ConcurrentWritersGaplessNoDup drives many real journaled +// mutations through independent connections at once and asserts the journal ends +// up with exactly one gapless, duplicate-free seq per committed mutation. It +// exercises the end-to-end safety property (a committed seq=N implies every +// seq 0 { + // Contiguous: gapless across the whole concurrent batch. + s.Equalf(got[i-1].Seq+1, r.Seq, "seqs must be gapless and ordered: %d then %d", got[i-1].Seq, r.Seq) + } } } diff --git a/internal/storage/embeddeddolt/journal_concurrency_test.go b/internal/storage/embeddeddolt/journal_concurrency_test.go new file mode 100644 index 000000000..8ee93b030 --- /dev/null +++ b/internal/storage/embeddeddolt/journal_concurrency_test.go @@ -0,0 +1,80 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestMutationsJournal_EmbeddedConcurrentGaplessNoDup is the FINDING 1 proof for +// the embedded engine (the default local workspace). Unlike the SQL server — +// which resolves concurrent writers optimistically with a serialization abort at +// commit — the embedded engine serializes writers on the counter row. Either way +// the counter-drawn seq must come out gapless, commit-ordered, and duplicate-free +// under concurrent real mutations. N goroutines each create an issue through the +// real store path (store.CreateIssue -> issueops.CreateIssueInTx -> +// insertMutationRow -> nextMutationSeq); the journal must end with exactly one +// contiguous seq per create. +func TestMutationsJournal_EmbeddedConcurrentGaplessNoDup(t *testing.T) { + env := newTestEnv(t, "ecw") + issueops.SetJournalEnabled(true) + t.Cleanup(func() { issueops.SetJournalEnabled(false) }) + store := env.store + + const writers = 12 + var wg sync.WaitGroup + errs := make([]error, writers) + done := make(chan struct{}) + go func() { + for i := 0; i < writers; i++ { + wg.Add(1) + go func(k int) { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + iss := &types.Issue{ID: fmt.Sprintf("ecw-%d", k), Title: "t", IssueType: types.TypeTask, Status: types.StatusOpen} + errs[k] = store.CreateIssue(ctx, iss, "actor") + }(i) + } + wg.Wait() + close(done) + }() + + select { + case <-done: + case <-time.After(90 * time.Second): + t.Fatal("concurrent CreateIssue timed out (deadlock?)") + } + for k, e := range errs { + if e != nil { + t.Fatalf("create %d failed: %v", k, e) + } + } + + rows, err := store.ReadMutationsJournal(context.Background(), 0, 0) + if err != nil { + t.Fatalf("read journal: %v", err) + } + if len(rows) != writers { + t.Fatalf("journal rows = %d, want %d", len(rows), writers) + } + seen := map[int64]bool{} + var prev int64 + for i, r := range rows { + if seen[r.Seq] { + t.Fatalf("duplicate seq %d", r.Seq) + } + seen[r.Seq] = true + if i > 0 && r.Seq != prev+1 { + t.Fatalf("seqs must be gapless and ordered: %d then %d", prev, r.Seq) + } + prev = r.Seq + } +} diff --git a/internal/storage/issueops/comments.go b/internal/storage/issueops/comments.go index 8ea5857ff..3c76a22fe 100644 --- a/internal/storage/issueops/comments.go +++ b/internal/storage/issueops/comments.go @@ -144,6 +144,12 @@ func ImportIssueCommentInTx(ctx context.Context, tx *sql.Tx, issueID, author, te return nil, fmt.Errorf("add comment to %s: %w", commentTable, err) } + // Journal the comment as an update to the issue in the same transaction — a + // comment mutates the issue's bead state, recorded like a label change. + if err := RecordMutationInTx(ctx, tx, MutationUpdate, issueID); err != nil { + return nil, err + } + return &types.Comment{ ID: id, IssueID: issueID, diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index ddc60a52b..35edd6f69 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -256,6 +256,10 @@ func DeleteIssuesInTx(ctx context.Context, tx *sql.Tx, ids []string, cascade boo } totalRegularsDeleted := 0 + // deletedRegularIDs is the set of ids that actually removed a row, so the + // journal records a delete only for those — never a phantom delete for an id + // that matched nothing (the batched DELETE reports only a per-batch total). + var deletedRegularIDs []string for i := 0; i < len(finalRegularIDs); i += deleteBatchSize { end := i + deleteBatchSize if end > len(finalRegularIDs) { @@ -264,6 +268,13 @@ func DeleteIssuesInTx(ctx context.Context, tx *sql.Tx, ids []string, cascade boo batch := finalRegularIDs[i:end] batchInClause, batchArgs := buildSQLInClause(batch) + // Which ids in this batch actually exist? In this transaction nothing else + // removes them, so the present set is exactly the set the DELETE removes. + present, err := existingIDsInTx(ctx, tx, "issues", batchInClause, batchArgs) + if err != nil { + return nil, err + } + deleteResult, err := tx.ExecContext(ctx, fmt.Sprintf(`DELETE FROM issues WHERE id IN (%s)`, batchInClause), batchArgs...) @@ -272,6 +283,7 @@ func DeleteIssuesInTx(ctx context.Context, tx *sql.Tx, ids []string, cascade boo } rowsAffected, _ := deleteResult.RowsAffected() totalRegularsDeleted += int(rowsAffected) + deletedRegularIDs = append(deletedRegularIDs, present...) // Deleted issues hold no leases. if _, err := tx.ExecContext(ctx, @@ -282,11 +294,11 @@ func DeleteIssuesInTx(ctx context.Context, tx *sql.Tx, ids []string, cascade boo } result.DeletedCount = totalRegularsDeleted + len(allWispIDs) - // Journal every regular issue deleted by this bulk/cascade operation. Wisps - // were deleted through deleteIssueRowInTx above, which journals each itself; - // finalRegularIDs is the cascade-expanded set of regular ids removed by the - // direct batched DELETE, so this records cascade deletes too. - for _, id := range finalRegularIDs { + // Journal every regular issue this bulk/cascade operation actually removed. + // Wisps were deleted through deleteIssueRowInTx above, which journals each + // itself; deletedRegularIDs is the cascade-expanded set of regular ids that + // truly had a row removed, so this records cascade deletes without phantoms. + for _, id := range deletedRegularIDs { if err := RecordDeleteInTx(ctx, tx, id); err != nil { return nil, err } @@ -405,6 +417,29 @@ func findExternalDependentsBatchedInTx(ctx context.Context, tx *sql.Tx, ids []st return result, nil } +// existingIDsInTx returns the subset of the batch (already rendered as inClause +// + args) whose ids exist in table, so a caller can journal a delete only for +// rows that were actually present. table is a hardcoded issue/wisp constant. +// +//nolint:gosec // G201: table is a hardcoded issue/wisp constant; inClause is ? placeholders. +func existingIDsInTx(ctx context.Context, tx DBTX, table, inClause string, args []any) ([]string, error) { + rows, err := tx.QueryContext(ctx, + fmt.Sprintf("SELECT id FROM %s WHERE id IN (%s)", table, inClause), args...) + if err != nil { + return nil, fmt.Errorf("check existing ids in %s: %w", table, err) + } + defer rows.Close() + var present []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scan existing id in %s: %w", table, err) + } + present = append(present, id) + } + return present, rows.Err() +} + //nolint:gosec // G201: table is selected by callers from fixed issue/wisp auxiliary tables. func CountRowsForIssueIDsInTx(ctx context.Context, tx DBTX, table string, ids []string) (int, error) { total := 0 diff --git a/internal/storage/issueops/journal.go b/internal/storage/issueops/journal.go index 1052ca8f0..7f1fe7261 100644 --- a/internal/storage/issueops/journal.go +++ b/internal/storage/issueops/journal.go @@ -16,9 +16,20 @@ import ( // The durable mutations journal records every committed issue mutation as a row // in the clone-local bd_mutations_journal table, written in the SAME // transaction as the mutation itself. Because the row and the mutation commit -// atomically, the journal can never lag the data, produce a false record, or -// collide on a sequence number: the seq is an engine-assigned AUTO_INCREMENT PK, -// serialized across every connection to the shared database. +// atomically, the journal can never lag the data or produce a false record. +// +// The seq is NOT an AUTO_INCREMENT. AUTO_INCREMENT assigns a value at INSERT, +// not at commit, so under concurrent transactions (the shared SQL server) +// commit-visibility order can invert seq order: a lower seq can commit after a +// higher seq is already visible, and a consumer tailing WHERE seq > cursor +// would permanently skip it. Instead each seq is drawn from the single-row +// counter table bd_mutations_seq inside the mutation's own transaction (see +// nextMutationSeq). The shared counter row makes concurrent allocators conflict, +// so exactly one commit order survives; the surviving seqs are gapless and +// commit-ordered (a rolled-back allocator burns no seq — the increment rolls +// back with it). This holds on both of bd's Dolt concurrency models: the SQL +// server aborts the losing commit with a serialization error (retried by the +// write path), while the embedded engine serializes writers on the counter row. // // External tooling reads the journal through `bd mutations tail --since ` // and `bd mutations export`, replaying the exact mutation history of the @@ -26,10 +37,12 @@ import ( // // Emission lives here, at the issueops seam, because both write plumbings — the // DoltStorage decorator chain and the unit-of-work path — bottom out in these -// *InTx functions. Instrumenting the seam makes coverage structural: every -// mutation path (including wisps, ready-claims, lease reclaim, renames, and -// cascade deletes) flows through it. TestEveryMutationFunctionJournals guards -// against a new mutation path silently skipping the journal. +// *InTx functions, and both funnel their INSERT through insertMutationRow, so +// the seq mechanism is shared by construction and cannot drift between plumbings. +// Instrumenting the seam makes coverage structural: every mutation path +// (including wisps, ready-claims, lease reclaim, renames, and cascade deletes) +// flows through it. TestEveryMutationFunctionJournals guards against a new +// mutation path silently skipping the journal. // MutationOp names the kind of mutation a journal row records. type MutationOp string @@ -109,8 +122,11 @@ func RecordDepMutationInTx(ctx context.Context, tx DBTX, op MutationOp, issueID, return insertMutationRow(ctx, tx, op, issueID, issue, &MutationDep{Kind: kind, Target: target}) } -// insertMutationRow performs the actual INSERT. A nil issue is stored as SQL -// NULL (deletes); a nil dep is stored as SQL NULL (non-dependency ops). +// insertMutationRow performs the actual INSERT. It is the ONE seam both write +// plumbings funnel through, so the seq mechanism cannot drift between them. A +// nil issue is stored as SQL NULL (deletes); a nil dep is stored as SQL NULL +// (non-dependency ops). ts is the insert time, stamped inside the committing +// transaction. func insertMutationRow(ctx context.Context, tx DBTX, op MutationOp, issueID string, issue *types.Issue, dep *MutationDep) error { var issueJSON any if issue != nil { @@ -128,16 +144,68 @@ func insertMutationRow(ctx context.Context, tx DBTX, op MutationOp, issueID stri } depJSON = string(b) } - _, err := tx.ExecContext(ctx, ` - INSERT INTO bd_mutations_journal (ts, op, issue_id, issue_json, dep_json) - VALUES (?, ?, ?, ?, ?) - `, time.Now().UTC(), string(op), issueID, issueJSON, depJSON) + seq, err := nextMutationSeq(ctx, tx) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, ` + INSERT INTO bd_mutations_journal (seq, ts, op, issue_id, issue_json, dep_json) + VALUES (?, ?, ?, ?, ?, ?) + `, seq, time.Now().UTC(), string(op), issueID, issueJSON, depJSON) if err != nil { return fmt.Errorf("journal: record %s for %s: %w", op, issueID, err) } return nil } +// nextMutationSeq allocates the next journal sequence number from the single-row +// bd_mutations_seq counter, INSIDE the caller's transaction. Incrementing the +// shared counter row is what serializes seq assignment: two transactions that +// both allocate a seq contend on the one row, so only one commit order survives. +// The value becomes the journal row's seq, yielding gapless, commit-ordered seqs +// (a rolled-back transaction rolls back its increment, burning no seq). The +// counter persists across restart and prune never touches it, so seq never +// resets. The seed row is created by migration 0056 / ignored 0014; the +// self-heal below re-creates it at the journal's high-water mark if it is ever +// missing, so a re-seed can never collide with an existing seq. +func nextMutationSeq(ctx context.Context, tx DBTX) (int64, error) { + advance := func() (int64, error) { + res, err := tx.ExecContext(ctx, "UPDATE bd_mutations_seq SET next_seq = next_seq + 1 WHERE id = 0") + if err != nil { + return 0, fmt.Errorf("journal: advance seq counter: %w", err) + } + return res.RowsAffected() + } + n, err := advance() + if err != nil { + return 0, err + } + if n == 0 { + // Seed the row (idempotent), then raise it past any existing journal seq so + // the re-seed can never collide. VALUES + GREATEST, not INSERT ... SELECT + // MAX(): in Dolt a literal+aggregate SELECT over an empty table yields zero + // rows, so an INSERT ... SELECT would seed nothing on a fresh journal. + if _, err := tx.ExecContext(ctx, "INSERT IGNORE INTO bd_mutations_seq (id, next_seq) VALUES (0, 0)"); err != nil { + return 0, fmt.Errorf("journal: seed seq counter: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE bd_mutations_seq + SET next_seq = GREATEST(next_seq, COALESCE((SELECT MAX(seq) FROM bd_mutations_journal), 0)) + WHERE id = 0 + `); err != nil { + return 0, fmt.Errorf("journal: raise seq counter to high-water mark: %w", err) + } + if _, err := advance(); err != nil { + return 0, err + } + } + var seq int64 + if err := tx.QueryRowContext(ctx, "SELECT next_seq FROM bd_mutations_seq WHERE id = 0").Scan(&seq); err != nil { + return 0, fmt.Errorf("journal: read seq counter: %w", err) + } + return seq, nil +} + // compile-time assurance that *sql.Tx satisfies DBTX (the emit helpers accept // both *sql.Tx and *sql.DB via DBTX). var _ DBTX = (*sql.Tx)(nil) diff --git a/internal/storage/issueops/journal_completeness_test.go b/internal/storage/issueops/journal_completeness_test.go index 090881663..7dbba067d 100644 --- a/internal/storage/issueops/journal_completeness_test.go +++ b/internal/storage/issueops/journal_completeness_test.go @@ -1,131 +1,118 @@ package issueops import ( - "go/ast" - "go/parser" - "go/token" - "io/fs" - "strings" "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" ) -// TestEveryMutationFunctionJournals is the completeness guard for the durable -// mutations journal. It parses this package's source, builds the intra-package -// call graph, and asserts that every mutation entry point either records a -// journal row directly (calls one of the Record*InTx emit helpers) or calls a -// function that transitively does. +// The emit helpers a function calls to journal a row directly. +var journalEmitHelpers = map[string]bool{ + "RecordMutationInTx": true, + "RecordDeleteInTx": true, + "RecordDepMutationInTx": true, + "insertMutationRow": true, +} + +// mutationEntryPoints are the issueops functions that must result in a journal +// row. Every write plumbing bottoms out in one of these. The structural +// cross-check below (TestEveryBeadMutatorJournalsOrIsExempt) guarantees this +// list stays complete: any exported function that writes a work-bead table must +// appear here or in beadDMLExemptions, so a new mutation path cannot be added +// without being accounted for. +var mutationEntryPoints = []string{ + "CreateIssueInTx", + "CreateIssueInTxWithResult", + "CreateIssuesInTx", + "CreateIssuesInTxWithResult", + "PersistDependenciesWithOptionsResult", // creation-time dependency edges + "UpdateIssueInTx", + "UpdateIssueWithoutEventInTx", + "CloseIssueInTx", + "CloseIssueWithoutEventInTx", + "ReopenIssueInTx", + "DeleteIssueInTx", + "DeleteIssuesInTx", + "DeleteIssuesBySourceRepoInTx", + "ClaimIssueInTx", + "ClaimReadyIssueInTx", + "UnclaimIssueInTx", + "UnclaimIssueIfAssigneeInTx", + "ReclaimExpiredLeasesInTx", + "PromoteFromEphemeralInTx", + "AddDependencyInTx", + "RemoveDependencyInTx", + "AddLabelInTx", + "RemoveLabelInTx", + "UpdateIssueIDInTx", + "AddIssueCommentInTx", + "ImportIssueCommentInTx", +} + +// beadDMLExemptions are exported functions the DML detector flags as writing a +// work-bead table but which legitimately do NOT journal, each with a reason. +// They fall into four buckets: (1) derived is_blocked maintenance; (2) aux-table +// writers the templated-%s heuristic can't distinguish from a bead table (events, +// child counters); (3) constituent sub-helpers of a create/rename/promote/delete +// whose top-level entry point journals the whole mutation once; (4) compaction +// maintenance outside the create/update/close/delete/dep/label op vocabulary. The +// staleness check fails if any stops being flagged, so an exemption cannot rot. +var beadDMLExemptions = map[string]string{ + // (1) is_blocked is a denormalized/derived column recomputed as a side effect + // of the graph mutation that changed it (which journals). + "MarkIsBlockedInTx": "maintains the derived is_blocked column, not a bead mutation", + "RecomputeIsBlockedInTx": "recomputes the derived is_blocked column, not a bead mutation", + "RecomputeAllIsBlockedInTx": "recomputes the derived is_blocked column, not a bead mutation", + "RecomputeIsBlockedForIDsInTx": "recomputes the derived is_blocked column, not a bead mutation", + "RecomputeIsBlockedForWispIDsInTx": "recomputes the derived is_blocked column, not a bead mutation", + "RecomputeIsBlockedAfterMergeInTx": "recomputes the derived is_blocked column, not a bead mutation", + + // (2) aux tables matched via templated %s, not work-bead state. + "RecordEventInTable": "writes the events audit table (templated %s), not work-bead state", + "RecordFullEventInTable": "writes the events audit table (templated %s), not work-bead state", + "AddCommentEventInTx": "writes the events audit table (templated %s), not work-bead state", + "GetNextChildIDTx": "writes the child_counters allocation table (templated %s), not work-bead state", + + // (3) constituent sub-helpers; the calling entry point journals the whole + // mutation once (a create/rename/promote/delete emits a single row). + "InsertIssueIntoTable": "raw issue insert; the calling create entry point journals the create", + "InsertIssueIfNew": "raw issue insert; the calling create entry point journals the create", + "PersistLabels": "constituent label write of a create; the create entry point journals it", + "PersistComments": "constituent comment write of a create; the create entry point journals it", + "UpdateWispIDInDependenciesInTx": "rewrites dep rows during a rename; UpdateIssueIDInTx journals the rename", + "UpdateIssueIDInDependenciesInTx": "rewrites dep rows during a rename; UpdateIssueIDInTx journals the rename", + "RetargetInboundDependenciesToWispInTx": "rewrites dep rows during promote; PromoteFromEphemeralInTx journals it", + "RetargetInboundDependenciesToIssueInTx": "rewrites dep rows during promote; PromoteFromEphemeralInTx journals it", + "DeleteWispFromDependenciesInTx": "cleans up dep rows during a delete that journals the delete", + "DeleteWispsFromDependenciesInTx": "cleans up dep rows during a delete that journals the delete", + + // (4) compaction maintenance — a lossy content rewrite outside the mutation + // op vocabulary; not currently journaled. + "ApplyCompactionInTx": "compaction content rewrite (maintenance), outside the mutation op vocabulary", + "RestoreFromSnapshotInTx": "restores a compacted issue (maintenance), outside the mutation op vocabulary", +} + +// TestEveryMutationFunctionJournals parses this package's source, builds the +// intra-package call graph, and asserts every mutation entry point either +// records a journal row directly (calls one of the Record*InTx emit helpers) or +// calls a function that transitively does. // // This kills the enumeration-drift class that sank the decorator design: there, // coverage was a hand-maintained list of overridden methods, and new mutation -// paths (ClaimReadyIssue, wisp ops, lease reclaim, cascade delete, …) silently -// slipped through. Here, if a listed mutation function stops emitting — directly -// or through its delegates — this test fails. When you add a NEW mutation entry -// point to issueops, add it to mutationEntryPoints below and make it emit. +// paths silently slipped through. Here, if a listed mutation function stops +// emitting — directly or through its delegates — this test fails. func TestEveryMutationFunctionJournals(t *testing.T) { - // The mutation entry points that must result in a journal row. Every write - // plumbing bottoms out in one of these; see the emission sites for where - // each records (some via a lowercase worker or a delegate, which the - // transitive check below follows). - mutationEntryPoints := []string{ - "CreateIssueInTx", - "CreateIssueInTxWithResult", - "CreateIssuesInTx", - "CreateIssuesInTxWithResult", - "PersistDependenciesWithOptionsResult", // creation-time dependency edges - "UpdateIssueInTx", - "UpdateIssueWithoutEventInTx", - "CloseIssueInTx", - "CloseIssueWithoutEventInTx", - "ReopenIssueInTx", - "DeleteIssueInTx", - "DeleteIssuesInTx", - "DeleteIssuesBySourceRepoInTx", - "ClaimIssueInTx", - "ClaimReadyIssueInTx", - "UnclaimIssueInTx", - "UnclaimIssueIfAssigneeInTx", - "ReclaimExpiredLeasesInTx", - "PromoteFromEphemeralInTx", - "AddDependencyInTx", - "RemoveDependencyInTx", - "AddLabelInTx", - "RemoveLabelInTx", - "UpdateIssueIDInTx", - } - - // The emit helpers a function calls to journal a row directly. - emitHelpers := map[string]bool{ - "RecordMutationInTx": true, - "RecordDeleteInTx": true, - "RecordDepMutationInTx": true, - "insertMutationRow": true, - } - - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { - return !strings.HasSuffix(fi.Name(), "_test.go") - }, 0) + fns, err := journalscan.ParsePackage(".") if err != nil { t.Fatalf("parse issueops package: %v", err) } - // calls[fn] = set of intra-package function names fn calls (including the - // emit helpers). directlyEmits[fn] = fn calls an emit helper. - calls := map[string]map[string]bool{} - directlyEmits := map[string]bool{} - - for _, pkg := range pkgs { - for _, file := range pkg.Files { - for _, decl := range file.Decls { - fn, ok := decl.(*ast.FuncDecl) - if !ok { - continue - } - name := fn.Name.Name - callset := map[string]bool{} - ast.Inspect(fn, func(n ast.Node) bool { - call, ok := n.(*ast.CallExpr) - if !ok { - return true - } - // Intra-package calls are bare identifiers: foo(...). - if ident, ok := call.Fun.(*ast.Ident); ok { - callset[ident.Name] = true - if emitHelpers[ident.Name] { - directlyEmits[name] = true - } - } - return true - }) - calls[name] = callset - } - } - } - - // Fixpoint: a function emits if it directly emits or calls a function that - // emits. - emits := map[string]bool{} - for name := range directlyEmits { - emits[name] = true - } - for changed := true; changed; { - changed = false - for name, callset := range calls { - if emits[name] { - continue - } - for callee := range callset { - if emits[callee] { - emits[name] = true - changed = true - break - } - } - } - } + emits := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.CallsAnyOf(journalEmitHelpers) }, + func(f *journalscan.FuncInfo) []string { return f.AllCallNames() }) for _, entry := range mutationEntryPoints { - if _, defined := calls[entry]; !defined { + if _, defined := fns[entry]; !defined { t.Errorf("mutation entry point %q not found in issueops — was it renamed? update mutationEntryPoints", entry) continue } @@ -134,3 +121,58 @@ func TestEveryMutationFunctionJournals(t *testing.T) { } } } + +// TestEveryBeadMutatorJournalsOrIsExempt is the STRUCTURAL completeness +// cross-check. It detects, by DML rather than by name, every EXPORTED function +// that writes a work-bead table (INSERT / UPDATE / DELETE against issues, wisps, +// dependencies, labels, comments, and their wisp variants — literal or templated +// table name), and asserts each one journals (calls an emit helper directly or +// transitively) OR is an explicitly-exempted derived-state / aux / sub-helper. +// A new exported mutator that writes a bead table therefore cannot ship without +// either journaling or being justified in beadDMLExemptions, closing the "named +// outside the pattern, silently un-journaled" hole the hand list alone left open. +func TestEveryBeadMutatorJournalsOrIsExempt(t *testing.T) { + fns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse issueops package: %v", err) + } + + // A function writes a bead table if its own body does, or a free function it + // calls transitively does. + beadDML := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.OwnBeadDML }, + func(f *journalscan.FuncInfo) []string { return f.IdentCalls }) + + // A function emits if it calls an emit helper directly or transitively. + emits := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.CallsAnyOf(journalEmitHelpers) }, + func(f *journalscan.FuncInfo) []string { return f.AllCallNames() }) + + seenExempt := map[string]bool{} + var checked int + for key, f := range fns { + if f.Recv != "" || !f.Exported || !beadDML[key] { + continue + } + if reason, ok := beadDMLExemptions[f.Name]; ok { + if reason == "" { + t.Errorf("%s has an empty exemption reason", f.Name) + } + seenExempt[f.Name] = true + continue + } + checked++ + if !emits[key] { + t.Errorf("exported function %q writes a work-bead table but does not journal (no Record*InTx emit helper directly or transitively) and is not exempted — make it journal, or add it to beadDMLExemptions with a reason", f.Name) + } + } + + if checked == 0 { + t.Fatal("cross-check found no exported bead mutators — DML detection or parsing changed; the guard is not actually running") + } + for m := range beadDMLExemptions { + if !seenExempt[m] { + t.Errorf("exemption %q no longer matches an exported bead-writing function — remove it", m) + } + } +} diff --git a/internal/storage/issueops/journal_prune.go b/internal/storage/issueops/journal_prune.go index 848c243bd..f3cb8bdcf 100644 --- a/internal/storage/issueops/journal_prune.go +++ b/internal/storage/issueops/journal_prune.go @@ -89,26 +89,57 @@ func ReadMutationsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([] return out, rows.Err() } -// PruneMutationsInTx deletes journal rows with seq below before, honoring the -// retain-days and retain-rows floors (0 disables a floor), and returns the -// number of rows deleted. It runs inside the caller's transaction. -func PruneMutationsInTx(ctx context.Context, tx DBTX, before int64, retainDays, retainRows int, now time.Time) (int64, error) { +// ComputeMutationsPruneWhere resolves the retain-rows floor via readCeil and +// returns the full DELETE predicate (without the "WHERE" keyword) plus its bind +// args, or skip=true when the whole journal is inside the retained window and +// nothing may be pruned. readCeil reports (ceil, found, err): found is false +// when the journal holds retainRows or fewer rows. It is only invoked when +// retainRows > 0. +// +// This is the ONE place the retain-floor orchestration lives. Both prune +// plumbings — the DBTX path (PruneMutationsInTx) and the proxied raw-SQL path in +// cmd/bd — call it, so the two can never drift on which rows a floor protects. +// Only the substrate-specific ceil read and the DELETE execution differ. +func ComputeMutationsPruneWhere(before int64, retainDays, retainRows int, now time.Time, readCeil func() (ceil int64, found bool, err error)) (where string, args []any, skip bool, err error) { var ( rowsCeil int64 rowsCeilOK bool ) if retainRows > 0 { - err := tx.QueryRowContext(ctx, MutationsPruneRowsCeilQuery(), retainRows).Scan(&rowsCeil) - if errors.Is(err, sql.ErrNoRows) { - // The whole journal is inside the retained window: nothing to prune. - return 0, nil + ceil, found, cerr := readCeil() + if cerr != nil { + return "", nil, false, cerr } - if err != nil { - return 0, fmt.Errorf("journal: compute retain-rows floor: %w", err) + if !found { + return "", nil, true, nil } - rowsCeilOK = true + rowsCeil, rowsCeilOK = ceil, true + } + where, args = BuildMutationsPruneWhere(before, retainDays, now, rowsCeil, rowsCeilOK) + return where, args, false, nil +} + +// PruneMutationsInTx deletes journal rows with seq below before, honoring the +// retain-days and retain-rows floors (0 disables a floor), and returns the +// number of rows deleted. It runs inside the caller's transaction. +func PruneMutationsInTx(ctx context.Context, tx DBTX, before int64, retainDays, retainRows int, now time.Time) (int64, error) { + where, args, skip, err := ComputeMutationsPruneWhere(before, retainDays, retainRows, now, func() (int64, bool, error) { + var ceil int64 + scanErr := tx.QueryRowContext(ctx, MutationsPruneRowsCeilQuery(), retainRows).Scan(&ceil) + if errors.Is(scanErr, sql.ErrNoRows) { + return 0, false, nil + } + if scanErr != nil { + return 0, false, fmt.Errorf("journal: compute retain-rows floor: %w", scanErr) + } + return ceil, true, nil + }) + if err != nil { + return 0, err + } + if skip { + return 0, nil } - where, args := BuildMutationsPruneWhere(before, retainDays, now, rowsCeil, rowsCeilOK) res, err := tx.ExecContext(ctx, "DELETE FROM bd_mutations_journal WHERE "+where, args...) if err != nil { return 0, fmt.Errorf("journal: prune below %d: %w", before, err) diff --git a/internal/storage/issueops/journal_prune_test.go b/internal/storage/issueops/journal_prune_test.go index eed15e571..f235d115c 100644 --- a/internal/storage/issueops/journal_prune_test.go +++ b/internal/storage/issueops/journal_prune_test.go @@ -1,6 +1,7 @@ package issueops import ( + "errors" "testing" "time" ) @@ -62,3 +63,62 @@ func TestBuildMutationsPruneWhere(t *testing.T) { }) } } + +// TestComputeMutationsPruneWhere pins the shared retain-floor orchestration both +// prune plumbings (the DBTX path and the proxied raw-SQL path) run through, so +// they can never drift on the retain-rows short-circuit or which rows a floor +// protects. +func TestComputeMutationsPruneWhere(t *testing.T) { + now := time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC) + + t.Run("no retain-rows skips ceil read", func(t *testing.T) { + called := false + where, args, skip, err := ComputeMutationsPruneWhere(500, 7, 0, now, func() (int64, bool, error) { + called = true + return 0, false, nil + }) + if err != nil || skip { + t.Fatalf("err=%v skip=%v, want nil/false", err, skip) + } + if called { + t.Error("readCeil must not be called when retainRows == 0") + } + if where != "seq < ? AND ts < ?" || len(args) != 2 { + t.Errorf("where=%q args=%v", where, args) + } + }) + + t.Run("retain-rows not found skips prune", func(t *testing.T) { + _, _, skip, err := ComputeMutationsPruneWhere(500, 0, 10, now, func() (int64, bool, error) { + return 0, false, nil + }) + if err != nil { + t.Fatalf("err=%v", err) + } + if !skip { + t.Error("skip must be true when the whole journal is inside the retained window") + } + }) + + t.Run("retain-rows floor applied", func(t *testing.T) { + where, args, skip, err := ComputeMutationsPruneWhere(500, 0, 10, now, func() (int64, bool, error) { + return 480, true, nil + }) + if err != nil || skip { + t.Fatalf("err=%v skip=%v", err, skip) + } + if where != "seq < ? AND seq <= ?" || len(args) != 2 || args[1] != int64(480) { + t.Errorf("where=%q args=%v", where, args) + } + }) + + t.Run("ceil read error propagates", func(t *testing.T) { + sentinel := errors.New("boom") + _, _, _, err := ComputeMutationsPruneWhere(500, 0, 10, now, func() (int64, bool, error) { + return 0, false, sentinel + }) + if !errors.Is(err, sentinel) { + t.Errorf("err = %v, want %v", err, sentinel) + } + }) +} diff --git a/internal/storage/journalscan/journalscan.go b/internal/storage/journalscan/journalscan.go new file mode 100644 index 000000000..37d1821ed --- /dev/null +++ b/internal/storage/journalscan/journalscan.go @@ -0,0 +1,181 @@ +// Package journalscan provides the static-analysis primitives the mutations +// journal completeness guards share. Both the issueops seam and the domain/db +// unit-of-work seam must journal every mutation that writes a work-bead table; +// their guard tests detect such mutators STRUCTURALLY — by the DML a function +// executes — rather than by matching on method-name prefixes, which could let a +// mutator named off-pattern ship un-journaled. This package holds the parsing, +// bead-table DML detection, and call-graph fixpoint those guards run. +package journalscan + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "strings" +) + +// FuncInfo captures the call/DML shape of one top-level function or method, +// keyed by a package-unique name (receiver-qualified for methods). +type FuncInfo struct { + Recv string // receiver type name ("" for free functions) + Name string // bare method/function name + Exported bool // the bare name is exported + IdentCalls []string // intra-package bare-identifier calls (free functions) + SelCalls []string // selector calls, by selector name (x.Foo -> "Foo") + OwnBeadDML bool // body issues INSERT/UPDATE/DELETE against a bead table +} + +// AllCallNames returns every called name, both bare-identifier and selector. +func (f *FuncInfo) AllCallNames() []string { + return append(append([]string{}, f.IdentCalls...), f.SelCalls...) +} + +// CallsAnyOf reports whether the function calls any name in set (bare or selector). +func (f *FuncInfo) CallsAnyOf(set map[string]bool) bool { + for _, c := range f.AllCallNames() { + if set[c] { + return true + } + } + return false +} + +// ReceiverTypeName returns the bare type name of a method receiver +// (e.g. *fooImpl -> fooImpl). +func ReceiverTypeName(expr ast.Expr) string { + if star, ok := expr.(*ast.StarExpr); ok { + expr = star.X + } + if ident, ok := expr.(*ast.Ident); ok { + return ident.Name + } + return "" +} + +// ParsePackage parses dir's non-test .go files and returns one FuncInfo per +// top-level function/method, keyed by "Recv.Name" (or "Name" for free funcs). +func ParsePackage(dir string) (map[string]*FuncInfo, error) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dir, func(fi fs.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + return nil, err + } + out := map[string]*FuncInfo{} + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + f := &FuncInfo{Name: fn.Name.Name, Exported: fn.Name.IsExported()} + if fn.Recv != nil && len(fn.Recv.List) > 0 { + f.Recv = ReceiverTypeName(fn.Recv.List[0].Type) + } + ast.Inspect(fn, func(n ast.Node) bool { + switch node := n.(type) { + case *ast.CallExpr: + switch fun := node.Fun.(type) { + case *ast.Ident: + f.IdentCalls = append(f.IdentCalls, fun.Name) + case *ast.SelectorExpr: + f.SelCalls = append(f.SelCalls, fun.Sel.Name) + } + case *ast.BasicLit: + if node.Kind == token.STRING && SQLWritesBeadTable(node.Value) { + f.OwnBeadDML = true + } + } + return true + }) + key := f.Name + if f.Recv != "" { + key = f.Recv + "." + f.Name + } + out[key] = f + } + } + } + return out, nil +} + +// Fixpoint returns the set of function keys for which seed is true or which +// (transitively) call a name for which it becomes true, following edges. A +// called bare name resolves to a free function of that name and to any method +// with that name (name-based resolution, sufficient for a guard). +func Fixpoint(fns map[string]*FuncInfo, seed func(*FuncInfo) bool, edges func(*FuncInfo) []string) map[string]bool { + resolve := func(name string) []string { + var keys []string + if _, ok := fns[name]; ok { + keys = append(keys, name) + } + for key, f := range fns { + if f.Recv != "" && f.Name == name { + keys = append(keys, key) + } + } + return keys + } + got := map[string]bool{} + for key, f := range fns { + if seed(f) { + got[key] = true + } + } + for changed := true; changed; { + changed = false + for key, f := range fns { + if got[key] { + continue + } + for _, callee := range edges(f) { + for _, ck := range resolve(callee) { + if got[ck] { + got[key] = true + changed = true + break + } + } + if got[key] { + break + } + } + } + } + return got +} + +// BeadTables are the work-bead tables a mutation must be journaled for. +var BeadTables = []string{ + "issues", "wisps", + "dependencies", "wisp_dependencies", + "labels", "wisp_labels", + "comments", "wisp_comments", +} + +// SQLWritesBeadTable reports whether a SQL string literal issues an +// INSERT / UPDATE / DELETE against a work-bead table, whether the table name is +// literal (INSERT INTO issues) or templated (INSERT INTO %s — which in the +// mutation seams always routes to a bead table via table-routing helpers). +func SQLWritesBeadTable(lit string) bool { + s := strings.ToUpper(lit) + s = strings.ReplaceAll(s, "`", "") + s = strings.Join(strings.Fields(s), " ") // collapse whitespace/newlines + targets := []string{"%S"} + for _, tbl := range BeadTables { + targets = append(targets, strings.ToUpper(tbl)) + } + for _, tbl := range targets { + if strings.Contains(s, "INSERT INTO "+tbl+" ") || + strings.Contains(s, "INSERT IGNORE INTO "+tbl+" ") || + strings.Contains(s, "REPLACE INTO "+tbl+" ") || + strings.Contains(s, "UPDATE "+tbl+" ") || + strings.Contains(s, "DELETE FROM "+tbl+" ") { + return true + } + } + return false +} diff --git a/internal/storage/schema/migrations/0056_create_mutations_journal.down.sql b/internal/storage/schema/migrations/0056_create_mutations_journal.down.sql index 5a90e4ea6..12dbd6994 100644 --- a/internal/storage/schema/migrations/0056_create_mutations_journal.down.sql +++ b/internal/storage/schema/migrations/0056_create_mutations_journal.down.sql @@ -1,3 +1,4 @@ --- Reverse migration 0056: drop the clone-local mutations journal table. --- The table is dolt_ignored, so this only touches the working set. +-- Reverse migration 0056: drop the clone-local mutations journal tables. +-- Both are dolt_ignored, so this only touches the working set. DROP TABLE IF EXISTS bd_mutations_journal; +DROP TABLE IF EXISTS bd_mutations_seq; diff --git a/internal/storage/schema/migrations/0056_create_mutations_journal.up.sql b/internal/storage/schema/migrations/0056_create_mutations_journal.up.sql index 948f7e757..8a2884cd4 100644 --- a/internal/storage/schema/migrations/0056_create_mutations_journal.up.sql +++ b/internal/storage/schema/migrations/0056_create_mutations_journal.up.sql @@ -8,18 +8,26 @@ -- -- The table is clone-local (registered dolt_ignored, seeded by MigrateUp's -- doltIgnorePatterns before this migration runs): it is operational state, never --- versioned or federated, so its AUTO_INCREMENT seq stays monotonic and --- collision-free without ever producing a merge conflict. That is exactly why --- the federation-conflict reasoning behind migration 0037 (which dropped --- AUTO_INCREMENT from the *versioned* tables) does not apply here. +-- versioned or federated, so its seq stays monotonic without ever producing a +-- merge conflict. +-- +-- seq is NOT AUTO_INCREMENT. AUTO_INCREMENT assigns at INSERT, not at commit, so +-- under concurrent transactions (the shared SQL server) a lower seq can commit +-- AFTER a higher seq becomes visible, and a consumer tailing WHERE seq > cursor +-- permanently skips it. Instead seq is drawn from the single-row counter table +-- bd_mutations_seq inside the mutation's own transaction (issueops.nextMutationSeq): +-- the shared counter row makes concurrent allocators conflict, so exactly one +-- commit order survives and the surviving seqs are gapless and commit-ordered +-- (a rolled-back allocator burns no seq). The counter table is dolt_ignored too, +-- so it shares the journal's working-set locality. -- -- Fresh clones never run this migration (the schema_migrations cursor arrives --- at-latest); they materialize the table via ignored migration 0014. The --- __temp__ + conditional RENAME dance keeps the CREATE idempotent so a re-run, --- or a run against a workspace that already has the table, is a no-op. +-- at-latest); they materialize both tables via ignored migration 0014. The +-- __temp__ + conditional RENAME dance keeps each CREATE idempotent so a re-run, +-- or a run against a workspace that already has the tables, is a no-op. DROP TABLE IF EXISTS __temp__bd_mutations_journal; CREATE TABLE __temp__bd_mutations_journal ( - seq BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + seq BIGINT NOT NULL PRIMARY KEY, ts DATETIME NOT NULL, op VARCHAR(32) NOT NULL, issue_id VARCHAR(255) NOT NULL, @@ -30,3 +38,21 @@ CREATE TABLE __temp__bd_mutations_journal ( SET @exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_mutations_journal'); SET @sql = IF(@exists = 0, 'RENAME TABLE __temp__bd_mutations_journal TO bd_mutations_journal', 'DROP TABLE __temp__bd_mutations_journal'); PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +DROP TABLE IF EXISTS __temp__bd_mutations_seq; +CREATE TABLE __temp__bd_mutations_seq ( + id INT NOT NULL PRIMARY KEY, + next_seq BIGINT NOT NULL +); +SET @seq_exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_mutations_seq'); +SET @seq_sql = IF(@seq_exists = 0, 'RENAME TABLE __temp__bd_mutations_seq TO bd_mutations_seq', 'DROP TABLE __temp__bd_mutations_seq'); +PREPARE stmt FROM @seq_sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; +-- Seed the counter row (idempotent), then raise it to the journal's current +-- high-water mark so seq never resets or collides (0 on a fresh table). Done as +-- VALUES + a GREATEST update rather than INSERT ... SELECT MAX(): in Dolt a +-- SELECT that mixes a literal with an aggregate over an EMPTY table yields zero +-- rows, so an INSERT ... SELECT would seed nothing on a fresh workspace. +INSERT IGNORE INTO bd_mutations_seq (id, next_seq) VALUES (0, 0); +UPDATE bd_mutations_seq + SET next_seq = GREATEST(next_seq, COALESCE((SELECT MAX(seq) FROM bd_mutations_journal), 0)) + WHERE id = 0; diff --git a/internal/storage/schema/migrations/ignored/0014_create_mutations_journal.up.sql b/internal/storage/schema/migrations/ignored/0014_create_mutations_journal.up.sql index b4f0a73e6..6b0bd9635 100644 --- a/internal/storage/schema/migrations/ignored/0014_create_mutations_journal.up.sql +++ b/internal/storage/schema/migrations/ignored/0014_create_mutations_journal.up.sql @@ -5,12 +5,15 @@ -- -- bd_mutations_journal is dolt_ignored (seeded by MigrateUp's doltIgnorePatterns -- before this migration runs): operational, never versioned or federated, so its --- AUTO_INCREMENT seq stays monotonic and collision-free with no merge conflict. --- Same __temp__ + conditional RENAME pattern: create only when absent, never --- touch an existing table. See issueops.RecordMutationInTx for the writer. +-- seq stays monotonic with no merge conflict. seq is NOT AUTO_INCREMENT — it is +-- drawn from the single-row bd_mutations_seq counter inside the mutation's own +-- transaction so concurrent allocators conflict and the surviving seqs are +-- gapless and commit-ordered (see main migration 0056 and issueops.nextMutationSeq). +-- Same __temp__ + conditional RENAME pattern for both tables: create only when +-- absent, never touch an existing table. See issueops.RecordMutationInTx for the writer. DROP TABLE IF EXISTS __temp__bd_mutations_journal; CREATE TABLE __temp__bd_mutations_journal ( - seq BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + seq BIGINT NOT NULL PRIMARY KEY, ts DATETIME NOT NULL, op VARCHAR(32) NOT NULL, issue_id VARCHAR(255) NOT NULL, @@ -21,3 +24,18 @@ CREATE TABLE __temp__bd_mutations_journal ( SET @exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_mutations_journal'); SET @sql = IF(@exists = 0, 'RENAME TABLE __temp__bd_mutations_journal TO bd_mutations_journal', 'DROP TABLE __temp__bd_mutations_journal'); PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +DROP TABLE IF EXISTS __temp__bd_mutations_seq; +CREATE TABLE __temp__bd_mutations_seq ( + id INT NOT NULL PRIMARY KEY, + next_seq BIGINT NOT NULL +); +SET @seq_exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_mutations_seq'); +SET @seq_sql = IF(@seq_exists = 0, 'RENAME TABLE __temp__bd_mutations_seq TO bd_mutations_seq', 'DROP TABLE __temp__bd_mutations_seq'); +PREPARE stmt FROM @seq_sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; +-- Seed (idempotent), then raise to the journal high-water mark. See main +-- migration 0056 for why this is VALUES + GREATEST rather than INSERT ... SELECT. +INSERT IGNORE INTO bd_mutations_seq (id, next_seq) VALUES (0, 0); +UPDATE bd_mutations_seq + SET next_seq = GREATEST(next_seq, COALESCE((SELECT MAX(seq) FROM bd_mutations_journal), 0)) + WHERE id = 0; diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 27dd46807..63587f6f8 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -221,6 +221,7 @@ var ( // re-asserts the full set idempotently at the top of every write-mode open. var doltIgnorePatterns = []string{ "bd_mutations_journal", + "bd_mutations_seq", "ignored_schema_migrations", "leases", "local_metadata", diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8f6acb54b..8e21105ca 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -242,8 +242,8 @@ type RawDBAccessor interface { // MutationsJournalRow is one raw bd_mutations_journal row surfaced to the // `bd mutations` CLI. IssueJSON is empty when the op is a delete (no surviving -// row); DepJSON is empty for non-dependency ops. TS is the committed-at -// timestamp normalized to a string. +// row); DepJSON is empty for non-dependency ops. TS is the insert-time timestamp +// (stamped inside the committing transaction) normalized to a string. type MutationsJournalRow struct { Seq int64 TS string