From 0103d268b6ed2b6f8823648f84db8ccda72b85eb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 03:26:05 -0700 Subject: [PATCH] fix(dolt): stage versioned-history tables on the delete/sweep role plane (post-merge #6675) (#6844) Post-merge review of #6675 (landed 53ce8b545..73702cc73) left one gating major: the server-backed delete role bypasses withVersionedHistoryTables. #6675 made every fixed-list DOLT_ADD path in internal/storage/dolt append the versioned-history tables, and said so in the helper's own doc comment ("Every fixed-list DOLT_ADD path in this package routes through here"). Two paths did not: deleter.Delete and sweeper.Sweep loop their hand-listed sweptTables directly. For the deleter that is a durability hole, not an inconsistency. A delete is a version-completeness exemption for the row it REMOVES, never for its transaction: DeleteInTx calls RewriteDeletedReferencesInTx, which updates every surviving neighbor that cites a deleted id through UpdateIssueInTx -- the minting entry point -- and commitWriteTx has scoped that transaction for minting. So a delete of a cited bead mints issue_versions and store_epoch rows inside a transaction that then stages only sweptTables and issues its own DOLT_COMMIT: the rows sit in the working set, outside the commit that describes them, unreplicated and liable to be swept into whatever unrelated commit stages next. For the two append-only tables that loss is the unrecoverable kind #6675's own hazard comment names. Both loops now route through s.withVersionedHistoryTables(sweptTables). Sweep mints nothing today (SweepInTx deletes through DeleteIssuesInTx, which rewrites no neighbor), so that half is the funnel's consistency half: it shares the table list and the minting-scoped withWriteTx with the deleter. Coverage, so the funnel comment is enforced rather than asserted: - TestEveryDoltAddStagesVersionedHistoryTables (new): every function in the package that passes a DOLT_ADD statement must route its staged set through withVersionedHistoryTables or MarkVersionedHistoryDirty, or carry an exemption reason. commitWorkingSet is the one exemption -- it stages the dolt_status-derived dirty set, so a dirty version table is already in it by construction. Same shape as TestEveryRawTxVersionScopeIsScopedOrExempt, including the exemption staleness check and the non-vacuity fatal. - TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit (new): drives the minting delete path no test drove -- a neighbor citing the deleted id, Force to orphan it -- and requires dolt_status clean for issue_versions, store_epoch and issues afterwards. History is enabled AFTER the fixture rows exist so the delete's own transaction takes the first mint and seeds the epoch singleton, which is what makes the store_epoch assertion load-bearing here. Mutation evidence: - Revert deleter.go's funnel call: the guard fails naming deleter.Delete ALONE, and the new contract case fails on issue_versions and store_epoch and NOT on issues -- the delete's commit happened, only the version rows were left behind. TestDualWriteVersionRowsRideTheMutationsDoltCommit stays green, so the new case covers an arm no incumbent covered. - Revert sweeper.go's funnel call: the guard fails naming sweeper.Sweep alone and both contract cases stay green. That asymmetry is why the sweeper's coverage is the structural guard and not a behavioral pin. Also adds store_epoch to TestDualWriteVersionRowsRideTheMutationsDoltCommit's dolt_status loop -- the review's other in-file minor: the table whose loss the change calls unrecoverable was the one table that test did not assert clean -- and names the delete/sweep planes in doltAddAndCommitInTx's HAZARD surface list, whose widened-staged-set paragraph this change makes true of them too. Deliberately not here: store_epoch.bumped_at is still DATETIME(0) after 0069 widened the paired version timestamps (non-gating minor, zero Go writers today). 0069 has landed and is frozen, so the code arm is a new migration slot; filed as bd-sor0q against the phase that gives bumped_at its first writer. No CHANGELOG entry: SetVersionedHistoryEnabled still has no non-test caller, so there is no user-visible behavior change -- the same reason #6675 itself added none. Co-authored-by: Claude Opus 5 --- internal/storage/dolt/deleter.go | 12 +- .../dolt/dualwrite_history_contract_test.go | 94 +++++++++++++++- internal/storage/dolt/ephemeral_routing.go | 23 ++-- internal/storage/dolt/sweeper.go | 9 +- .../dolt/version_staging_completeness_test.go | 104 ++++++++++++++++++ 5 files changed, 229 insertions(+), 13 deletions(-) create mode 100644 internal/storage/dolt/version_staging_completeness_test.go diff --git a/internal/storage/dolt/deleter.go b/internal/storage/dolt/deleter.go index cb56798bb..9aa079497 100644 --- a/internal/storage/dolt/deleter.go +++ b/internal/storage/dolt/deleter.go @@ -83,7 +83,17 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue } // The same tables a sweep stages; the neighbor rewrite lands in // `issues`, which is already on the list. - for _, table := range sweptTables { + // + // THE VERSIONED-HISTORY TABLES ARE NOT on that list, and the rewrite + // writes them: RewriteDeletedReferencesInTx updates every surviving + // neighbor through UpdateIssueInTx, the seam's minting entry point, and + // this transaction is scoped for minting because commitWriteTx scopes + // the transaction withWriteTx hands to the body above. Staging the + // fixed list alone would leave those rows in the working set, outside + // the commit that describes them — the exact loss withVersionedHistoryTables + // exists to prevent, and for store_epoch the unrecoverable kind (see the + // append-only note in doltAddAndCommitInTx's HAZARD block). + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted) diff --git a/internal/storage/dolt/dualwrite_history_contract_test.go b/internal/storage/dolt/dualwrite_history_contract_test.go index 64788658d..b8baf11b5 100644 --- a/internal/storage/dolt/dualwrite_history_contract_test.go +++ b/internal/storage/dolt/dualwrite_history_contract_test.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/backend/conformance" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // TestDualWriteContract runs the dual-write history contract with @@ -282,7 +283,12 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { t.Fatalf("issues.current_revision for %s = %d, want 1", id, currentRevision) } - for _, table := range []string{"issue_versions", "issues"} { + // All three tables issueops.VersionedHistoryStagedTables names, not just the + // two the version rows land in. store_epoch is the one whose loss this PR + // calls unrecoverable, and the first mint seeds its singleton row + // (version_history.go), so in this scenario it is guaranteed dirty-then-staged: + // a regression that dropped it from the staged set would otherwise pass here. + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { var dirty int if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { @@ -293,3 +299,89 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { } } } + +// TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit covers the +// staging plane neither DualWriteFixture nor the RunInTransaction case above can +// reach: the server-backed delete role. +// +// A delete is a version-completeness EXEMPTION for the row it removes — there is +// no surviving row to version — which is why issueops' own exemption table lists +// DeleteInTx. It is not an exemption for the TRANSACTION. DeleteInTx rewrites +// every surviving neighbor that cites a deleted id through UpdateIssueInTx, the +// minting entry point, so a delete with a neighbor really does mint; and until +// the fix this test arrived with, deleter.Delete staged its hand-listed +// sweptTables and nothing else, leaving those version rows in the working set +// while issuing its own DOLT_COMMIT. +// +// The delete path is also the only one of the three where the store_epoch +// assertion is load-bearing without arranging for it: history is turned on AFTER +// the fixture rows exist, so the delete's own transaction takes the first mint in +// this store and seeds the epoch singleton inside it. +func TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit(t *testing.T) { + store, storeCleanup := setupTestStore(t) + defer storeCleanup() + ctx, cancel := testContext(t) + defer cancel() + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + + const target, neighbor = "dwdel-target", "dwdel-neighbor" + fixtures := []*types.Issue{ + {ID: target, Title: "doomed", IssueType: types.TypeTask, Status: types.StatusOpen}, + // The citation is what makes this a MINTING delete: the rewrite only + // touches a neighbor whose text names a deleted id. + {ID: neighbor, Title: "survivor", Description: "blocked by " + target, + IssueType: types.TypeTask, Status: types.StatusOpen}, + } + for _, issue := range fixtures { + if err := store.CreateIssue(ctx, issue, "creator"); err != nil { + t.Fatalf("create %s: %v", issue.ID, err) + } + } + // An INBOUND edge, so the neighborhood read finds the survivor at all; + // Force is then what lets the delete orphan it instead of refusing. + if err := store.AddDependency(ctx, &types.Dependency{ + IssueID: neighbor, DependsOnID: target, Type: types.DepBlocks, + }, "linker"); err != nil { + t.Fatalf("add dep: %v", err) + } + + configurer.SetVersionedHistoryEnabled(true) + defer configurer.SetVersionedHistoryEnabled(false) + + deleter, err := store.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + result, err := deleter.Delete(ctx, publicops.DeleteRequest{ + IDs: []string{target}, Force: true, Actor: "deleter", + }) + if err != nil { + t.Fatalf("delete %s: %v", target, err) + } + if result.ReferencesUpdated != 1 { + t.Fatalf("ReferencesUpdated = %d, want 1 — the citation rewrite did not run, so this case is not exercising the minting delete path it exists for", result.ReferencesUpdated) + } + + var versions int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?`, neighbor).Scan(&versions); err != nil { + t.Fatalf("count issue_versions for %s: %v", neighbor, err) + } + if versions != 1 { + t.Fatalf("issue_versions rows for the rewritten neighbor %s = %d, want 1 — the delete's own transaction is not scoped for minting", neighbor, versions) + } + + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { + var dirty int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { + t.Fatalf("read dolt_status for %s: %v", table, err) + } + if dirty != 0 { + t.Errorf("dolt_status still reports %s dirty after the delete committed — the neighbor's version rows are outside the delete's own Dolt commit: unreplicated, and waiting to be swept into whatever unrelated commit stages next", table) + } + } +} diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index 74bbd20ac..d745aa427 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -348,7 +348,8 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The hazard remains LIVE everywhere the in-tx ordering survives — a larger // surface than this helper's callers: wisp promote/demote (this file), // legacy reopen (issues.go), RunInIssueLifecycleTransaction -// (transaction.go), AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern +// (transaction.go), the role deleter and sweeper (deleter.go, sweeper.go), +// AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern // inlined directly in the legacy DoltStore write methods in issues.go and // slots.go (UpdateIssue, UpdateIssueChecked, ClaimIssue, ClaimReadyIssue, // UnclaimIssue, UnclaimIssueIfAssignee, ReclaimExpiredLeases, CloseIssue*, @@ -361,15 +362,17 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The STAGED SET is the hazard's other axis, and it widened. When versioned // history is active, withVersionedHistoryTables adds issue_versions and // store_epoch to whatever fixed list a caller passed (every caller here -// already stages issues), so those two tables are now exposed to the lost -// update above on every in-tx plane named in the paragraph above. The -// exposure is not equivalent to the one issues already had: a reverted issues -// row is rewritten by the next mutation of that bead, while issue_versions -// and store_epoch are append-only, so a row reverted to its BEGIN-time value -// is never rewritten -- it is gone, leaving a hole in the history no later -// write fills. Migrating these planes to the post-tx ordering closes this -// along with the rest of the hazard; it is written down here so the -// activation phase inherits a known hazard rather than discovering it. +// already stages issues), and the role deleter and sweeper call it on their +// own fixed list for the same durability reason, so those two tables are now +// exposed to the lost update above on every in-tx plane named in the paragraph +// above. The exposure is not equivalent to the one issues already had: a +// reverted issues row is rewritten by the next mutation of that bead, while +// issue_versions and store_epoch are append-only, so a row reverted to its +// BEGIN-time value is never rewritten -- it is gone, leaving a hole in the +// history no later write fills. Migrating these planes to the post-tx +// ordering closes this along with the rest of the hazard; it is written down +// here so the activation phase inherits a known hazard rather than +// discovering it. func (s *DoltStore) doltAddAndCommitInTx(ctx context.Context, tx *sql.Tx, tables []string, commitMsg string) error { // Batch/off auto-commit (bd-4wamg): leave the writes in the working set // for a later explicit commit point (bd dolt commit / CommitPending) diff --git a/internal/storage/dolt/sweeper.go b/internal/storage/dolt/sweeper.go index 8d6bae809..d1f6d81d8 100644 --- a/internal/storage/dolt/sweeper.go +++ b/internal/storage/dolt/sweeper.go @@ -73,7 +73,14 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop if result.Swept == 0 { return nil } - for _, table := range sweptTables { + // Through the same funnel the deleter uses. A sweep mints nothing today + // — SweepInTx deletes through DeleteIssuesInTx, which rewrites no + // neighbor, so there is no surviving row to version — but it shares + // this table list and the minting-scoped withWriteTx with the deleter, + // and a staging list that carries the version tables on one and not the + // other is the asymmetry the wisp-arm scope comment refused: inert + // today, and silently wrong the day a sweep gains a surviving-row write. + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier) diff --git a/internal/storage/dolt/version_staging_completeness_test.go b/internal/storage/dolt/version_staging_completeness_test.go new file mode 100644 index 000000000..0702751ba --- /dev/null +++ b/internal/storage/dolt/version_staging_completeness_test.go @@ -0,0 +1,104 @@ +package dolt + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// versionStagingFunnels are the ways a staging path picks up the tables the +// versioned-history seam writes: the store helper that appends them to a +// caller's fixed list, or the dirty-table tracker that marks them for +// versioncontrolops.StageAndCommit. +var versionStagingFunnels = map[string]bool{ + "withVersionedHistoryTables": true, + "MarkVersionedHistoryDirty": true, +} + +// doltAddStagingExemptions are functions that issue a DOLT_ADD and legitimately +// do NOT route their staged set through a funnel, each with a reason. The +// staleness check below fails if one stops being flagged, so an exemption +// cannot rot. +var doltAddStagingExemptions = map[string]string{ + "DoltStore.commitWorkingSet": "stages the dolt_status-derived dirty set rather than a fixed list: a dirty issue_versions or store_epoch is already in that set by construction, so there is nothing for withVersionedHistoryTables to append", +} + +// TestEveryDoltAddStagesVersionedHistoryTables is the staging twin of +// TestEveryRawTxVersionScopeIsScopedOrExempt, and exists for the same reason at +// one step further down the path: those guards prove the seam MINTS and that +// the store turns minting ON for the transaction it mints in; nothing proved +// the minted rows are STAGED into the Dolt commit the mutation makes. Until +// this guard existed, withVersionedHistoryTables' own "Every fixed-list +// DOLT_ADD path in this package routes through here" was an asserted claim, and +// it was false: the role deleter and sweeper looped their hand-listed +// sweptTables directly, so a delete that rewrote a surviving neighbor's +// citations minted version rows inside a transaction that then staged +// everything except them. +// +// Unstaged version rows are the quiet failure this whole phase exists to close. +// They read back perfectly from SQL while sitting in the working set, outside +// the commit that describes them: unreplicated, and liable to be swept into +// whatever unrelated commit stages next. For the append-only tables the loss is +// the unrecoverable kind — no later write refills a hole in issue_versions or +// store_epoch the way the next mutation of a bead rewrites its issues row. +// +// KNOWN LIMIT: the detector reads the string literals a function passes as call +// ARGUMENTS, which is the form every DOLT_ADD in this package uses. A DOLT_ADD +// assembled in a variable or through fmt.Sprintf would be invisible to it — the +// non-vacuity check below is what keeps that from silently emptying the guard, +// the way the scope guards' own "the guard is not actually running" fatals do. +func TestEveryDoltAddStagesVersionedHistoryTables(t *testing.T) { + doltFns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + seenExempt := map[string]bool{} + var checked int + for key, f := range doltFns { + if !callsDoltAdd(f) { + continue + } + if reason, ok := doltAddStagingExemptions[key]; ok { + if reason == "" { + t.Errorf("%s has an empty DOLT_ADD staging exemption reason", key) + } + seenExempt[key] = true + continue + } + checked++ + if !f.CallsAnyOf(versionStagingFunnels) { + t.Errorf("%s stages tables with DOLT_ADD but never routes the set through withVersionedHistoryTables "+ + "(or marks it with MarkVersionedHistoryDirty) — when versioned history is active the version rows this "+ + "transaction minted stay in the working set, outside the Dolt commit that describes them: unreplicated, "+ + "and for the append-only tables unrecoverable. Route the list through s.withVersionedHistoryTables, "+ + "or add it to doltAddStagingExemptions with a reason.", key) + } + } + + if checked == 0 { + t.Fatal("guard found no DOLT_ADD staging path — literal detection or parsing changed; the guard is not actually running") + } + for key := range doltAddStagingExemptions { + if !seenExempt[key] { + t.Errorf("DOLT_ADD staging exemption %q no longer matches a DOLT_ADD path — remove it", key) + } + } +} + +// callsDoltAdd reports whether f passes a DOLT_ADD statement to anything. It +// deliberately asks nothing about the SHAPE of the call: a whole-working-set +// DOLT_ADD('-A') needs no funnel, but making that an automatic pass would let a +// future fixed-list site hide behind a shape test. Such a site takes an +// exemption line naming the reason instead. +func callsDoltAdd(f *journalscan.FuncInfo) bool { + for _, call := range f.Calls { + for _, arg := range call.Args { + if strings.Contains(arg, "DOLT_ADD") { + return true + } + } + } + return false +}