diff --git a/internal/storage/dolt/deleter.go b/internal/storage/dolt/deleter.go index cb56798bb..9aa079497 100644 --- a/internal/storage/dolt/deleter.go +++ b/internal/storage/dolt/deleter.go @@ -83,7 +83,17 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue } // The same tables a sweep stages; the neighbor rewrite lands in // `issues`, which is already on the list. - for _, table := range sweptTables { + // + // THE VERSIONED-HISTORY TABLES ARE NOT on that list, and the rewrite + // writes them: RewriteDeletedReferencesInTx updates every surviving + // neighbor through UpdateIssueInTx, the seam's minting entry point, and + // this transaction is scoped for minting because commitWriteTx scopes + // the transaction withWriteTx hands to the body above. Staging the + // fixed list alone would leave those rows in the working set, outside + // the commit that describes them — the exact loss withVersionedHistoryTables + // exists to prevent, and for store_epoch the unrecoverable kind (see the + // append-only note in doltAddAndCommitInTx's HAZARD block). + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted) diff --git a/internal/storage/dolt/dualwrite_history_contract_test.go b/internal/storage/dolt/dualwrite_history_contract_test.go index 64788658d..b8baf11b5 100644 --- a/internal/storage/dolt/dualwrite_history_contract_test.go +++ b/internal/storage/dolt/dualwrite_history_contract_test.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/backend/conformance" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // TestDualWriteContract runs the dual-write history contract with @@ -282,7 +283,12 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { t.Fatalf("issues.current_revision for %s = %d, want 1", id, currentRevision) } - for _, table := range []string{"issue_versions", "issues"} { + // All three tables issueops.VersionedHistoryStagedTables names, not just the + // two the version rows land in. store_epoch is the one whose loss this PR + // calls unrecoverable, and the first mint seeds its singleton row + // (version_history.go), so in this scenario it is guaranteed dirty-then-staged: + // a regression that dropped it from the staged set would otherwise pass here. + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { var dirty int if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { @@ -293,3 +299,89 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { } } } + +// TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit covers the +// staging plane neither DualWriteFixture nor the RunInTransaction case above can +// reach: the server-backed delete role. +// +// A delete is a version-completeness EXEMPTION for the row it removes — there is +// no surviving row to version — which is why issueops' own exemption table lists +// DeleteInTx. It is not an exemption for the TRANSACTION. DeleteInTx rewrites +// every surviving neighbor that cites a deleted id through UpdateIssueInTx, the +// minting entry point, so a delete with a neighbor really does mint; and until +// the fix this test arrived with, deleter.Delete staged its hand-listed +// sweptTables and nothing else, leaving those version rows in the working set +// while issuing its own DOLT_COMMIT. +// +// The delete path is also the only one of the three where the store_epoch +// assertion is load-bearing without arranging for it: history is turned on AFTER +// the fixture rows exist, so the delete's own transaction takes the first mint in +// this store and seeds the epoch singleton inside it. +func TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit(t *testing.T) { + store, storeCleanup := setupTestStore(t) + defer storeCleanup() + ctx, cancel := testContext(t) + defer cancel() + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + + const target, neighbor = "dwdel-target", "dwdel-neighbor" + fixtures := []*types.Issue{ + {ID: target, Title: "doomed", IssueType: types.TypeTask, Status: types.StatusOpen}, + // The citation is what makes this a MINTING delete: the rewrite only + // touches a neighbor whose text names a deleted id. + {ID: neighbor, Title: "survivor", Description: "blocked by " + target, + IssueType: types.TypeTask, Status: types.StatusOpen}, + } + for _, issue := range fixtures { + if err := store.CreateIssue(ctx, issue, "creator"); err != nil { + t.Fatalf("create %s: %v", issue.ID, err) + } + } + // An INBOUND edge, so the neighborhood read finds the survivor at all; + // Force is then what lets the delete orphan it instead of refusing. + if err := store.AddDependency(ctx, &types.Dependency{ + IssueID: neighbor, DependsOnID: target, Type: types.DepBlocks, + }, "linker"); err != nil { + t.Fatalf("add dep: %v", err) + } + + configurer.SetVersionedHistoryEnabled(true) + defer configurer.SetVersionedHistoryEnabled(false) + + deleter, err := store.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + result, err := deleter.Delete(ctx, publicops.DeleteRequest{ + IDs: []string{target}, Force: true, Actor: "deleter", + }) + if err != nil { + t.Fatalf("delete %s: %v", target, err) + } + if result.ReferencesUpdated != 1 { + t.Fatalf("ReferencesUpdated = %d, want 1 — the citation rewrite did not run, so this case is not exercising the minting delete path it exists for", result.ReferencesUpdated) + } + + var versions int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?`, neighbor).Scan(&versions); err != nil { + t.Fatalf("count issue_versions for %s: %v", neighbor, err) + } + if versions != 1 { + t.Fatalf("issue_versions rows for the rewritten neighbor %s = %d, want 1 — the delete's own transaction is not scoped for minting", neighbor, versions) + } + + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { + var dirty int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { + t.Fatalf("read dolt_status for %s: %v", table, err) + } + if dirty != 0 { + t.Errorf("dolt_status still reports %s dirty after the delete committed — the neighbor's version rows are outside the delete's own Dolt commit: unreplicated, and waiting to be swept into whatever unrelated commit stages next", table) + } + } +} diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index 74bbd20ac..d745aa427 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -348,7 +348,8 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The hazard remains LIVE everywhere the in-tx ordering survives — a larger // surface than this helper's callers: wisp promote/demote (this file), // legacy reopen (issues.go), RunInIssueLifecycleTransaction -// (transaction.go), AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern +// (transaction.go), the role deleter and sweeper (deleter.go, sweeper.go), +// AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern // inlined directly in the legacy DoltStore write methods in issues.go and // slots.go (UpdateIssue, UpdateIssueChecked, ClaimIssue, ClaimReadyIssue, // UnclaimIssue, UnclaimIssueIfAssignee, ReclaimExpiredLeases, CloseIssue*, @@ -361,15 +362,17 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The STAGED SET is the hazard's other axis, and it widened. When versioned // history is active, withVersionedHistoryTables adds issue_versions and // store_epoch to whatever fixed list a caller passed (every caller here -// already stages issues), so those two tables are now exposed to the lost -// update above on every in-tx plane named in the paragraph above. The -// exposure is not equivalent to the one issues already had: a reverted issues -// row is rewritten by the next mutation of that bead, while issue_versions -// and store_epoch are append-only, so a row reverted to its BEGIN-time value -// is never rewritten -- it is gone, leaving a hole in the history no later -// write fills. Migrating these planes to the post-tx ordering closes this -// along with the rest of the hazard; it is written down here so the -// activation phase inherits a known hazard rather than discovering it. +// already stages issues), and the role deleter and sweeper call it on their +// own fixed list for the same durability reason, so those two tables are now +// exposed to the lost update above on every in-tx plane named in the paragraph +// above. The exposure is not equivalent to the one issues already had: a +// reverted issues row is rewritten by the next mutation of that bead, while +// issue_versions and store_epoch are append-only, so a row reverted to its +// BEGIN-time value is never rewritten -- it is gone, leaving a hole in the +// history no later write fills. Migrating these planes to the post-tx +// ordering closes this along with the rest of the hazard; it is written down +// here so the activation phase inherits a known hazard rather than +// discovering it. func (s *DoltStore) doltAddAndCommitInTx(ctx context.Context, tx *sql.Tx, tables []string, commitMsg string) error { // Batch/off auto-commit (bd-4wamg): leave the writes in the working set // for a later explicit commit point (bd dolt commit / CommitPending) diff --git a/internal/storage/dolt/sweeper.go b/internal/storage/dolt/sweeper.go index 8d6bae809..d1f6d81d8 100644 --- a/internal/storage/dolt/sweeper.go +++ b/internal/storage/dolt/sweeper.go @@ -73,7 +73,14 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop if result.Swept == 0 { return nil } - for _, table := range sweptTables { + // Through the same funnel the deleter uses. A sweep mints nothing today + // — SweepInTx deletes through DeleteIssuesInTx, which rewrites no + // neighbor, so there is no surviving row to version — but it shares + // this table list and the minting-scoped withWriteTx with the deleter, + // and a staging list that carries the version tables on one and not the + // other is the asymmetry the wisp-arm scope comment refused: inert + // today, and silently wrong the day a sweep gains a surviving-row write. + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier) diff --git a/internal/storage/dolt/version_staging_completeness_test.go b/internal/storage/dolt/version_staging_completeness_test.go new file mode 100644 index 000000000..0702751ba --- /dev/null +++ b/internal/storage/dolt/version_staging_completeness_test.go @@ -0,0 +1,104 @@ +package dolt + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// versionStagingFunnels are the ways a staging path picks up the tables the +// versioned-history seam writes: the store helper that appends them to a +// caller's fixed list, or the dirty-table tracker that marks them for +// versioncontrolops.StageAndCommit. +var versionStagingFunnels = map[string]bool{ + "withVersionedHistoryTables": true, + "MarkVersionedHistoryDirty": true, +} + +// doltAddStagingExemptions are functions that issue a DOLT_ADD and legitimately +// do NOT route their staged set through a funnel, each with a reason. The +// staleness check below fails if one stops being flagged, so an exemption +// cannot rot. +var doltAddStagingExemptions = map[string]string{ + "DoltStore.commitWorkingSet": "stages the dolt_status-derived dirty set rather than a fixed list: a dirty issue_versions or store_epoch is already in that set by construction, so there is nothing for withVersionedHistoryTables to append", +} + +// TestEveryDoltAddStagesVersionedHistoryTables is the staging twin of +// TestEveryRawTxVersionScopeIsScopedOrExempt, and exists for the same reason at +// one step further down the path: those guards prove the seam MINTS and that +// the store turns minting ON for the transaction it mints in; nothing proved +// the minted rows are STAGED into the Dolt commit the mutation makes. Until +// this guard existed, withVersionedHistoryTables' own "Every fixed-list +// DOLT_ADD path in this package routes through here" was an asserted claim, and +// it was false: the role deleter and sweeper looped their hand-listed +// sweptTables directly, so a delete that rewrote a surviving neighbor's +// citations minted version rows inside a transaction that then staged +// everything except them. +// +// Unstaged version rows are the quiet failure this whole phase exists to close. +// They read back perfectly from SQL while sitting in the working set, outside +// the commit that describes them: unreplicated, and liable to be swept into +// whatever unrelated commit stages next. For the append-only tables the loss is +// the unrecoverable kind — no later write refills a hole in issue_versions or +// store_epoch the way the next mutation of a bead rewrites its issues row. +// +// KNOWN LIMIT: the detector reads the string literals a function passes as call +// ARGUMENTS, which is the form every DOLT_ADD in this package uses. A DOLT_ADD +// assembled in a variable or through fmt.Sprintf would be invisible to it — the +// non-vacuity check below is what keeps that from silently emptying the guard, +// the way the scope guards' own "the guard is not actually running" fatals do. +func TestEveryDoltAddStagesVersionedHistoryTables(t *testing.T) { + doltFns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + seenExempt := map[string]bool{} + var checked int + for key, f := range doltFns { + if !callsDoltAdd(f) { + continue + } + if reason, ok := doltAddStagingExemptions[key]; ok { + if reason == "" { + t.Errorf("%s has an empty DOLT_ADD staging exemption reason", key) + } + seenExempt[key] = true + continue + } + checked++ + if !f.CallsAnyOf(versionStagingFunnels) { + t.Errorf("%s stages tables with DOLT_ADD but never routes the set through withVersionedHistoryTables "+ + "(or marks it with MarkVersionedHistoryDirty) — when versioned history is active the version rows this "+ + "transaction minted stay in the working set, outside the Dolt commit that describes them: unreplicated, "+ + "and for the append-only tables unrecoverable. Route the list through s.withVersionedHistoryTables, "+ + "or add it to doltAddStagingExemptions with a reason.", key) + } + } + + if checked == 0 { + t.Fatal("guard found no DOLT_ADD staging path — literal detection or parsing changed; the guard is not actually running") + } + for key := range doltAddStagingExemptions { + if !seenExempt[key] { + t.Errorf("DOLT_ADD staging exemption %q no longer matches a DOLT_ADD path — remove it", key) + } + } +} + +// callsDoltAdd reports whether f passes a DOLT_ADD statement to anything. It +// deliberately asks nothing about the SHAPE of the call: a whole-working-set +// DOLT_ADD('-A') needs no funnel, but making that an automatic pass would let a +// future fixed-list site hide behind a shape test. Such a site takes an +// exemption line naming the reason instead. +func callsDoltAdd(f *journalscan.FuncInfo) bool { + for _, call := range f.Calls { + for _, arg := range call.Args { + if strings.Contains(arg, "DOLT_ADD") { + return true + } + } + } + return false +}