From 5a808e3eef0cdb1b1852a6c4abaeb8e74feb81eb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:12:43 +0000 Subject: [PATCH 1/5] feat(storage): shape-adaptive events journal writes and reads (PR A1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bd_events_journal writes and reads now probe the table's actual column set on each store open and adapt accordingly, instead of assuming every clone has comment_json and actor. Older table shapes (e.g. a clone that predates those columns) no longer break every journaled mutation; the adaptive path drops only the unsupported payload fields and logs once, rather than failing every write. This is PR A1 from BEADS-JOURNAL-PLAN.md §4 (T2.1-T2.9): the shape probe runs once per store open (internal/storage/issueops/journal_shape.go) and the dolt/embeddeddolt write and read paths, plus the uow layer that wraps them, consult it. PR A2 (the converging ignored-plane migration and `bd doctor` check) lands separately on top of this, since it depends on the adaptive I/O introduced here. Co-Authored-By: Claude Sonnet 5 --- .../scripts/embedded-storage-test-shards.txt | 4 + internal/eventsjournal/activation.go | 11 + internal/storage/dolt/store.go | 65 ++++- internal/storage/dolt/transaction.go | 19 +- internal/storage/embeddeddolt/BUILD.bazel | 1 + .../journal_shape_adaptive_test.go | 218 +++++++++++++++ internal/storage/embeddeddolt/store.go | 64 ++++- .../storage/externaldeps/uow_decorator.go | 15 + internal/storage/issueops/BUILD.bazel | 1 + internal/storage/issueops/journal.go | 48 +++- internal/storage/issueops/journal_prune.go | 42 ++- internal/storage/issueops/journal_shape.go | 257 ++++++++++++++++++ internal/storage/storage.go | 26 ++ internal/storage/uow/dolt_sql_provider.go | 53 +++- internal/storage/uow/doltserver_tx.go | 15 + internal/storage/uow/journal_scope_test.go | 16 ++ internal/storage/uow/notifying.go | 12 + internal/storage/uow/notifying_parity_test.go | 10 +- 18 files changed, 863 insertions(+), 14 deletions(-) create mode 100644 internal/storage/embeddeddolt/journal_shape_adaptive_test.go create mode 100644 internal/storage/issueops/journal_shape.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index f336184d7..53b387095 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -156,6 +156,7 @@ 15 1 TestMergeWithStrategy_OursKeepsLocal 15 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase +15 2 TestAdaptiveInsert_GciShape 15 2 TestCleanGitRemoteCacheGarbage 15 2 TestCreateBatchFastPathsMatchPerRow_Embedded 15 2 TestDepRekeyMarkerStillInIgnoredSeries @@ -229,6 +230,7 @@ 15 4 TestIgnoredCursorHealNeverResurrectsRolledBackVersions 15 4 TestMain 15 4 TestMemoriesContract +15 4 TestMissingRequiredColumnRefusesAtOpen 15 4 TestPromoteClearsNoHistory 15 4 TestRemoteVerbPlainRemoteKeepsAmbientEnv 15 4 TestReopenIssue @@ -236,6 +238,7 @@ 15 4 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials 15 5 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase +15 5 TestAdaptiveRead_GciShape 15 5 TestBlockingAnnotatorContract 15 5 TestBootstrapperContract 15 5 TestBuildDSN_WithDatabase @@ -403,6 +406,7 @@ 15 10 TestGetReadyWorkMoleculeFilter 15 10 TestHelperProcess 15 10 TestIgnoredCursorHealResumeRespectsOperatorOverride +15 10 TestJournalShapeProbeOncePerOpen 15 10 TestReclaimRefusesForeignReplicaLease 15 10 TestRemoteEntryPointsUseVersionControlOps 15 10 TestTransactionCleanupBeforeSQLCommitRemainsDefinite diff --git a/internal/eventsjournal/activation.go b/internal/eventsjournal/activation.go index 1af97cd4c..ad98baf83 100644 --- a/internal/eventsjournal/activation.go +++ b/internal/eventsjournal/activation.go @@ -146,6 +146,12 @@ func ActivateStore(beadsDir string, s storage.DoltStorage, err error) (storage.D // the journal is that a consumer can trust its cursor, and a plumbing that // records nothing while reporting success is the one outcome that breaks that // trust invisibly. A disabled workspace accepts any plumbing. +// +// It also fails there when the configurer DOES support the journal but its +// table's shape does not (storage.EventsJournalShapeChecker, PR A1): a missing +// required column refuses at open, the same way a nil configurer does, rather +// than letting SetEventsJournalEnabled succeed and every write fail +// afterward — the gci failure mode this check replaces. func Apply(configurer storage.EventsJournalConfigurer, enabled bool) error { if configurer == nil { if enabled { @@ -154,5 +160,10 @@ func Apply(configurer storage.EventsJournalConfigurer, enabled bool) error { return nil } configurer.SetEventsJournalEnabled(enabled) + if checker, ok := configurer.(storage.EventsJournalShapeChecker); ok { + if err := checker.EventsJournalActivationError(); err != nil { + return err + } + } return nil } diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index c05633734..2a6b638cb 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -308,6 +308,7 @@ var _ storage.Compactor = (*DoltStore)(nil) var _ storage.SchemaMigrator = (*DoltStore)(nil) var _ storage.ExternalRefHistoryQuerier = (*DoltStore)(nil) var _ storage.EventsJournalConfigurer = (*DoltStore)(nil) +var _ storage.EventsJournalShapeChecker = (*DoltStore)(nil) var _ storage.VersionedHistoryConfigurer = (*DoltStore)(nil) // DoltStore implements the Storage interface using Dolt @@ -320,6 +321,17 @@ type DoltStore struct { // eventsJournalEnabled activates the durable events journal for THIS store // instance only (storage.EventsJournalConfigurer); never process-global. eventsJournalEnabled atomic.Bool + // journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled + // runs on activation (issueops.ProbeJournalShape, PR A1). Nil means either + // the journal is disabled or the probe failed; scopeEventsJournalTransaction + // treats nil the same as "do not journal this transaction" — see + // journalActivationErr. + journalShape atomic.Pointer[issueops.JournalShape] + // journalActivationErr holds the probe's error when SetEventsJournalEnabled(true) + // found a table shape the journal cannot run against (a missing required + // column, or a missing table). Nil whenever journalShape is non-nil or the + // journal is disabled. See storage.EventsJournalShapeChecker. + journalActivationErr atomic.Pointer[error] // versionedHistoryEnabled activates dual-write issue-version history for // THIS store instance only (storage.VersionedHistoryConfigurer); never // process-global. @@ -1083,6 +1095,13 @@ func (s *DoltStore) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) e return fmt.Errorf("begin read tx: %w", err) } defer func() { _ = tx.Rollback() }() + // ReadEventsJournal/ReadEventsJournalPage read through this helper, and + // the column list they SELECT depends on the probed shape, not on + // whether the journal is enabled for writes — scope the shape only, so + // readEventsRowsInTx sees this store's actual table instead of + // defaulting to canonicalJournalShape (PR A1). + clearShape := issueops.ScopeEventsJournalShape(tx, s.journalShape.Load()) + defer clearShape() return fn(tx) }) } @@ -1379,12 +1398,56 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche } // SetEventsJournalEnabled activates the journal for this store instance only. +// +// Activating probes bd_events_journal's actual shape ONCE +// (issueops.ProbeJournalShape, PR A1) and caches the result for this +// instance's lifetime; EventsJournalActivationError reports whether that +// probe found a shape the journal can actually run against. Deactivating +// clears both and skips the probe entirely — a disabled store accepts any +// shape. func (s *DoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) + if !enabled { + s.journalShape.Store(nil) + s.journalActivationErr.Store(nil) + return + } + shape, err := issueops.ProbeJournalShape(context.Background(), s.db) + if err != nil { + s.journalShape.Store(nil) + s.journalActivationErr.Store(&err) + return + } + s.journalShape.Store(shape) + s.journalActivationErr.Store(nil) } +// EventsJournalActivationError implements storage.EventsJournalShapeChecker: +// nil unless the last SetEventsJournalEnabled(true) call found a table shape +// the journal cannot run against. +func (s *DoltStore) EventsJournalActivationError() error { + if p := s.journalActivationErr.Load(); p != nil { + return *p + } + return nil +} + +// scopeEventsJournalTransaction binds this store's cached activation state and +// probed shape to tx. The activation switch folds in the shape probe's +// success: a probe that found an unsupported table (journalShape nil) scopes +// the transaction as DISABLED regardless of what SetEventsJournalEnabled was +// called with, so a caller that enables the journal directly and ignores the +// returned EventsJournalActivationError still never attempts a write against +// a shape that cannot support it (storage.EventsJournalShapeChecker). func (s *DoltStore) scopeEventsJournalTransaction(tx *sql.Tx) func() { - return issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) + shape := s.journalShape.Load() + enabled := s.eventsJournalEnabled.Load() && shape != nil + clearActivation := issueops.ScopeEventsJournalTransaction(tx, enabled) + clearShape := issueops.ScopeEventsJournalShape(tx, shape) + return func() { + clearShape() + clearActivation() + } } // SetVersionedHistoryEnabled activates dual-write issue-version history for diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index 8bebf0b9f..d6e9528ba 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -222,7 +222,14 @@ func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg s // exists to make impossible. In journal mode both planes therefore share the // pinned regular transaction. The default journal-off path keeps the // established split transactions untouched. - journalEnabled := s.eventsJournalEnabled.Load() + // journalShape is folded into journalEnabled the same way + // scopeEventsJournalTransaction folds it in for every other raw-tx mutator + // (store.go): a probe that found an unsupported shape (journalShape nil) + // must pin neither plane together nor attempt a write here, even if a + // caller enabled the journal directly and ignored the returned + // EventsJournalActivationError (storage.EventsJournalShapeChecker, PR A1). + journalShape := s.journalShape.Load() + journalEnabled := s.eventsJournalEnabled.Load() && journalShape != nil ignoredTx := regularTx if !journalEnabled { // NOTE (GH#3140 metrics skew): the pool-wait bracket above measures only @@ -241,6 +248,16 @@ func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg s } clearJournalScope := issueops.ScopeEventsJournalTransaction(regularTx, journalEnabled) defer clearJournalScope() + // Shape counterpart of the activation switch just above (PR A1): + // insertEventRow/readEventsRowsInTx reach this store's probed shape only + // through a tx-scoped value, never through the store itself, so without + // this call every write on this path — CreateIssue, UpdateIssue, comments, + // the whole doltTransaction mutator surface reached from + // runDoltTransactionRecording — would silently fall back to + // canonicalJournalShape and defeat adaptive I/O for this store's main + // write path. + clearJournalShape := issueops.ScopeEventsJournalShape(regularTx, journalShape) + defer clearJournalShape() // Versioned history binds to the SAME regular transaction the mutation runs // in, for the same reason the journal does: RecordVersionInTx no-ops unless // this scope is set. Without it every issues-plane mutation routed through diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 9a4057072..2ec779e4d 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -156,6 +156,7 @@ go_test( "journal_concurrency_test.go", "journal_contract_test.go", "journal_retention_test.go", + "journal_shape_adaptive_test.go", "journal_truncation_test.go", "label_list_all_benchmark_test.go", "label_rename_test.go", diff --git a/internal/storage/embeddeddolt/journal_shape_adaptive_test.go b/internal/storage/embeddeddolt/journal_shape_adaptive_test.go new file mode 100644 index 000000000..81ce8e56a --- /dev/null +++ b/internal/storage/embeddeddolt/journal_shape_adaptive_test.go @@ -0,0 +1,218 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// Adaptive I/O for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2a/§4.3, PR +// A1). These tests reproduce gas-city-inc's exact shape — a table built +// before `comment_json` and `actor` ever existed — directly on an +// otherwise-ordinary embedded store, and prove the writer and reader adapt to +// it instead of failing every journaled write (T2.1/T2.2), that adaptation +// probes the shape once per activation rather than once per write (T2.3), +// and that a table missing a REQUIRED column refuses at open rather than +// silently disabling the journal or failing per write (T2.4). + +// newGciShapeStore returns a store whose bd_events_journal has had +// comment_json and actor dropped, reproducing gas-city-inc's shape +// (BEADS-JOURNAL-PLAN.md §1.1: "gci.bd_events_journal has the columns seq, +// ts, op, issue_id, issue_json, dep_json" — no actor, no comment_json). The +// journal is NOT activated; callers that want to observe activation itself +// (the probe, its warning, its cached result) call SetEventsJournalEnabled +// themselves. +func newGciShapeStore(t *testing.T, prefix string) *testEnv { + t.Helper() + te := newTestEnv(t, prefix) + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json") + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor") + return te +} + +// TestAdaptiveInsert_GciShape is T2.1: on the gci shape, create, update and +// comment all succeed, the rows land, and the comment payload is dropped with +// exactly one warning. A fixed INSERT column list (the pre-A1 behavior) fails +// every one of these three writes, because each one names either actor or +// comment_json. +func TestAdaptiveInsert_GciShape(t *testing.T) { + te := newGciShapeStore(t, "ai") + ctx := context.Background() + + stderr := captureStderr(t, func() { + te.store.SetEventsJournalEnabled(true) + }) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation on the gci shape (missing only OPTIONAL columns) must succeed, got: %v", err) + } + if n := strings.Count(stderr, "events journal:"); n != 1 { + t.Fatalf("activation warning count = %d, want exactly 1 (one warning per store, not per write):\n%s", n, stderr) + } + for _, want := range []string{"actor", "comment_json"} { + if !strings.Contains(stderr, want) { + t.Errorf("activation warning does not name missing column %q:\n%s", want, stderr) + } + } + + issue := &types.Issue{Title: "gci create", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue on gci shape: %v", err) + } + if err := te.store.UpdateIssue(ctx, issue.ID, map[string]interface{}{"title": "gci updated"}, "tester"); err != nil { + t.Fatalf("UpdateIssue on gci shape: %v", err) + } + if _, err := te.store.AddIssueComment(ctx, issue.ID, "tester", "gci comment"); err != nil { + t.Fatalf("AddIssueComment on gci shape: %v", err) + } + + rows, err := te.store.ReadEventsJournal(ctx, 0, 0) + if err != nil { + t.Fatalf("ReadEventsJournal on gci shape: %v", err) + } + var ops []string + for _, r := range rows { + if r.IssueID != issue.ID { + continue + } + ops = append(ops, r.Op) + // Dropped, not merely blank: the column does not exist on this shape, + // so the adaptive reader must never have tried to select it. + if r.Actor != "" { + t.Errorf("row op=%s: Actor = %q, want \"\" (column does not exist on this shape)", r.Op, r.Actor) + } + if r.CommentJSON != "" { + t.Errorf("row op=%s: CommentJSON = %q, want \"\" (column does not exist on this shape)", r.Op, r.CommentJSON) + } + } + wantOps := []string{"create", "update", "comment"} + if len(ops) != len(wantOps) { + t.Fatalf("journaled ops for %s = %v, want %v (a fixed INSERT column list fails every one of these on the gci shape)", issue.ID, ops, wantOps) + } + for i, want := range wantOps { + if ops[i] != want { + t.Errorf("journaled op[%d] = %q, want %q", i, ops[i], want) + } + } +} + +// TestAdaptiveRead_GciShape is T2.2: a page read on the gci shape succeeds, +// and actor/comment come back empty rather than failing the SELECT. A fixed +// SELECT list (the pre-A1 behavior) fails the read outright, because it names +// both actor and comment_json. +func TestAdaptiveRead_GciShape(t *testing.T) { + te := newGciShapeStore(t, "ar") + ctx := context.Background() + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation on the gci shape: %v", err) + } + + issue := &types.Issue{ID: "ar-1", Title: "gci read", Status: types.StatusOpen, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue on gci shape: %v", err) + } + + page, err := te.store.ReadEventsJournalPage(ctx, 0, 0) + if err != nil { + t.Fatalf("ReadEventsJournalPage on gci shape: %v", err) + } + if len(page.Rows) == 0 { + t.Fatal("ReadEventsJournalPage on gci shape returned no rows") + } + for _, r := range page.Rows { + if r.Actor != "" { + t.Errorf("row op=%s: Actor = %q, want \"\" on a shape with no actor column", r.Op, r.Actor) + } + if r.CommentJSON != "" { + t.Errorf("row op=%s: CommentJSON = %q, want \"\" on a shape with no comment_json column", r.Op, r.CommentJSON) + } + } +} + +// TestJournalShapeProbeOncePerOpen is T2.3: 100 writes against one already- +// activated store issue no INFORMATION_SCHEMA probes of their own — only the +// ONE SetEventsJournalEnabled ran. A per-write probe would show up here as a +// nonzero delta across the 100 writes. +func TestJournalShapeProbeOncePerOpen(t *testing.T) { + te := newTestEnv(t, "pr") + ctx := context.Background() + + before := issueops.JournalShapeProbeCountForTest() + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation on canonical shape: %v", err) + } + afterActivate := issueops.JournalShapeProbeCountForTest() + if delta := afterActivate - before; delta != 1 { + t.Fatalf("probes issued by SetEventsJournalEnabled = %d, want exactly 1", delta) + } + + for i := 0; i < 100; i++ { + issue := &types.Issue{ + Title: fmt.Sprintf("probe-once %d", i), Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, + } + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue %d: %v", i, err) + } + } + + after := issueops.JournalShapeProbeCountForTest() + if delta := after - afterActivate; delta != 0 { + t.Fatalf("probes issued by 100 writes against an already-activated store = %d, want 0 (a per-write probe, not a per-open probe)", delta) + } +} + +// TestMissingRequiredColumnRefusesAtOpen is T2.4: a table missing a REQUIRED +// column (op) makes activation fail with a typed error when the journal is +// enabled, but leaves it succeeding when disabled — a disabled workspace +// accepts any shape. It also pins that a caller who enables the journal +// anyway (ignoring the returned activation error) still never attempts the +// INSERT: the mutation itself must not fail, which is the whole point of +// moving this failure to open time instead of per write. +func TestMissingRequiredColumnRefusesAtOpen(t *testing.T) { + te := newTestEnv(t, "mr") + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN op") + + te.store.SetEventsJournalEnabled(true) + err := te.store.EventsJournalActivationError() + if err == nil { + t.Fatal("activation with the journal enabled against a table missing a required column (op) succeeded; want a typed error") + } + if !errors.Is(err, issueops.ErrJournalShapeUnsupported) { + t.Errorf("activation error = %v, does not match issueops.ErrJournalShapeUnsupported", err) + } + var shapeErr *issueops.JournalShapeError + if !errors.As(err, &shapeErr) { + t.Fatalf("activation error is not a *issueops.JournalShapeError: %v (%T)", err, err) + } + if len(shapeErr.Missing) != 1 || shapeErr.Missing[0] != "op" { + t.Errorf("JournalShapeError.Missing = %v, want [op]", shapeErr.Missing) + } + + // Disabling must still succeed: a disabled workspace accepts any shape, and + // never runs the probe that would otherwise fail it. + te.store.SetEventsJournalEnabled(false) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Errorf("activation with the journal disabled against the same broken shape failed: %v", err) + } + + // Re-enable (probe fails again, same cached nil shape) and prove the + // mutation itself still lands: this is the gci failure mode A1 exists to + // close, where a per-write INSERT against a missing column rolled back the + // user's own write. + te.store.SetEventsJournalEnabled(true) + issue := &types.Issue{Title: "no op column", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue must still succeed even though the journal cannot activate on this shape: %v", err) + } +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 4e8a121a9..92449f2e3 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -32,6 +32,7 @@ var _ storage.Flattener = (*EmbeddedDoltStore)(nil) var _ storage.Compactor = (*EmbeddedDoltStore)(nil) var _ storage.SchemaMigrator = (*EmbeddedDoltStore)(nil) var _ storage.EventsJournalConfigurer = (*EmbeddedDoltStore)(nil) +var _ storage.EventsJournalShapeChecker = (*EmbeddedDoltStore)(nil) var _ storage.VersionedHistoryConfigurer = (*EmbeddedDoltStore)(nil) var _ storage.ExternalRefHistoryQuerier = (*EmbeddedDoltStore)(nil) @@ -53,6 +54,16 @@ type EmbeddedDoltStore struct { // eventsJournalEnabled activates the durable events journal for THIS store // instance only (storage.EventsJournalConfigurer); never process-global. eventsJournalEnabled atomic.Bool + // journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled + // runs on activation (issueops.ProbeJournalShape, PR A1), over a short-lived + // connection since this store has no persistent db handle. Nil means either + // the journal is disabled or the probe failed — see journalActivationErr. + journalShape atomic.Pointer[issueops.JournalShape] + // journalActivationErr holds the probe's error when SetEventsJournalEnabled(true) + // found a table shape the journal cannot run against (a missing required + // column, or a missing table). Nil whenever journalShape is non-nil or the + // journal is disabled. See storage.EventsJournalShapeChecker. + journalActivationErr atomic.Pointer[error] // versionedHistoryEnabled activates dual-write issue-version history for // THIS store instance only (storage.VersionedHistoryConfigurer); never // process-global. @@ -457,8 +468,20 @@ func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func err = fmt.Errorf("embeddeddolt: begin tx: %w", err) return } - clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) + // The activation switch folds in the shape probe's success: a probe that + // found an unsupported table (journalShape nil) scopes this transaction as + // DISABLED regardless of what SetEventsJournalEnabled was called with, so a + // caller that enables the journal directly and ignores the returned + // EventsJournalActivationError still never attempts a write against a shape + // that cannot support it (storage.EventsJournalShapeChecker). Reads + // (commit=false callers) are never gated by this switch — only by the + // shape — since ReadEventsJournal has no enabled/disabled concept of its + // own. + journalShape := s.journalShape.Load() + clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load() && journalShape != nil) defer clearJournalScope() + clearJournalShapeScope := issueops.ScopeEventsJournalShape(tx, journalShape) + defer clearJournalShapeScope() clearVersionScope := issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load()) defer clearVersionScope() clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) @@ -484,8 +507,47 @@ func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func } // SetEventsJournalEnabled activates the journal for this store instance only. +// +// Activating probes bd_events_journal's actual shape ONCE +// (issueops.ProbeJournalShape, PR A1), over a short-lived connection opened +// and closed just for the probe since this store keeps no persistent db +// handle, and caches the result for this instance's lifetime; +// EventsJournalActivationError reports whether that probe found a shape the +// journal can actually run against. Deactivating clears both and skips the +// probe entirely — a disabled store accepts any shape. func (s *EmbeddedDoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) + if !enabled { + s.journalShape.Store(nil) + s.journalActivationErr.Store(nil) + return + } + db, cleanup, err := OpenSQL(context.Background(), s.dataDir, s.database, s.branch) + if err != nil { + s.journalShape.Store(nil) + wrapped := fmt.Errorf("journal: open connection to probe shape: %w", err) + s.journalActivationErr.Store(&wrapped) + return + } + defer func() { _ = cleanup() }() + shape, probeErr := issueops.ProbeJournalShape(context.Background(), db) + if probeErr != nil { + s.journalShape.Store(nil) + s.journalActivationErr.Store(&probeErr) + return + } + s.journalShape.Store(shape) + s.journalActivationErr.Store(nil) +} + +// EventsJournalActivationError implements storage.EventsJournalShapeChecker: +// nil unless the last SetEventsJournalEnabled(true) call found a table shape +// the journal cannot run against. +func (s *EmbeddedDoltStore) EventsJournalActivationError() error { + if p := s.journalActivationErr.Load(); p != nil { + return *p + } + return nil } // SetVersionedHistoryEnabled activates dual-write issue-version history for diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 9db47cd44..247f0211f 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -134,6 +134,20 @@ func (p *uowProvider) SetEventsJournalEnabled(enabled bool) { } } +// EventsJournalActivationError forwards the inner provider's shape-probe +// result (storage.EventsJournalShapeChecker, PR A1). This wrapper is the +// OUTERMOST provider on both real chains (see SetVersionedHistoryEnabled), so +// without this forwarder eventsjournal.Apply's type assertion for the checker +// would fail here even when the inner provider's own probe found an +// unsupported table shape — the open would then succeed against a shape it +// cannot journal, the gci failure mode this interface exists to avoid. +func (p *uowProvider) EventsJournalActivationError() error { + if checker, ok := p.UnitOfWorkProvider.(storage.EventsJournalShapeChecker); ok { + return checker.EventsJournalActivationError() + } + return nil +} + // SetVersionedHistoryEnabled forwards dual-write issue-version activation // inward, for the same reason SetEventsJournalEnabled does. // @@ -162,6 +176,7 @@ func (p *uowProvider) RunEventsMaintenanceTx(ctx context.Context, fn func(contex var ( _ uow.PoolTuner = (*uowProvider)(nil) _ storage.EventsJournalConfigurer = (*uowProvider)(nil) + _ storage.EventsJournalShapeChecker = (*uowProvider)(nil) _ storage.VersionedHistoryConfigurer = (*uowProvider)(nil) _ issueops.EventsMaintenanceRunner = (*uowProvider)(nil) _ uow.IssueLifecycleSource = (*uowProvider)(nil) diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index d774d7aff..0fd8ec5d4 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -63,6 +63,7 @@ go_library( "journal_autoprune.go", "journal_dependencies.go", "journal_prune.go", + "journal_shape.go", "labels.go", "lease.go", "metadata.go", diff --git a/internal/storage/issueops/journal.go b/internal/storage/issueops/journal.go index a4dc140e3..10be59552 100644 --- a/internal/storage/issueops/journal.go +++ b/internal/storage/issueops/journal.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "sort" + "strings" "sync" "time" @@ -420,6 +421,15 @@ func getJournalIssueInTx(ctx context.Context, tx DBTX, issueID string) (*types.I // (non-dependency ops). ts is the insert time, stamped inside the committing // transaction. actor is stored as-is — "" for the genuinely unattributable // paths — in the NOT NULL DEFAULT ” actor column. +// +// The column list is shape-adaptive (journal_shape.go, PR A1): seq, ts, op +// and issue_id are always named, and each optional column — actor, +// issue_json, dep_json, comment_json — is named only when journalShapeFor(tx) +// says this store's table actually has it. On a canonical table that is every +// column, in the same order this INSERT has always used; on an old-lineage +// table missing one, the write still lands and that one payload is simply not +// recorded, rather than failing the whole mutation the journal row shares a +// transaction with. func insertEventRow(ctx context.Context, tx DBTX, op EventOp, issueID string, issue *types.Issue, dep *EventDep, comment *EventComment, actor string) error { var issueJSON any if issue != nil { @@ -445,11 +455,41 @@ func insertEventRow(ctx context.Context, tx DBTX, op EventOp, issueID string, is } commentJSON = string(b) } + + shape := journalShapeFor(tx) + type optionalJournalValue struct { + column string + value any + } + var optionalValues []optionalJournalValue + if shape.HasActor { + optionalValues = append(optionalValues, optionalJournalValue{"actor", actor}) + } + if shape.HasIssueJSON { + optionalValues = append(optionalValues, optionalJournalValue{"issue_json", issueJSON}) + } + if shape.HasDepJSON { + optionalValues = append(optionalValues, optionalJournalValue{"dep_json", depJSON}) + } + if shape.HasCommentJSON { + optionalValues = append(optionalValues, optionalJournalValue{"comment_json", commentJSON}) + } + columns := make([]string, 0, 4+len(optionalValues)) + columns = append(columns, "seq", "ts", "op", "issue_id") + for _, ov := range optionalValues { + columns = append(columns, ov.column) + } + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(columns)), ", ") + //nolint:gosec // columns is built entirely from the fixed name literals above, never from caller input. + query := fmt.Sprintf("INSERT INTO bd_events_journal (%s) VALUES (%s)", strings.Join(columns, ", "), placeholders) + insert := func(seq int64) error { - _, err := tx.ExecContext(ctx, ` - INSERT INTO bd_events_journal (seq, ts, op, issue_id, actor, issue_json, dep_json, comment_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - `, seq, time.Now().UTC(), string(op), issueID, actor, issueJSON, depJSON, commentJSON) + args := make([]any, 0, len(columns)) + args = append(args, seq, time.Now().UTC(), string(op), issueID) + for _, ov := range optionalValues { + args = append(args, ov.value) + } + _, err := tx.ExecContext(ctx, query, args...) return err } diff --git a/internal/storage/issueops/journal_prune.go b/internal/storage/issueops/journal_prune.go index 50658f8ce..d3617162c 100644 --- a/internal/storage/issueops/journal_prune.go +++ b/internal/storage/issueops/journal_prune.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strconv" + "strings" "time" "github.com/steveyegge/beads/internal/storage" @@ -251,10 +252,30 @@ func readEventsHeadInTx(ctx context.Context, tx DBTX) (int64, error) { return head, nil } +// readEventsRowsInTx's column list is shape-adaptive (journal_shape.go, PR +// A1): seq, ts, op and issue_id are always selected, and each optional column +// is selected only when journalShapeFor(tx) says this store's table actually +// has it. A column the probe did not find is simply never requested, so the +// corresponding field on storage.EventsJournalRow reads as "" — never a scan +// error — for a consumer on an old-lineage table. func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([]storage.EventsJournalRow, error) { + shape := journalShapeFor(tx) // CAST(ts AS CHAR) normalizes the DATETIME to a stable string across drivers. - q := `SELECT seq, CAST(ts AS CHAR), op, issue_id, actor, issue_json, dep_json, comment_json - FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC` + columns := []string{"seq", "CAST(ts AS CHAR)", "op", "issue_id"} + if shape.HasActor { + columns = append(columns, "actor") + } + if shape.HasIssueJSON { + columns = append(columns, "issue_json") + } + if shape.HasDepJSON { + columns = append(columns, "dep_json") + } + if shape.HasCommentJSON { + columns = append(columns, "comment_json") + } + //nolint:gosec // columns is built entirely from the fixed name literals above, never from caller input. + q := "SELECT " + strings.Join(columns, ", ") + " FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC" if limit > 0 { q += " LIMIT " + strconv.Itoa(limit) } @@ -268,14 +289,29 @@ func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([ for rows.Next() { var ( r storage.EventsJournalRow + actor sql.NullString issueJS sql.NullString depJS sql.NullString commentJS sql.NullString ) - if err := rows.Scan(&r.Seq, &r.TS, &r.Op, &r.IssueID, &r.Actor, &issueJS, &depJS, &commentJS); err != nil { + dest := []any{&r.Seq, &r.TS, &r.Op, &r.IssueID} + if shape.HasActor { + dest = append(dest, &actor) + } + if shape.HasIssueJSON { + dest = append(dest, &issueJS) + } + if shape.HasDepJSON { + dest = append(dest, &depJS) + } + if shape.HasCommentJSON { + dest = append(dest, &commentJS) + } + if err := rows.Scan(dest...); err != nil { return nil, fmt.Errorf("journal: scan row: %w", err) } r.TS = normalizeEventsTimestamp(r.TS) + r.Actor = actor.String r.IssueJSON = issueJS.String r.DepJSON = depJS.String r.CommentJSON = commentJS.String diff --git a/internal/storage/issueops/journal_shape.go b/internal/storage/issueops/journal_shape.go new file mode 100644 index 000000000..a5b2d26d5 --- /dev/null +++ b/internal/storage/issueops/journal_shape.go @@ -0,0 +1,257 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + + "github.com/steveyegge/beads/internal/debug" +) + +// Adaptive I/O for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2a, PR A1). +// +// The writer and reader used to name a fixed column list. That is fine on a +// table built by the current migrations, but a table built by an earlier +// lineage — gas-city-inc's is the live case — can lack an optional column +// forever: the ignored series only ever creates the table when it is wholly +// absent, and its one repair (ignored/0023) probes column TYPES, never +// EXISTENCE. A fixed INSERT naming a missing column then fails every +// journaled write, and because the journal row commits in the SAME +// transaction as the mutation it records, that failure rolls back the user's +// write along with it. +// +// The fix probes once, at activation, rather than guessing or failing per +// write: +// - REQUIRED columns {seq, ts, op, issue_id} have existed on every lineage +// this table has ever shipped under; a table missing one of them, or +// missing entirely, cannot journal at all. ProbeJournalShape refuses with +// a typed error so a store's "enable the journal" step fails at open, +// before any mutation is attempted — never as a per-write failure after +// the work is otherwise done, which is the gci failure mode above. +// - OPTIONAL columns {actor, issue_json, dep_json, comment_json} may be +// missing on an old-lineage table. The writer drops whichever of them the +// probe did not find; the reader returns "" for them. One warning is +// logged per probe (i.e. once per store instance, not once per write), +// and ignored/0028 (PR A2) is what actually converges the shape — this +// file only keeps a drifted table WRITABLE and READABLE in the meantime. +// +// A store instance probes exactly once, when storage.EventsJournalConfigurer +// activates the journal (see each backend's SetEventsJournalEnabled), and +// caches the *JournalShape for its own lifetime. ScopeEventsJournalShape +// carries that cached value onto one transaction, next to +// ScopeEventsJournalTransaction's activation switch, so insertEventRow and +// readEventsRowsInTx — both of which only ever see a tx, never the store — +// can reach it without a per-write probe (TestJournalShapeProbeOncePerOpen). + +// journalTableName is the clone-local table every probe, insert and read in +// this file targets. +const journalTableName = "bd_events_journal" + +// requiredJournalColumns are the columns ProbeJournalShape treats as +// non-negotiable: every lineage of bd_events_journal has always carried them, +// and the seq/ts/op/issue_id contract (nextEventSeq, the reader's ordering) +// is meaningless without them. +var requiredJournalColumns = []string{"seq", "ts", "op", "issue_id"} + +// optionalJournalColumns are the columns a pre-merge lineage may lack. Order +// matches the canonical INSERT column order a healthy table has carried since +// migration 0066/ignored 0025 added actor. +var optionalJournalColumns = []string{"actor", "issue_json", "dep_json", "comment_json"} + +// JournalShape records which OPTIONAL bd_events_journal columns exist on one +// store's table. A nil *JournalShape is never handed to a caller; probing a +// table missing a REQUIRED column returns an error instead (see +// ProbeJournalShape). +type JournalShape struct { + HasActor bool + HasIssueJSON bool + HasDepJSON bool + HasCommentJSON bool +} + +// missingOptional names the optional columns this shape does not have, in +// optionalJournalColumns order, for the doctor finding and the one-per-probe +// warning. +func (s *JournalShape) missingOptional() []string { + if s == nil { + return nil + } + var missing []string + if !s.HasActor { + missing = append(missing, "actor") + } + if !s.HasIssueJSON { + missing = append(missing, "issue_json") + } + if !s.HasDepJSON { + missing = append(missing, "dep_json") + } + if !s.HasCommentJSON { + missing = append(missing, "comment_json") + } + return missing +} + +// canonicalJournalShape is every optional column present — the shape every +// workspace created from main migration 0064 plus ignored 0022/0025 has, and +// the default journalShapeFor returns for a transaction nobody scoped. That +// default matters: it is what every call site written before this file +// existed (and every test that activates the journal through the ctx-only +// WithEventsJournal test helper rather than a real store) keeps getting, so +// adaptive I/O is additive and changes nothing for a canonical table. +var canonicalJournalShape = &JournalShape{ + HasActor: true, + HasIssueJSON: true, + HasDepJSON: true, + HasCommentJSON: true, +} + +// ErrJournalShapeUnsupported is the sentinel errors.Is matches against any +// *JournalShapeError returned by ProbeJournalShape, regardless of which +// required columns a particular table happens to be missing. +var ErrJournalShapeUnsupported = errors.New("events journal: table shape does not support the events journal") + +// JournalShapeError reports that bd_events_journal (or the table itself) is +// missing one or more REQUIRED columns. It is returned by ProbeJournalShape +// and is the typed error storage.EventsJournalConfigurer implementations +// surface from activation, so "enable the journal" fails at open rather than +// the gci failure mode: succeeding at open and then failing every write thereafter. +type JournalShapeError struct { + Table string + Missing []string +} + +func (e *JournalShapeError) Error() string { + return fmt.Sprintf( + "events journal: table %s is missing required column(s) %s; the events journal cannot be enabled against this shape (see `bd doctor`)", + e.Table, strings.Join(e.Missing, ", "), + ) +} + +// Is makes errors.Is(err, ErrJournalShapeUnsupported) match any +// *JournalShapeError, so a caller can test for the class of failure without +// caring which columns a particular table lacks. +func (e *JournalShapeError) Is(target error) bool { + return target == ErrJournalShapeUnsupported //nolint:errorlint // sentinel identity comparison is the documented contract of Is. +} + +// journalShapeProbeCount counts how many times ProbeJournalShape has actually +// queried INFORMATION_SCHEMA in this process. It exists for +// TestJournalShapeProbeOncePerOpen, which asserts that a batch of writes +// against one already-activated store issues none of its own. +var journalShapeProbeCount atomic.Int64 + +// JournalShapeProbeCountForTest reports journalShapeProbeCount's current +// value. Test-only instrumentation, read by TestJournalShapeProbeOncePerOpen; +// production code never calls it. +func JournalShapeProbeCountForTest() int64 { return journalShapeProbeCount.Load() } + +// ProbeJournalShape runs the one INFORMATION_SCHEMA.COLUMNS query that +// determines what q's bd_events_journal table actually supports. Callers run +// it exactly once per store/provider instance, from +// storage.EventsJournalConfigurer.SetEventsJournalEnabled, and cache the +// result for that instance's lifetime — never per write or per transaction. +// +// A table missing any required column, including a wholly absent table +// (which reads as every column missing, since a COLUMNS query against a +// nonexistent table returns zero rows — the same shape ignored/0023's +// existing probes rely on), returns *JournalShapeError instead of a shape. +func ProbeJournalShape(ctx context.Context, q DBTX) (*JournalShape, error) { + journalShapeProbeCount.Add(1) + + rows, err := q.QueryContext(ctx, ` + SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`, journalTableName) + if err != nil { + return nil, fmt.Errorf("journal: probe %s shape: %w", journalTableName, err) + } + defer rows.Close() + + present := make(map[string]bool, len(requiredJournalColumns)+len(optionalJournalColumns)) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("journal: scan %s column: %w", journalTableName, err) + } + present[strings.ToLower(name)] = true + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("journal: read %s shape: %w", journalTableName, err) + } + + var missing []string + for _, col := range requiredJournalColumns { + if !present[col] { + missing = append(missing, col) + } + } + if len(missing) > 0 { + return nil, &JournalShapeError{Table: journalTableName, Missing: missing} + } + + shape := &JournalShape{ + HasActor: present["actor"], + HasIssueJSON: present["issue_json"], + HasDepJSON: present["dep_json"], + HasCommentJSON: present["comment_json"], + } + warnJournalShapeDegraded(shape) + return shape, nil +} + +// warnJournalShapeDegraded writes one audit line to STDERR, suppressed only +// by quiet mode, the first (and every) time a store probes a table missing an +// optional column — one call per ProbeJournalShape invocation, which is one +// per store activation, which is "one warning per store" regardless of how +// many writes that store goes on to make. +func warnJournalShapeDegraded(shape *JournalShape) { + missing := shape.missingOptional() + if len(missing) == 0 { + return + } + if debug.IsQuiet() { + return + } + fmt.Fprintf(os.Stderr, + "events journal: %s is missing optional column(s) %s; affected payload(s) will be dropped until it converges (see `bd doctor`)\n", + journalTableName, strings.Join(missing, ", ")) +} + +// journalShapes associates a probed *JournalShape with ONE concrete +// transaction, the same per-tx lifetime journalTransactions gives the +// activation switch. Entries live for one transaction only. +var journalShapes sync.Map // map[DBTX]*JournalShape + +// ScopeEventsJournalShape binds shape to tx so insertEventRow and +// readEventsRowsInTx can reach the store's already-cached probe result +// without taking the store itself as a parameter. Store implementations call +// it immediately after BeginTx, alongside ScopeEventsJournalTransaction, +// passing the *JournalShape SetEventsJournalEnabled cached at activation — +// never a fresh probe. +// +// A nil tx or nil shape is a no-op; journalShapeFor then returns +// canonicalJournalShape, today's fixed eight-column behavior. +func ScopeEventsJournalShape(tx DBTX, shape *JournalShape) func() { + if tx == nil || shape == nil { + return func() {} + } + journalShapes.Store(tx, shape) + return func() { journalShapes.Delete(tx) } +} + +// journalShapeFor returns the shape scoped to tx, or canonicalJournalShape +// when nobody scoped one — which is every call site that predates this file, +// and every test driving the journal through the ctx-only WithEventsJournal +// helper rather than a real store's activation. +func journalShapeFor(tx DBTX) *JournalShape { + if v, ok := journalShapes.Load(tx); ok { + if shape, ok := v.(*JournalShape); ok && shape != nil { + return shape + } + } + return canonicalJournalShape +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index ab27581d8..6ac8539cb 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -915,6 +915,32 @@ type EventsJournalConfigurer interface { SetEventsJournalEnabled(enabled bool) } +// EventsJournalShapeChecker reports whether the LAST SetEventsJournalEnabled +// call on this instance was able to activate the journal. +// +// It exists because SetEventsJournalEnabled itself stays void: dozens of +// tests across three backends already call it as a bare statement, and +// adaptive I/O (PR A1) must not force every one of them to start checking an +// error it never had before. Instead, enabling the journal probes the table's +// shape internally and caches any failure; this is the seam a factory's open +// path checks immediately afterward to decide whether activation actually +// succeeded. A missing or unsupported table makes SetEventsJournalEnabled(true) +// a no-op for WRITES, from that store's perspective: nothing panics, no +// INSERT is attempted, and no write is lost or corrupted — but +// eventsjournal.Apply uses this checker to fail the overall open with a typed +// error (issueops.JournalShapeError) instead of letting every subsequent +// write fail one at a time, which is the gci failure mode this interface +// exists to avoid. A caller that enables the journal directly, bypassing +// Apply, and then reads it anyway gets a loud SQL error from the read itself +// rather than a silently wrong answer — reads are never gated by the enabled +// flag, so there is no "no-op" to fall back to there. +// +// A store that does not implement it is assumed never to fail activation +// (today, every implementation does). +type EventsJournalShapeChecker interface { + EventsJournalActivationError() error +} + // VersionedHistoryConfigurer controls dual-write issue-version history // activation on ONE storage instance. Implementations must never use // process-global state, for the same reason as EventsJournalConfigurer: diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index e7681d800..608601488 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -57,6 +57,15 @@ type doltSQLProvider struct { // eventsJournalEnabled activates the durable events journal for THIS // provider instance only. See SetEventsJournalEnabled. eventsJournalEnabled atomic.Bool + // journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled + // runs on activation (issueops.ProbeJournalShape, PR A1). Nil means either + // the journal is disabled or the probe failed — see journalActivationErr. + journalShape atomic.Pointer[issueops.JournalShape] + // journalActivationErr holds the probe's error when SetEventsJournalEnabled(true) + // found a table shape the journal cannot run against (a missing required + // column, or a missing table). Nil whenever journalShape is non-nil or the + // journal is disabled. See storage.EventsJournalShapeChecker. + journalActivationErr atomic.Pointer[error] // versionedHistoryEnabled activates dual-write issue-version history for // THIS provider instance only. See SetVersionedHistoryEnabled. versionedHistoryEnabled atomic.Bool @@ -72,8 +81,38 @@ type doltSQLProvider struct { // BeginTx). Without that binding the uow plumbing writes mutations while // journaling nothing — the failure is invisible, because the code runs and the // write lands and the journal is simply empty. +// +// Activating also probes bd_events_journal's actual shape ONCE +// (issueops.ProbeJournalShape, PR A1) and caches the result for this +// instance's lifetime; EventsJournalActivationError reports whether that +// probe found a shape the journal can actually run against. Deactivating +// clears both and skips the probe entirely — a disabled provider accepts any +// shape. func (p *doltSQLProvider) SetEventsJournalEnabled(enabled bool) { p.eventsJournalEnabled.Store(enabled) + if !enabled { + p.journalShape.Store(nil) + p.journalActivationErr.Store(nil) + return + } + shape, err := issueops.ProbeJournalShape(context.Background(), p.db) + if err != nil { + p.journalShape.Store(nil) + p.journalActivationErr.Store(&err) + return + } + p.journalShape.Store(shape) + p.journalActivationErr.Store(nil) +} + +// EventsJournalActivationError implements storage.EventsJournalShapeChecker: +// nil unless the last SetEventsJournalEnabled(true) call found a table shape +// the journal cannot run against (PR A1). +func (p *doltSQLProvider) EventsJournalActivationError() error { + if ptr := p.journalActivationErr.Load(); ptr != nil { + return *ptr + } + return nil } // SetVersionedHistoryEnabled activates dual-write issue-version history for @@ -160,6 +199,7 @@ var ( _ UnitOfWorkProvider = (*doltSQLProvider)(nil) _ TxProvider = (*doltSQLProvider)(nil) _ storage.EventsJournalConfigurer = (*doltSQLProvider)(nil) + _ storage.EventsJournalShapeChecker = (*doltSQLProvider)(nil) _ storage.VersionedHistoryConfigurer = (*doltSQLProvider)(nil) ) @@ -195,9 +235,20 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { // (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its // transaction. The blocked-recheck scope is bound and released the same // way; Commit takes what it recorded once the transaction has committed. + // The activation switch folds in the shape probe's success: a probe that + // found an unsupported table (journalShape nil) scopes this transaction as + // DISABLED regardless of what SetEventsJournalEnabled was called with, so a + // caller that enables the journal directly and ignores the returned + // EventsJournalActivationError still never attempts a write against a shape + // that cannot support it (storage.EventsJournalShapeChecker). Reads share + // this same BeginTx, and are never gated by the activation switch — only by + // the shape — since EventsJournalCursor has no enabled/disabled concept of + // its own. + journalShape := p.journalShape.Load() return &doltServerTx{ conn: conn, - clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), + clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load() && journalShape != nil), + clearJournalShape: issueops.ScopeEventsJournalShape(conn, journalShape), clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()), clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn), }, nil diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index 4fd396e7a..6c6f81702 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -18,6 +18,10 @@ type doltServerTx struct { // releaseConn and poisonConn — so the activation entry cannot outlive the // transaction it describes, whichever way the transaction ends. clearJournalScope func() + // clearJournalShape releases the probed *issueops.JournalShape BeginTx bound + // to conn (PR A1), the shape counterpart of clearJournalScope's enabled/disabled + // switch. Same binding at BeginTx, same release at releaseConn/poisonConn. + clearJournalShape func() // clearVersionScope is clearJournalScope's counterpart for dual-write // issue-version history: same binding at BeginTx, same release at // releaseConn/poisonConn. @@ -197,6 +201,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() + t.releaseJournalShapeScope() t.releaseVersionScope() t.releaseRecheckScope() if t.conn != nil { @@ -215,6 +220,15 @@ func (t *doltServerTx) releaseJournalScope() { } } +// releaseJournalShapeScope is releaseJournalScope's counterpart for the probed +// *issueops.JournalShape (PR A1). Idempotent for the same reason. +func (t *doltServerTx) releaseJournalShapeScope() { + if t.clearJournalShape != nil { + t.clearJournalShape() + t.clearJournalShape = nil + } +} + // releaseVersionScope is releaseJournalScope's counterpart for dual-write // issue-version history. Idempotent for the same reason. func (t *doltServerTx) releaseVersionScope() { @@ -241,6 +255,7 @@ func (t *doltServerTx) releaseRecheckScope() { // database/sql close the connection and drop it from the pool. func (t *doltServerTx) poisonConn() { t.releaseJournalScope() + t.releaseJournalShapeScope() t.releaseVersionScope() t.releaseRecheckScope() if t.conn == nil { diff --git a/internal/storage/uow/journal_scope_test.go b/internal/storage/uow/journal_scope_test.go index e41504494..05ccc37bc 100644 --- a/internal/storage/uow/journal_scope_test.go +++ b/internal/storage/uow/journal_scope_test.go @@ -22,6 +22,19 @@ import ( // commits, and the journal is simply empty — so it must be pinned from both // directions: enabled emits, disabled does not. +// expectCanonicalJournalShapeProbe arms the one INFORMATION_SCHEMA.COLUMNS +// query SetEventsJournalEnabled(true) now issues to probe bd_events_journal's +// shape (issueops.ProbeJournalShape, PR A1), returning every column a +// canonical table carries so activation succeeds with the shape every test in +// this file was written against before shape probing existed. +func expectCanonicalJournalShapeProbe(mock sqlmock.Sqlmock) { + mock.ExpectQuery("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS"). + WithArgs("bd_events_journal"). + WillReturnRows(sqlmock.NewRows([]string{"COLUMN_NAME"}). + AddRow("seq").AddRow("ts").AddRow("op").AddRow("issue_id"). + AddRow("actor").AddRow("issue_json").AddRow("dep_json").AddRow("comment_json")) +} + func TestProviderImplementsEventsJournalConfigurer(t *testing.T) { p, _ := newMockTxProvider(t) var configurer storage.EventsJournalConfigurer = p @@ -37,7 +50,10 @@ func TestProviderImplementsEventsJournalConfigurer(t *testing.T) { // internal bookkeeping. func TestBeginTxScopesJournalActivationToThePinnedConn(t *testing.T) { p, mock := newMockTxProvider(t) + expectCanonicalJournalShapeProbe(mock) p.SetEventsJournalEnabled(true) + require.NoError(t, p.EventsJournalActivationError(), + "activation against the canonical shape the probe expectation below describes must succeed") mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("UPDATE bd_events_seq SET next_seq").WillReturnResult(sqlmock.NewResult(0, 1)) diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index d61b68529..29bb5ee8c 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -352,6 +352,18 @@ func (p *notifyingProvider) SetEventsJournalEnabled(enabled bool) { } } +// EventsJournalActivationError forwards the inner provider's shape-probe +// result (storage.EventsJournalShapeChecker, PR A1), for the same reason +// SetEventsJournalEnabled forwards activation itself: the wrapper holds no +// journal state of its own, and eventsjournal.Apply must see the inner +// provider's real probe result through whichever wrapper sits on top of it. +func (p *notifyingProvider) EventsJournalActivationError() error { + if checker, ok := p.inner.(storage.EventsJournalShapeChecker); ok { + return checker.EventsJournalActivationError() + } + return nil +} + // SetVersionedHistoryEnabled forwards dual-write issue-version history // activation to the provider that actually binds it to a transaction // (doltSQLProvider.BeginTx), for the same reason and in the same shape as diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 5bccc063e..3ccc611cc 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -299,9 +299,13 @@ func TestNotifyingProviderBuildsRolesOnItself(t *testing.T) { // for both binds on the inner provider's transactions, and a retention // pass writes no bead. None of these build a role, so none of them can // build one "on itself". - "SetEventsJournalEnabled": true, - "RunEventsMaintenanceTx": true, - "SetVersionedHistoryEnabled": true, + "SetEventsJournalEnabled": true, + // EventsJournalActivationError (PR A1) forwards the inner provider's + // shape-probe result the same way SetEventsJournalEnabled forwards + // activation: a query, not a role constructor. + "EventsJournalActivationError": true, + "RunEventsMaintenanceTx": true, + "SetVersionedHistoryEnabled": true, } var checked int From b2ad36a797e87f998d026b483b8363ca11302cc4 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:23:39 +0000 Subject: [PATCH 2/5] feat(schema,doctor): converge bd_events_journal shape on the ignored plane (PR A2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ignored/0028, a converging migration that backfills comment_json and actor onto any bd_events_journal table missing them and raises bd_events_seq past MAX(seq) if it has fallen behind, so a store with an older journal shape heals on its next writable open instead of staying broken. Journal schema changes land only on the ignored plane, which older binaries never version-check, so this is safe for mixed-version clones. Registers bd_events_journal.comment_json and .actor as sentinel columns (floor 21) alongside the existing leases sentinel, so a clone whose cursor claims to be past 28 but is missing either column also gets floored back into the replay range and heals the same way. Adds `bd doctor`'s "Events Journal Shape" check (cmd/bd/doctor/journal_shape.go, T2.12), wired into runDiagnostics, plus `bd doctor --fix` support for the missing-column finding. The counter-behind-MAX(seq) finding is reported but intentionally not claimed fixable in isolation: 0028 never touches bd_events_seq or a journal row on its own, so repairing it requires a pending migration to run; this finding is surfaced for manual investigation instead of being promised a fix. Per the plan's design note (f), the outbox-depth and seq-epoch findings it also describes are deferred — `bd_events_outbox` and `bd_events_seq.epoch` don't exist yet; they belong to a later plan item (PRs A4-A6, §5), not this one. Adds a new migration-hygiene check ("Check F") that rejects any new main-plane migration touching bd_events_journal or bd_events_seq, so future journal schema work can't accidentally leave the ignored plane. The plan's §4.2(c)/T2.12 text calls this "hygiene check E" and names its test TestHygieneCheckE, but the script's existing check E is unrelated (no PREPARE'd DML in new main-plane migrations); this adds Check F instead and the new tests are named TestHygieneCheckFRejects... with a doc comment cross-referencing the plan's naming. Depends on PR A1 (shape-adaptive journal I/O), which this builds on. Co-Authored-By: Claude Sonnet 5 --- .../scripts/embedded-storage-test-shards.txt | 4 + cmd/bd/BUILD.bazel | 2 + cmd/bd/doctor.go | 7 + cmd/bd/doctor/BUILD.bazel | 1 + cmd/bd/doctor/fix/migrate.go | 10 + cmd/bd/doctor/fix_gate.go | 2 + cmd/bd/doctor/journal_shape.go | 237 +++++++++ cmd/bd/doctor_fix.go | 2 + cmd/bd/doctor_journal_shape_test.go | 177 +++++++ internal/storage/embeddeddolt/BUILD.bazel | 1 + .../journal_shape_converge_test.go | 487 ++++++++++++++++++ internal/storage/schema/BUILD.bazel | 3 + .../storage/schema/cursor_reality_test.go | 27 +- .../0028_converge_events_journal_shape.up.sql | 163 ++++++ .../schema/old_ignored_ceiling_test.go | 124 +++++ internal/storage/schema/schema.go | 16 +- .../0024_placeholder.up.sql | 8 + scripts/BUILD.bazel | 1 + scripts/check-migration-hygiene.sh | 49 +- ...ck_migration_hygiene_journal_plane_test.go | 139 +++++ 20 files changed, 1452 insertions(+), 8 deletions(-) create mode 100644 cmd/bd/doctor/journal_shape.go create mode 100644 cmd/bd/doctor_journal_shape_test.go create mode 100644 internal/storage/embeddeddolt/journal_shape_converge_test.go create mode 100644 internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql create mode 100644 internal/storage/schema/old_ignored_ceiling_test.go create mode 100644 internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql create mode 100644 scripts/check_migration_hygiene_journal_plane_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 53b387095..c00ff8f61 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -154,6 +154,7 @@ 15 1 TestEmbeddedRoleFixtureKitHooksAreUsable 15 1 TestLifecycleCreateContract 15 1 TestMergeWithStrategy_OursKeepsLocal +15 1 TestStoreAtIgnored28WritesAndJournals 15 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase 15 2 TestAdaptiveInsert_GciShape @@ -349,6 +350,7 @@ 15 8 TestEmbeddedSQLCommitResponseLossIsIndeterminate 15 8 TestGetReadyWorkIncludeEphemeralAppliesWispFilters 15 8 TestHasRepository +15 8 TestIgnored0028ConvergesAllShapes 15 8 TestLenientGateWarning_ComposedOutput 15 8 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle 15 8 TestReclaimExpiredLeaseSurvivesRestartEmbedded @@ -378,6 +380,7 @@ 15 9 TestMetadataCASContract 15 9 TestPromoteStillWorksWithCollisionGuard 15 9 TestReadyWorkWithMissingDeferredParentTables +15 9 TestReplay22To28NoOpOnHealthyStore 15 9 TestUpdateMergeOps_DoltEngine 15 10 TestBuildDSN_SpacesInPath @@ -543,3 +546,4 @@ 15 15 TestIgnoredCursorHealIsIdempotent 15 15 TestReadyWorkStatementBudget 15 15 TestReleaserContract +15 15 TestSentinelJournalColumnsReplay diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index e6eec5dbd..928c22ffe 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -682,6 +682,7 @@ go_test( "doctor_health_smoke_test.go", "doctor_health_test.go", "doctor_init_diagnostics_cgo_test.go", + "doctor_journal_shape_test.go", "doctor_max_rows_test.go", "doctor_migrate_fix_test.go", "doctor_mutation_gate_test.go", @@ -1249,6 +1250,7 @@ go_test( deps = [ "//:beads", "//cmd/bd/doctor", + "//cmd/bd/doctor/fix", "//internal/ado", "//internal/beads", "//internal/ceiling", diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index 0640d3a9e..81ffd1c26 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -748,6 +748,13 @@ func runDiagnostics(path string) doctorResult { skewCheck := convertWithCategory(doctor.CheckMigrationContentSkew(sharedStore), doctor.CategoryData) result.Checks = append(result.Checks, skewCheck) + // Check 7g: Events journal shape (BEADS-JOURNAL-PLAN.md §4.2f, PR A2, T2.12). + journalShapeCheck := convertDoctorCheck(doctor.CheckEventsJournalShape(path)) + result.Checks = append(result.Checks, journalShapeCheck) + if journalShapeCheck.Status == statusError { + result.OverallOK = false + } + // Dolt health checks (connection, schema, issue count, status). for _, dc := range doctor.RunDoltHealthChecks(path) { result.Checks = append(result.Checks, convertDoctorCheck(dc)) diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index 5f541e3e8..0052f6ced 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -26,6 +26,7 @@ go_library( "hooks_migration.go", "installation.go", "integrity.go", + "journal_shape.go", "kv.go", "label_whitespace.go", "legacy.go", diff --git a/cmd/bd/doctor/fix/migrate.go b/cmd/bd/doctor/fix/migrate.go index d81cf3868..612c54b10 100644 --- a/cmd/bd/doctor/fix/migrate.go +++ b/cmd/bd/doctor/fix/migrate.go @@ -136,6 +136,16 @@ func SchemaCompatibility(path string) error { return DatabaseVersion(path) } +// EventsJournalShape fixes the "Events Journal Shape" doctor finding +// (BEADS-JOURNAL-PLAN.md §4.2f, PR A2: missing optional bd_events_journal +// columns, or a bd_events_seq counter behind bd_events_journal's MAX(seq)). +// The converging ignored-plane migration (ignored/0028) is applied as a side +// effect of DatabaseVersion's writable store open, the same as every other +// pending-migration fix in this file. +func EventsJournalShape(path string) error { + return DatabaseVersion(path) +} + // FreshCloneImport handles the "Fresh Clone" fix: imports JSONL issues into an // existing (possibly empty) Dolt store. This covers the case where the Database // fix already created the store but a prior version didn't import. diff --git a/cmd/bd/doctor/fix_gate.go b/cmd/bd/doctor/fix_gate.go index 213f2a9bb..d0e5410f4 100644 --- a/cmd/bd/doctor/fix_gate.go +++ b/cmd/bd/doctor/fix_gate.go @@ -243,6 +243,8 @@ func SanitizeFixRecommendation(fix string, gate FixGate) string { // audited against that rule. The ones deliberately left out open a store, so // being withheld under a blocked gate is correct rather than an omission: // "Database", "Database Integrity", "Fresh Clone", "Schema Compatibility", +// "Events Journal Shape" (opens the store, applying the converging +// ignored-plane migration as a side effect, same as Schema Compatibility), // "Repo Fingerprint" (fix.RepoFingerprint and fix.FixMissingMetadata both write // through the store), "Dolt Schema", "Pending Migrations", and the row-level // data repairs. "Sync Divergence", "JSONL Config" and "Untracked Files" are diff --git a/cmd/bd/doctor/journal_shape.go b/cmd/bd/doctor/journal_shape.go new file mode 100644 index 000000000..682527a6d --- /dev/null +++ b/cmd/bd/doctor/journal_shape.go @@ -0,0 +1,237 @@ +package doctor + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// journalOptionalColumns mirrors the optional-column list +// internal/storage/issueops/journal_shape.go's ProbeJournalShape treats as +// optional (BEADS-JOURNAL-PLAN.md §4.2a, PR A1). It is duplicated here +// rather than imported: this check reports over doctor's own plain +// *sql.DB (openDoltConn), and ProbeJournalShape prints its own +// once-per-probe STDERR warning as a side effect, which would duplicate +// this check's finding on every `bd doctor` run. +var journalOptionalColumns = []string{"actor", "issue_json", "dep_json", "comment_json"} + +// CheckEventsJournalShape is the standalone entry point for the "Events +// Journal Shape" doctor check (T2.12, BEADS-JOURNAL-PLAN.md §4.2f, PR A2). +// It opens its own connection rather than joining RunDoltHealthChecks' +// shared one, the same standalone shape as CheckDoltRemoteGitOrigin and +// CheckMigrationContentSkew, so it can be registered in doctor.go beside +// them without touching runDoltHealthChecksInternal's fixed-shape branches. +func CheckEventsJournalShape(path string) DoctorCheck { + beadsDir := ResolveBeadsDirForRepo(path) + + if !IsDoltBackend(beadsDir) { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "N/A (non-Dolt backend)", + Category: CategoryDolt, + } + } + + conn, err := openDoltConn(beadsDir) + if err != nil { + // Matches runDoltHealthChecksInternal's own no-server-running + // tolerance: a store that hasn't been opened yet (server not started) + // is not a journal-shape finding. + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "Skipped (no server running; will auto-start on next bd command)", + Category: CategoryDolt, + } + } + defer conn.Close() + + return checkEventsJournalShapeWithDB(conn) +} + +// checkEventsJournalShapeWithDB is T2.12 (BEADS-JOURNAL-PLAN.md §4.2f, PR +// A2): "bd doctor 'events journal shape'" reports: +// - optional columns bd_events_journal is still missing. Adaptive I/O (PR +// A1) keeps a table in this state writable and readable, dropping only +// the affected payloads, but a dropped payload is still a finding — the +// plan's own risk note says to "keep the doctor finding and a +// once-per-store warning, never silence". +// - the bd_events_seq counter sitting behind bd_events_journal's own +// MAX(seq): the counter must never trail what has already been durably +// written (nextEventSeq increments it inside the same transaction as the +// row that carries its new value), so this can only mean the counter was +// lost and reseeded (e.g. restored from a stale backup) without the rows +// that came after it. +// +// The plan's design note (f) also lists "outbox depth and the age of its +// oldest row" and "a NULL epoch". Those refer to bd_events_outbox and +// bd_events_seq.epoch, introduced by PRs A5/A6 (§4.3's own table scopes +// T2.10/T2.11 to Postgres/PR B1 and T2.13 to PR A12; A5/A6 are likewise a +// separate, later plan item, §5) — neither object exists in this schema yet, +// so there is nothing to probe. Once they land, this is the function to +// extend. +func checkEventsJournalShapeWithDB(conn *doltConn) DoctorCheck { + ctx := context.Background() + + exists, err := tableExists(ctx, conn.db, "bd_events_journal") + if err != nil { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusWarning, + Message: "Could not probe bd_events_journal", + Detail: err.Error(), + Category: CategoryDolt, + } + } + if !exists { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "N/A (bd_events_journal not created yet)", + Category: CategoryDolt, + } + } + + var findings []string + var hasMissingColumns, hasUnfixableFinding bool + + missingOptional, err := missingJournalOptionalColumns(ctx, conn.db) + if err != nil { + findings = append(findings, fmt.Sprintf("could not read bd_events_journal's columns: %v", err)) + hasUnfixableFinding = true + } else if len(missingOptional) > 0 { + findings = append(findings, fmt.Sprintf( + "bd_events_journal is missing optional column(s): %s (affected payloads are dropped on write)", + strings.Join(missingOptional, ", "))) + hasMissingColumns = true + } + + // The counter-behind finding is deliberately NOT claimed as --fix-able: + // ignored/0028's own header states it "never reads or writes + // bd_events_seq or a single journal row", so reopening the store (what + // fix.EventsJournalShape/DatabaseVersion does) cannot repair it. Reseeding + // the counter is a data correction, not a DDL convergence, and the plan's + // design note (f) only promises "--fix applies the converging DDL" — it + // says nothing about reseeding a counter, so this stays a report-only + // finding pending an explicit decision on safe reseed semantics. + counterBehind, counterDetail, err := journalCounterBehindMaxSeq(ctx, conn.db) + if err != nil { + findings = append(findings, fmt.Sprintf("could not compare bd_events_seq to MAX(seq): %v", err)) + hasUnfixableFinding = true + } else if counterBehind { + findings = append(findings, counterDetail+" (not repaired by 'bd doctor --fix'; needs manual investigation)") + hasUnfixableFinding = true + } + + if len(findings) == 0 { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "bd_events_journal shape converged", + Category: CategoryDolt, + } + } + + check := DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusWarning, + Message: fmt.Sprintf("%d finding(s)", len(findings)), + Detail: strings.Join(findings, "\n"), + Category: CategoryDolt, + } + switch { + case hasMissingColumns && hasUnfixableFinding: + check.Fix = "Run 'bd doctor --fix' to add the missing column(s); the remaining finding(s) above need manual investigation" + case hasMissingColumns: + check.Fix = "Run 'bd doctor --fix' to apply the converging ignored-plane migration (adds comment_json/actor, widens columns, creates indexes)" + } + return check +} + +// tableExists reports whether table exists in the connected database. A +// read-only probe (COUNT with LIMIT 1), matching checkSchemaWithDB's idiom +// elsewhere in this package. +func tableExists(ctx context.Context, db *sql.DB, table string) (bool, error) { + var count int + err := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s LIMIT 1", table)).Scan(&count) //nolint:gosec // table is one of this file's own constant strings, never user input + if err != nil { + // A missing table is not this function's error to report: every other + // SQL failure is. + if strings.Contains(strings.ToLower(err.Error()), "doesn't exist") || + strings.Contains(strings.ToLower(err.Error()), "not found") || + strings.Contains(strings.ToLower(err.Error()), "unknown table") { + return false, nil + } + return false, err + } + return true, nil +} + +// missingJournalOptionalColumns runs the same INFORMATION_SCHEMA.COLUMNS +// probe internal/storage/issueops.ProbeJournalShape uses (over a plain +// connection, with no side-effecting warning), and returns which of +// journalOptionalColumns bd_events_journal does not carry. +func missingJournalOptionalColumns(ctx context.Context, db *sql.DB) ([]string, error) { + rows, err := db.QueryContext(ctx, ` + SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'`) + if err != nil { + return nil, err + } + defer rows.Close() + + present := make(map[string]bool, len(journalOptionalColumns)) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + present[strings.ToLower(name)] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + + var missing []string + for _, col := range journalOptionalColumns { + if !present[col] { + missing = append(missing, col) + } + } + return missing, nil +} + +// journalCounterBehindMaxSeq reports whether bd_events_seq's counter trails +// the highest seq actually recorded in bd_events_journal. bd_events_seq may +// not exist on a database that predates ignored/0019 (the events table) or +// has never journaled; that reads as "not behind" (nothing to compare), +// never as a finding. +func journalCounterBehindMaxSeq(ctx context.Context, db *sql.DB) (bool, string, error) { + counterExists, err := tableExists(ctx, db, "bd_events_seq") + if err != nil { + return false, "", err + } + if !counterExists { + return false, "", nil + } + + var counter, maxSeq int64 + err = db.QueryRowContext(ctx, ` + SELECT + (SELECT COALESCE(next_seq, 0) FROM bd_events_seq WHERE id = 0), + (SELECT COALESCE(MAX(seq), 0) FROM bd_events_journal) + `).Scan(&counter, &maxSeq) + if err != nil { + return false, "", err + } + + if counter < maxSeq { + return true, fmt.Sprintf( + "bd_events_seq counter (%d) is behind bd_events_journal's MAX(seq) (%d): "+ + "the counter was likely lost and reseeded without the rows written after it", + counter, maxSeq), nil + } + return false, "", nil +} diff --git a/cmd/bd/doctor_fix.go b/cmd/bd/doctor_fix.go index 89b73532c..33ba323fd 100644 --- a/cmd/bd/doctor_fix.go +++ b/cmd/bd/doctor_fix.go @@ -403,6 +403,8 @@ func applyFixList(path string, fixes []doctorCheck) { err = fix.DatabaseIntegrity(path) case "Schema Compatibility": err = fix.SchemaCompatibility(path) + case "Events Journal Shape": + err = fix.EventsJournalShape(path) case "Repo Fingerprint": err = fix.RepoFingerprint(path, doctorYes) // Also repair any other missing metadata fields (bd_version, repo_id, clone_id) diff --git a/cmd/bd/doctor_journal_shape_test.go b/cmd/bd/doctor_journal_shape_test.go new file mode 100644 index 000000000..584fb98c2 --- /dev/null +++ b/cmd/bd/doctor_journal_shape_test.go @@ -0,0 +1,177 @@ +//go:build cgo + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/steveyegge/beads/cmd/bd/doctor" + "github.com/steveyegge/beads/cmd/bd/doctor/fix" + "github.com/steveyegge/beads/internal/types" +) + +// findDoctorCheck returns the check named name from checks, or nil. +func findDoctorCheck(checks []doctorCheck, name string) *doctorCheck { + for i := range checks { + if checks[i].Name == name { + return &checks[i] + } + } + return nil +} + +// TestDoctorJournalShape is T2.12 (BEADS-JOURNAL-PLAN.md §4.2f, PR A2): +// "bd doctor" raises an "Events Journal Shape" finding for a bd_events_journal +// missing an optional column, and `--fix` (fix.EventsJournalShape) clears it +// by applying the converging ignored-plane migration (ignored/0028). A +// counter behind bd_events_journal's own MAX(seq) is also raised, but is +// deliberately NOT claimed as --fix-able: ignored/0028's own header states it +// "never reads or writes bd_events_seq or a single journal row", and when no +// column is also missing there is no sentinel contradiction to float the +// ignored cursor's floor and force the replay that would otherwise touch it +// (ignored/0022's own idempotent high-water-mark raise, re-run only as part +// of that replay). +// +// Scope: this test covers only what A1/A2 actually built and what exists in +// this schema today. The plan's design note (f) also lists "outbox depth and +// the age of its oldest row" and "a NULL epoch"; those reference +// bd_events_outbox and bd_events_seq.epoch, introduced by PRs A5/A6 (§5, a +// separate, later plan item) — neither object exists yet, so there is +// nothing for this test (or the check itself) to exercise for them. +// +// The two sub-findings are tested from independently damaged stores (not one +// store carrying both faults at once): dropping comment_json ALSO floors the +// ignored cursor via its sentinel and forces a 22..28 replay on the next +// writable open, and that replay's pass through ignored/0022 re-raises +// bd_events_seq to GREATEST(current, MAX(seq)) as an unrelated side effect — +// entangling the two findings' fixability would make the counter case look +// fixable only because a *different* finding's repair happened to touch it. +// +// Kills: the check not wired into doctor (runDiagnostics never calling +// doctor.CheckEventsJournalShape) — confirmed via the clean-store sub-test +// below, which goes through runDiagnostics itself rather than calling the +// check function directly, and manually by temporarily removing doctor.go's +// wiring call and observing it fail with "check not found" before restoring +// it; the check function itself going unimplemented (a compile failure); +// and --fix not actually converging the column shape (the post-fix re-probe +// would still see missing comment_json). +func TestDoctorJournalShape(t *testing.T) { + t.Run("wired into runDiagnostics on a healthy store", func(t *testing.T) { + tmpDir, store := setupValidateTestDB(t, "jnlok") + ctx := context.Background() + store.SetEventsJournalEnabled(true) + if err := store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation: %v", err) + } + issue := &types.Issue{Title: "t2.12 wiring seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "jnlok"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + store.Close() + + result := runDiagnostics(tmpDir) + check := findDoctorCheck(result.Checks, "Events Journal Shape") + if check == nil { + t.Fatal("Events Journal Shape check not found in runDiagnostics' output") + } + if check.Status != statusOK { + t.Errorf("status = %q, want %q (message: %s, detail: %s)", check.Status, statusOK, check.Message, check.Detail) + } + }) + + t.Run("missing optional column is raised and fixed", func(t *testing.T) { + tmpDir, store := setupValidateTestDB(t, "jnlcol") + ctx := context.Background() + store.SetEventsJournalEnabled(true) + if err := store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation: %v", err) + } + issue := &types.Issue{Title: "t2.12 column seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "jnlcol"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + // Damage exactly as the plan's evidence base found it (gas-city-inc: + // comment_json absent outright). + if _, err := store.DB().ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil { + t.Fatalf("drop comment_json: %v", err) + } + store.Close() + + check := doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusWarning { + t.Fatalf("after damage: status = %q, want %q (message: %s)", check.Status, statusWarning, check.Message) + } + if !strings.Contains(check.Detail, "comment_json") { + t.Errorf("after damage: Detail = %q, want it to mention comment_json", check.Detail) + } + if strings.Contains(check.Detail, "bd_events_seq counter") { + t.Errorf("after damage: Detail = %q, unexpectedly mentions the counter (this store never touched it)", check.Detail) + } + if check.Fix == "" || !strings.Contains(check.Fix, "bd doctor --fix") { + t.Errorf("after damage: Fix = %q, want guidance to run 'bd doctor --fix'", check.Fix) + } + + if err := fix.EventsJournalShape(tmpDir); err != nil { + t.Fatalf("fix.EventsJournalShape: %v", err) + } + + check = doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusOK { + t.Fatalf("after fix: status = %q, want %q (message: %s, detail: %s)", check.Status, statusOK, check.Message, check.Detail) + } + }) + + t.Run("counter behind MAX(seq) is raised and not claimed fixable", func(t *testing.T) { + tmpDir, store := setupValidateTestDB(t, "jnlseq") + ctx := context.Background() + store.SetEventsJournalEnabled(true) + if err := store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation: %v", err) + } + issue := &types.Issue{Title: "t2.12 counter seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "jnlseq"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + // Damage ONLY the counter (no column touched, so no sentinel + // contradiction floors the ignored cursor and nothing replays + // ignored/0022 on the next writable open). + if _, err := store.DB().ExecContext(ctx, "UPDATE bd_events_seq SET next_seq = 0 WHERE id = 0"); err != nil { + t.Fatalf("reseed next_seq behind MAX(seq): %v", err) + } + store.Close() + + check := doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusWarning { + t.Fatalf("after damage: status = %q, want %q (message: %s)", check.Status, statusWarning, check.Message) + } + if !strings.Contains(check.Detail, "bd_events_seq counter") { + t.Errorf("after damage: Detail = %q, want it to mention the counter behind MAX(seq)", check.Detail) + } + if strings.Contains(check.Detail, "comment_json") { + t.Errorf("after damage: Detail = %q, unexpectedly mentions comment_json (this store never touched it)", check.Detail) + } + if !strings.Contains(check.Detail, "not repaired by 'bd doctor --fix'") { + t.Errorf("after damage: Detail = %q, want it to disclose this finding is not --fix-able", check.Detail) + } + if strings.Contains(check.Fix, "bd doctor --fix") { + t.Errorf("after damage: Fix = %q, must not claim 'bd doctor --fix' resolves a counter-behind finding", check.Fix) + } + + if err := fix.EventsJournalShape(tmpDir); err != nil { + t.Fatalf("fix.EventsJournalShape: %v", err) + } + + check = doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusWarning { + t.Fatalf("after fix: status = %q, want %q (no pending migration exists to repair a counter-only fault, so --fix is a no-op here): message=%s detail=%s", + check.Status, statusWarning, check.Message, check.Detail) + } + if !strings.Contains(check.Detail, "bd_events_seq counter") { + t.Errorf("after fix: Detail = %q, want the still-unrepaired counter finding to remain", check.Detail) + } + }) +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 2ec779e4d..cd3b9a6b0 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -157,6 +157,7 @@ go_test( "journal_contract_test.go", "journal_retention_test.go", "journal_shape_adaptive_test.go", + "journal_shape_converge_test.go", "journal_truncation_test.go", "label_list_all_benchmark_test.go", "label_rename_test.go", diff --git a/internal/storage/embeddeddolt/journal_shape_converge_test.go b/internal/storage/embeddeddolt/journal_shape_converge_test.go new file mode 100644 index 000000000..67753caf5 --- /dev/null +++ b/internal/storage/embeddeddolt/journal_shape_converge_test.go @@ -0,0 +1,487 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/types" +) + +// Shape convergence for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2b/§4.3, +// PR A2). ignored/0028 closes the exact gap ignored/0023 left open (probing +// DATA_TYPE alone cannot see a column that is entirely absent): these tests +// reproduce every historical shape the plan's evidence base found, prove each +// one converges on canonical without perturbing a single row or the seq +// counter (T2.5), pin the sentinel-triggered replay's floor at exactly 21 +// (T2.6), and pin that the same replay is a true no-op everywhere outside +// bd_events_journal itself when nothing else is damaged (T2.7). + +// newShape3666e5026Store reproduces the events-journal-3666e5026 release +// shape (BEADS-JOURNAL-PLAN.md §1.1, "where that shape came from"): +// issue_json LONGTEXT, dep_json TEXT, no comment_json, no actor, no ts index. +// This predates both comment_json and actor outright. +func newShape3666e5026Store(t *testing.T, prefix string) *testEnv { + t.Helper() + te := newTestEnv(t, prefix) + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json") + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor") + te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN dep_json TEXT") + te.exec(t, ctx, "DROP INDEX idx_bd_events_journal_ts ON bd_events_journal") + return te +} + +// newForkIgnored0017ShapeStore reproduces the enterprise fork's +// ignored/0017_create_events_journal shape (0023's own header, bd-t9ovd): +// both payload columns TEXT, no actor (actor postdates this shape; it is +// bd OSS migration 0025), no ts index. ignored/0023 already fully heals this +// one — dropping actor is what forces the sentinel-triggered replay that +// exercises 0023's widen-if-TEXT step here, the same mechanism a lineage +// that genuinely never ran 0023/0025 before reaching this shape would hit. +func newForkIgnored0017ShapeStore(t *testing.T, prefix string) *testEnv { + t.Helper() + te := newTestEnv(t, prefix) + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor") + te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN dep_json TEXT") + te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN comment_json TEXT") + te.exec(t, ctx, "DROP INDEX idx_bd_events_journal_ts ON bd_events_journal") + return te +} + +// openDB opens a short-lived raw SQL connection to te's store and registers +// its cleanup with t.Cleanup, for tests that need to hold the connection +// across several calls (schema.CurrentIgnoredVersion, schema.MigrateUp, +// direct probes) rather than te.exec's one-shot open/close. +func (te *testEnv) openDB(t *testing.T, ctx context.Context) *sql.DB { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + t.Cleanup(func() { _ = cleanup() }) + return db +} + +// journalCoreRow is the subset of bd_events_journal's columns every observed +// shape (canonical or historical) carries. comment_json and actor are +// deliberately excluded: they are exactly what this migration adds back, so +// a before/after comparison that is scoped to the columns every shape shares +// is what proves the healing touched only the columns it should. +type journalCoreRow struct { + Seq int64 + Ts string + Op string + IssueID string + IssueJSON sql.NullString + DepJSON sql.NullString +} + +// snapshotJournalCore captures every bd_events_journal row (core columns +// only, see journalCoreRow) in seq order, plus bd_events_seq's counter. +func snapshotJournalCore(t *testing.T, ctx context.Context, db *sql.DB) ([]journalCoreRow, int64) { + t.Helper() + rows, err := db.QueryContext(ctx, "SELECT seq, ts, op, issue_id, issue_json, dep_json FROM bd_events_journal ORDER BY seq") + if err != nil { + t.Fatalf("snapshot bd_events_journal: %v", err) + } + defer rows.Close() + var out []journalCoreRow + for rows.Next() { + var r journalCoreRow + if err := rows.Scan(&r.Seq, &r.Ts, &r.Op, &r.IssueID, &r.IssueJSON, &r.DepJSON); err != nil { + t.Fatalf("scan bd_events_journal row: %v", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate bd_events_journal: %v", err) + } + var nextSeq int64 + if err := db.QueryRowContext(ctx, "SELECT next_seq FROM bd_events_seq WHERE id = 0").Scan(&nextSeq); err != nil { + t.Fatalf("snapshot bd_events_seq: %v", err) + } + return out, nextSeq +} + +// assertIndexExists fails the test if the named index is missing. +func assertIndexExists(t *testing.T, ctx context.Context, db *sql.DB, table, index string) { + t.Helper() + var count int + if err := db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?`, + table, index).Scan(&count); err != nil { + t.Fatalf("probe index %s on %s: %v", index, table, err) + } + if count == 0 { + t.Errorf("%s: index %s missing after convergence", table, index) + } +} + +// assertColumnShapeConverged diffs got against want (both from +// clonePlaneTableShape, migrate_ignored_plane_shape_test.go) with messaging +// specific to this file's "did this fixture converge on canonical" question, +// rather than that helper's own fresh-init-vs-fresh-clone framing. +func assertColumnShapeConverged(t *testing.T, fixtureName, table string, want, got map[string]clonePlaneColumn) { + t.Helper() + names := map[string]bool{} + for name := range want { + names[name] = true + } + for name := range got { + names[name] = true + } + for name := range names { + w, inWant := want[name] + g, inGot := got[name] + switch { + case !inGot: + t.Errorf("%s: %s.%s present on the canonical reference but missing after convergence", fixtureName, table, name) + case !inWant: + t.Errorf("%s: %s.%s present after convergence but not on the canonical reference", fixtureName, table, name) + case w != g: + t.Errorf("%s: %s.%s shape differs from canonical: want %s, got %s", + fixtureName, table, name, formatClonePlaneColumn(w), formatClonePlaneColumn(g)) + } + } +} + +// TestIgnored0028ConvergesAllShapes is T2.5: every historical shape the +// plan's evidence base found ends up byte-identical to canonical after +// MigrateUp, and not a single row or the seq counter moved getting there. +// Kills: probing only DATA_TYPE (0023's gap — a dropped fixture here would +// stay missing comment_json/actor and fail the shape diff); any write to the +// counter or rows (caught by the before/after snapshot). +func TestIgnored0028ConvergesAllShapes(t *testing.T) { + // dbPrefix is separate from name (rather than derived from it): embedded + // mode database names reject hyphens, but "fork-ignored-0017" is the + // plan's own name for that fixture and reads better in -v output and + // failure messages than a mangled version of it would. + fixtures := []struct { + name string + dbPrefix string + build func(t *testing.T, prefix string) *testEnv + damaged bool + }{ + {name: "3666e5026", dbPrefix: "cv3666e5026", build: newShape3666e5026Store, damaged: true}, + {name: "gci", dbPrefix: "cvgci", build: newGciShapeStore, damaged: true}, + {name: "fork-ignored-0017", dbPrefix: "cvfork0017", build: newForkIgnored0017ShapeStore, damaged: true}, + {name: "canonical", dbPrefix: "cvcanon", build: func(t *testing.T, prefix string) *testEnv { return newTestEnv(t, prefix) }, damaged: false}, + } + + ctx := context.Background() + reference := newTestEnv(t, "cvref") + refDB := reference.openDB(t, ctx) + refConn, err := refDB.Conn(ctx) + if err != nil { + t.Fatalf("reference Conn: %v", err) + } + defer refConn.Close() + wantShape := clonePlaneTableShape(t, ctx, refConn, "bd_events_journal") + + for _, fx := range fixtures { + t.Run(fx.name, func(t *testing.T) { + te := fx.build(t, fx.dbPrefix) + + // One real journaled event while the shape is still canonical (every + // builder above damages bd_events_journal only AFTER newTestEnv + // returns), so the snapshot below compares real data, not an empty + // table. + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation before damaging the shape: %v", err) + } + issue := &types.Issue{Title: fx.name + " seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("seed CreateIssue: %v", err) + } + + db := te.openDB(t, ctx) + beforeRows, beforeSeq := snapshotJournalCore(t, ctx, db) + + // MigrateUp's own return value cannot prove convergence here: it + // reports only the main-plane applied count (plus a reconcile + // flag), never the ignored-plane count, so the sentinel + // contradiction and its heal are both checked through + // CurrentIgnoredVersion instead (see TestSentinelJournalColumnsReplay). + if fx.damaged { + before, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (before MigrateUp): %v", err) + } + if want := schema.LatestIgnoredVersion(); before >= want { + t.Fatalf("CurrentIgnoredVersion before MigrateUp = %d, want below latest (%d): this fixture's damage did not float a sentinel contradiction", before, want) + } + } + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp: %v", err) + } + + if after, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after MigrateUp): %v", err) + } else if want := schema.LatestIgnoredVersion(); after != want { + t.Fatalf("CurrentIgnoredVersion after MigrateUp = %d, want latest (%d): this fixture did not converge", after, want) + } + + afterRows, afterSeq := snapshotJournalCore(t, ctx, db) + if beforeSeq != afterSeq { + t.Errorf("bd_events_seq.next_seq changed: before=%d after=%d", beforeSeq, afterSeq) + } + if !reflect.DeepEqual(beforeRows, afterRows) { + t.Errorf("bd_events_journal rows changed across MigrateUp:\nbefore=%+v\nafter=%+v", beforeRows, afterRows) + } + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Conn: %v", err) + } + defer conn.Close() + gotShape := clonePlaneTableShape(t, ctx, conn, "bd_events_journal") + assertColumnShapeConverged(t, fx.name, "bd_events_journal", wantShape, gotShape) + assertIndexExists(t, ctx, db, "bd_events_journal", "idx_bd_events_journal_ts") + assertIndexExists(t, ctx, db, "bd_events_journal", "idx_bd_events_journal_issue") + }) + } +} + +// TestSentinelJournalColumnsReplay is T2.6: a store already at the latest +// ignored cursor that loses comment_json out of band (a backup restored +// between doors, or damage out of band) floors at exactly 21 and heals on the +// next writable open. MigrateUp's own return value cannot prove this: it +// reports only the main-plane applied count (plus a reconcile flag), never +// the ignored-plane count, so every assertion here goes through +// CurrentIgnoredVersion instead. Kills: no sentinel registered (the floor +// read would stay at latest instead of dropping to 21, caught by the +// exact-21 assertion below); a floor of 22 or more (same assertion — 22 +// would mean 0022, bd_events_journal's own creator, never replays, which +// cannot heal a shape where the table already exists but a column does +// not); a replay that runs but does not actually add comment_json back +// (caught by the final post-heal probe). +func TestSentinelJournalColumnsReplay(t *testing.T) { + te := newTestEnv(t, "sj") + ctx := context.Background() + db := te.openDB(t, ctx) + + before, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (before damage): %v", err) + } + if want := schema.LatestIgnoredVersion(); before != want { + t.Fatalf("CurrentIgnoredVersion before damage = %d, want latest (%d)", before, want) + } + + // Damage through the SAME connection db already holds open, not a second + // embeddeddolt.OpenSQL handle (te.exec's own short-lived one): embedded + // mode's engine-open path serializes on the directory, and a second + // concurrent handle to it deadlocks against the first instead of simply + // queuing. + if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil { + t.Fatalf("drop comment_json: %v", err) + } + + floored, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (after dropping comment_json): %v", err) + } + if floored != 21 { + t.Fatalf("CurrentIgnoredVersion after dropping comment_json = %d, want exactly 21", floored) + } + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp (forced replay): %v", err) + } + + healed, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (after heal): %v", err) + } + if want := schema.LatestIgnoredVersion(); healed != want { + t.Fatalf("CurrentIgnoredVersion after heal = %d, want latest (%d)", healed, want) + } + + var commentType sql.NullString + if err := db.QueryRowContext(ctx, ` + SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = 'comment_json'`, + ).Scan(&commentType); err != nil { + t.Fatalf("probe comment_json after heal: %v", err) + } + if !commentType.Valid { + t.Fatal("comment_json is still missing after the forced replay") + } +} + +// ignoredPlaneTablesExceptJournal lists every clone-local (dolt_ignore) table +// other than bd_events_journal itself, matching +// TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape's table list +// (migrate_ignored_plane_shape_test.go). bd_events_journal's own shape +// legitimately changes during the replay this test forces; every one of +// these must not. +var ignoredPlaneTablesExceptJournal = []string{ + "wisps", "wisp_labels", "wisp_dependencies", "wisp_events", + "wisp_comments", "wisp_child_counters", "events", "leases", + "repo_mtimes", "local_metadata", "bd_events_seq", +} + +// dumpTable renders every row of table as an order-stable, type-agnostic +// snapshot: each value read as text (or ""), ordered by the table's +// first column. It exists so TestReplay22To28NoOpOnHealthyStore can compare +// whole tables it otherwise knows nothing about the schema of, rather than +// hand-maintaining a column list per table. +func dumpTable(t *testing.T, ctx context.Context, db *sql.DB, table string) []map[string]string { + t.Helper() + rows, err := db.QueryContext(ctx, "SELECT * FROM "+table+" ORDER BY 1") + if err != nil { + t.Fatalf("dump %s: %v", table, err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("dump %s columns: %v", table, err) + } + var out []map[string]string + for rows.Next() { + vals := make([]sql.NullString, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + t.Fatalf("dump %s row: %v", table, err) + } + row := make(map[string]string, len(cols)) + for i, c := range cols { + if vals[i].Valid { + row[c] = vals[i].String + } else { + row[c] = "" + } + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate %s: %v", table, err) + } + return out +} + +func snapshotIgnoredPlaneTablesExceptJournal(t *testing.T, ctx context.Context, db *sql.DB) map[string][]map[string]string { + t.Helper() + snap := make(map[string][]map[string]string, len(ignoredPlaneTablesExceptJournal)) + for _, table := range ignoredPlaneTablesExceptJournal { + snap[table] = dumpTable(t, ctx, db, table) + } + return snap +} + +// TestReplay22To28NoOpOnHealthyStore is T2.7: forcing the sentinel-triggered +// replay of ignored 22..28 must perturb nothing outside bd_events_journal +// itself — not a single row in any other clone-local table, and critically +// not wisps.updated_at, which an unguarded statement in this range would +// restamp via ON UPDATE CURRENT_TIMESTAMP (the #5981 harm class the +// replay-floor's two placement constraints exist to prevent; see schema.go's +// schemaSentinelColumn doc comment). Kills: an unguarded statement above the +// floor touching a table or row it should not. +func TestReplay22To28NoOpOnHealthyStore(t *testing.T) { + te := newTestEnv(t, "rp") + ctx := context.Background() + + wispID := "wisp-rp-1" + te.exec(t, ctx, + "INSERT INTO wisps (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type, ephemeral) VALUES (?, 'w', '', '', '', '', 'open', 2, 'task', 1)", + wispID) + + db := te.openDB(t, ctx) + before := snapshotIgnoredPlaneTablesExceptJournal(t, ctx, db) + + // Force the replay the only way it ever happens in production: contradict + // a sentinel, which floors the cursor to 21 and makes 22..28 pending again + // (TestSentinelJournalColumnsReplay proves the floor and the heal; this + // test only cares what else the resulting replay does or does not touch). + // Damage through db itself, not a second embeddeddolt.OpenSQL handle: see + // the matching comment in TestSentinelJournalColumnsReplay. + if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil { + t.Fatalf("drop comment_json: %v", err) + } + // MigrateUp's own return value cannot prove the replay ran: it reports + // only the main-plane applied count, never the ignored-plane count (see + // TestSentinelJournalColumnsReplay), so the contradiction and its heal are + // both checked through CurrentIgnoredVersion instead. + if floored, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after damage): %v", err) + } else if floored != 21 { + t.Fatalf("CurrentIgnoredVersion after dropping comment_json = %d, want exactly 21", floored) + } + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp (forced 22..28 replay): %v", err) + } + if healed, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after replay): %v", err) + } else if want := schema.LatestIgnoredVersion(); healed != want { + t.Fatalf("CurrentIgnoredVersion after replay = %d, want latest (%d): the replay did not run", healed, want) + } + + after := snapshotIgnoredPlaneTablesExceptJournal(t, ctx, db) + if !reflect.DeepEqual(before, after) { + t.Errorf("replaying ignored 22..28 perturbed a table it should never touch:\nbefore=%+v\nafter=%+v", before, after) + } +} + +// TestStoreAtIgnored28WritesAndJournals is the real-store half of T2.9 +// (BEADS-JOURNAL-PLAN.md §4.3, PR A2; schema.TestOldIgnoredCeilingOpensAfter0028 +// is the other half, pinning architecturally that no open-path guard compares +// a binary's own ignored ceiling against the stored cursor). A fresh store +// migrates to ignored 28 on its own, with no ignored-ceiling input from the +// caller at all; this just confirms that store opens, writes, and journals +// normally once there, matching the plan's own framing of the claim. +func TestStoreAtIgnored28WritesAndJournals(t *testing.T) { + te := newTestEnv(t, "oldceil") + ctx := context.Background() + + // Short-lived connection for the pre-check only: te.store's own calls + // below (SetEventsJournalEnabled, CreateIssue) each open and close their + // own connection to the same directory, and OpenSQL's backoff waits + // until ctx cancellation (never), so holding a second connection open + // concurrently via te.openDB here would deadlock them forever instead of + // failing. te.openDB is only safe to call once nothing else still needs + // to open its own connection to this store (see T2.5/T2.6/T2.7 above, + // which all finish their te.store.* calls first). + func() { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + if got, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion: %v", err) + } else if want := schema.LatestIgnoredVersion(); got != want { + t.Fatalf("CurrentIgnoredVersion = %d, want latest (%d) on a freshly initialized store", got, want) + } + }() + + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation at ignored 28: %v", err) + } + issue := &types.Issue{Title: "t2.9 seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue at ignored 28: %v", err) + } + + db := te.openDB(t, ctx) + var journaled int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ?", issue.ID).Scan(&journaled); err != nil { + t.Fatalf("probe bd_events_journal: %v", err) + } + if journaled == 0 { + t.Error("CreateIssue at ignored 28 did not journal") + } +} diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 5108ff37d..5acd23138 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -117,6 +117,7 @@ go_library( "migrations/ignored/0026_dep_rekey_dedup_marker.up.sql", "migrations/ignored/0027_add_wisps_current_revision.up.sql", "migrations/0069_widen_issue_versions_datetime_precision.up.sql", + "migrations/ignored/0028_converge_events_journal_shape.up.sql", ], importpath = "github.com/steveyegge/beads/internal/storage/schema", visibility = ["//:__subpackages__"], @@ -151,6 +152,7 @@ go_test( "migration_0069_datetime_precision_test.go", "migration_body_drain_test.go", "migration_watchdog_test.go", + "old_ignored_ceiling_test.go", "progress_test.go", "remote_migrate_gate_test.go", "schema_skew_test.go", @@ -160,6 +162,7 @@ go_test( ], data = [":migrations"], embed = [":schema"], + embedsrcs = ["testdata/old_ignored_ceiling_24/0024_placeholder.up.sql"], deps = [ "//internal/debug", "//internal/storage/depid", diff --git a/internal/storage/schema/cursor_reality_test.go b/internal/storage/schema/cursor_reality_test.go index c41144fab..e57110f8f 100644 --- a/internal/storage/schema/cursor_reality_test.go +++ b/internal/storage/schema/cursor_reality_test.go @@ -133,9 +133,13 @@ func TestCursorRealityFloor(t *testing.T) { wantLimited: true, }, { - name: "schema corroborates the cursor", - present: allSentinelTables(), - columns: map[string]bool{"leases.granted_node": true}, + name: "schema corroborates the cursor", + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + "bd_events_journal.actor": true, + }, wantFloor: 0, wantLimited: false, }, @@ -252,8 +256,12 @@ func TestCurrentVersionClampsToSentinelFloor(t *testing.T) { name: "fully corroborated cursor is believed as read", raw: latest, present: allSentinelTables(), - columns: map[string]bool{"leases.granted_node": true}, - want: latest, + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + "bd_events_journal.actor": true, + }, + want: latest, }, } { t.Run(tc.name, func(t *testing.T) { @@ -583,7 +591,10 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { AddRow("content_hash", "char(64)", "YES", "", nil, "")) // The guarded read: cursor claims at-latest, both table sentinels - // corroborate it, and only the leases column does not. + // corroborate it, and only the leases column does not — the + // bd_events_journal sentinels (PR A2) are probed too, in the same + // sentinelColumns order, and corroborate here so the leases contradiction + // stays the only thing driving the floor. expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) for range ignoredSource.sentinelTables { @@ -592,6 +603,10 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { } mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) + for range 2 { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } boom := errors.New("stop here: the applier resumed above the floor") mock.ExpectExec(".*").WillReturnError(boom) diff --git a/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql new file mode 100644 index 000000000..e19b5b0c5 --- /dev/null +++ b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql @@ -0,0 +1,163 @@ +-- Converge every observed shape of bd_events_journal on the canonical one +-- (BEADS-JOURNAL-PLAN.md §4.2b, PR A2). +-- +-- ignored/0023 repairs this table too, but only by probing DATA_TYPE: a +-- column that is entirely ABSENT returns no row, DATA_TYPE reads NULL, and +-- `NULL = 1` takes the no-op branch. That gap is exactly what let three +-- distinct field shapes survive 0023 unrepaired: +-- +-- - gas-city-inc (gci): seq, ts, op, issue_id, issue_json, dep_json — +-- comment_json and actor both absent outright (plan §1.1). 0023 widened +-- dep_json and added the ts index (both already correct, i.e. both were +-- present with some type), so neither of the two missing-column probes +-- ever fired. +-- - the events-journal-3666e5026 release shape: issue_json LONGTEXT, +-- dep_json TEXT, no comment_json, no ts index (plan §1.1, "where that +-- shape came from"). 0023's dep_json widen DOES fire here, but the +-- missing comment_json still does not, for the same reason as gci. +-- - the enterprise fork's ignored/0017 shape (0023's own header, +-- bd-t9ovd): dep_json and comment_json both TEXT (present, so 0023's +-- widen fires for both), no ts index, no idx_wisps_defer_until. This one +-- IS fully healed by 0023 already; it is included here as a fixture +-- (TestIgnored0028ConvergesAllShapes) precisely to pin that this file +-- stays a no-op on a shape 0023 already fixed, not to add new coverage +-- for it. +-- +-- This file closes the gap left open above by probing EXISTENCE first for +-- the two columns that can be missing outright, then falling through to +-- 0023's widen-if-TEXT check for both payload columns and both indexes — +-- each guarded step probes its own drift independently and does nothing when +-- it is absent, so a healthy workspace through either door (fresh init, +-- fresh clone) runs every probe below and issues no DDL. See 0023's header +-- for why that independence matters (crash-replay of an interrupted pass) +-- and for the no-DROP, no-rename invariant this file also upholds. +-- +-- It never reads or writes bd_events_seq or a single journal row: the +-- counter and every row must be byte-identical before and after +-- (TestIgnored0028ConvergesAllShapes). +-- +-- Two of these columns are registered as sentinels in schema.go's +-- ignoredSource (`{bd_events_journal, comment_json, replayFloor: 21}` and +-- `{bd_events_journal, actor, replayFloor: 21}`): a store whose cursor already +-- reads past 28 but is missing either column — e.g. restored from a backup +-- taken between the two doors, or damaged out of band — heals itself on the +-- next writable open instead of staying silently broken forever +-- (TestSentinelJournalColumnsReplay). Floor 21 sits below this file (28) and +-- below the table's creator (ignored/0022), per schema.go's two replay-floor +-- constraints, and above every unguarded statement in the series +-- (TestReplay22To28NoOpOnHealthyStore). +-- +-- Journal tables change only on this (ignored) plane: scripts/check- +-- migration-hygiene.sh fails a main-plane migration that carries DDL against +-- any bd_events_* table, unconditionally, so this file — not a main-plane +-- twin — is the only place bd_events_journal's shape can converge. + +-- comment_json: add as LONGTEXT if missing entirely; widen if it is TEXT. +-- The NULL branch is what 0023 never had: DATA_TYPE reads NULL only when the +-- column (or the table) is absent, and the table-absent case is excluded by +-- the replay-floor constraint above (0022, the table's creator, always runs +-- first in any replay this migration participates in). +SET @comment_json_type = ( + SELECT DATA_TYPE + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'comment_json' +); +SET @sql = IF(@comment_json_type IS NULL, + 'ALTER TABLE bd_events_journal ADD COLUMN comment_json LONGTEXT', + IF(@comment_json_type = 'text', + 'ALTER TABLE bd_events_journal MODIFY COLUMN comment_json LONGTEXT', + 'SELECT 1')); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- actor: add VARCHAR(255) NOT NULL DEFAULT '' if missing. ignored/0025 +-- already does exactly this on the lineage that carries it; this is a second, +-- idempotent guard — identical definition, explicit table-exists probe as in +-- 0023/0025 — for any lineage that reaches 0028 without having run 0025 (the +-- sentinel above is what makes that heal rather than strand). +SET @needs_actor = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'actor') = 0, + 1, 0 +); +SET @sql = IF(@needs_actor = 1, + 'ALTER TABLE bd_events_journal ADD COLUMN actor VARCHAR(255) NOT NULL DEFAULT ''''', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- dep_json: widen if TEXT. 0023's own step, repeated here (not removed there: +-- shipped migrations are frozen, check C) for any lineage that reaches 0028 +-- without ever having run it. Every creator of this table this repo or its +-- fork lineage has ever shipped declares dep_json with SOME type, so NULL +-- here can only mean "no table", which the replay floor rules out. +SET @dep_json_is_text = ( + SELECT IF(DATA_TYPE = 'text', 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'dep_json' +); +SET @sql = IF(@dep_json_is_text = 1, + 'ALTER TABLE bd_events_journal MODIFY COLUMN dep_json LONGTEXT', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- issue_json: widen if TEXT, same reasoning as dep_json. No observed shape +-- (gci, 3666e5026, the enterprise fork) has ever shipped this column as TEXT, +-- so this is a no-op everywhere it has been checked; included for the same +-- completeness reason as idx_bd_events_journal_issue below. +SET @issue_json_is_text = ( + SELECT IF(DATA_TYPE = 'text', 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'issue_json' +); +SET @sql = IF(@issue_json_is_text = 1, + 'ALTER TABLE bd_events_journal MODIFY COLUMN issue_json LONGTEXT', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- idx_bd_events_journal_ts: create if missing. 0023's own step, repeated for +-- the same reason as actor above. +SET @needs_ts_index = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND INDEX_NAME = 'idx_bd_events_journal_ts') = 0, + 1, 0 +); +SET @sql = IF(@needs_ts_index = 1, + 'CREATE INDEX idx_bd_events_journal_ts ON bd_events_journal(ts)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- idx_bd_events_journal_issue: create if missing. Every observed shape +-- (gci, 3666e5026, the enterprise fork) already carries this index — it is +-- declared inline in every CREATE this repo or its fork lineage has ever +-- shipped — but an unconditional guard costs one more INFORMATION_SCHEMA +-- round trip and closes the same class of gap 0023 left open for +-- idx_bd_events_journal_ts, against a shape nobody has observed yet. +SET @needs_issue_index = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND INDEX_NAME = 'idx_bd_events_journal_issue') = 0, + 1, 0 +); +SET @sql = IF(@needs_issue_index = 1, + 'CREATE INDEX idx_bd_events_journal_issue ON bd_events_journal(issue_id)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/old_ignored_ceiling_test.go b/internal/storage/schema/old_ignored_ceiling_test.go new file mode 100644 index 000000000..00b587ab8 --- /dev/null +++ b/internal/storage/schema/old_ignored_ceiling_test.go @@ -0,0 +1,124 @@ +package schema + +import ( + "context" + "embed" + "regexp" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" +) + +// oldIgnoredCeiling24Files stands in for the ignored-migration set a binary +// built before BEADS-JOURNAL-PLAN.md's PR A2 would have embedded: 0025 +// (actor), 0026 (dep rekey marker), 0027 (wisps.current_revision) and 0028 +// (this plan's journal-shape convergence) all postdate it, so its ceiling is +// 24 — the d3ab CLI's measured ignored ceiling in the plan's evidence base +// (§4.1). Only the filename's version prefix is ever read +// (migrationSource.list/.latest), so one placeholder file is enough; it need +// not match the real ignored/0024 migration's SQL. +// +//go:embed testdata/old_ignored_ceiling_24/*.up.sql +var oldIgnoredCeiling24Files embed.FS + +// TestOldIgnoredCeilingOpensAfter0028 is T2.9 (BEADS-JOURNAL-PLAN.md §4.3, PR +// A2): a binary that only knows the ignored series through 24 opens a store +// whose ignored cursor is already at 28 without refusing, because nothing on +// the open path compares a binary's ignored ceiling against the stored +// cursor — CurrentIgnoredVersion/LatestIgnoredVersion feed no skew guard the +// way CurrentVersion/LatestVersion do for the main plane (checkSchemaSkew). +// +// Kills: +// - adding an ignored-plane skew check: oldIgnored (ceiling 24) reporting +// atLatest()==true and migrate() applying nothing against a mocked cursor +// of 28 pins that an old-ceiling source does not refuse or try to "catch +// up" just because the stored cursor is ahead of what it knows; and the +// checkSchemaSkew/CheckForwardDrift/CheckBehindDrift sub-tests below use a +// strict sqlmock (ExpectationsWereMet) that would fail if any of them +// issued a query against ignored_schema_migrations at all. +// - adding a main-plane twin for the journal migrations: that would move +// mainSource's own cursor, which the same checkSchemaSkew mock (driven +// only by schema_migrations, unaffected by the ignored plane) would catch +// as a changed main version the moment the twin's migration shipped. +func TestOldIgnoredCeilingOpensAfter0028(t *testing.T) { + oldIgnored := migrationSource{ + files: oldIgnoredCeiling24Files, + dir: "testdata/old_ignored_ceiling_24", + cursorTable: ignoredSource.cursorTable, + } + if got := oldIgnored.latest(); got != 24 { + t.Fatalf("oldIgnored.latest() = %d, want 24", got) + } + + t.Run("old-ceiling source sees the store as at-latest, not behind", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + expectCursorProbe(mock, "ignored_schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", 28) + + if !oldIgnored.atLatest(context.Background(), db) { + t.Error("oldIgnored.atLatest() = false for a cursor (28) past its own ceiling (24); an old binary must not see this store as behind") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } + }) + + t.Run("old-ceiling source applies nothing against a cursor past its ceiling", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CREATE TABLE IF NOT EXISTS ignored_schema_migrations")). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(regexp.QuoteMeta("SHOW COLUMNS FROM ignored_schema_migrations LIKE 'content_hash'")). + WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}). + AddRow("content_hash", "char(64)", "YES", "", nil, "")) + expectCursorProbe(mock, "ignored_schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", 28) + // No sentinelTables/sentinelColumns on oldIgnored (zero value), so + // cursorRealityFloor has nothing further to probe, and no + // ExecContext should run at all: migrate must see current(28) >= + // target(24) and return immediately. + + applied, _, err := oldIgnored.migrate(context.Background(), db, 0) + if err != nil { + t.Fatalf("oldIgnored.migrate: %v", err) + } + if applied != 0 { + t.Errorf("oldIgnored.migrate applied = %d, want 0 (a cursor past this source's ceiling has nothing pending)", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations (an old-ceiling binary must not touch the database at all here): %v", err) + } + }) + + t.Run("the real open-path guards never consult the ignored cursor", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + // Main plane only: at the binary's own latest, unaffected by the + // ignored cursor being 28. If checkSchemaSkew/CheckForwardDrift ever + // queried ignored_schema_migrations, the strict mock below (no such + // expectation registered) would fail the call. + expectCursorProbe(mock, "schema_migrations", true) + mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(version), 0) FROM schema_migrations")). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(LatestVersion())) + + if err := CheckForwardDrift(context.Background(), db); err != nil { + t.Errorf("CheckForwardDrift = %v, want nil (ignored=28 must not drive a main-plane forward-drift refusal)", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } + }) +} diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index b9dae90b6..253af5b80 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -405,7 +405,21 @@ var ( // 0001–0025 series replays, dragging in ignored/0007 and restamping // wisps.updated_at across the plane. Believing the cursor up to 11 // applies exactly the pending tail 0012–0025 instead. - sentinelColumns: []schemaSentinelColumn{{table: "leases", column: "granted_node", replayFloor: 11}}, + // + // bd_events_journal carries two more, registered together: a store + // whose cursor already reads past 28 but is missing comment_json or + // actor (restored from a backup taken between doors, or damaged out + // of band) heals itself on the next writable open instead of staying + // silently broken forever (BEADS-JOURNAL-PLAN.md §4.2b, PR A2). Floor + // 21 satisfies the same two constraints as the leases floor above: + // 0022 (the table's creator) and 0028 (comment_json's adder; actor's + // adder is 0025, also above 21) both sit strictly above it, and no + // unguarded statement in the series sits at or below it. + sentinelColumns: []schemaSentinelColumn{ + {table: "leases", column: "granted_node", replayFloor: 11}, + {table: "bd_events_journal", column: "comment_json", replayFloor: 21}, + {table: "bd_events_journal", column: "actor", replayFloor: 21}, + }, } ) diff --git a/internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql b/internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql new file mode 100644 index 000000000..6f9e51c14 --- /dev/null +++ b/internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql @@ -0,0 +1,8 @@ +-- Placeholder for TestOldIgnoredCeilingOpensAfter0028 (T2.9, +-- BEADS-JOURNAL-PLAN.md PR A2): stands in for the newest ignored migration a +-- binary built before this plan's 0025-0028 would have embedded (the d3ab +-- CLI's measured ignored ceiling in the plan's evidence base, §4.1). Its +-- content is never executed by the test -- migrationSource.latest() and +-- .list() only parse the filename for a version number -- so this file need +-- not match the real ignored/0024 migration's SQL. +SELECT 1; diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index f7aa0b8cb..e7ae732b8 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -26,6 +26,7 @@ go_test( "bazel_skip_allowlist_test.go", "check_build_tags_test.go", "check_doc_freshness_test.go", + "check_migration_hygiene_journal_plane_test.go", "check_release_cross_compile_test.go", "check_testing_short_test.go", "check_winget_portable_alias_test.go", diff --git a/scripts/check-migration-hygiene.sh b/scripts/check-migration-hygiene.sh index 7a1db5af4..50250abea 100755 --- a/scripts/check-migration-hygiene.sh +++ b/scripts/check-migration-hygiene.sh @@ -86,7 +86,23 @@ # point of the check is that the idiom stops spreading into migrations # where the write does matter. # -# Checks C, D, and E compare against $BASE_SHA if set (CI passes the PR base), +# F. A new main-plane migration must not contain DDL (CREATE/ALTER/RENAME/ +# DROP TABLE) against a journal table (`bd_events_journal`, +# `bd_events_seq`). Unlike check D, there is no twin exemption: journal +# tables change only on the ignored plane (Dolt) or in Postgres DDL +# (BEADS-JOURNAL-PLAN.md §4.2c, PR A2) — an ignored-series twin does not +# excuse a main-plane DDL statement here the way it does for the other +# clone-local tables check D covers, because a main-plane change to +# these specific tables bumps the skew-checked main cursor (schema.go's +# forward-drift guard) for a table whose whole point is to stay +# invisible to binaries that predate it. BEADS-JOURNAL-PLAN.md names +# this rule "hygiene check E" and its test `TestHygieneCheckE`; it is +# implemented here as check F and the test is named for what it checks +# rather than for the plan's letter, because this repo's own check E +# already means something else (no PREPARE'd DML, above) and reusing the +# letter would misdescribe both. +# +# Checks C, D, E, and F compare against $BASE_SHA if set (CI passes the PR base), # else origin/main, else main; they are skipped with a warning when no base # is resolvable (e.g. shallow clone without the base commit). @@ -439,6 +455,37 @@ EOF done fi +# --- Check F: no main-plane DDL on journal tables --------------------------- +# bd_events_journal/bd_events_seq change only on the ignored plane (Dolt) or +# in Postgres DDL (BEADS-JOURNAL-PLAN.md §4.2c, PR A2). Scoped to added_main +# the same as checks D and E; unlike D, there is no twin exemption here — +# see this check's doc comment at the top of the file for why. +if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then + echo "WARN (journal plane) no usable base ref; skipping check F." >&2 +else + journal_tables='bd_events_journal|bd_events_seq' + ddl_re_f="(alter|create|rename|drop)[[:space:]]+table([[:space:]]+if[[:space:]]+(not[[:space:]]+)?exists)?[[:space:]]+[\`']?(${journal_tables})\b" + for f in $added_main; do + [ -f "$f" ] || continue + stripped=$(sed 's/--.*$//' "$f" | tr '[:upper:]' '[:lower:]') + hits=$(grep -n -E "$ddl_re_f" <<<"$stripped" || true) + if [ -n "$hits" ]; then + fail=1 + echo "FAIL (journal plane) $f contains main-plane DDL on a journal table:" + echo "$hits" | sed 's/^/ line /' + cat <<'EOF' + bd_events_journal and bd_events_seq change only on the ignored plane: a + main-plane migration touching either one bumps the skew-checked main + cursor for a table whose whole point is to stay invisible to binaries that + predate it (BEADS-JOURNAL-PLAN.md §4.2c). Move this DDL to a new file under + internal/storage/schema/migrations/ignored/ instead; there is no twin + exemption for these two tables the way check D allows for other + clone-local tables. +EOF + fi + done +fi + if [ "$fail" -ne 0 ]; then echo echo "Migration hygiene check failed. See docs in scripts/check-migration-hygiene.sh." diff --git a/scripts/check_migration_hygiene_journal_plane_test.go b/scripts/check_migration_hygiene_journal_plane_test.go new file mode 100644 index 000000000..0d99258aa --- /dev/null +++ b/scripts/check_migration_hygiene_journal_plane_test.go @@ -0,0 +1,139 @@ +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// This file is T2.8 (BEADS-JOURNAL-PLAN.md §4.3, PR A2): a main-plane +// migration with DDL on bd_events_* fails check-migration-hygiene.sh. The +// plan names the rule "hygiene check E" and the test TestHygieneCheckE, but +// this repo's own check E already means something else (no PREPARE'd DML in +// new main-plane migrations — see that check's doc comment in +// check-migration-hygiene.sh). The rule is implemented here as a new check F +// instead, and these tests are named for what they check rather than for the +// plan's letter, to avoid misdescribing either check. + +// runCheckMigrationHygiene runs check-migration-hygiene.sh in a throwaway git +// repository: baseFiles are committed on main (standing in for the PR's +// base); newFiles are then added to the working tree uncommitted (standing in +// for the PR's diff — checks C, D, E and F all diff the working tree against +// this base commit). +func runCheckMigrationHygiene(t *testing.T, baseFiles, newFiles map[string]string) (string, error) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("checker is a Bash boundary") + } + requireHostTool(t, "git") + requireHostTool(t, "bash") + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "check-migration-hygiene.sh")) + if err != nil { + t.Fatalf("read check-migration-hygiene.sh: %v", err) + } + dir := t.TempDir() + write := func(files map[string]string) { + for rel, content := range files { + full := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(full), err) + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + } + all := map[string]string{"scripts/check-migration-hygiene.sh": string(script)} + for rel, content := range baseFiles { + all[rel] = content + } + write(all) + env := append(os.Environ(), + "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"add", "-A"}, {"commit", "-q", "-m", "base"}} { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = env + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + write(newFiles) + cmd := exec.Command("bash", filepath.Join(dir, "scripts", "check-migration-hygiene.sh")) + cmd.Dir = dir + cmd.Env = env + out, err := cmd.CombinedOutput() + return string(out), err +} + +// TestHygieneCheckFRejectsMainPlaneJournalDDL is T2.8: a new main-plane +// migration altering bd_events_journal fails the script even though it ships +// an ignored-series twin in the same change — proving this is check F (no +// twin exemption, ever) and not merely check D (twin required) passing +// through by accident. Kills: removing check F, or weakening it to accept a +// twin the way check D does. +func TestHygieneCheckFRejectsMainPlaneJournalDDL(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/0100_add_journal_widget.up.sql": "ALTER TABLE bd_events_journal ADD COLUMN widget VARCHAR(32);\n", + "internal/storage/schema/migrations/ignored/0100_add_journal_widget.up.sql": "-- twin: not what check F looks at; its presence must not save the main-plane file.\nSELECT 1;\n", + }) + if err == nil { + t.Fatalf("main-plane DDL on bd_events_journal was accepted even with an ignored twin:\n%s", out) + } + if !strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("expected a \"FAIL (journal plane)\" report, got:\n%s", out) + } + if !strings.Contains(out, "0100_add_journal_widget.up.sql") { + t.Errorf("expected the offending file named in output, got:\n%s", out) + } +} + +// TestHygieneCheckFRejectsMainPlaneEventsSeqDDL pins that the rule covers +// bd_events_seq too, not only bd_events_journal. +func TestHygieneCheckFRejectsMainPlaneEventsSeqDDL(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/0100_rename_events_seq.up.sql": "ALTER TABLE bd_events_seq ADD COLUMN note VARCHAR(8);\n", + }) + if err == nil { + t.Fatalf("main-plane DDL on bd_events_seq was accepted:\n%s", out) + } + if !strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("expected a \"FAIL (journal plane)\" report, got:\n%s", out) + } +} + +// TestHygieneCheckFAllowsMainPlaneDDLOnNonJournalTables is the negative +// control: a main-plane migration that never mentions a journal table passes +// cleanly, so check F is not a blanket rejection of main-plane DDL. +func TestHygieneCheckFAllowsMainPlaneDDLOnNonJournalTables(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/0100_add_issues_widget.up.sql": "ALTER TABLE issues ADD COLUMN widget VARCHAR(32);\n", + }) + if err != nil { + t.Fatalf("main-plane DDL on a non-journal table was rejected: %v\n%s", err, out) + } + if strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("did not expect a journal-plane failure, got:\n%s", out) + } +} + +// TestHygieneCheckFIgnoresIgnoredPlaneJournalDDL confirms the rule is +// main-plane-only: the same DDL shipped as an ignored-series migration (the +// only place journal tables are allowed to change, PR A2) passes. +func TestHygieneCheckFIgnoresIgnoredPlaneJournalDDL(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/ignored/0100_add_journal_widget.up.sql": "ALTER TABLE bd_events_journal ADD COLUMN widget VARCHAR(32);\n", + }) + if err != nil { + t.Fatalf("ignored-plane DDL on bd_events_journal was rejected: %v\n%s", err, out) + } + if strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("did not expect a journal-plane failure for an ignored-series migration, got:\n%s", out) + } +} From 28ea285662182d4b345f5008d7be24ee5b7d0d5a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 03:39:49 +0000 Subject: [PATCH 3/5] test(schema): re-pin #5496's counter canary to 0023 alone; pin sentinel-replay counter convergence ignored/0023's own counter-canary assertion pinned next_seq to a literal 1, which only held by coincidence of fixture ordering; isolate the "0023 never writes bd_events_seq" contract against 0023's frozen bytes directly and assert next_seq == GREATEST(seeded, MAX(seq)) instead. Add the matching isolation test for 0028, and pin that a sentinel-triggered replay of ignored 22-28 converges the counter from every observed crash/restore state (behind, equal, ahead-from-an-older-restore, empty-journal, and a mid-sequence hole) regardless of which sentinel column fired it. Also pins the static invariant that only ignored/0022's two statements may touch bd_events_journal data in the replayed 22-28 range, and adds the corresponding unit floor cases to TestCursorRealityFloor and TestCurrentVersionClampsToSentinelFloor. Co-Authored-By: Claude Sonnet 5 --- .../scripts/embedded-storage-test-shards.txt | 3 + .../journal_shape_converge_test.go | 325 +++++++++++++++++- .../migrate_journal_shape_repair_test.go | 128 +++++-- .../storage/schema/cursor_reality_test.go | 176 ++++++++++ 4 files changed, 596 insertions(+), 36 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index c00ff8f61..0933d102d 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -294,6 +294,7 @@ 15 6 TestRenameLabel 15 6 TestRenameLabelMintsOneVersionPerTouchedBead 15 6 TestRunInTransactionPostCommitStageFailureIsIndeterminate +15 6 TestSentinelReplayCounterConverges 15 6 TestStatsReporterContract 15 6 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials 15 6 TestWithPeerAuth_RestoresEnvWhenCallbackFails @@ -308,6 +309,7 @@ 15 7 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType 15 7 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane 15 7 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp +15 7 TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched 15 7 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender 15 7 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext 15 7 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet @@ -459,6 +461,7 @@ 15 12 TestEmbeddedRecomputeAllBlockedWiring 15 12 TestEmbeddedRelationsResolvesAWispAnchor 15 12 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges +15 12 TestIgnored0028AloneLeavesCounterAndRowsUntouched 15 12 TestLargeBatchApplyStatementCounts712_Embedded 15 12 TestLifecycleUpdateContract 15 12 TestPreviewStoreRefusesLocalMetadataWrite diff --git a/internal/storage/embeddeddolt/journal_shape_converge_test.go b/internal/storage/embeddeddolt/journal_shape_converge_test.go index 67753caf5..3f3910ad2 100644 --- a/internal/storage/embeddeddolt/journal_shape_converge_test.go +++ b/internal/storage/embeddeddolt/journal_shape_converge_test.go @@ -5,6 +5,7 @@ package embeddeddolt_test import ( "context" "database/sql" + "fmt" "reflect" "testing" @@ -157,8 +158,11 @@ func assertColumnShapeConverged(t *testing.T, fixtureName, table string, want, g // plan's evidence base found ends up byte-identical to canonical after // MigrateUp, and not a single row or the seq counter moved getting there. // Kills: probing only DATA_TYPE (0023's gap — a dropped fixture here would -// stay missing comment_json/actor and fail the shape diff); any write to the -// counter or rows (caught by the before/after snapshot). +// stay missing comment_json/actor and fail the shape diff); any non-monotone +// write to the counter or rows (caught by the before/after snapshot; monotone +// writes are TestIgnored0028AloneLeavesCounterAndRowsUntouched's job — this +// test's forced replay always re-runs ignored/0022 too, whose own GREATEST +// raise would mask one). func TestIgnored0028ConvergesAllShapes(t *testing.T) { // dbPrefix is separate from name (rather than derived from it): embedded // mode database names reject hyphens, but "fork-ignored-0017" is the @@ -252,6 +256,139 @@ func TestIgnored0028ConvergesAllShapes(t *testing.T) { } } +// isolationRow is the row shape TestIgnored0028AloneLeavesCounterAndRowsUntouched +// compares before and after running ignored/0028 directly: the four columns +// every damaged fixture in that test shares, plus comment_json only on the +// one fixture (fork-ignored-0017) that still has it when 0028 starts. +type isolationRow struct { + Seq int64 + Ts, Op, IssueID string + IssueJSON, DepJSON, CommentJSON sql.NullString +} + +// readIsolationRows reads isolationRows in seq order. comment_json is only +// selected when hasCommentJSON is true: on two of the three fixtures the +// column does not exist yet when this runs, so selecting it unconditionally +// would fail with "column not found" rather than compare anything. +func readIsolationRows(t *testing.T, ctx context.Context, conn *sql.Conn, hasCommentJSON bool) []isolationRow { + t.Helper() + query := "SELECT seq, ts, op, issue_id, issue_json, dep_json FROM bd_events_journal ORDER BY seq" + if hasCommentJSON { + query = "SELECT seq, ts, op, issue_id, issue_json, dep_json, comment_json FROM bd_events_journal ORDER BY seq" + } + rows, err := conn.QueryContext(ctx, query) + if err != nil { + t.Fatalf("read bd_events_journal: %v", err) + } + defer rows.Close() + var out []isolationRow + for rows.Next() { + var r isolationRow + dest := []any{&r.Seq, &r.Ts, &r.Op, &r.IssueID, &r.IssueJSON, &r.DepJSON} + if hasCommentJSON { + dest = append(dest, &r.CommentJSON) + } + if err := rows.Scan(dest...); err != nil { + t.Fatalf("scan bd_events_journal row: %v", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate bd_events_journal: %v", err) + } + return out +} + +func assertIsolationRowsEqual(t *testing.T, before, after []isolationRow) { + t.Helper() + if len(before) != len(after) { + t.Fatalf("journal row count = %d, want %d (0028 alone must not touch data)", len(after), len(before)) + } + for i := range before { + if before[i] != after[i] { + t.Errorf("journal row seq %d changed: before %+v, after %+v", before[i].Seq, before[i], after[i]) + } + } +} + +// TestIgnored0028AloneLeavesCounterAndRowsUntouched is T2.5b. T2.5 forces its +// replay through a sentinel contradiction, which always re-runs ignored/0022 +// too (every fixture there drops actor), so a monotone GREATEST write to the +// counter by 0028 itself would be invisible: 0022's own identical GREATEST +// raise on the same pass would mask it. Running 0028's frozen bytes directly, +// with the counter seeded below MAX(seq), closes that gap: any counter write +// at all, monotone or not, shows. Kills: any counter or row write in 0028, +// including a monotone GREATEST. +func TestIgnored0028AloneLeavesCounterAndRowsUntouched(t *testing.T) { + fixtures := []struct { + name string + dbPrefix string + build func(t *testing.T, prefix string) *testEnv + hasCommentJSON bool + }{ + {name: "3666e5026", dbPrefix: "i28a3666e5026", build: newShape3666e5026Store, hasCommentJSON: false}, + {name: "gci", dbPrefix: "i28agci", build: newGciShapeStore, hasCommentJSON: false}, + {name: "fork-ignored-0017", dbPrefix: "i28afork0017", build: newForkIgnored0017ShapeStore, hasCommentJSON: true}, + } + + for _, fx := range fixtures { + t.Run(fx.name, func(t *testing.T) { + te := fx.build(t, fx.dbPrefix) + ctx := context.Background() + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Conn: %v", err) + } + defer conn.Close() + + seed := []int64{1, 2, 3} + for _, seq := range seed { + ts := tsForSeq(seq) + query := "INSERT INTO bd_events_journal (seq, ts, op, issue_id, issue_json, dep_json) VALUES (?, ?, 'create', 'bd-i28a', ?, ?)" + args := []any{seq, ts, `{"id":"bd-i28a"}`, `[]`} + if fx.hasCommentJSON { + query = "INSERT INTO bd_events_journal (seq, ts, op, issue_id, issue_json, dep_json, comment_json) VALUES (?, ?, 'create', 'bd-i28a', ?, ?, ?)" + args = append(args, `{"text":"hi"}`) + } + if _, err := conn.ExecContext(ctx, query, args...); err != nil { + t.Fatalf("seed journal row %d: %v", seq, err) + } + } + execFrozenGuard(t, ctx, conn, "UPDATE bd_events_seq SET next_seq = 1 WHERE id = 0") + rowsBefore := readIsolationRows(t, ctx, conn, fx.hasCommentJSON) + if len(rowsBefore) != len(seed) { + t.Fatalf("seeded journal rows = %d, want %d", len(rowsBefore), len(seed)) + } + + sqlText, err := schema.IgnoredMigrationSQL("0028_converge_events_journal_shape.up.sql") + if err != nil { + t.Fatalf("read ignored 0028 migration: %v", err) + } + execFrozenGuard(t, ctx, conn, sqlText) + + if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 { + t.Errorf("next_seq = %d after applying 0028 alone, want 1: 0028 must never write the counter, not even a monotone raise", got) + } + assertIsolationRowsEqual(t, rowsBefore, readIsolationRows(t, ctx, conn, fx.hasCommentJSON)) + + for _, column := range []string{"comment_json", "dep_json"} { + if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" { + t.Errorf("bd_events_journal.%s DATA_TYPE = %q after 0028, want %q", column, got, "longtext") + } + } + requireColumnCount(t, ctx, conn, "bd_events_journal", "actor", 1) + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1) + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1) + }) + } +} + // TestSentinelJournalColumnsReplay is T2.6: a store already at the latest // ignored cursor that loses comment_json out of band (a backup restored // between doors, or damage out of band) floors at exactly 21 and heals on the @@ -319,6 +456,190 @@ func TestSentinelJournalColumnsReplay(t *testing.T) { } } +// tsForSeq gives TestSentinelReplayCounterConverges's and +// TestIgnored0028AloneLeavesCounterAndRowsUntouched's seeded rows distinct, +// ordered timestamps without threading one more field through every fixture +// for a value no assertion in either test reads. +func tsForSeq(seq int64) string { + return fmt.Sprintf("2026-08-01 00:00:%02d", seq) +} + +// counterConvergeState is one bd_events_journal/bd_events_seq relationship +// TestSentinelReplayCounterConverges seeds: the rows to leave in the journal, +// the counter value to seed, and the counter the sentinel-triggered replay +// must converge to (GREATEST(before, MAX(seq))). +type counterConvergeState struct { + name string + code string // short, hyphen/underscore-free token for database name prefixes + seqs []int64 + before int64 + want int64 +} + +var counterConvergeStates = []counterConvergeState{ + {name: "behind", code: "behind", seqs: []int64{1, 2, 3}, before: 1, want: 3}, + {name: "equal", code: "equal", seqs: []int64{1, 2, 3}, before: 3, want: 3}, + {name: "ahead_journal_restored_older", code: "ajro", seqs: []int64{1, 2, 3}, before: 10, want: 10}, + {name: "empty_journal_counter_ahead", code: "ejca", seqs: nil, before: 7, want: 7}, + {name: "hole_above_counter", code: "hac", seqs: []int64{1, 3}, before: 1, want: 3}, +} + +// TestSentinelReplayCounterConverges is T2.6b: across every counter/journal +// relationship #5496's evidence base found, a sentinel-triggered replay of +// ignored 22..28 must leave bd_events_seq at exactly +// GREATEST(seeded, MAX(seq)) — never lower, never past the high-water mark — +// and then the very next real write must land contiguously right after it. +// Run against both sentinel columns: dropping actor across all five states, +// plus dropping comment_json on "behind" alone to prove that sentinel also +// reaches the counter (it is registered on the same table, same floor). +// +// Kills: the sentinel floor sitting at 22 or above (behind stays at 1, and +// hole_above_counter's next write lands at the stale seq 2, a late arrival); +// a replayed statement that lowers the counter, e.g. a future +// `SET next_seq = COALESCE(MAX(seq), 0)` with no GREATEST (the ahead_* and +// empty_* cases fail); an over-raise such as MAX(seq)+1 (behind and equal +// read 4, and the first new row is 5, a gap); dropping the actor sentinel +// (actor/behind stays at 1 and the column is never restored). +func TestSentinelReplayCounterConverges(t *testing.T) { + type subtest struct { + name string + dbPrefix string + state counterConvergeState + column string + } + var subtests []subtest + for _, state := range counterConvergeStates { + subtests = append(subtests, subtest{ + name: state.name + "/actor", + dbPrefix: "scc" + state.code + "a", + state: state, + column: "actor", + }) + } + subtests = append(subtests, subtest{ + name: "behind/comment_json", + dbPrefix: "sccbehindc", + state: counterConvergeStates[0], + column: "comment_json", + }) + + for _, st := range subtests { + t.Run(st.name, func(t *testing.T) { + te := newTestEnv(t, st.dbPrefix) + ctx := context.Background() + + // A raw handle with an explicit cleanup, not te.openDB: this test + // closes it early (below) and reopens afterward, which te.openDB's + // t.Cleanup-deferred-to-test-end handle cannot do. + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + closed := false + closeDB := func() { + if !closed { + _ = cleanup() + closed = true + } + } + defer closeDB() + + if _, err := db.ExecContext(ctx, "DELETE FROM bd_events_journal"); err != nil { + t.Fatalf("clear bd_events_journal: %v", err) + } + for _, seq := range st.state.seqs { + if _, err := db.ExecContext(ctx, + "INSERT INTO bd_events_journal (seq, ts, op, issue_id) VALUES (?, ?, 'create', 'bd-scc')", + seq, tsForSeq(seq)); err != nil { + t.Fatalf("seed journal row %d: %v", seq, err) + } + } + if _, err := db.ExecContext(ctx, "UPDATE bd_events_seq SET next_seq = ? WHERE id = 0", st.state.before); err != nil { + t.Fatalf("seed next_seq: %v", err) + } + //nolint:gosec // G201: st.column is one of two test-local literals ("actor", "comment_json"). + if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN "+st.column); err != nil { + t.Fatalf("drop %s: %v", st.column, err) + } + + if floored, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after dropping %s): %v", st.column, err) + } else if floored != 21 { + t.Fatalf("CurrentIgnoredVersion after dropping %s = %d, want exactly 21", st.column, floored) + } + rowsBefore, _ := snapshotJournalCore(t, ctx, db) + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp (forced replay): %v", err) + } + + if after, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after replay): %v", err) + } else if want := schema.LatestIgnoredVersion(); after != want { + t.Fatalf("CurrentIgnoredVersion after replay = %d, want latest (%d)", after, want) + } + var colCount int + if err := db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = ?`, st.column).Scan(&colCount); err != nil { + t.Fatalf("probe %s after replay: %v", st.column, err) + } + if colCount == 0 { + t.Errorf("bd_events_journal.%s is still missing after the forced replay", st.column) + } + + rowsAfter, nextSeq := snapshotJournalCore(t, ctx, db) + if nextSeq != st.state.want { + t.Errorf("next_seq = %d after replay, want %d (GREATEST(seeded %d, MAX(seq)))", nextSeq, st.state.want, st.state.before) + } + if !reflect.DeepEqual(rowsBefore, rowsAfter) { + t.Errorf("bd_events_journal rows changed across the replay:\nbefore=%+v\nafter=%+v", rowsBefore, rowsAfter) + } + + // Close the raw handle before any te.store call: embedded opens + // serialize on the directory and a second concurrent handle + // deadlocks against this one instead of queuing (see + // TestStoreAtIgnored28WritesAndJournals). + closeDB() + + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation after convergence: %v", err) + } + issue := &types.Issue{Title: st.name + " seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue after convergence: %v", err) + } + + reopened := te.openDB(t, ctx) + var minSeq, maxSeq, count int64 + if err := reopened.QueryRowContext(ctx, + "SELECT COALESCE(MIN(seq),0), COALESCE(MAX(seq),0), COUNT(*) FROM bd_events_journal WHERE seq > ?", + st.state.want).Scan(&minSeq, &maxSeq, &count); err != nil { + t.Fatalf("probe new journal rows: %v", err) + } + if count == 0 { + t.Fatal("CreateIssue after convergence did not journal a row above the converged counter") + } + if minSeq != st.state.want+1 { + t.Errorf("lowest new seq = %d, want %d (want + 1)", minSeq, st.state.want+1) + } + if maxSeq-minSeq+1 != count { + t.Errorf("new seqs are not contiguous: min=%d max=%d count=%d", minSeq, maxSeq, count) + } + if st.state.name == "hole_above_counter" { + var staleCount int64 + if err := reopened.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal WHERE seq = 2").Scan(&staleCount); err != nil { + t.Fatalf("probe seq 2: %v", err) + } + if staleCount != 0 { + t.Errorf("seq 2 (the hole below the converged counter) has a row; a late arrival filled it instead of the new write landing above the counter") + } + } + }) + } +} + // ignoredPlaneTablesExceptJournal lists every clone-local (dolt_ignore) table // other than bd_events_journal itself, matching // TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape's table list diff --git a/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go b/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go index 1b86dd3f5..6643504ec 100644 --- a/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go +++ b/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go @@ -62,6 +62,72 @@ func TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape(t *testing.T conn, closeConn := newWorkspaceBeforeMigration0023(t, ctx) defer closeConn() + rowsBefore := seedForkLineageDrift(t, ctx, conn) + + if _, err := schema.MigrateUp(ctx, conn); err != nil { + t.Fatalf("repair MigrateUp: %v", err) + } + requireIgnoredCursorAtLatest(t, ctx, conn) + + // Shape converged. + for _, column := range []string{"dep_json", "comment_json"} { + if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" { + t.Errorf("bd_events_journal.%s DATA_TYPE = %q after repair, want %q", column, got, "longtext") + } + } + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1) + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1) + requireIndex(t, ctx, conn, "wisps", "idx_wisps_defer_until", 1) + + // Data survived byte for byte. + assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) + // The pass may raise a counter that is behind the journal to its + // high-water mark (ignored/0022's tail, re-run because this fork shape has + // no actor column and trips the bd_events_journal.actor sentinel at floor + // 21) and must leave it nowhere else: GREATEST(seeded, MAX(seq)). + const seeded = 1 + high := int(rowsBefore[len(rowsBefore)-1].seq) + if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != max(seeded, high) { + t.Errorf("bd_events_seq.next_seq = %d after repair, want GREATEST(seeded %d, MAX(seq) %d) = %d: a pass may only raise a behind counter to the high-water mark, never lower it or move it past MAX(seq)", + got, seeded, high, max(seeded, high)) + } + + // The repair is what makes the write that used to roll back a user's + // mutation land instead. + if _, err := conn.ExecContext(ctx, + "INSERT INTO bd_events_journal (seq, ts, op, issue_id, comment_json) VALUES (99, '2026-08-01 00:00:03', 'comment', 'bd-t9ovd', ?)", + strings.Repeat("x", 70000)); err != nil { + t.Fatalf("oversized comment_json insert still refused after repair: %v", err) + } + if got := scalarInt(t, ctx, conn, "SELECT LENGTH(comment_json) FROM bd_events_journal WHERE seq = 99"); got != 70000 { + t.Errorf("stored oversized comment_json length = %d, want 70000", got) + } + execFrozenGuard(t, ctx, conn, "DELETE FROM bd_events_journal WHERE seq = 99") + + // Crash-replay: the pass can be killed and re-run, so the frozen bytes must + // be a clean no-op against the state they just produced. Re-arm the + // canary first: if replaying 0023 alone ever wrote the counter, letting it + // start already at the high-water mark would hide that. + createBefore := showCreateTable(t, ctx, conn, "bd_events_journal") + execFrozenGuard(t, ctx, conn, "UPDATE bd_events_seq SET next_seq = 1 WHERE id = 0") + execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t)) + if createAfter := showCreateTable(t, ctx, conn, "bd_events_journal"); createAfter != createBefore { + t.Errorf("replaying 0023 changed bd_events_journal:\nbefore:\n%s\nafter:\n%s", createBefore, createAfter) + } + if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 { + t.Errorf("next_seq = %d after replaying 0023, want 1: replaying 0023 wrote the counter", got) + } + assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) +} + +// seedForkLineageDrift builds the drifted fork-lineage journal shape 0023 +// repairs: the fork's TEXT-column DDL, idx_wisps_defer_until dropped, three +// journal rows, and a seq counter deliberately set BELOW the journal's own +// MAX(seq). It also proves the fixture really is drifted before handing it +// back: an oversized comment_json insert must be refused by the TEXT column. +func seedForkLineageDrift(t *testing.T, ctx context.Context, conn *sql.Conn) []journalRow { + t.Helper() + // The drift. bd_events_journal and wisps are both clone-local and absent // from HEAD here, so replacing the table and dropping the index leaves the // working set exactly as clean as it was. @@ -71,9 +137,14 @@ func TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape(t *testing.T // Data the repair must not touch: three journal rows whose payloads carry // quotes, backslashes, NULLs, and a multi-KB blob (bound as parameters, so // the column holds the literal Go string), plus a seq counter deliberately - // set BELOW the journal's own MAX(seq). Only 0022's seeding raises it to - // the high-water mark, so next_seq staying at 1 is proof that 0022 did not - // re-run and that 0023 left the counter alone. + // set BELOW the journal's own MAX(seq). This seeds two separate claims: + // that 0023 alone never touches the counter or a row + // (TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched runs + // its frozen bytes directly to check that), and that the full MigrateUp + // pass converges the counter to GREATEST(seeded, MAX(seq)) — this fork + // shape has no actor column, so it trips the bd_events_journal.actor + // sentinel at floor 21, which re-runs ignored/0022, whose tail is the same + // statement as issueops.healEventSeqCounter. bigPayload := strings.Repeat(`{"k":"v'\"\\","n":0},`, 400) + `{"end":true}` seed := []struct { seq int64 @@ -107,47 +178,36 @@ VALUES (?, ?, ?, 'bd-t9ovd', ?, ?, ?)`, s.seq, s.ts, s.op, s.issueJSON, s.depJSO t.Fatal("oversized comment_json insert succeeded on the drifted TEXT column; the fixture is not reproducing the drift this migration repairs") } - if _, err := schema.MigrateUp(ctx, conn); err != nil { - t.Fatalf("repair MigrateUp: %v", err) - } - requireIgnoredCursorAtLatest(t, ctx, conn) + return rowsBefore +} + +// TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched pins +// 0023's own contract ("neither bd_events_seq nor a single journal row is +// read or written", its header) with the runner out of the picture: the +// frozen bytes run directly against the drifted fork shape, so every repair +// branch fires and no sentinel, floor, or neighbouring migration can cause or +// mask a counter write. The counter is seeded below MAX(seq), so even a +// monotone GREATEST-style write would show. +func TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched(t *testing.T) { + requireEmbedded(t) + ctx := t.Context() + conn, closeConn := newWorkspaceBeforeMigration0023(t, ctx) + defer closeConn() + rowsBefore := seedForkLineageDrift(t, ctx, conn) + + execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t)) - // Shape converged. for _, column := range []string{"dep_json", "comment_json"} { if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" { - t.Errorf("bd_events_journal.%s DATA_TYPE = %q after repair, want %q", column, got, "longtext") + t.Fatalf("0023 did not widen %s (got %q): the fixture no longer exercises its branches", column, got) } } requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1) - requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1) requireIndex(t, ctx, conn, "wisps", "idx_wisps_defer_until", 1) - - // Data survived byte for byte, and the counter was not touched. assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 { - t.Errorf("bd_events_seq.next_seq = %d after repair, want 1 (0023 must touch neither the counter nor any data)", got) + t.Errorf("next_seq = %d after applying 0023 alone, want 1: 0023 must never write the counter", got) } - - // The repair is what makes the write that used to roll back a user's - // mutation land instead. - if _, err := conn.ExecContext(ctx, - "INSERT INTO bd_events_journal (seq, ts, op, issue_id, comment_json) VALUES (99, '2026-08-01 00:00:03', 'comment', 'bd-t9ovd', ?)", - strings.Repeat("x", 70000)); err != nil { - t.Fatalf("oversized comment_json insert still refused after repair: %v", err) - } - if got := scalarInt(t, ctx, conn, "SELECT LENGTH(comment_json) FROM bd_events_journal WHERE seq = 99"); got != 70000 { - t.Errorf("stored oversized comment_json length = %d, want 70000", got) - } - execFrozenGuard(t, ctx, conn, "DELETE FROM bd_events_journal WHERE seq = 99") - - // Crash-replay: the pass can be killed and re-run, so the frozen bytes must - // be a clean no-op against the state they just produced. - createBefore := showCreateTable(t, ctx, conn, "bd_events_journal") - execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t)) - if createAfter := showCreateTable(t, ctx, conn, "bd_events_journal"); createAfter != createBefore { - t.Errorf("replaying 0023 changed bd_events_journal:\nbefore:\n%s\nafter:\n%s", createBefore, createAfter) - } - assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) } // TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace is the insurance diff --git a/internal/storage/schema/cursor_reality_test.go b/internal/storage/schema/cursor_reality_test.go index e57110f8f..9afb70877 100644 --- a/internal/storage/schema/cursor_reality_test.go +++ b/internal/storage/schema/cursor_reality_test.go @@ -143,6 +143,30 @@ func TestCursorRealityFloor(t *testing.T) { wantFloor: 0, wantLimited: false, }, + { + // T2.6c: the journal floor limits on its own, with leases + // corroborated and the OTHER journal column present. Moving actor + // off floor 21 would read 24 here instead (see + // TestCurrentVersionClampsToSentinelFloor's matching case). + name: "journal actor absent alone", + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + }, + wantFloor: 21, + wantLimited: true, + }, + { + name: "journal comment_json absent alone", + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.actor": true, + }, + wantFloor: 21, + wantLimited: true, + }, } { t.Run(tc.name, func(t *testing.T) { stubSentinelTables(t, tc.present, nil) @@ -263,6 +287,56 @@ func TestCurrentVersionClampsToSentinelFloor(t *testing.T) { }, want: latest, }, + { + // T2.6c: the fleet's shape at the first A2 open. Every production + // Dolt store sits at ignored cursor 24 (BEADS-JOURNAL-PLAN.md §4.1) + // and predates actor outright, so this clamps to 21 and 22..28 + // replay once. + name: "fleet first open: cursor 24, journal actor absent", + raw: 24, + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + }, + want: 21, + }, + { + // #5496's fork-lineage fixture: cursor 22, actor absent (that shape + // predates ignored/0025, which adds actor, outright). + name: "fork-lineage fixture: cursor 22, actor absent", + raw: 22, + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + }, + want: 21, + }, + { + // A raw cursor already sitting exactly at the journal floor: the + // clamp is a ceiling, not a promotion, so a contradiction at or + // below it changes nothing. + name: "cursor at the journal floor is untouched", + raw: 21, + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + }, + want: 21, + }, + { + // Taking the MAXIMUM floor instead of the minimum in + // cursorRealityFloor would read 21 here (the journal floor); the + // minimum reads 11 (the leases floor). + name: "leases and journal actor both absent", + raw: latest, + present: allSentinelTables(), + columns: map[string]bool{ + "bd_events_journal.comment_json": true, + }, + want: 11, + }, } { t.Run(tc.name, func(t *testing.T) { stubSentinelTables(t, tc.present, nil) @@ -668,3 +742,105 @@ func TestPendingVersionsUnderFloorDoNotReArmAuxMarkers(t *testing.T) { t.Error("ignored/0007 is pending on a cursor-11 store; its unguarded UPDATE would restamp wisps.updated_at") } } + +// journalSentinelReplayDML matches an (?i)-insensitive statement that +// mutates data: UPDATE ... SET, DELETE FROM, INSERT [IGNORE] INTO, +// REPLACE INTO, or TRUNCATE TABLE. It is deliberately loose about what +// follows — this test's job is to flag every candidate and then check it +// against an allowlist, not to parse SQL. +var journalSentinelReplayDML = regexp.MustCompile(`(?i)\b(update\s+\S+\s+set|delete\s+from|insert\s+(ignore\s+)?into|replace\s+into|truncate\s+table)\b`) + +var ( + journalSentinelCommentLineRE = regexp.MustCompile(`(?m)^\s*--.*$`) + journalSentinelOnUpdateRE = regexp.MustCompile(`(?i)on\s+update\s+current_timestamp(\s*\(\s*\))?`) + journalSentinelOnDupKeyRE = regexp.MustCompile(`(?i)on\s+duplicate\s+key\s+update`) + journalSentinelWhitespaceRE = regexp.MustCompile(`\s+`) +) + +// journalSentinelReplayAllowlist is every statement, by migration version, +// that the bd_events_journal sentinels' replay range (today: above floor 21) +// is permitted to carry, keyed by its whitespace-collapsed lowercase form. +// Each entry needs a justification here before it is added. +var journalSentinelReplayAllowlist = map[int][]string{ + 22: { + // Idempotent: INSERT IGNORE no-ops once bd_events_seq's single row + // (id=0) exists, on every replay after the first. + "insert ignore into bd_events_seq (id, next_seq) values (0, 0)", + // Monotone: identical in effect to issueops.healEventSeqCounter — a + // no-op when the counter is already at or ahead of the journal's + // high-water mark, and a raise to exactly MAX(seq) when it is behind. + // See TestSentinelReplayCounterConverges (embeddeddolt package) for + // the behavioral proof. + "update bd_events_seq set next_seq = greatest(next_seq, coalesce((select max(seq) from bd_events_journal), 0)) where id = 0", + }, +} + +// TestJournalSentinelReplayRangeDMLIsAllowlisted is T2.14: every ignored +// migration above the bd_events_journal sentinels' floor is a migration a +// sentinel contradiction can force to replay against a store that already +// has real data in it (section 3 of the A2 conflict decision). A DML +// statement in that range is therefore not merely "guarded" by an IF — it +// must also be safe to re-run against live rows, which an IF alone does not +// guarantee (an unconditional UPDATE inside a guarded ALTER branch would +// still re-fire every time the branch's condition holds). This pins the +// range to exactly the DML this decision reviewed, so a future migration +// added above the floor cannot silently introduce an unreviewed one. +// +// The floor is read from the registry, not hard-coded, so a future change to +// the sentinel values updates this test's scope automatically. +// +// Kills: any future ignored migration above the floor that adds DML which +// every journal-sentinel replay would silently re-run — for example a +// jv/txn backfill `UPDATE bd_events_journal ...` or an `UPDATE wisps ...` +// restamp (A4-A6 add ignored/0029-0031 inside this range). +func TestJournalSentinelReplayRangeDMLIsAllowlisted(t *testing.T) { + floor := 0 + haveFloor := false + for _, sc := range ignoredSource.sentinelColumns { + if sc.table != "bd_events_journal" { + continue + } + if !haveFloor || sc.replayFloor < floor { + floor = sc.replayFloor + } + haveFloor = true + } + if !haveFloor { + t.Fatal("no bd_events_journal sentinel column is registered in ignoredSource; this test has nothing to pin") + } + + for _, mf := range ignoredSource.list() { + if mf.version <= floor { + continue + } + blob, err := ignoredSource.files.ReadFile(ignoredSource.dir + "/" + mf.name) + if err != nil { + t.Fatalf("read %s: %v", mf.name, err) + } + + body := journalSentinelCommentLineRE.ReplaceAllString(string(blob), "") + body = journalSentinelOnUpdateRE.ReplaceAllString(body, " ") + body = journalSentinelOnDupKeyRE.ReplaceAllString(body, " ") + + for _, stmt := range strings.Split(body, ";") { + if !journalSentinelReplayDML.MatchString(stmt) { + continue + } + normalized := strings.ToLower(strings.TrimSpace(journalSentinelWhitespaceRE.ReplaceAllString(stmt, " "))) + if normalized == "" { + continue + } + allowed := false + for _, want := range journalSentinelReplayAllowlist[mf.version] { + if normalized == want { + allowed = true + break + } + } + if !allowed { + t.Errorf("ignored/%04d (above the bd_events_journal replay floor %d) carries DML not on the replay-safety allowlist:\n %s\nEvery journal-sentinel-triggered replay re-runs this statement against a live store; add it to journalSentinelReplayAllowlist with a justification, or make it a no-op above the floor.", + mf.version, floor, normalized) + } + } + } +} From 851db307ddae4726e1b5c53107475dd8ef897822 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 03:42:24 +0000 Subject: [PATCH 4/5] fix(schema): ignored/0028 comment_json step guards on table existence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ignored/0028's comment_json ALTER/MODIFY step probed only INFORMATION_SCHEMA.COLUMNS for comment_json's DATA_TYPE, which reads NULL both when the column is absent AND when bd_events_journal itself does not exist yet — the latter case fires a PREPARE/EXECUTE against a nonexistent table instead of taking the no-op branch. Guard the step on the table's own existence first, matching the explicit table-exists conjuncts the actor and index steps already use. TestIgnored0028NoopsWithoutItsTables (T2.5c) pins this: it drops bd_events_journal before running 0028 and requires the migration survive without creating the table out of nothing. Fails on the pre-fix migration with "table not found: bd_events_journal"; passes after the guard. Co-Authored-By: Claude Sonnet 5 --- .../scripts/embedded-storage-test-shards.txt | 1 + .../journal_shape_converge_test.go | 40 +++++++++++++++++++ .../0028_converge_events_journal_shape.up.sql | 27 +++++++------ 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 0933d102d..c7b767ee9 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -516,6 +516,7 @@ 15 14 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits 15 14 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor 15 14 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff +15 14 TestIgnored0028NoopsWithoutItsTables 15 14 TestMergeWithStrategy_TheirsResolvesAndCommits 15 14 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint 15 14 TestOpenAfterCloseCreatesNewStore diff --git a/internal/storage/embeddeddolt/journal_shape_converge_test.go b/internal/storage/embeddeddolt/journal_shape_converge_test.go index 3f3910ad2..eb141ba6b 100644 --- a/internal/storage/embeddeddolt/journal_shape_converge_test.go +++ b/internal/storage/embeddeddolt/journal_shape_converge_test.go @@ -389,6 +389,46 @@ func TestIgnored0028AloneLeavesCounterAndRowsUntouched(t *testing.T) { } } +// TestIgnored0028NoopsWithoutItsTables is the 0028 analogue of 0023's +// TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables +// (migrate_journal_shape_repair_test.go): 0028 must survive running against a +// workspace with no bd_events_journal table at all, the same intermediate +// state 0023 already has to survive. Kills: removing or omitting the +// table-exists conjunct from 0028's comment_json step. +func TestIgnored0028NoopsWithoutItsTables(t *testing.T) { + te := newTestEnv(t, "i28notab") + ctx := context.Background() + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Conn: %v", err) + } + defer conn.Close() + + execFrozenGuard(t, ctx, conn, "DROP TABLE IF EXISTS bd_events_journal") + + sqlText, err := schema.IgnoredMigrationSQL("0028_converge_events_journal_shape.up.sql") + if err != nil { + t.Fatalf("read ignored 0028 migration: %v", err) + } + execFrozenGuard(t, ctx, conn, sqlText) + + var got int + if err := conn.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'`).Scan(&got); err != nil { + t.Fatalf("probe bd_events_journal presence: %v", err) + } + if got != 0 { + t.Errorf("0028 created bd_events_journal out of nothing (count = %d); it must only ever repair an existing table", got) + } +} + // TestSentinelJournalColumnsReplay is T2.6: a store already at the latest // ignored cursor that loses comment_json out of band (a backup restored // between doors, or damage out of band) floors at exactly 21 and heals on the diff --git a/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql index e19b5b0c5..1b26d7ff5 100644 --- a/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql +++ b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql @@ -52,11 +52,12 @@ -- any bd_events_* table, unconditionally, so this file — not a main-plane -- twin — is the only place bd_events_journal's shape can converge. --- comment_json: add as LONGTEXT if missing entirely; widen if it is TEXT. --- The NULL branch is what 0023 never had: DATA_TYPE reads NULL only when the --- column (or the table) is absent, and the table-absent case is excluded by --- the replay-floor constraint above (0022, the table's creator, always runs --- first in any replay this migration participates in). +-- comment_json: add as LONGTEXT if missing; widen if TEXT. An absent TABLE +-- takes the no-op branch explicitly (a COLUMNS probe alone cannot tell "no +-- column" from "no table"), so this step never depends on 0022 having run +-- earlier in the same pass. +SET @journal_exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'); SET @comment_json_type = ( SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS @@ -64,11 +65,12 @@ SET @comment_json_type = ( AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = 'comment_json' ); -SET @sql = IF(@comment_json_type IS NULL, - 'ALTER TABLE bd_events_journal ADD COLUMN comment_json LONGTEXT', - IF(@comment_json_type = 'text', - 'ALTER TABLE bd_events_journal MODIFY COLUMN comment_json LONGTEXT', - 'SELECT 1')); +SET @sql = IF(@journal_exists = 0, 'SELECT 1', + IF(@comment_json_type IS NULL, + 'ALTER TABLE bd_events_journal ADD COLUMN comment_json LONGTEXT', + IF(@comment_json_type = 'text', + 'ALTER TABLE bd_events_journal MODIFY COLUMN comment_json LONGTEXT', + 'SELECT 1'))); PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; -- actor: add VARCHAR(255) NOT NULL DEFAULT '' if missing. ignored/0025 @@ -94,8 +96,9 @@ PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; -- dep_json: widen if TEXT. 0023's own step, repeated here (not removed there: -- shipped migrations are frozen, check C) for any lineage that reaches 0028 -- without ever having run it. Every creator of this table this repo or its --- fork lineage has ever shipped declares dep_json with SOME type, so NULL --- here can only mean "no table", which the replay floor rules out. +-- fork lineage has ever shipped declares dep_json with SOME type, so a +-- missing table yields NULL here too, which takes the no-op branch (the +-- comment_json step above makes the same table-absence case explicit). SET @dep_json_is_text = ( SELECT IF(DATA_TYPE = 'text', 1, 0) FROM INFORMATION_SCHEMA.COLUMNS From 9895023d85b2543fcbefe97fb1e9b514088365ee Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 03:45:57 +0000 Subject: [PATCH 5/5] docs(schema,doctor): state the actor sentinel's real blast radius and 0022's replay-safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three corrections, no behaviour change: - schema.go's sentinelColumns doc comment had the floor-21 safety argument backwards ("no unguarded statement in the series sits at or below it" — the floor's job is to keep 0007 OUT of the replay, not to claim nothing above it is unguarded). State the real invariant (0022's INSERT IGNORE seed and GREATEST raise are the only DML above the floor, and both are replay-safe) and the real blast radius: the actor sentinel alone fires on every production Dolt store at ignored cursor 24, replaying 22-28 once. - schema.go's floor constraint (i) and 0028's header get the same replay-safety note and a pointer to TestJournalSentinelReplayRangeDMLIsAllowlisted, which now pins it. - cmd/bd/doctor/journal_shape.go's counter-behind comment claimed a reopen "cannot repair" the finding; it is in fact repaired as a side effect whenever the same open replays ignored/0022 under any sentinel contradiction. The user-facing report-only text is unchanged in A2. Co-Authored-By: Claude Sonnet 5 --- cmd/bd/doctor/journal_shape.go | 19 +++++++++++-------- .../0028_converge_events_journal_shape.up.sql | 14 ++++++++++---- internal/storage/schema/schema.go | 18 +++++++++++++++--- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/cmd/bd/doctor/journal_shape.go b/cmd/bd/doctor/journal_shape.go index 682527a6d..c44667db4 100644 --- a/cmd/bd/doctor/journal_shape.go +++ b/cmd/bd/doctor/journal_shape.go @@ -108,14 +108,17 @@ func checkEventsJournalShapeWithDB(conn *doltConn) DoctorCheck { hasMissingColumns = true } - // The counter-behind finding is deliberately NOT claimed as --fix-able: - // ignored/0028's own header states it "never reads or writes - // bd_events_seq or a single journal row", so reopening the store (what - // fix.EventsJournalShape/DatabaseVersion does) cannot repair it. Reseeding - // the counter is a data correction, not a DDL convergence, and the plan's - // design note (f) only promises "--fix applies the converging DDL" — it - // says nothing about reseeding a counter, so this stays a report-only - // finding pending an explicit decision on safe reseed semantics. + // The counter-behind finding is deliberately NOT claimed as --fix-able, + // but that is a scoping choice, not a capability gap: `--fix` (what + // fix.EventsJournalShape/DatabaseVersion does) does not target this + // finding directly, yet it is repaired as a side effect whenever the same + // open replays ignored/0022 (any sentinel contradiction — see + // schema.go's bd_events_journal sentinels), by the same GREATEST raise + // the writer's own heal uses. A finding about the counter alone, with no + // column also missing, triggers no such replay. The plan's design note + // (f) only promises "--fix applies the converging DDL" — it says nothing + // about reseeding a counter on its own, so this stays a report-only + // finding in A2. Making it fixable on its own is an optional follow-up. counterBehind, counterDetail, err := journalCounterBehindMaxSeq(ctx, conn.db) if err != nil { findings = append(findings, fmt.Sprintf("could not compare bd_events_seq to MAX(seq): %v", err)) diff --git a/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql index 1b26d7ff5..be319e19a 100644 --- a/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql +++ b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql @@ -42,10 +42,16 @@ -- reads past 28 but is missing either column — e.g. restored from a backup -- taken between the two doors, or damaged out of band — heals itself on the -- next writable open instead of staying silently broken forever --- (TestSentinelJournalColumnsReplay). Floor 21 sits below this file (28) and --- below the table's creator (ignored/0022), per schema.go's two replay-floor --- constraints, and above every unguarded statement in the series --- (TestReplay22To28NoOpOnHealthyStore). +-- (TestSentinelJournalColumnsReplay). The actor sentinel alone also fires on +-- every store sitting at ignored cursor 22-27 without it, which is every +-- production Dolt store measured at cursor 24 (BEADS-JOURNAL-PLAN.md §4.1): +-- those stores replay 22-28 once, on their first open by a binary carrying +-- this sentinel. Floor 21 sits below this file (28) and below the table's +-- creator (ignored/0022), per schema.go's two replay-floor constraints, and +-- the only DML above it is 0022's INSERT IGNORE seed and GREATEST raise +-- (TestJournalSentinelReplayRangeDMLIsAllowlisted; +-- TestReplay22To28NoOpOnHealthyStore pins that nothing else in the range +-- touches any other table or row). -- -- Journal tables change only on this (ignored) plane: scripts/check- -- migration-hygiene.sh fails a main-plane migration that carries DDL against diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 253af5b80..a06a0e6ad 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -363,7 +363,9 @@ type schemaSentinelColumn struct { // restamping every wisp on a plane with no history to restore from // (#5981 harm class). ignored/0015 is the guarded successor and is // genuinely pending on those stores, so the recompute still happens - // — just without the timestamp damage. + // — just without the timestamp damage. Statements above the floor + // must be replay-safe, not merely guarded; see + // TestJournalSentinelReplayRangeDMLIsAllowlisted. // (ii) The migration that creates the column AND the migration that // creates the table carrying it must both be ABOVE the floor, so the // replay can still heal both shapes the single INFORMATION_SCHEMA @@ -413,8 +415,18 @@ var ( // silently broken forever (BEADS-JOURNAL-PLAN.md §4.2b, PR A2). Floor // 21 satisfies the same two constraints as the leases floor above: // 0022 (the table's creator) and 0028 (comment_json's adder; actor's - // adder is 0025, also above 21) both sit strictly above it, and no - // unguarded statement in the series sits at or below it. + // adder is 0025, also above 21) both sit strictly above it; + // ignored/0007's unguarded restamp sits below it, and the only DML + // above it is 0022's INSERT IGNORE seed and GREATEST raise + // (TestJournalSentinelReplayRangeDMLIsAllowlisted). + // + // The actor sentinel alone also fires on every store sitting at + // ignored cursor 22-27 that predates 0025, which is every production + // Dolt store measured at cursor 24 (BEADS-JOURNAL-PLAN.md §4.1): + // those stores replay 22-28 once, on their first open by a binary + // carrying this sentinel. 0022's tail is the healEventSeqCounter + // statement: a no-op on a counter already at or ahead of the + // journal, and a raise to exactly MAX(seq) on one that is behind. sentinelColumns: []schemaSentinelColumn{ {table: "leases", column: "granted_node", replayFloor: 11}, {table: "bd_events_journal", column: "comment_json", replayFloor: 21},