diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index f336184d7..c7b767ee9 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -154,8 +154,10 @@ 15 1 TestEmbeddedRoleFixtureKitHooksAreUsable 15 1 TestLifecycleCreateContract 15 1 TestMergeWithStrategy_OursKeepsLocal +15 1 TestStoreAtIgnored28WritesAndJournals 15 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase +15 2 TestAdaptiveInsert_GciShape 15 2 TestCleanGitRemoteCacheGarbage 15 2 TestCreateBatchFastPathsMatchPerRow_Embedded 15 2 TestDepRekeyMarkerStillInIgnoredSeries @@ -229,6 +231,7 @@ 15 4 TestIgnoredCursorHealNeverResurrectsRolledBackVersions 15 4 TestMain 15 4 TestMemoriesContract +15 4 TestMissingRequiredColumnRefusesAtOpen 15 4 TestPromoteClearsNoHistory 15 4 TestRemoteVerbPlainRemoteKeepsAmbientEnv 15 4 TestReopenIssue @@ -236,6 +239,7 @@ 15 4 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials 15 5 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase +15 5 TestAdaptiveRead_GciShape 15 5 TestBlockingAnnotatorContract 15 5 TestBootstrapperContract 15 5 TestBuildDSN_WithDatabase @@ -290,6 +294,7 @@ 15 6 TestRenameLabel 15 6 TestRenameLabelMintsOneVersionPerTouchedBead 15 6 TestRunInTransactionPostCommitStageFailureIsIndeterminate +15 6 TestSentinelReplayCounterConverges 15 6 TestStatsReporterContract 15 6 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials 15 6 TestWithPeerAuth_RestoresEnvWhenCallbackFails @@ -304,6 +309,7 @@ 15 7 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType 15 7 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane 15 7 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp +15 7 TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched 15 7 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender 15 7 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext 15 7 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet @@ -346,6 +352,7 @@ 15 8 TestEmbeddedSQLCommitResponseLossIsIndeterminate 15 8 TestGetReadyWorkIncludeEphemeralAppliesWispFilters 15 8 TestHasRepository +15 8 TestIgnored0028ConvergesAllShapes 15 8 TestLenientGateWarning_ComposedOutput 15 8 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle 15 8 TestReclaimExpiredLeaseSurvivesRestartEmbedded @@ -375,6 +382,7 @@ 15 9 TestMetadataCASContract 15 9 TestPromoteStillWorksWithCollisionGuard 15 9 TestReadyWorkWithMissingDeferredParentTables +15 9 TestReplay22To28NoOpOnHealthyStore 15 9 TestUpdateMergeOps_DoltEngine 15 10 TestBuildDSN_SpacesInPath @@ -403,6 +411,7 @@ 15 10 TestGetReadyWorkMoleculeFilter 15 10 TestHelperProcess 15 10 TestIgnoredCursorHealResumeRespectsOperatorOverride +15 10 TestJournalShapeProbeOncePerOpen 15 10 TestReclaimRefusesForeignReplicaLease 15 10 TestRemoteEntryPointsUseVersionControlOps 15 10 TestTransactionCleanupBeforeSQLCommitRemainsDefinite @@ -452,6 +461,7 @@ 15 12 TestEmbeddedRecomputeAllBlockedWiring 15 12 TestEmbeddedRelationsResolvesAWispAnchor 15 12 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges +15 12 TestIgnored0028AloneLeavesCounterAndRowsUntouched 15 12 TestLargeBatchApplyStatementCounts712_Embedded 15 12 TestLifecycleUpdateContract 15 12 TestPreviewStoreRefusesLocalMetadataWrite @@ -506,6 +516,7 @@ 15 14 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits 15 14 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor 15 14 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff +15 14 TestIgnored0028NoopsWithoutItsTables 15 14 TestMergeWithStrategy_TheirsResolvesAndCommits 15 14 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint 15 14 TestOpenAfterCloseCreatesNewStore @@ -539,3 +550,4 @@ 15 15 TestIgnoredCursorHealIsIdempotent 15 15 TestReadyWorkStatementBudget 15 15 TestReleaserContract +15 15 TestSentinelJournalColumnsReplay diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 69dd70cdf..7bf14a858 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -683,6 +683,7 @@ go_test( "doctor_health_smoke_test.go", "doctor_health_test.go", "doctor_init_diagnostics_cgo_test.go", + "doctor_journal_shape_test.go", "doctor_max_rows_test.go", "doctor_migrate_fix_test.go", "doctor_mutation_gate_test.go", @@ -1250,6 +1251,7 @@ go_test( deps = [ "//:beads", "//cmd/bd/doctor", + "//cmd/bd/doctor/fix", "//internal/ado", "//internal/beads", "//internal/ceiling", diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index 0640d3a9e..81ffd1c26 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -748,6 +748,13 @@ func runDiagnostics(path string) doctorResult { skewCheck := convertWithCategory(doctor.CheckMigrationContentSkew(sharedStore), doctor.CategoryData) result.Checks = append(result.Checks, skewCheck) + // Check 7g: Events journal shape (BEADS-JOURNAL-PLAN.md §4.2f, PR A2, T2.12). + journalShapeCheck := convertDoctorCheck(doctor.CheckEventsJournalShape(path)) + result.Checks = append(result.Checks, journalShapeCheck) + if journalShapeCheck.Status == statusError { + result.OverallOK = false + } + // Dolt health checks (connection, schema, issue count, status). for _, dc := range doctor.RunDoltHealthChecks(path) { result.Checks = append(result.Checks, convertDoctorCheck(dc)) diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index 5f541e3e8..0052f6ced 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -26,6 +26,7 @@ go_library( "hooks_migration.go", "installation.go", "integrity.go", + "journal_shape.go", "kv.go", "label_whitespace.go", "legacy.go", diff --git a/cmd/bd/doctor/fix/migrate.go b/cmd/bd/doctor/fix/migrate.go index d81cf3868..612c54b10 100644 --- a/cmd/bd/doctor/fix/migrate.go +++ b/cmd/bd/doctor/fix/migrate.go @@ -136,6 +136,16 @@ func SchemaCompatibility(path string) error { return DatabaseVersion(path) } +// EventsJournalShape fixes the "Events Journal Shape" doctor finding +// (BEADS-JOURNAL-PLAN.md §4.2f, PR A2: missing optional bd_events_journal +// columns, or a bd_events_seq counter behind bd_events_journal's MAX(seq)). +// The converging ignored-plane migration (ignored/0028) is applied as a side +// effect of DatabaseVersion's writable store open, the same as every other +// pending-migration fix in this file. +func EventsJournalShape(path string) error { + return DatabaseVersion(path) +} + // FreshCloneImport handles the "Fresh Clone" fix: imports JSONL issues into an // existing (possibly empty) Dolt store. This covers the case where the Database // fix already created the store but a prior version didn't import. diff --git a/cmd/bd/doctor/fix_gate.go b/cmd/bd/doctor/fix_gate.go index 213f2a9bb..d0e5410f4 100644 --- a/cmd/bd/doctor/fix_gate.go +++ b/cmd/bd/doctor/fix_gate.go @@ -243,6 +243,8 @@ func SanitizeFixRecommendation(fix string, gate FixGate) string { // audited against that rule. The ones deliberately left out open a store, so // being withheld under a blocked gate is correct rather than an omission: // "Database", "Database Integrity", "Fresh Clone", "Schema Compatibility", +// "Events Journal Shape" (opens the store, applying the converging +// ignored-plane migration as a side effect, same as Schema Compatibility), // "Repo Fingerprint" (fix.RepoFingerprint and fix.FixMissingMetadata both write // through the store), "Dolt Schema", "Pending Migrations", and the row-level // data repairs. "Sync Divergence", "JSONL Config" and "Untracked Files" are diff --git a/cmd/bd/doctor/journal_shape.go b/cmd/bd/doctor/journal_shape.go new file mode 100644 index 000000000..c44667db4 --- /dev/null +++ b/cmd/bd/doctor/journal_shape.go @@ -0,0 +1,240 @@ +package doctor + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// journalOptionalColumns mirrors the optional-column list +// internal/storage/issueops/journal_shape.go's ProbeJournalShape treats as +// optional (BEADS-JOURNAL-PLAN.md §4.2a, PR A1). It is duplicated here +// rather than imported: this check reports over doctor's own plain +// *sql.DB (openDoltConn), and ProbeJournalShape prints its own +// once-per-probe STDERR warning as a side effect, which would duplicate +// this check's finding on every `bd doctor` run. +var journalOptionalColumns = []string{"actor", "issue_json", "dep_json", "comment_json"} + +// CheckEventsJournalShape is the standalone entry point for the "Events +// Journal Shape" doctor check (T2.12, BEADS-JOURNAL-PLAN.md §4.2f, PR A2). +// It opens its own connection rather than joining RunDoltHealthChecks' +// shared one, the same standalone shape as CheckDoltRemoteGitOrigin and +// CheckMigrationContentSkew, so it can be registered in doctor.go beside +// them without touching runDoltHealthChecksInternal's fixed-shape branches. +func CheckEventsJournalShape(path string) DoctorCheck { + beadsDir := ResolveBeadsDirForRepo(path) + + if !IsDoltBackend(beadsDir) { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "N/A (non-Dolt backend)", + Category: CategoryDolt, + } + } + + conn, err := openDoltConn(beadsDir) + if err != nil { + // Matches runDoltHealthChecksInternal's own no-server-running + // tolerance: a store that hasn't been opened yet (server not started) + // is not a journal-shape finding. + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "Skipped (no server running; will auto-start on next bd command)", + Category: CategoryDolt, + } + } + defer conn.Close() + + return checkEventsJournalShapeWithDB(conn) +} + +// checkEventsJournalShapeWithDB is T2.12 (BEADS-JOURNAL-PLAN.md §4.2f, PR +// A2): "bd doctor 'events journal shape'" reports: +// - optional columns bd_events_journal is still missing. Adaptive I/O (PR +// A1) keeps a table in this state writable and readable, dropping only +// the affected payloads, but a dropped payload is still a finding — the +// plan's own risk note says to "keep the doctor finding and a +// once-per-store warning, never silence". +// - the bd_events_seq counter sitting behind bd_events_journal's own +// MAX(seq): the counter must never trail what has already been durably +// written (nextEventSeq increments it inside the same transaction as the +// row that carries its new value), so this can only mean the counter was +// lost and reseeded (e.g. restored from a stale backup) without the rows +// that came after it. +// +// The plan's design note (f) also lists "outbox depth and the age of its +// oldest row" and "a NULL epoch". Those refer to bd_events_outbox and +// bd_events_seq.epoch, introduced by PRs A5/A6 (§4.3's own table scopes +// T2.10/T2.11 to Postgres/PR B1 and T2.13 to PR A12; A5/A6 are likewise a +// separate, later plan item, §5) — neither object exists in this schema yet, +// so there is nothing to probe. Once they land, this is the function to +// extend. +func checkEventsJournalShapeWithDB(conn *doltConn) DoctorCheck { + ctx := context.Background() + + exists, err := tableExists(ctx, conn.db, "bd_events_journal") + if err != nil { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusWarning, + Message: "Could not probe bd_events_journal", + Detail: err.Error(), + Category: CategoryDolt, + } + } + if !exists { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "N/A (bd_events_journal not created yet)", + Category: CategoryDolt, + } + } + + var findings []string + var hasMissingColumns, hasUnfixableFinding bool + + missingOptional, err := missingJournalOptionalColumns(ctx, conn.db) + if err != nil { + findings = append(findings, fmt.Sprintf("could not read bd_events_journal's columns: %v", err)) + hasUnfixableFinding = true + } else if len(missingOptional) > 0 { + findings = append(findings, fmt.Sprintf( + "bd_events_journal is missing optional column(s): %s (affected payloads are dropped on write)", + strings.Join(missingOptional, ", "))) + hasMissingColumns = true + } + + // The counter-behind finding is deliberately NOT claimed as --fix-able, + // but that is a scoping choice, not a capability gap: `--fix` (what + // fix.EventsJournalShape/DatabaseVersion does) does not target this + // finding directly, yet it is repaired as a side effect whenever the same + // open replays ignored/0022 (any sentinel contradiction — see + // schema.go's bd_events_journal sentinels), by the same GREATEST raise + // the writer's own heal uses. A finding about the counter alone, with no + // column also missing, triggers no such replay. The plan's design note + // (f) only promises "--fix applies the converging DDL" — it says nothing + // about reseeding a counter on its own, so this stays a report-only + // finding in A2. Making it fixable on its own is an optional follow-up. + counterBehind, counterDetail, err := journalCounterBehindMaxSeq(ctx, conn.db) + if err != nil { + findings = append(findings, fmt.Sprintf("could not compare bd_events_seq to MAX(seq): %v", err)) + hasUnfixableFinding = true + } else if counterBehind { + findings = append(findings, counterDetail+" (not repaired by 'bd doctor --fix'; needs manual investigation)") + hasUnfixableFinding = true + } + + if len(findings) == 0 { + return DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusOK, + Message: "bd_events_journal shape converged", + Category: CategoryDolt, + } + } + + check := DoctorCheck{ + Name: "Events Journal Shape", + Status: StatusWarning, + Message: fmt.Sprintf("%d finding(s)", len(findings)), + Detail: strings.Join(findings, "\n"), + Category: CategoryDolt, + } + switch { + case hasMissingColumns && hasUnfixableFinding: + check.Fix = "Run 'bd doctor --fix' to add the missing column(s); the remaining finding(s) above need manual investigation" + case hasMissingColumns: + check.Fix = "Run 'bd doctor --fix' to apply the converging ignored-plane migration (adds comment_json/actor, widens columns, creates indexes)" + } + return check +} + +// tableExists reports whether table exists in the connected database. A +// read-only probe (COUNT with LIMIT 1), matching checkSchemaWithDB's idiom +// elsewhere in this package. +func tableExists(ctx context.Context, db *sql.DB, table string) (bool, error) { + var count int + err := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s LIMIT 1", table)).Scan(&count) //nolint:gosec // table is one of this file's own constant strings, never user input + if err != nil { + // A missing table is not this function's error to report: every other + // SQL failure is. + if strings.Contains(strings.ToLower(err.Error()), "doesn't exist") || + strings.Contains(strings.ToLower(err.Error()), "not found") || + strings.Contains(strings.ToLower(err.Error()), "unknown table") { + return false, nil + } + return false, err + } + return true, nil +} + +// missingJournalOptionalColumns runs the same INFORMATION_SCHEMA.COLUMNS +// probe internal/storage/issueops.ProbeJournalShape uses (over a plain +// connection, with no side-effecting warning), and returns which of +// journalOptionalColumns bd_events_journal does not carry. +func missingJournalOptionalColumns(ctx context.Context, db *sql.DB) ([]string, error) { + rows, err := db.QueryContext(ctx, ` + SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'`) + if err != nil { + return nil, err + } + defer rows.Close() + + present := make(map[string]bool, len(journalOptionalColumns)) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + present[strings.ToLower(name)] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + + var missing []string + for _, col := range journalOptionalColumns { + if !present[col] { + missing = append(missing, col) + } + } + return missing, nil +} + +// journalCounterBehindMaxSeq reports whether bd_events_seq's counter trails +// the highest seq actually recorded in bd_events_journal. bd_events_seq may +// not exist on a database that predates ignored/0019 (the events table) or +// has never journaled; that reads as "not behind" (nothing to compare), +// never as a finding. +func journalCounterBehindMaxSeq(ctx context.Context, db *sql.DB) (bool, string, error) { + counterExists, err := tableExists(ctx, db, "bd_events_seq") + if err != nil { + return false, "", err + } + if !counterExists { + return false, "", nil + } + + var counter, maxSeq int64 + err = db.QueryRowContext(ctx, ` + SELECT + (SELECT COALESCE(next_seq, 0) FROM bd_events_seq WHERE id = 0), + (SELECT COALESCE(MAX(seq), 0) FROM bd_events_journal) + `).Scan(&counter, &maxSeq) + if err != nil { + return false, "", err + } + + if counter < maxSeq { + return true, fmt.Sprintf( + "bd_events_seq counter (%d) is behind bd_events_journal's MAX(seq) (%d): "+ + "the counter was likely lost and reseeded without the rows written after it", + counter, maxSeq), nil + } + return false, "", nil +} diff --git a/cmd/bd/doctor_fix.go b/cmd/bd/doctor_fix.go index 89b73532c..33ba323fd 100644 --- a/cmd/bd/doctor_fix.go +++ b/cmd/bd/doctor_fix.go @@ -403,6 +403,8 @@ func applyFixList(path string, fixes []doctorCheck) { err = fix.DatabaseIntegrity(path) case "Schema Compatibility": err = fix.SchemaCompatibility(path) + case "Events Journal Shape": + err = fix.EventsJournalShape(path) case "Repo Fingerprint": err = fix.RepoFingerprint(path, doctorYes) // Also repair any other missing metadata fields (bd_version, repo_id, clone_id) diff --git a/cmd/bd/doctor_journal_shape_test.go b/cmd/bd/doctor_journal_shape_test.go new file mode 100644 index 000000000..584fb98c2 --- /dev/null +++ b/cmd/bd/doctor_journal_shape_test.go @@ -0,0 +1,177 @@ +//go:build cgo + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/steveyegge/beads/cmd/bd/doctor" + "github.com/steveyegge/beads/cmd/bd/doctor/fix" + "github.com/steveyegge/beads/internal/types" +) + +// findDoctorCheck returns the check named name from checks, or nil. +func findDoctorCheck(checks []doctorCheck, name string) *doctorCheck { + for i := range checks { + if checks[i].Name == name { + return &checks[i] + } + } + return nil +} + +// TestDoctorJournalShape is T2.12 (BEADS-JOURNAL-PLAN.md §4.2f, PR A2): +// "bd doctor" raises an "Events Journal Shape" finding for a bd_events_journal +// missing an optional column, and `--fix` (fix.EventsJournalShape) clears it +// by applying the converging ignored-plane migration (ignored/0028). A +// counter behind bd_events_journal's own MAX(seq) is also raised, but is +// deliberately NOT claimed as --fix-able: ignored/0028's own header states it +// "never reads or writes bd_events_seq or a single journal row", and when no +// column is also missing there is no sentinel contradiction to float the +// ignored cursor's floor and force the replay that would otherwise touch it +// (ignored/0022's own idempotent high-water-mark raise, re-run only as part +// of that replay). +// +// Scope: this test covers only what A1/A2 actually built and what exists in +// this schema today. The plan's design note (f) also lists "outbox depth and +// the age of its oldest row" and "a NULL epoch"; those reference +// bd_events_outbox and bd_events_seq.epoch, introduced by PRs A5/A6 (§5, a +// separate, later plan item) — neither object exists yet, so there is +// nothing for this test (or the check itself) to exercise for them. +// +// The two sub-findings are tested from independently damaged stores (not one +// store carrying both faults at once): dropping comment_json ALSO floors the +// ignored cursor via its sentinel and forces a 22..28 replay on the next +// writable open, and that replay's pass through ignored/0022 re-raises +// bd_events_seq to GREATEST(current, MAX(seq)) as an unrelated side effect — +// entangling the two findings' fixability would make the counter case look +// fixable only because a *different* finding's repair happened to touch it. +// +// Kills: the check not wired into doctor (runDiagnostics never calling +// doctor.CheckEventsJournalShape) — confirmed via the clean-store sub-test +// below, which goes through runDiagnostics itself rather than calling the +// check function directly, and manually by temporarily removing doctor.go's +// wiring call and observing it fail with "check not found" before restoring +// it; the check function itself going unimplemented (a compile failure); +// and --fix not actually converging the column shape (the post-fix re-probe +// would still see missing comment_json). +func TestDoctorJournalShape(t *testing.T) { + t.Run("wired into runDiagnostics on a healthy store", func(t *testing.T) { + tmpDir, store := setupValidateTestDB(t, "jnlok") + ctx := context.Background() + store.SetEventsJournalEnabled(true) + if err := store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation: %v", err) + } + issue := &types.Issue{Title: "t2.12 wiring seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "jnlok"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + store.Close() + + result := runDiagnostics(tmpDir) + check := findDoctorCheck(result.Checks, "Events Journal Shape") + if check == nil { + t.Fatal("Events Journal Shape check not found in runDiagnostics' output") + } + if check.Status != statusOK { + t.Errorf("status = %q, want %q (message: %s, detail: %s)", check.Status, statusOK, check.Message, check.Detail) + } + }) + + t.Run("missing optional column is raised and fixed", func(t *testing.T) { + tmpDir, store := setupValidateTestDB(t, "jnlcol") + ctx := context.Background() + store.SetEventsJournalEnabled(true) + if err := store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation: %v", err) + } + issue := &types.Issue{Title: "t2.12 column seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "jnlcol"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + // Damage exactly as the plan's evidence base found it (gas-city-inc: + // comment_json absent outright). + if _, err := store.DB().ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil { + t.Fatalf("drop comment_json: %v", err) + } + store.Close() + + check := doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusWarning { + t.Fatalf("after damage: status = %q, want %q (message: %s)", check.Status, statusWarning, check.Message) + } + if !strings.Contains(check.Detail, "comment_json") { + t.Errorf("after damage: Detail = %q, want it to mention comment_json", check.Detail) + } + if strings.Contains(check.Detail, "bd_events_seq counter") { + t.Errorf("after damage: Detail = %q, unexpectedly mentions the counter (this store never touched it)", check.Detail) + } + if check.Fix == "" || !strings.Contains(check.Fix, "bd doctor --fix") { + t.Errorf("after damage: Fix = %q, want guidance to run 'bd doctor --fix'", check.Fix) + } + + if err := fix.EventsJournalShape(tmpDir); err != nil { + t.Fatalf("fix.EventsJournalShape: %v", err) + } + + check = doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusOK { + t.Fatalf("after fix: status = %q, want %q (message: %s, detail: %s)", check.Status, statusOK, check.Message, check.Detail) + } + }) + + t.Run("counter behind MAX(seq) is raised and not claimed fixable", func(t *testing.T) { + tmpDir, store := setupValidateTestDB(t, "jnlseq") + ctx := context.Background() + store.SetEventsJournalEnabled(true) + if err := store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation: %v", err) + } + issue := &types.Issue{Title: "t2.12 counter seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "jnlseq"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + // Damage ONLY the counter (no column touched, so no sentinel + // contradiction floors the ignored cursor and nothing replays + // ignored/0022 on the next writable open). + if _, err := store.DB().ExecContext(ctx, "UPDATE bd_events_seq SET next_seq = 0 WHERE id = 0"); err != nil { + t.Fatalf("reseed next_seq behind MAX(seq): %v", err) + } + store.Close() + + check := doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusWarning { + t.Fatalf("after damage: status = %q, want %q (message: %s)", check.Status, statusWarning, check.Message) + } + if !strings.Contains(check.Detail, "bd_events_seq counter") { + t.Errorf("after damage: Detail = %q, want it to mention the counter behind MAX(seq)", check.Detail) + } + if strings.Contains(check.Detail, "comment_json") { + t.Errorf("after damage: Detail = %q, unexpectedly mentions comment_json (this store never touched it)", check.Detail) + } + if !strings.Contains(check.Detail, "not repaired by 'bd doctor --fix'") { + t.Errorf("after damage: Detail = %q, want it to disclose this finding is not --fix-able", check.Detail) + } + if strings.Contains(check.Fix, "bd doctor --fix") { + t.Errorf("after damage: Fix = %q, must not claim 'bd doctor --fix' resolves a counter-behind finding", check.Fix) + } + + if err := fix.EventsJournalShape(tmpDir); err != nil { + t.Fatalf("fix.EventsJournalShape: %v", err) + } + + check = doctor.CheckEventsJournalShape(tmpDir) + if check.Status != statusWarning { + t.Fatalf("after fix: status = %q, want %q (no pending migration exists to repair a counter-only fault, so --fix is a no-op here): message=%s detail=%s", + check.Status, statusWarning, check.Message, check.Detail) + } + if !strings.Contains(check.Detail, "bd_events_seq counter") { + t.Errorf("after fix: Detail = %q, want the still-unrepaired counter finding to remain", check.Detail) + } + }) +} diff --git a/internal/eventsjournal/activation.go b/internal/eventsjournal/activation.go index 1af97cd4c..ad98baf83 100644 --- a/internal/eventsjournal/activation.go +++ b/internal/eventsjournal/activation.go @@ -146,6 +146,12 @@ func ActivateStore(beadsDir string, s storage.DoltStorage, err error) (storage.D // the journal is that a consumer can trust its cursor, and a plumbing that // records nothing while reporting success is the one outcome that breaks that // trust invisibly. A disabled workspace accepts any plumbing. +// +// It also fails there when the configurer DOES support the journal but its +// table's shape does not (storage.EventsJournalShapeChecker, PR A1): a missing +// required column refuses at open, the same way a nil configurer does, rather +// than letting SetEventsJournalEnabled succeed and every write fail +// afterward — the gci failure mode this check replaces. func Apply(configurer storage.EventsJournalConfigurer, enabled bool) error { if configurer == nil { if enabled { @@ -154,5 +160,10 @@ func Apply(configurer storage.EventsJournalConfigurer, enabled bool) error { return nil } configurer.SetEventsJournalEnabled(enabled) + if checker, ok := configurer.(storage.EventsJournalShapeChecker); ok { + if err := checker.EventsJournalActivationError(); err != nil { + return err + } + } return nil } diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index c05633734..2a6b638cb 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -308,6 +308,7 @@ var _ storage.Compactor = (*DoltStore)(nil) var _ storage.SchemaMigrator = (*DoltStore)(nil) var _ storage.ExternalRefHistoryQuerier = (*DoltStore)(nil) var _ storage.EventsJournalConfigurer = (*DoltStore)(nil) +var _ storage.EventsJournalShapeChecker = (*DoltStore)(nil) var _ storage.VersionedHistoryConfigurer = (*DoltStore)(nil) // DoltStore implements the Storage interface using Dolt @@ -320,6 +321,17 @@ type DoltStore struct { // eventsJournalEnabled activates the durable events journal for THIS store // instance only (storage.EventsJournalConfigurer); never process-global. eventsJournalEnabled atomic.Bool + // journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled + // runs on activation (issueops.ProbeJournalShape, PR A1). Nil means either + // the journal is disabled or the probe failed; scopeEventsJournalTransaction + // treats nil the same as "do not journal this transaction" — see + // journalActivationErr. + journalShape atomic.Pointer[issueops.JournalShape] + // journalActivationErr holds the probe's error when SetEventsJournalEnabled(true) + // found a table shape the journal cannot run against (a missing required + // column, or a missing table). Nil whenever journalShape is non-nil or the + // journal is disabled. See storage.EventsJournalShapeChecker. + journalActivationErr atomic.Pointer[error] // versionedHistoryEnabled activates dual-write issue-version history for // THIS store instance only (storage.VersionedHistoryConfigurer); never // process-global. @@ -1083,6 +1095,13 @@ func (s *DoltStore) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) e return fmt.Errorf("begin read tx: %w", err) } defer func() { _ = tx.Rollback() }() + // ReadEventsJournal/ReadEventsJournalPage read through this helper, and + // the column list they SELECT depends on the probed shape, not on + // whether the journal is enabled for writes — scope the shape only, so + // readEventsRowsInTx sees this store's actual table instead of + // defaulting to canonicalJournalShape (PR A1). + clearShape := issueops.ScopeEventsJournalShape(tx, s.journalShape.Load()) + defer clearShape() return fn(tx) }) } @@ -1379,12 +1398,56 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche } // SetEventsJournalEnabled activates the journal for this store instance only. +// +// Activating probes bd_events_journal's actual shape ONCE +// (issueops.ProbeJournalShape, PR A1) and caches the result for this +// instance's lifetime; EventsJournalActivationError reports whether that +// probe found a shape the journal can actually run against. Deactivating +// clears both and skips the probe entirely — a disabled store accepts any +// shape. func (s *DoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) + if !enabled { + s.journalShape.Store(nil) + s.journalActivationErr.Store(nil) + return + } + shape, err := issueops.ProbeJournalShape(context.Background(), s.db) + if err != nil { + s.journalShape.Store(nil) + s.journalActivationErr.Store(&err) + return + } + s.journalShape.Store(shape) + s.journalActivationErr.Store(nil) } +// EventsJournalActivationError implements storage.EventsJournalShapeChecker: +// nil unless the last SetEventsJournalEnabled(true) call found a table shape +// the journal cannot run against. +func (s *DoltStore) EventsJournalActivationError() error { + if p := s.journalActivationErr.Load(); p != nil { + return *p + } + return nil +} + +// scopeEventsJournalTransaction binds this store's cached activation state and +// probed shape to tx. The activation switch folds in the shape probe's +// success: a probe that found an unsupported table (journalShape nil) scopes +// the transaction as DISABLED regardless of what SetEventsJournalEnabled was +// called with, so a caller that enables the journal directly and ignores the +// returned EventsJournalActivationError still never attempts a write against +// a shape that cannot support it (storage.EventsJournalShapeChecker). func (s *DoltStore) scopeEventsJournalTransaction(tx *sql.Tx) func() { - return issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) + shape := s.journalShape.Load() + enabled := s.eventsJournalEnabled.Load() && shape != nil + clearActivation := issueops.ScopeEventsJournalTransaction(tx, enabled) + clearShape := issueops.ScopeEventsJournalShape(tx, shape) + return func() { + clearShape() + clearActivation() + } } // SetVersionedHistoryEnabled activates dual-write issue-version history for diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index 8bebf0b9f..d6e9528ba 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -222,7 +222,14 @@ func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg s // exists to make impossible. In journal mode both planes therefore share the // pinned regular transaction. The default journal-off path keeps the // established split transactions untouched. - journalEnabled := s.eventsJournalEnabled.Load() + // journalShape is folded into journalEnabled the same way + // scopeEventsJournalTransaction folds it in for every other raw-tx mutator + // (store.go): a probe that found an unsupported shape (journalShape nil) + // must pin neither plane together nor attempt a write here, even if a + // caller enabled the journal directly and ignored the returned + // EventsJournalActivationError (storage.EventsJournalShapeChecker, PR A1). + journalShape := s.journalShape.Load() + journalEnabled := s.eventsJournalEnabled.Load() && journalShape != nil ignoredTx := regularTx if !journalEnabled { // NOTE (GH#3140 metrics skew): the pool-wait bracket above measures only @@ -241,6 +248,16 @@ func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg s } clearJournalScope := issueops.ScopeEventsJournalTransaction(regularTx, journalEnabled) defer clearJournalScope() + // Shape counterpart of the activation switch just above (PR A1): + // insertEventRow/readEventsRowsInTx reach this store's probed shape only + // through a tx-scoped value, never through the store itself, so without + // this call every write on this path — CreateIssue, UpdateIssue, comments, + // the whole doltTransaction mutator surface reached from + // runDoltTransactionRecording — would silently fall back to + // canonicalJournalShape and defeat adaptive I/O for this store's main + // write path. + clearJournalShape := issueops.ScopeEventsJournalShape(regularTx, journalShape) + defer clearJournalShape() // Versioned history binds to the SAME regular transaction the mutation runs // in, for the same reason the journal does: RecordVersionInTx no-ops unless // this scope is set. Without it every issues-plane mutation routed through diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 9a4057072..cd3b9a6b0 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -156,6 +156,8 @@ go_test( "journal_concurrency_test.go", "journal_contract_test.go", "journal_retention_test.go", + "journal_shape_adaptive_test.go", + "journal_shape_converge_test.go", "journal_truncation_test.go", "label_list_all_benchmark_test.go", "label_rename_test.go", diff --git a/internal/storage/embeddeddolt/journal_shape_adaptive_test.go b/internal/storage/embeddeddolt/journal_shape_adaptive_test.go new file mode 100644 index 000000000..81ce8e56a --- /dev/null +++ b/internal/storage/embeddeddolt/journal_shape_adaptive_test.go @@ -0,0 +1,218 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// Adaptive I/O for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2a/§4.3, PR +// A1). These tests reproduce gas-city-inc's exact shape — a table built +// before `comment_json` and `actor` ever existed — directly on an +// otherwise-ordinary embedded store, and prove the writer and reader adapt to +// it instead of failing every journaled write (T2.1/T2.2), that adaptation +// probes the shape once per activation rather than once per write (T2.3), +// and that a table missing a REQUIRED column refuses at open rather than +// silently disabling the journal or failing per write (T2.4). + +// newGciShapeStore returns a store whose bd_events_journal has had +// comment_json and actor dropped, reproducing gas-city-inc's shape +// (BEADS-JOURNAL-PLAN.md §1.1: "gci.bd_events_journal has the columns seq, +// ts, op, issue_id, issue_json, dep_json" — no actor, no comment_json). The +// journal is NOT activated; callers that want to observe activation itself +// (the probe, its warning, its cached result) call SetEventsJournalEnabled +// themselves. +func newGciShapeStore(t *testing.T, prefix string) *testEnv { + t.Helper() + te := newTestEnv(t, prefix) + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json") + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor") + return te +} + +// TestAdaptiveInsert_GciShape is T2.1: on the gci shape, create, update and +// comment all succeed, the rows land, and the comment payload is dropped with +// exactly one warning. A fixed INSERT column list (the pre-A1 behavior) fails +// every one of these three writes, because each one names either actor or +// comment_json. +func TestAdaptiveInsert_GciShape(t *testing.T) { + te := newGciShapeStore(t, "ai") + ctx := context.Background() + + stderr := captureStderr(t, func() { + te.store.SetEventsJournalEnabled(true) + }) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation on the gci shape (missing only OPTIONAL columns) must succeed, got: %v", err) + } + if n := strings.Count(stderr, "events journal:"); n != 1 { + t.Fatalf("activation warning count = %d, want exactly 1 (one warning per store, not per write):\n%s", n, stderr) + } + for _, want := range []string{"actor", "comment_json"} { + if !strings.Contains(stderr, want) { + t.Errorf("activation warning does not name missing column %q:\n%s", want, stderr) + } + } + + issue := &types.Issue{Title: "gci create", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue on gci shape: %v", err) + } + if err := te.store.UpdateIssue(ctx, issue.ID, map[string]interface{}{"title": "gci updated"}, "tester"); err != nil { + t.Fatalf("UpdateIssue on gci shape: %v", err) + } + if _, err := te.store.AddIssueComment(ctx, issue.ID, "tester", "gci comment"); err != nil { + t.Fatalf("AddIssueComment on gci shape: %v", err) + } + + rows, err := te.store.ReadEventsJournal(ctx, 0, 0) + if err != nil { + t.Fatalf("ReadEventsJournal on gci shape: %v", err) + } + var ops []string + for _, r := range rows { + if r.IssueID != issue.ID { + continue + } + ops = append(ops, r.Op) + // Dropped, not merely blank: the column does not exist on this shape, + // so the adaptive reader must never have tried to select it. + if r.Actor != "" { + t.Errorf("row op=%s: Actor = %q, want \"\" (column does not exist on this shape)", r.Op, r.Actor) + } + if r.CommentJSON != "" { + t.Errorf("row op=%s: CommentJSON = %q, want \"\" (column does not exist on this shape)", r.Op, r.CommentJSON) + } + } + wantOps := []string{"create", "update", "comment"} + if len(ops) != len(wantOps) { + t.Fatalf("journaled ops for %s = %v, want %v (a fixed INSERT column list fails every one of these on the gci shape)", issue.ID, ops, wantOps) + } + for i, want := range wantOps { + if ops[i] != want { + t.Errorf("journaled op[%d] = %q, want %q", i, ops[i], want) + } + } +} + +// TestAdaptiveRead_GciShape is T2.2: a page read on the gci shape succeeds, +// and actor/comment come back empty rather than failing the SELECT. A fixed +// SELECT list (the pre-A1 behavior) fails the read outright, because it names +// both actor and comment_json. +func TestAdaptiveRead_GciShape(t *testing.T) { + te := newGciShapeStore(t, "ar") + ctx := context.Background() + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation on the gci shape: %v", err) + } + + issue := &types.Issue{ID: "ar-1", Title: "gci read", Status: types.StatusOpen, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue on gci shape: %v", err) + } + + page, err := te.store.ReadEventsJournalPage(ctx, 0, 0) + if err != nil { + t.Fatalf("ReadEventsJournalPage on gci shape: %v", err) + } + if len(page.Rows) == 0 { + t.Fatal("ReadEventsJournalPage on gci shape returned no rows") + } + for _, r := range page.Rows { + if r.Actor != "" { + t.Errorf("row op=%s: Actor = %q, want \"\" on a shape with no actor column", r.Op, r.Actor) + } + if r.CommentJSON != "" { + t.Errorf("row op=%s: CommentJSON = %q, want \"\" on a shape with no comment_json column", r.Op, r.CommentJSON) + } + } +} + +// TestJournalShapeProbeOncePerOpen is T2.3: 100 writes against one already- +// activated store issue no INFORMATION_SCHEMA probes of their own — only the +// ONE SetEventsJournalEnabled ran. A per-write probe would show up here as a +// nonzero delta across the 100 writes. +func TestJournalShapeProbeOncePerOpen(t *testing.T) { + te := newTestEnv(t, "pr") + ctx := context.Background() + + before := issueops.JournalShapeProbeCountForTest() + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation on canonical shape: %v", err) + } + afterActivate := issueops.JournalShapeProbeCountForTest() + if delta := afterActivate - before; delta != 1 { + t.Fatalf("probes issued by SetEventsJournalEnabled = %d, want exactly 1", delta) + } + + for i := 0; i < 100; i++ { + issue := &types.Issue{ + Title: fmt.Sprintf("probe-once %d", i), Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, + } + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue %d: %v", i, err) + } + } + + after := issueops.JournalShapeProbeCountForTest() + if delta := after - afterActivate; delta != 0 { + t.Fatalf("probes issued by 100 writes against an already-activated store = %d, want 0 (a per-write probe, not a per-open probe)", delta) + } +} + +// TestMissingRequiredColumnRefusesAtOpen is T2.4: a table missing a REQUIRED +// column (op) makes activation fail with a typed error when the journal is +// enabled, but leaves it succeeding when disabled — a disabled workspace +// accepts any shape. It also pins that a caller who enables the journal +// anyway (ignoring the returned activation error) still never attempts the +// INSERT: the mutation itself must not fail, which is the whole point of +// moving this failure to open time instead of per write. +func TestMissingRequiredColumnRefusesAtOpen(t *testing.T) { + te := newTestEnv(t, "mr") + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN op") + + te.store.SetEventsJournalEnabled(true) + err := te.store.EventsJournalActivationError() + if err == nil { + t.Fatal("activation with the journal enabled against a table missing a required column (op) succeeded; want a typed error") + } + if !errors.Is(err, issueops.ErrJournalShapeUnsupported) { + t.Errorf("activation error = %v, does not match issueops.ErrJournalShapeUnsupported", err) + } + var shapeErr *issueops.JournalShapeError + if !errors.As(err, &shapeErr) { + t.Fatalf("activation error is not a *issueops.JournalShapeError: %v (%T)", err, err) + } + if len(shapeErr.Missing) != 1 || shapeErr.Missing[0] != "op" { + t.Errorf("JournalShapeError.Missing = %v, want [op]", shapeErr.Missing) + } + + // Disabling must still succeed: a disabled workspace accepts any shape, and + // never runs the probe that would otherwise fail it. + te.store.SetEventsJournalEnabled(false) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Errorf("activation with the journal disabled against the same broken shape failed: %v", err) + } + + // Re-enable (probe fails again, same cached nil shape) and prove the + // mutation itself still lands: this is the gci failure mode A1 exists to + // close, where a per-write INSERT against a missing column rolled back the + // user's own write. + te.store.SetEventsJournalEnabled(true) + issue := &types.Issue{Title: "no op column", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue must still succeed even though the journal cannot activate on this shape: %v", err) + } +} diff --git a/internal/storage/embeddeddolt/journal_shape_converge_test.go b/internal/storage/embeddeddolt/journal_shape_converge_test.go new file mode 100644 index 000000000..eb141ba6b --- /dev/null +++ b/internal/storage/embeddeddolt/journal_shape_converge_test.go @@ -0,0 +1,848 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "fmt" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/types" +) + +// Shape convergence for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2b/§4.3, +// PR A2). ignored/0028 closes the exact gap ignored/0023 left open (probing +// DATA_TYPE alone cannot see a column that is entirely absent): these tests +// reproduce every historical shape the plan's evidence base found, prove each +// one converges on canonical without perturbing a single row or the seq +// counter (T2.5), pin the sentinel-triggered replay's floor at exactly 21 +// (T2.6), and pin that the same replay is a true no-op everywhere outside +// bd_events_journal itself when nothing else is damaged (T2.7). + +// newShape3666e5026Store reproduces the events-journal-3666e5026 release +// shape (BEADS-JOURNAL-PLAN.md §1.1, "where that shape came from"): +// issue_json LONGTEXT, dep_json TEXT, no comment_json, no actor, no ts index. +// This predates both comment_json and actor outright. +func newShape3666e5026Store(t *testing.T, prefix string) *testEnv { + t.Helper() + te := newTestEnv(t, prefix) + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json") + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor") + te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN dep_json TEXT") + te.exec(t, ctx, "DROP INDEX idx_bd_events_journal_ts ON bd_events_journal") + return te +} + +// newForkIgnored0017ShapeStore reproduces the enterprise fork's +// ignored/0017_create_events_journal shape (0023's own header, bd-t9ovd): +// both payload columns TEXT, no actor (actor postdates this shape; it is +// bd OSS migration 0025), no ts index. ignored/0023 already fully heals this +// one — dropping actor is what forces the sentinel-triggered replay that +// exercises 0023's widen-if-TEXT step here, the same mechanism a lineage +// that genuinely never ran 0023/0025 before reaching this shape would hit. +func newForkIgnored0017ShapeStore(t *testing.T, prefix string) *testEnv { + t.Helper() + te := newTestEnv(t, prefix) + ctx := context.Background() + te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor") + te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN dep_json TEXT") + te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN comment_json TEXT") + te.exec(t, ctx, "DROP INDEX idx_bd_events_journal_ts ON bd_events_journal") + return te +} + +// openDB opens a short-lived raw SQL connection to te's store and registers +// its cleanup with t.Cleanup, for tests that need to hold the connection +// across several calls (schema.CurrentIgnoredVersion, schema.MigrateUp, +// direct probes) rather than te.exec's one-shot open/close. +func (te *testEnv) openDB(t *testing.T, ctx context.Context) *sql.DB { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + t.Cleanup(func() { _ = cleanup() }) + return db +} + +// journalCoreRow is the subset of bd_events_journal's columns every observed +// shape (canonical or historical) carries. comment_json and actor are +// deliberately excluded: they are exactly what this migration adds back, so +// a before/after comparison that is scoped to the columns every shape shares +// is what proves the healing touched only the columns it should. +type journalCoreRow struct { + Seq int64 + Ts string + Op string + IssueID string + IssueJSON sql.NullString + DepJSON sql.NullString +} + +// snapshotJournalCore captures every bd_events_journal row (core columns +// only, see journalCoreRow) in seq order, plus bd_events_seq's counter. +func snapshotJournalCore(t *testing.T, ctx context.Context, db *sql.DB) ([]journalCoreRow, int64) { + t.Helper() + rows, err := db.QueryContext(ctx, "SELECT seq, ts, op, issue_id, issue_json, dep_json FROM bd_events_journal ORDER BY seq") + if err != nil { + t.Fatalf("snapshot bd_events_journal: %v", err) + } + defer rows.Close() + var out []journalCoreRow + for rows.Next() { + var r journalCoreRow + if err := rows.Scan(&r.Seq, &r.Ts, &r.Op, &r.IssueID, &r.IssueJSON, &r.DepJSON); err != nil { + t.Fatalf("scan bd_events_journal row: %v", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate bd_events_journal: %v", err) + } + var nextSeq int64 + if err := db.QueryRowContext(ctx, "SELECT next_seq FROM bd_events_seq WHERE id = 0").Scan(&nextSeq); err != nil { + t.Fatalf("snapshot bd_events_seq: %v", err) + } + return out, nextSeq +} + +// assertIndexExists fails the test if the named index is missing. +func assertIndexExists(t *testing.T, ctx context.Context, db *sql.DB, table, index string) { + t.Helper() + var count int + if err := db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?`, + table, index).Scan(&count); err != nil { + t.Fatalf("probe index %s on %s: %v", index, table, err) + } + if count == 0 { + t.Errorf("%s: index %s missing after convergence", table, index) + } +} + +// assertColumnShapeConverged diffs got against want (both from +// clonePlaneTableShape, migrate_ignored_plane_shape_test.go) with messaging +// specific to this file's "did this fixture converge on canonical" question, +// rather than that helper's own fresh-init-vs-fresh-clone framing. +func assertColumnShapeConverged(t *testing.T, fixtureName, table string, want, got map[string]clonePlaneColumn) { + t.Helper() + names := map[string]bool{} + for name := range want { + names[name] = true + } + for name := range got { + names[name] = true + } + for name := range names { + w, inWant := want[name] + g, inGot := got[name] + switch { + case !inGot: + t.Errorf("%s: %s.%s present on the canonical reference but missing after convergence", fixtureName, table, name) + case !inWant: + t.Errorf("%s: %s.%s present after convergence but not on the canonical reference", fixtureName, table, name) + case w != g: + t.Errorf("%s: %s.%s shape differs from canonical: want %s, got %s", + fixtureName, table, name, formatClonePlaneColumn(w), formatClonePlaneColumn(g)) + } + } +} + +// TestIgnored0028ConvergesAllShapes is T2.5: every historical shape the +// plan's evidence base found ends up byte-identical to canonical after +// MigrateUp, and not a single row or the seq counter moved getting there. +// Kills: probing only DATA_TYPE (0023's gap — a dropped fixture here would +// stay missing comment_json/actor and fail the shape diff); any non-monotone +// write to the counter or rows (caught by the before/after snapshot; monotone +// writes are TestIgnored0028AloneLeavesCounterAndRowsUntouched's job — this +// test's forced replay always re-runs ignored/0022 too, whose own GREATEST +// raise would mask one). +func TestIgnored0028ConvergesAllShapes(t *testing.T) { + // dbPrefix is separate from name (rather than derived from it): embedded + // mode database names reject hyphens, but "fork-ignored-0017" is the + // plan's own name for that fixture and reads better in -v output and + // failure messages than a mangled version of it would. + fixtures := []struct { + name string + dbPrefix string + build func(t *testing.T, prefix string) *testEnv + damaged bool + }{ + {name: "3666e5026", dbPrefix: "cv3666e5026", build: newShape3666e5026Store, damaged: true}, + {name: "gci", dbPrefix: "cvgci", build: newGciShapeStore, damaged: true}, + {name: "fork-ignored-0017", dbPrefix: "cvfork0017", build: newForkIgnored0017ShapeStore, damaged: true}, + {name: "canonical", dbPrefix: "cvcanon", build: func(t *testing.T, prefix string) *testEnv { return newTestEnv(t, prefix) }, damaged: false}, + } + + ctx := context.Background() + reference := newTestEnv(t, "cvref") + refDB := reference.openDB(t, ctx) + refConn, err := refDB.Conn(ctx) + if err != nil { + t.Fatalf("reference Conn: %v", err) + } + defer refConn.Close() + wantShape := clonePlaneTableShape(t, ctx, refConn, "bd_events_journal") + + for _, fx := range fixtures { + t.Run(fx.name, func(t *testing.T) { + te := fx.build(t, fx.dbPrefix) + + // One real journaled event while the shape is still canonical (every + // builder above damages bd_events_journal only AFTER newTestEnv + // returns), so the snapshot below compares real data, not an empty + // table. + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation before damaging the shape: %v", err) + } + issue := &types.Issue{Title: fx.name + " seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("seed CreateIssue: %v", err) + } + + db := te.openDB(t, ctx) + beforeRows, beforeSeq := snapshotJournalCore(t, ctx, db) + + // MigrateUp's own return value cannot prove convergence here: it + // reports only the main-plane applied count (plus a reconcile + // flag), never the ignored-plane count, so the sentinel + // contradiction and its heal are both checked through + // CurrentIgnoredVersion instead (see TestSentinelJournalColumnsReplay). + if fx.damaged { + before, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (before MigrateUp): %v", err) + } + if want := schema.LatestIgnoredVersion(); before >= want { + t.Fatalf("CurrentIgnoredVersion before MigrateUp = %d, want below latest (%d): this fixture's damage did not float a sentinel contradiction", before, want) + } + } + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp: %v", err) + } + + if after, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after MigrateUp): %v", err) + } else if want := schema.LatestIgnoredVersion(); after != want { + t.Fatalf("CurrentIgnoredVersion after MigrateUp = %d, want latest (%d): this fixture did not converge", after, want) + } + + afterRows, afterSeq := snapshotJournalCore(t, ctx, db) + if beforeSeq != afterSeq { + t.Errorf("bd_events_seq.next_seq changed: before=%d after=%d", beforeSeq, afterSeq) + } + if !reflect.DeepEqual(beforeRows, afterRows) { + t.Errorf("bd_events_journal rows changed across MigrateUp:\nbefore=%+v\nafter=%+v", beforeRows, afterRows) + } + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Conn: %v", err) + } + defer conn.Close() + gotShape := clonePlaneTableShape(t, ctx, conn, "bd_events_journal") + assertColumnShapeConverged(t, fx.name, "bd_events_journal", wantShape, gotShape) + assertIndexExists(t, ctx, db, "bd_events_journal", "idx_bd_events_journal_ts") + assertIndexExists(t, ctx, db, "bd_events_journal", "idx_bd_events_journal_issue") + }) + } +} + +// isolationRow is the row shape TestIgnored0028AloneLeavesCounterAndRowsUntouched +// compares before and after running ignored/0028 directly: the four columns +// every damaged fixture in that test shares, plus comment_json only on the +// one fixture (fork-ignored-0017) that still has it when 0028 starts. +type isolationRow struct { + Seq int64 + Ts, Op, IssueID string + IssueJSON, DepJSON, CommentJSON sql.NullString +} + +// readIsolationRows reads isolationRows in seq order. comment_json is only +// selected when hasCommentJSON is true: on two of the three fixtures the +// column does not exist yet when this runs, so selecting it unconditionally +// would fail with "column not found" rather than compare anything. +func readIsolationRows(t *testing.T, ctx context.Context, conn *sql.Conn, hasCommentJSON bool) []isolationRow { + t.Helper() + query := "SELECT seq, ts, op, issue_id, issue_json, dep_json FROM bd_events_journal ORDER BY seq" + if hasCommentJSON { + query = "SELECT seq, ts, op, issue_id, issue_json, dep_json, comment_json FROM bd_events_journal ORDER BY seq" + } + rows, err := conn.QueryContext(ctx, query) + if err != nil { + t.Fatalf("read bd_events_journal: %v", err) + } + defer rows.Close() + var out []isolationRow + for rows.Next() { + var r isolationRow + dest := []any{&r.Seq, &r.Ts, &r.Op, &r.IssueID, &r.IssueJSON, &r.DepJSON} + if hasCommentJSON { + dest = append(dest, &r.CommentJSON) + } + if err := rows.Scan(dest...); err != nil { + t.Fatalf("scan bd_events_journal row: %v", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate bd_events_journal: %v", err) + } + return out +} + +func assertIsolationRowsEqual(t *testing.T, before, after []isolationRow) { + t.Helper() + if len(before) != len(after) { + t.Fatalf("journal row count = %d, want %d (0028 alone must not touch data)", len(after), len(before)) + } + for i := range before { + if before[i] != after[i] { + t.Errorf("journal row seq %d changed: before %+v, after %+v", before[i].Seq, before[i], after[i]) + } + } +} + +// TestIgnored0028AloneLeavesCounterAndRowsUntouched is T2.5b. T2.5 forces its +// replay through a sentinel contradiction, which always re-runs ignored/0022 +// too (every fixture there drops actor), so a monotone GREATEST write to the +// counter by 0028 itself would be invisible: 0022's own identical GREATEST +// raise on the same pass would mask it. Running 0028's frozen bytes directly, +// with the counter seeded below MAX(seq), closes that gap: any counter write +// at all, monotone or not, shows. Kills: any counter or row write in 0028, +// including a monotone GREATEST. +func TestIgnored0028AloneLeavesCounterAndRowsUntouched(t *testing.T) { + fixtures := []struct { + name string + dbPrefix string + build func(t *testing.T, prefix string) *testEnv + hasCommentJSON bool + }{ + {name: "3666e5026", dbPrefix: "i28a3666e5026", build: newShape3666e5026Store, hasCommentJSON: false}, + {name: "gci", dbPrefix: "i28agci", build: newGciShapeStore, hasCommentJSON: false}, + {name: "fork-ignored-0017", dbPrefix: "i28afork0017", build: newForkIgnored0017ShapeStore, hasCommentJSON: true}, + } + + for _, fx := range fixtures { + t.Run(fx.name, func(t *testing.T) { + te := fx.build(t, fx.dbPrefix) + ctx := context.Background() + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Conn: %v", err) + } + defer conn.Close() + + seed := []int64{1, 2, 3} + for _, seq := range seed { + ts := tsForSeq(seq) + query := "INSERT INTO bd_events_journal (seq, ts, op, issue_id, issue_json, dep_json) VALUES (?, ?, 'create', 'bd-i28a', ?, ?)" + args := []any{seq, ts, `{"id":"bd-i28a"}`, `[]`} + if fx.hasCommentJSON { + query = "INSERT INTO bd_events_journal (seq, ts, op, issue_id, issue_json, dep_json, comment_json) VALUES (?, ?, 'create', 'bd-i28a', ?, ?, ?)" + args = append(args, `{"text":"hi"}`) + } + if _, err := conn.ExecContext(ctx, query, args...); err != nil { + t.Fatalf("seed journal row %d: %v", seq, err) + } + } + execFrozenGuard(t, ctx, conn, "UPDATE bd_events_seq SET next_seq = 1 WHERE id = 0") + rowsBefore := readIsolationRows(t, ctx, conn, fx.hasCommentJSON) + if len(rowsBefore) != len(seed) { + t.Fatalf("seeded journal rows = %d, want %d", len(rowsBefore), len(seed)) + } + + sqlText, err := schema.IgnoredMigrationSQL("0028_converge_events_journal_shape.up.sql") + if err != nil { + t.Fatalf("read ignored 0028 migration: %v", err) + } + execFrozenGuard(t, ctx, conn, sqlText) + + if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 { + t.Errorf("next_seq = %d after applying 0028 alone, want 1: 0028 must never write the counter, not even a monotone raise", got) + } + assertIsolationRowsEqual(t, rowsBefore, readIsolationRows(t, ctx, conn, fx.hasCommentJSON)) + + for _, column := range []string{"comment_json", "dep_json"} { + if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" { + t.Errorf("bd_events_journal.%s DATA_TYPE = %q after 0028, want %q", column, got, "longtext") + } + } + requireColumnCount(t, ctx, conn, "bd_events_journal", "actor", 1) + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1) + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1) + }) + } +} + +// TestIgnored0028NoopsWithoutItsTables is the 0028 analogue of 0023's +// TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables +// (migrate_journal_shape_repair_test.go): 0028 must survive running against a +// workspace with no bd_events_journal table at all, the same intermediate +// state 0023 already has to survive. Kills: removing or omitting the +// table-exists conjunct from 0028's comment_json step. +func TestIgnored0028NoopsWithoutItsTables(t *testing.T) { + te := newTestEnv(t, "i28notab") + ctx := context.Background() + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Conn: %v", err) + } + defer conn.Close() + + execFrozenGuard(t, ctx, conn, "DROP TABLE IF EXISTS bd_events_journal") + + sqlText, err := schema.IgnoredMigrationSQL("0028_converge_events_journal_shape.up.sql") + if err != nil { + t.Fatalf("read ignored 0028 migration: %v", err) + } + execFrozenGuard(t, ctx, conn, sqlText) + + var got int + if err := conn.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'`).Scan(&got); err != nil { + t.Fatalf("probe bd_events_journal presence: %v", err) + } + if got != 0 { + t.Errorf("0028 created bd_events_journal out of nothing (count = %d); it must only ever repair an existing table", got) + } +} + +// TestSentinelJournalColumnsReplay is T2.6: a store already at the latest +// ignored cursor that loses comment_json out of band (a backup restored +// between doors, or damage out of band) floors at exactly 21 and heals on the +// next writable open. MigrateUp's own return value cannot prove this: it +// reports only the main-plane applied count (plus a reconcile flag), never +// the ignored-plane count, so every assertion here goes through +// CurrentIgnoredVersion instead. Kills: no sentinel registered (the floor +// read would stay at latest instead of dropping to 21, caught by the +// exact-21 assertion below); a floor of 22 or more (same assertion — 22 +// would mean 0022, bd_events_journal's own creator, never replays, which +// cannot heal a shape where the table already exists but a column does +// not); a replay that runs but does not actually add comment_json back +// (caught by the final post-heal probe). +func TestSentinelJournalColumnsReplay(t *testing.T) { + te := newTestEnv(t, "sj") + ctx := context.Background() + db := te.openDB(t, ctx) + + before, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (before damage): %v", err) + } + if want := schema.LatestIgnoredVersion(); before != want { + t.Fatalf("CurrentIgnoredVersion before damage = %d, want latest (%d)", before, want) + } + + // Damage through the SAME connection db already holds open, not a second + // embeddeddolt.OpenSQL handle (te.exec's own short-lived one): embedded + // mode's engine-open path serializes on the directory, and a second + // concurrent handle to it deadlocks against the first instead of simply + // queuing. + if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil { + t.Fatalf("drop comment_json: %v", err) + } + + floored, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (after dropping comment_json): %v", err) + } + if floored != 21 { + t.Fatalf("CurrentIgnoredVersion after dropping comment_json = %d, want exactly 21", floored) + } + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp (forced replay): %v", err) + } + + healed, err := schema.CurrentIgnoredVersion(ctx, db) + if err != nil { + t.Fatalf("CurrentIgnoredVersion (after heal): %v", err) + } + if want := schema.LatestIgnoredVersion(); healed != want { + t.Fatalf("CurrentIgnoredVersion after heal = %d, want latest (%d)", healed, want) + } + + var commentType sql.NullString + if err := db.QueryRowContext(ctx, ` + SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = 'comment_json'`, + ).Scan(&commentType); err != nil { + t.Fatalf("probe comment_json after heal: %v", err) + } + if !commentType.Valid { + t.Fatal("comment_json is still missing after the forced replay") + } +} + +// tsForSeq gives TestSentinelReplayCounterConverges's and +// TestIgnored0028AloneLeavesCounterAndRowsUntouched's seeded rows distinct, +// ordered timestamps without threading one more field through every fixture +// for a value no assertion in either test reads. +func tsForSeq(seq int64) string { + return fmt.Sprintf("2026-08-01 00:00:%02d", seq) +} + +// counterConvergeState is one bd_events_journal/bd_events_seq relationship +// TestSentinelReplayCounterConverges seeds: the rows to leave in the journal, +// the counter value to seed, and the counter the sentinel-triggered replay +// must converge to (GREATEST(before, MAX(seq))). +type counterConvergeState struct { + name string + code string // short, hyphen/underscore-free token for database name prefixes + seqs []int64 + before int64 + want int64 +} + +var counterConvergeStates = []counterConvergeState{ + {name: "behind", code: "behind", seqs: []int64{1, 2, 3}, before: 1, want: 3}, + {name: "equal", code: "equal", seqs: []int64{1, 2, 3}, before: 3, want: 3}, + {name: "ahead_journal_restored_older", code: "ajro", seqs: []int64{1, 2, 3}, before: 10, want: 10}, + {name: "empty_journal_counter_ahead", code: "ejca", seqs: nil, before: 7, want: 7}, + {name: "hole_above_counter", code: "hac", seqs: []int64{1, 3}, before: 1, want: 3}, +} + +// TestSentinelReplayCounterConverges is T2.6b: across every counter/journal +// relationship #5496's evidence base found, a sentinel-triggered replay of +// ignored 22..28 must leave bd_events_seq at exactly +// GREATEST(seeded, MAX(seq)) — never lower, never past the high-water mark — +// and then the very next real write must land contiguously right after it. +// Run against both sentinel columns: dropping actor across all five states, +// plus dropping comment_json on "behind" alone to prove that sentinel also +// reaches the counter (it is registered on the same table, same floor). +// +// Kills: the sentinel floor sitting at 22 or above (behind stays at 1, and +// hole_above_counter's next write lands at the stale seq 2, a late arrival); +// a replayed statement that lowers the counter, e.g. a future +// `SET next_seq = COALESCE(MAX(seq), 0)` with no GREATEST (the ahead_* and +// empty_* cases fail); an over-raise such as MAX(seq)+1 (behind and equal +// read 4, and the first new row is 5, a gap); dropping the actor sentinel +// (actor/behind stays at 1 and the column is never restored). +func TestSentinelReplayCounterConverges(t *testing.T) { + type subtest struct { + name string + dbPrefix string + state counterConvergeState + column string + } + var subtests []subtest + for _, state := range counterConvergeStates { + subtests = append(subtests, subtest{ + name: state.name + "/actor", + dbPrefix: "scc" + state.code + "a", + state: state, + column: "actor", + }) + } + subtests = append(subtests, subtest{ + name: "behind/comment_json", + dbPrefix: "sccbehindc", + state: counterConvergeStates[0], + column: "comment_json", + }) + + for _, st := range subtests { + t.Run(st.name, func(t *testing.T) { + te := newTestEnv(t, st.dbPrefix) + ctx := context.Background() + + // A raw handle with an explicit cleanup, not te.openDB: this test + // closes it early (below) and reopens afterward, which te.openDB's + // t.Cleanup-deferred-to-test-end handle cannot do. + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + closed := false + closeDB := func() { + if !closed { + _ = cleanup() + closed = true + } + } + defer closeDB() + + if _, err := db.ExecContext(ctx, "DELETE FROM bd_events_journal"); err != nil { + t.Fatalf("clear bd_events_journal: %v", err) + } + for _, seq := range st.state.seqs { + if _, err := db.ExecContext(ctx, + "INSERT INTO bd_events_journal (seq, ts, op, issue_id) VALUES (?, ?, 'create', 'bd-scc')", + seq, tsForSeq(seq)); err != nil { + t.Fatalf("seed journal row %d: %v", seq, err) + } + } + if _, err := db.ExecContext(ctx, "UPDATE bd_events_seq SET next_seq = ? WHERE id = 0", st.state.before); err != nil { + t.Fatalf("seed next_seq: %v", err) + } + //nolint:gosec // G201: st.column is one of two test-local literals ("actor", "comment_json"). + if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN "+st.column); err != nil { + t.Fatalf("drop %s: %v", st.column, err) + } + + if floored, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after dropping %s): %v", st.column, err) + } else if floored != 21 { + t.Fatalf("CurrentIgnoredVersion after dropping %s = %d, want exactly 21", st.column, floored) + } + rowsBefore, _ := snapshotJournalCore(t, ctx, db) + + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp (forced replay): %v", err) + } + + if after, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after replay): %v", err) + } else if want := schema.LatestIgnoredVersion(); after != want { + t.Fatalf("CurrentIgnoredVersion after replay = %d, want latest (%d)", after, want) + } + var colCount int + if err := db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = ?`, st.column).Scan(&colCount); err != nil { + t.Fatalf("probe %s after replay: %v", st.column, err) + } + if colCount == 0 { + t.Errorf("bd_events_journal.%s is still missing after the forced replay", st.column) + } + + rowsAfter, nextSeq := snapshotJournalCore(t, ctx, db) + if nextSeq != st.state.want { + t.Errorf("next_seq = %d after replay, want %d (GREATEST(seeded %d, MAX(seq)))", nextSeq, st.state.want, st.state.before) + } + if !reflect.DeepEqual(rowsBefore, rowsAfter) { + t.Errorf("bd_events_journal rows changed across the replay:\nbefore=%+v\nafter=%+v", rowsBefore, rowsAfter) + } + + // Close the raw handle before any te.store call: embedded opens + // serialize on the directory and a second concurrent handle + // deadlocks against this one instead of queuing (see + // TestStoreAtIgnored28WritesAndJournals). + closeDB() + + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation after convergence: %v", err) + } + issue := &types.Issue{Title: st.name + " seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue after convergence: %v", err) + } + + reopened := te.openDB(t, ctx) + var minSeq, maxSeq, count int64 + if err := reopened.QueryRowContext(ctx, + "SELECT COALESCE(MIN(seq),0), COALESCE(MAX(seq),0), COUNT(*) FROM bd_events_journal WHERE seq > ?", + st.state.want).Scan(&minSeq, &maxSeq, &count); err != nil { + t.Fatalf("probe new journal rows: %v", err) + } + if count == 0 { + t.Fatal("CreateIssue after convergence did not journal a row above the converged counter") + } + if minSeq != st.state.want+1 { + t.Errorf("lowest new seq = %d, want %d (want + 1)", minSeq, st.state.want+1) + } + if maxSeq-minSeq+1 != count { + t.Errorf("new seqs are not contiguous: min=%d max=%d count=%d", minSeq, maxSeq, count) + } + if st.state.name == "hole_above_counter" { + var staleCount int64 + if err := reopened.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal WHERE seq = 2").Scan(&staleCount); err != nil { + t.Fatalf("probe seq 2: %v", err) + } + if staleCount != 0 { + t.Errorf("seq 2 (the hole below the converged counter) has a row; a late arrival filled it instead of the new write landing above the counter") + } + } + }) + } +} + +// ignoredPlaneTablesExceptJournal lists every clone-local (dolt_ignore) table +// other than bd_events_journal itself, matching +// TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape's table list +// (migrate_ignored_plane_shape_test.go). bd_events_journal's own shape +// legitimately changes during the replay this test forces; every one of +// these must not. +var ignoredPlaneTablesExceptJournal = []string{ + "wisps", "wisp_labels", "wisp_dependencies", "wisp_events", + "wisp_comments", "wisp_child_counters", "events", "leases", + "repo_mtimes", "local_metadata", "bd_events_seq", +} + +// dumpTable renders every row of table as an order-stable, type-agnostic +// snapshot: each value read as text (or ""), ordered by the table's +// first column. It exists so TestReplay22To28NoOpOnHealthyStore can compare +// whole tables it otherwise knows nothing about the schema of, rather than +// hand-maintaining a column list per table. +func dumpTable(t *testing.T, ctx context.Context, db *sql.DB, table string) []map[string]string { + t.Helper() + rows, err := db.QueryContext(ctx, "SELECT * FROM "+table+" ORDER BY 1") + if err != nil { + t.Fatalf("dump %s: %v", table, err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("dump %s columns: %v", table, err) + } + var out []map[string]string + for rows.Next() { + vals := make([]sql.NullString, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + t.Fatalf("dump %s row: %v", table, err) + } + row := make(map[string]string, len(cols)) + for i, c := range cols { + if vals[i].Valid { + row[c] = vals[i].String + } else { + row[c] = "" + } + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate %s: %v", table, err) + } + return out +} + +func snapshotIgnoredPlaneTablesExceptJournal(t *testing.T, ctx context.Context, db *sql.DB) map[string][]map[string]string { + t.Helper() + snap := make(map[string][]map[string]string, len(ignoredPlaneTablesExceptJournal)) + for _, table := range ignoredPlaneTablesExceptJournal { + snap[table] = dumpTable(t, ctx, db, table) + } + return snap +} + +// TestReplay22To28NoOpOnHealthyStore is T2.7: forcing the sentinel-triggered +// replay of ignored 22..28 must perturb nothing outside bd_events_journal +// itself — not a single row in any other clone-local table, and critically +// not wisps.updated_at, which an unguarded statement in this range would +// restamp via ON UPDATE CURRENT_TIMESTAMP (the #5981 harm class the +// replay-floor's two placement constraints exist to prevent; see schema.go's +// schemaSentinelColumn doc comment). Kills: an unguarded statement above the +// floor touching a table or row it should not. +func TestReplay22To28NoOpOnHealthyStore(t *testing.T) { + te := newTestEnv(t, "rp") + ctx := context.Background() + + wispID := "wisp-rp-1" + te.exec(t, ctx, + "INSERT INTO wisps (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type, ephemeral) VALUES (?, 'w', '', '', '', '', 'open', 2, 'task', 1)", + wispID) + + db := te.openDB(t, ctx) + before := snapshotIgnoredPlaneTablesExceptJournal(t, ctx, db) + + // Force the replay the only way it ever happens in production: contradict + // a sentinel, which floors the cursor to 21 and makes 22..28 pending again + // (TestSentinelJournalColumnsReplay proves the floor and the heal; this + // test only cares what else the resulting replay does or does not touch). + // Damage through db itself, not a second embeddeddolt.OpenSQL handle: see + // the matching comment in TestSentinelJournalColumnsReplay. + if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil { + t.Fatalf("drop comment_json: %v", err) + } + // MigrateUp's own return value cannot prove the replay ran: it reports + // only the main-plane applied count, never the ignored-plane count (see + // TestSentinelJournalColumnsReplay), so the contradiction and its heal are + // both checked through CurrentIgnoredVersion instead. + if floored, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after damage): %v", err) + } else if floored != 21 { + t.Fatalf("CurrentIgnoredVersion after dropping comment_json = %d, want exactly 21", floored) + } + if _, err := schema.MigrateUp(ctx, db); err != nil { + t.Fatalf("MigrateUp (forced 22..28 replay): %v", err) + } + if healed, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion (after replay): %v", err) + } else if want := schema.LatestIgnoredVersion(); healed != want { + t.Fatalf("CurrentIgnoredVersion after replay = %d, want latest (%d): the replay did not run", healed, want) + } + + after := snapshotIgnoredPlaneTablesExceptJournal(t, ctx, db) + if !reflect.DeepEqual(before, after) { + t.Errorf("replaying ignored 22..28 perturbed a table it should never touch:\nbefore=%+v\nafter=%+v", before, after) + } +} + +// TestStoreAtIgnored28WritesAndJournals is the real-store half of T2.9 +// (BEADS-JOURNAL-PLAN.md §4.3, PR A2; schema.TestOldIgnoredCeilingOpensAfter0028 +// is the other half, pinning architecturally that no open-path guard compares +// a binary's own ignored ceiling against the stored cursor). A fresh store +// migrates to ignored 28 on its own, with no ignored-ceiling input from the +// caller at all; this just confirms that store opens, writes, and journals +// normally once there, matching the plan's own framing of the claim. +func TestStoreAtIgnored28WritesAndJournals(t *testing.T) { + te := newTestEnv(t, "oldceil") + ctx := context.Background() + + // Short-lived connection for the pre-check only: te.store's own calls + // below (SetEventsJournalEnabled, CreateIssue) each open and close their + // own connection to the same directory, and OpenSQL's backoff waits + // until ctx cancellation (never), so holding a second connection open + // concurrently via te.openDB here would deadlock them forever instead of + // failing. te.openDB is only safe to call once nothing else still needs + // to open its own connection to this store (see T2.5/T2.6/T2.7 above, + // which all finish their te.store.* calls first). + func() { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + if got, err := schema.CurrentIgnoredVersion(ctx, db); err != nil { + t.Fatalf("CurrentIgnoredVersion: %v", err) + } else if want := schema.LatestIgnoredVersion(); got != want { + t.Fatalf("CurrentIgnoredVersion = %d, want latest (%d) on a freshly initialized store", got, want) + } + }() + + te.store.SetEventsJournalEnabled(true) + if err := te.store.EventsJournalActivationError(); err != nil { + t.Fatalf("activation at ignored 28: %v", err) + } + issue := &types.Issue{Title: "t2.9 seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue at ignored 28: %v", err) + } + + db := te.openDB(t, ctx) + var journaled int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ?", issue.ID).Scan(&journaled); err != nil { + t.Fatalf("probe bd_events_journal: %v", err) + } + if journaled == 0 { + t.Error("CreateIssue at ignored 28 did not journal") + } +} diff --git a/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go b/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go index 1b86dd3f5..6643504ec 100644 --- a/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go +++ b/internal/storage/embeddeddolt/migrate_journal_shape_repair_test.go @@ -62,6 +62,72 @@ func TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape(t *testing.T conn, closeConn := newWorkspaceBeforeMigration0023(t, ctx) defer closeConn() + rowsBefore := seedForkLineageDrift(t, ctx, conn) + + if _, err := schema.MigrateUp(ctx, conn); err != nil { + t.Fatalf("repair MigrateUp: %v", err) + } + requireIgnoredCursorAtLatest(t, ctx, conn) + + // Shape converged. + for _, column := range []string{"dep_json", "comment_json"} { + if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" { + t.Errorf("bd_events_journal.%s DATA_TYPE = %q after repair, want %q", column, got, "longtext") + } + } + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1) + requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1) + requireIndex(t, ctx, conn, "wisps", "idx_wisps_defer_until", 1) + + // Data survived byte for byte. + assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) + // The pass may raise a counter that is behind the journal to its + // high-water mark (ignored/0022's tail, re-run because this fork shape has + // no actor column and trips the bd_events_journal.actor sentinel at floor + // 21) and must leave it nowhere else: GREATEST(seeded, MAX(seq)). + const seeded = 1 + high := int(rowsBefore[len(rowsBefore)-1].seq) + if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != max(seeded, high) { + t.Errorf("bd_events_seq.next_seq = %d after repair, want GREATEST(seeded %d, MAX(seq) %d) = %d: a pass may only raise a behind counter to the high-water mark, never lower it or move it past MAX(seq)", + got, seeded, high, max(seeded, high)) + } + + // The repair is what makes the write that used to roll back a user's + // mutation land instead. + if _, err := conn.ExecContext(ctx, + "INSERT INTO bd_events_journal (seq, ts, op, issue_id, comment_json) VALUES (99, '2026-08-01 00:00:03', 'comment', 'bd-t9ovd', ?)", + strings.Repeat("x", 70000)); err != nil { + t.Fatalf("oversized comment_json insert still refused after repair: %v", err) + } + if got := scalarInt(t, ctx, conn, "SELECT LENGTH(comment_json) FROM bd_events_journal WHERE seq = 99"); got != 70000 { + t.Errorf("stored oversized comment_json length = %d, want 70000", got) + } + execFrozenGuard(t, ctx, conn, "DELETE FROM bd_events_journal WHERE seq = 99") + + // Crash-replay: the pass can be killed and re-run, so the frozen bytes must + // be a clean no-op against the state they just produced. Re-arm the + // canary first: if replaying 0023 alone ever wrote the counter, letting it + // start already at the high-water mark would hide that. + createBefore := showCreateTable(t, ctx, conn, "bd_events_journal") + execFrozenGuard(t, ctx, conn, "UPDATE bd_events_seq SET next_seq = 1 WHERE id = 0") + execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t)) + if createAfter := showCreateTable(t, ctx, conn, "bd_events_journal"); createAfter != createBefore { + t.Errorf("replaying 0023 changed bd_events_journal:\nbefore:\n%s\nafter:\n%s", createBefore, createAfter) + } + if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 { + t.Errorf("next_seq = %d after replaying 0023, want 1: replaying 0023 wrote the counter", got) + } + assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) +} + +// seedForkLineageDrift builds the drifted fork-lineage journal shape 0023 +// repairs: the fork's TEXT-column DDL, idx_wisps_defer_until dropped, three +// journal rows, and a seq counter deliberately set BELOW the journal's own +// MAX(seq). It also proves the fixture really is drifted before handing it +// back: an oversized comment_json insert must be refused by the TEXT column. +func seedForkLineageDrift(t *testing.T, ctx context.Context, conn *sql.Conn) []journalRow { + t.Helper() + // The drift. bd_events_journal and wisps are both clone-local and absent // from HEAD here, so replacing the table and dropping the index leaves the // working set exactly as clean as it was. @@ -71,9 +137,14 @@ func TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape(t *testing.T // Data the repair must not touch: three journal rows whose payloads carry // quotes, backslashes, NULLs, and a multi-KB blob (bound as parameters, so // the column holds the literal Go string), plus a seq counter deliberately - // set BELOW the journal's own MAX(seq). Only 0022's seeding raises it to - // the high-water mark, so next_seq staying at 1 is proof that 0022 did not - // re-run and that 0023 left the counter alone. + // set BELOW the journal's own MAX(seq). This seeds two separate claims: + // that 0023 alone never touches the counter or a row + // (TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched runs + // its frozen bytes directly to check that), and that the full MigrateUp + // pass converges the counter to GREATEST(seeded, MAX(seq)) — this fork + // shape has no actor column, so it trips the bd_events_journal.actor + // sentinel at floor 21, which re-runs ignored/0022, whose tail is the same + // statement as issueops.healEventSeqCounter. bigPayload := strings.Repeat(`{"k":"v'\"\\","n":0},`, 400) + `{"end":true}` seed := []struct { seq int64 @@ -107,47 +178,36 @@ VALUES (?, ?, ?, 'bd-t9ovd', ?, ?, ?)`, s.seq, s.ts, s.op, s.issueJSON, s.depJSO t.Fatal("oversized comment_json insert succeeded on the drifted TEXT column; the fixture is not reproducing the drift this migration repairs") } - if _, err := schema.MigrateUp(ctx, conn); err != nil { - t.Fatalf("repair MigrateUp: %v", err) - } - requireIgnoredCursorAtLatest(t, ctx, conn) + return rowsBefore +} + +// TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched pins +// 0023's own contract ("neither bd_events_seq nor a single journal row is +// read or written", its header) with the runner out of the picture: the +// frozen bytes run directly against the drifted fork shape, so every repair +// branch fires and no sentinel, floor, or neighbouring migration can cause or +// mask a counter write. The counter is seeded below MAX(seq), so even a +// monotone GREATEST-style write would show. +func TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched(t *testing.T) { + requireEmbedded(t) + ctx := t.Context() + conn, closeConn := newWorkspaceBeforeMigration0023(t, ctx) + defer closeConn() + rowsBefore := seedForkLineageDrift(t, ctx, conn) + + execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t)) - // Shape converged. for _, column := range []string{"dep_json", "comment_json"} { if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" { - t.Errorf("bd_events_journal.%s DATA_TYPE = %q after repair, want %q", column, got, "longtext") + t.Fatalf("0023 did not widen %s (got %q): the fixture no longer exercises its branches", column, got) } } requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1) - requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1) requireIndex(t, ctx, conn, "wisps", "idx_wisps_defer_until", 1) - - // Data survived byte for byte, and the counter was not touched. assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 { - t.Errorf("bd_events_seq.next_seq = %d after repair, want 1 (0023 must touch neither the counter nor any data)", got) + t.Errorf("next_seq = %d after applying 0023 alone, want 1: 0023 must never write the counter", got) } - - // The repair is what makes the write that used to roll back a user's - // mutation land instead. - if _, err := conn.ExecContext(ctx, - "INSERT INTO bd_events_journal (seq, ts, op, issue_id, comment_json) VALUES (99, '2026-08-01 00:00:03', 'comment', 'bd-t9ovd', ?)", - strings.Repeat("x", 70000)); err != nil { - t.Fatalf("oversized comment_json insert still refused after repair: %v", err) - } - if got := scalarInt(t, ctx, conn, "SELECT LENGTH(comment_json) FROM bd_events_journal WHERE seq = 99"); got != 70000 { - t.Errorf("stored oversized comment_json length = %d, want 70000", got) - } - execFrozenGuard(t, ctx, conn, "DELETE FROM bd_events_journal WHERE seq = 99") - - // Crash-replay: the pass can be killed and re-run, so the frozen bytes must - // be a clean no-op against the state they just produced. - createBefore := showCreateTable(t, ctx, conn, "bd_events_journal") - execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t)) - if createAfter := showCreateTable(t, ctx, conn, "bd_events_journal"); createAfter != createBefore { - t.Errorf("replaying 0023 changed bd_events_journal:\nbefore:\n%s\nafter:\n%s", createBefore, createAfter) - } - assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn)) } // TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace is the insurance diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 4e8a121a9..92449f2e3 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -32,6 +32,7 @@ var _ storage.Flattener = (*EmbeddedDoltStore)(nil) var _ storage.Compactor = (*EmbeddedDoltStore)(nil) var _ storage.SchemaMigrator = (*EmbeddedDoltStore)(nil) var _ storage.EventsJournalConfigurer = (*EmbeddedDoltStore)(nil) +var _ storage.EventsJournalShapeChecker = (*EmbeddedDoltStore)(nil) var _ storage.VersionedHistoryConfigurer = (*EmbeddedDoltStore)(nil) var _ storage.ExternalRefHistoryQuerier = (*EmbeddedDoltStore)(nil) @@ -53,6 +54,16 @@ type EmbeddedDoltStore struct { // eventsJournalEnabled activates the durable events journal for THIS store // instance only (storage.EventsJournalConfigurer); never process-global. eventsJournalEnabled atomic.Bool + // journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled + // runs on activation (issueops.ProbeJournalShape, PR A1), over a short-lived + // connection since this store has no persistent db handle. Nil means either + // the journal is disabled or the probe failed — see journalActivationErr. + journalShape atomic.Pointer[issueops.JournalShape] + // journalActivationErr holds the probe's error when SetEventsJournalEnabled(true) + // found a table shape the journal cannot run against (a missing required + // column, or a missing table). Nil whenever journalShape is non-nil or the + // journal is disabled. See storage.EventsJournalShapeChecker. + journalActivationErr atomic.Pointer[error] // versionedHistoryEnabled activates dual-write issue-version history for // THIS store instance only (storage.VersionedHistoryConfigurer); never // process-global. @@ -457,8 +468,20 @@ func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func err = fmt.Errorf("embeddeddolt: begin tx: %w", err) return } - clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) + // The activation switch folds in the shape probe's success: a probe that + // found an unsupported table (journalShape nil) scopes this transaction as + // DISABLED regardless of what SetEventsJournalEnabled was called with, so a + // caller that enables the journal directly and ignores the returned + // EventsJournalActivationError still never attempts a write against a shape + // that cannot support it (storage.EventsJournalShapeChecker). Reads + // (commit=false callers) are never gated by this switch — only by the + // shape — since ReadEventsJournal has no enabled/disabled concept of its + // own. + journalShape := s.journalShape.Load() + clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load() && journalShape != nil) defer clearJournalScope() + clearJournalShapeScope := issueops.ScopeEventsJournalShape(tx, journalShape) + defer clearJournalShapeScope() clearVersionScope := issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load()) defer clearVersionScope() clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) @@ -484,8 +507,47 @@ func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func } // SetEventsJournalEnabled activates the journal for this store instance only. +// +// Activating probes bd_events_journal's actual shape ONCE +// (issueops.ProbeJournalShape, PR A1), over a short-lived connection opened +// and closed just for the probe since this store keeps no persistent db +// handle, and caches the result for this instance's lifetime; +// EventsJournalActivationError reports whether that probe found a shape the +// journal can actually run against. Deactivating clears both and skips the +// probe entirely — a disabled store accepts any shape. func (s *EmbeddedDoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) + if !enabled { + s.journalShape.Store(nil) + s.journalActivationErr.Store(nil) + return + } + db, cleanup, err := OpenSQL(context.Background(), s.dataDir, s.database, s.branch) + if err != nil { + s.journalShape.Store(nil) + wrapped := fmt.Errorf("journal: open connection to probe shape: %w", err) + s.journalActivationErr.Store(&wrapped) + return + } + defer func() { _ = cleanup() }() + shape, probeErr := issueops.ProbeJournalShape(context.Background(), db) + if probeErr != nil { + s.journalShape.Store(nil) + s.journalActivationErr.Store(&probeErr) + return + } + s.journalShape.Store(shape) + s.journalActivationErr.Store(nil) +} + +// EventsJournalActivationError implements storage.EventsJournalShapeChecker: +// nil unless the last SetEventsJournalEnabled(true) call found a table shape +// the journal cannot run against. +func (s *EmbeddedDoltStore) EventsJournalActivationError() error { + if p := s.journalActivationErr.Load(); p != nil { + return *p + } + return nil } // SetVersionedHistoryEnabled activates dual-write issue-version history for diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 9db47cd44..247f0211f 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -134,6 +134,20 @@ func (p *uowProvider) SetEventsJournalEnabled(enabled bool) { } } +// EventsJournalActivationError forwards the inner provider's shape-probe +// result (storage.EventsJournalShapeChecker, PR A1). This wrapper is the +// OUTERMOST provider on both real chains (see SetVersionedHistoryEnabled), so +// without this forwarder eventsjournal.Apply's type assertion for the checker +// would fail here even when the inner provider's own probe found an +// unsupported table shape — the open would then succeed against a shape it +// cannot journal, the gci failure mode this interface exists to avoid. +func (p *uowProvider) EventsJournalActivationError() error { + if checker, ok := p.UnitOfWorkProvider.(storage.EventsJournalShapeChecker); ok { + return checker.EventsJournalActivationError() + } + return nil +} + // SetVersionedHistoryEnabled forwards dual-write issue-version activation // inward, for the same reason SetEventsJournalEnabled does. // @@ -162,6 +176,7 @@ func (p *uowProvider) RunEventsMaintenanceTx(ctx context.Context, fn func(contex var ( _ uow.PoolTuner = (*uowProvider)(nil) _ storage.EventsJournalConfigurer = (*uowProvider)(nil) + _ storage.EventsJournalShapeChecker = (*uowProvider)(nil) _ storage.VersionedHistoryConfigurer = (*uowProvider)(nil) _ issueops.EventsMaintenanceRunner = (*uowProvider)(nil) _ uow.IssueLifecycleSource = (*uowProvider)(nil) diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index d774d7aff..0fd8ec5d4 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -63,6 +63,7 @@ go_library( "journal_autoprune.go", "journal_dependencies.go", "journal_prune.go", + "journal_shape.go", "labels.go", "lease.go", "metadata.go", diff --git a/internal/storage/issueops/journal.go b/internal/storage/issueops/journal.go index a4dc140e3..10be59552 100644 --- a/internal/storage/issueops/journal.go +++ b/internal/storage/issueops/journal.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "sort" + "strings" "sync" "time" @@ -420,6 +421,15 @@ func getJournalIssueInTx(ctx context.Context, tx DBTX, issueID string) (*types.I // (non-dependency ops). ts is the insert time, stamped inside the committing // transaction. actor is stored as-is — "" for the genuinely unattributable // paths — in the NOT NULL DEFAULT ” actor column. +// +// The column list is shape-adaptive (journal_shape.go, PR A1): seq, ts, op +// and issue_id are always named, and each optional column — actor, +// issue_json, dep_json, comment_json — is named only when journalShapeFor(tx) +// says this store's table actually has it. On a canonical table that is every +// column, in the same order this INSERT has always used; on an old-lineage +// table missing one, the write still lands and that one payload is simply not +// recorded, rather than failing the whole mutation the journal row shares a +// transaction with. func insertEventRow(ctx context.Context, tx DBTX, op EventOp, issueID string, issue *types.Issue, dep *EventDep, comment *EventComment, actor string) error { var issueJSON any if issue != nil { @@ -445,11 +455,41 @@ func insertEventRow(ctx context.Context, tx DBTX, op EventOp, issueID string, is } commentJSON = string(b) } + + shape := journalShapeFor(tx) + type optionalJournalValue struct { + column string + value any + } + var optionalValues []optionalJournalValue + if shape.HasActor { + optionalValues = append(optionalValues, optionalJournalValue{"actor", actor}) + } + if shape.HasIssueJSON { + optionalValues = append(optionalValues, optionalJournalValue{"issue_json", issueJSON}) + } + if shape.HasDepJSON { + optionalValues = append(optionalValues, optionalJournalValue{"dep_json", depJSON}) + } + if shape.HasCommentJSON { + optionalValues = append(optionalValues, optionalJournalValue{"comment_json", commentJSON}) + } + columns := make([]string, 0, 4+len(optionalValues)) + columns = append(columns, "seq", "ts", "op", "issue_id") + for _, ov := range optionalValues { + columns = append(columns, ov.column) + } + placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(columns)), ", ") + //nolint:gosec // columns is built entirely from the fixed name literals above, never from caller input. + query := fmt.Sprintf("INSERT INTO bd_events_journal (%s) VALUES (%s)", strings.Join(columns, ", "), placeholders) + insert := func(seq int64) error { - _, err := tx.ExecContext(ctx, ` - INSERT INTO bd_events_journal (seq, ts, op, issue_id, actor, issue_json, dep_json, comment_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - `, seq, time.Now().UTC(), string(op), issueID, actor, issueJSON, depJSON, commentJSON) + args := make([]any, 0, len(columns)) + args = append(args, seq, time.Now().UTC(), string(op), issueID) + for _, ov := range optionalValues { + args = append(args, ov.value) + } + _, err := tx.ExecContext(ctx, query, args...) return err } diff --git a/internal/storage/issueops/journal_prune.go b/internal/storage/issueops/journal_prune.go index 50658f8ce..d3617162c 100644 --- a/internal/storage/issueops/journal_prune.go +++ b/internal/storage/issueops/journal_prune.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strconv" + "strings" "time" "github.com/steveyegge/beads/internal/storage" @@ -251,10 +252,30 @@ func readEventsHeadInTx(ctx context.Context, tx DBTX) (int64, error) { return head, nil } +// readEventsRowsInTx's column list is shape-adaptive (journal_shape.go, PR +// A1): seq, ts, op and issue_id are always selected, and each optional column +// is selected only when journalShapeFor(tx) says this store's table actually +// has it. A column the probe did not find is simply never requested, so the +// corresponding field on storage.EventsJournalRow reads as "" — never a scan +// error — for a consumer on an old-lineage table. func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([]storage.EventsJournalRow, error) { + shape := journalShapeFor(tx) // CAST(ts AS CHAR) normalizes the DATETIME to a stable string across drivers. - q := `SELECT seq, CAST(ts AS CHAR), op, issue_id, actor, issue_json, dep_json, comment_json - FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC` + columns := []string{"seq", "CAST(ts AS CHAR)", "op", "issue_id"} + if shape.HasActor { + columns = append(columns, "actor") + } + if shape.HasIssueJSON { + columns = append(columns, "issue_json") + } + if shape.HasDepJSON { + columns = append(columns, "dep_json") + } + if shape.HasCommentJSON { + columns = append(columns, "comment_json") + } + //nolint:gosec // columns is built entirely from the fixed name literals above, never from caller input. + q := "SELECT " + strings.Join(columns, ", ") + " FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC" if limit > 0 { q += " LIMIT " + strconv.Itoa(limit) } @@ -268,14 +289,29 @@ func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([ for rows.Next() { var ( r storage.EventsJournalRow + actor sql.NullString issueJS sql.NullString depJS sql.NullString commentJS sql.NullString ) - if err := rows.Scan(&r.Seq, &r.TS, &r.Op, &r.IssueID, &r.Actor, &issueJS, &depJS, &commentJS); err != nil { + dest := []any{&r.Seq, &r.TS, &r.Op, &r.IssueID} + if shape.HasActor { + dest = append(dest, &actor) + } + if shape.HasIssueJSON { + dest = append(dest, &issueJS) + } + if shape.HasDepJSON { + dest = append(dest, &depJS) + } + if shape.HasCommentJSON { + dest = append(dest, &commentJS) + } + if err := rows.Scan(dest...); err != nil { return nil, fmt.Errorf("journal: scan row: %w", err) } r.TS = normalizeEventsTimestamp(r.TS) + r.Actor = actor.String r.IssueJSON = issueJS.String r.DepJSON = depJS.String r.CommentJSON = commentJS.String diff --git a/internal/storage/issueops/journal_shape.go b/internal/storage/issueops/journal_shape.go new file mode 100644 index 000000000..a5b2d26d5 --- /dev/null +++ b/internal/storage/issueops/journal_shape.go @@ -0,0 +1,257 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + + "github.com/steveyegge/beads/internal/debug" +) + +// Adaptive I/O for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2a, PR A1). +// +// The writer and reader used to name a fixed column list. That is fine on a +// table built by the current migrations, but a table built by an earlier +// lineage — gas-city-inc's is the live case — can lack an optional column +// forever: the ignored series only ever creates the table when it is wholly +// absent, and its one repair (ignored/0023) probes column TYPES, never +// EXISTENCE. A fixed INSERT naming a missing column then fails every +// journaled write, and because the journal row commits in the SAME +// transaction as the mutation it records, that failure rolls back the user's +// write along with it. +// +// The fix probes once, at activation, rather than guessing or failing per +// write: +// - REQUIRED columns {seq, ts, op, issue_id} have existed on every lineage +// this table has ever shipped under; a table missing one of them, or +// missing entirely, cannot journal at all. ProbeJournalShape refuses with +// a typed error so a store's "enable the journal" step fails at open, +// before any mutation is attempted — never as a per-write failure after +// the work is otherwise done, which is the gci failure mode above. +// - OPTIONAL columns {actor, issue_json, dep_json, comment_json} may be +// missing on an old-lineage table. The writer drops whichever of them the +// probe did not find; the reader returns "" for them. One warning is +// logged per probe (i.e. once per store instance, not once per write), +// and ignored/0028 (PR A2) is what actually converges the shape — this +// file only keeps a drifted table WRITABLE and READABLE in the meantime. +// +// A store instance probes exactly once, when storage.EventsJournalConfigurer +// activates the journal (see each backend's SetEventsJournalEnabled), and +// caches the *JournalShape for its own lifetime. ScopeEventsJournalShape +// carries that cached value onto one transaction, next to +// ScopeEventsJournalTransaction's activation switch, so insertEventRow and +// readEventsRowsInTx — both of which only ever see a tx, never the store — +// can reach it without a per-write probe (TestJournalShapeProbeOncePerOpen). + +// journalTableName is the clone-local table every probe, insert and read in +// this file targets. +const journalTableName = "bd_events_journal" + +// requiredJournalColumns are the columns ProbeJournalShape treats as +// non-negotiable: every lineage of bd_events_journal has always carried them, +// and the seq/ts/op/issue_id contract (nextEventSeq, the reader's ordering) +// is meaningless without them. +var requiredJournalColumns = []string{"seq", "ts", "op", "issue_id"} + +// optionalJournalColumns are the columns a pre-merge lineage may lack. Order +// matches the canonical INSERT column order a healthy table has carried since +// migration 0066/ignored 0025 added actor. +var optionalJournalColumns = []string{"actor", "issue_json", "dep_json", "comment_json"} + +// JournalShape records which OPTIONAL bd_events_journal columns exist on one +// store's table. A nil *JournalShape is never handed to a caller; probing a +// table missing a REQUIRED column returns an error instead (see +// ProbeJournalShape). +type JournalShape struct { + HasActor bool + HasIssueJSON bool + HasDepJSON bool + HasCommentJSON bool +} + +// missingOptional names the optional columns this shape does not have, in +// optionalJournalColumns order, for the doctor finding and the one-per-probe +// warning. +func (s *JournalShape) missingOptional() []string { + if s == nil { + return nil + } + var missing []string + if !s.HasActor { + missing = append(missing, "actor") + } + if !s.HasIssueJSON { + missing = append(missing, "issue_json") + } + if !s.HasDepJSON { + missing = append(missing, "dep_json") + } + if !s.HasCommentJSON { + missing = append(missing, "comment_json") + } + return missing +} + +// canonicalJournalShape is every optional column present — the shape every +// workspace created from main migration 0064 plus ignored 0022/0025 has, and +// the default journalShapeFor returns for a transaction nobody scoped. That +// default matters: it is what every call site written before this file +// existed (and every test that activates the journal through the ctx-only +// WithEventsJournal test helper rather than a real store) keeps getting, so +// adaptive I/O is additive and changes nothing for a canonical table. +var canonicalJournalShape = &JournalShape{ + HasActor: true, + HasIssueJSON: true, + HasDepJSON: true, + HasCommentJSON: true, +} + +// ErrJournalShapeUnsupported is the sentinel errors.Is matches against any +// *JournalShapeError returned by ProbeJournalShape, regardless of which +// required columns a particular table happens to be missing. +var ErrJournalShapeUnsupported = errors.New("events journal: table shape does not support the events journal") + +// JournalShapeError reports that bd_events_journal (or the table itself) is +// missing one or more REQUIRED columns. It is returned by ProbeJournalShape +// and is the typed error storage.EventsJournalConfigurer implementations +// surface from activation, so "enable the journal" fails at open rather than +// the gci failure mode: succeeding at open and then failing every write thereafter. +type JournalShapeError struct { + Table string + Missing []string +} + +func (e *JournalShapeError) Error() string { + return fmt.Sprintf( + "events journal: table %s is missing required column(s) %s; the events journal cannot be enabled against this shape (see `bd doctor`)", + e.Table, strings.Join(e.Missing, ", "), + ) +} + +// Is makes errors.Is(err, ErrJournalShapeUnsupported) match any +// *JournalShapeError, so a caller can test for the class of failure without +// caring which columns a particular table lacks. +func (e *JournalShapeError) Is(target error) bool { + return target == ErrJournalShapeUnsupported //nolint:errorlint // sentinel identity comparison is the documented contract of Is. +} + +// journalShapeProbeCount counts how many times ProbeJournalShape has actually +// queried INFORMATION_SCHEMA in this process. It exists for +// TestJournalShapeProbeOncePerOpen, which asserts that a batch of writes +// against one already-activated store issues none of its own. +var journalShapeProbeCount atomic.Int64 + +// JournalShapeProbeCountForTest reports journalShapeProbeCount's current +// value. Test-only instrumentation, read by TestJournalShapeProbeOncePerOpen; +// production code never calls it. +func JournalShapeProbeCountForTest() int64 { return journalShapeProbeCount.Load() } + +// ProbeJournalShape runs the one INFORMATION_SCHEMA.COLUMNS query that +// determines what q's bd_events_journal table actually supports. Callers run +// it exactly once per store/provider instance, from +// storage.EventsJournalConfigurer.SetEventsJournalEnabled, and cache the +// result for that instance's lifetime — never per write or per transaction. +// +// A table missing any required column, including a wholly absent table +// (which reads as every column missing, since a COLUMNS query against a +// nonexistent table returns zero rows — the same shape ignored/0023's +// existing probes rely on), returns *JournalShapeError instead of a shape. +func ProbeJournalShape(ctx context.Context, q DBTX) (*JournalShape, error) { + journalShapeProbeCount.Add(1) + + rows, err := q.QueryContext(ctx, ` + SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`, journalTableName) + if err != nil { + return nil, fmt.Errorf("journal: probe %s shape: %w", journalTableName, err) + } + defer rows.Close() + + present := make(map[string]bool, len(requiredJournalColumns)+len(optionalJournalColumns)) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("journal: scan %s column: %w", journalTableName, err) + } + present[strings.ToLower(name)] = true + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("journal: read %s shape: %w", journalTableName, err) + } + + var missing []string + for _, col := range requiredJournalColumns { + if !present[col] { + missing = append(missing, col) + } + } + if len(missing) > 0 { + return nil, &JournalShapeError{Table: journalTableName, Missing: missing} + } + + shape := &JournalShape{ + HasActor: present["actor"], + HasIssueJSON: present["issue_json"], + HasDepJSON: present["dep_json"], + HasCommentJSON: present["comment_json"], + } + warnJournalShapeDegraded(shape) + return shape, nil +} + +// warnJournalShapeDegraded writes one audit line to STDERR, suppressed only +// by quiet mode, the first (and every) time a store probes a table missing an +// optional column — one call per ProbeJournalShape invocation, which is one +// per store activation, which is "one warning per store" regardless of how +// many writes that store goes on to make. +func warnJournalShapeDegraded(shape *JournalShape) { + missing := shape.missingOptional() + if len(missing) == 0 { + return + } + if debug.IsQuiet() { + return + } + fmt.Fprintf(os.Stderr, + "events journal: %s is missing optional column(s) %s; affected payload(s) will be dropped until it converges (see `bd doctor`)\n", + journalTableName, strings.Join(missing, ", ")) +} + +// journalShapes associates a probed *JournalShape with ONE concrete +// transaction, the same per-tx lifetime journalTransactions gives the +// activation switch. Entries live for one transaction only. +var journalShapes sync.Map // map[DBTX]*JournalShape + +// ScopeEventsJournalShape binds shape to tx so insertEventRow and +// readEventsRowsInTx can reach the store's already-cached probe result +// without taking the store itself as a parameter. Store implementations call +// it immediately after BeginTx, alongside ScopeEventsJournalTransaction, +// passing the *JournalShape SetEventsJournalEnabled cached at activation — +// never a fresh probe. +// +// A nil tx or nil shape is a no-op; journalShapeFor then returns +// canonicalJournalShape, today's fixed eight-column behavior. +func ScopeEventsJournalShape(tx DBTX, shape *JournalShape) func() { + if tx == nil || shape == nil { + return func() {} + } + journalShapes.Store(tx, shape) + return func() { journalShapes.Delete(tx) } +} + +// journalShapeFor returns the shape scoped to tx, or canonicalJournalShape +// when nobody scoped one — which is every call site that predates this file, +// and every test driving the journal through the ctx-only WithEventsJournal +// helper rather than a real store's activation. +func journalShapeFor(tx DBTX) *JournalShape { + if v, ok := journalShapes.Load(tx); ok { + if shape, ok := v.(*JournalShape); ok && shape != nil { + return shape + } + } + return canonicalJournalShape +} diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 5108ff37d..5acd23138 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -117,6 +117,7 @@ go_library( "migrations/ignored/0026_dep_rekey_dedup_marker.up.sql", "migrations/ignored/0027_add_wisps_current_revision.up.sql", "migrations/0069_widen_issue_versions_datetime_precision.up.sql", + "migrations/ignored/0028_converge_events_journal_shape.up.sql", ], importpath = "github.com/steveyegge/beads/internal/storage/schema", visibility = ["//:__subpackages__"], @@ -151,6 +152,7 @@ go_test( "migration_0069_datetime_precision_test.go", "migration_body_drain_test.go", "migration_watchdog_test.go", + "old_ignored_ceiling_test.go", "progress_test.go", "remote_migrate_gate_test.go", "schema_skew_test.go", @@ -160,6 +162,7 @@ go_test( ], data = [":migrations"], embed = [":schema"], + embedsrcs = ["testdata/old_ignored_ceiling_24/0024_placeholder.up.sql"], deps = [ "//internal/debug", "//internal/storage/depid", diff --git a/internal/storage/schema/cursor_reality_test.go b/internal/storage/schema/cursor_reality_test.go index c41144fab..9afb70877 100644 --- a/internal/storage/schema/cursor_reality_test.go +++ b/internal/storage/schema/cursor_reality_test.go @@ -133,12 +133,40 @@ func TestCursorRealityFloor(t *testing.T) { wantLimited: true, }, { - name: "schema corroborates the cursor", - present: allSentinelTables(), - columns: map[string]bool{"leases.granted_node": true}, + name: "schema corroborates the cursor", + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + "bd_events_journal.actor": true, + }, wantFloor: 0, wantLimited: false, }, + { + // T2.6c: the journal floor limits on its own, with leases + // corroborated and the OTHER journal column present. Moving actor + // off floor 21 would read 24 here instead (see + // TestCurrentVersionClampsToSentinelFloor's matching case). + name: "journal actor absent alone", + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + }, + wantFloor: 21, + wantLimited: true, + }, + { + name: "journal comment_json absent alone", + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.actor": true, + }, + wantFloor: 21, + wantLimited: true, + }, } { t.Run(tc.name, func(t *testing.T) { stubSentinelTables(t, tc.present, nil) @@ -252,8 +280,62 @@ func TestCurrentVersionClampsToSentinelFloor(t *testing.T) { name: "fully corroborated cursor is believed as read", raw: latest, present: allSentinelTables(), - columns: map[string]bool{"leases.granted_node": true}, - want: latest, + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + "bd_events_journal.actor": true, + }, + want: latest, + }, + { + // T2.6c: the fleet's shape at the first A2 open. Every production + // Dolt store sits at ignored cursor 24 (BEADS-JOURNAL-PLAN.md §4.1) + // and predates actor outright, so this clamps to 21 and 22..28 + // replay once. + name: "fleet first open: cursor 24, journal actor absent", + raw: 24, + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + }, + want: 21, + }, + { + // #5496's fork-lineage fixture: cursor 22, actor absent (that shape + // predates ignored/0025, which adds actor, outright). + name: "fork-lineage fixture: cursor 22, actor absent", + raw: 22, + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + "bd_events_journal.comment_json": true, + }, + want: 21, + }, + { + // A raw cursor already sitting exactly at the journal floor: the + // clamp is a ceiling, not a promotion, so a contradiction at or + // below it changes nothing. + name: "cursor at the journal floor is untouched", + raw: 21, + present: allSentinelTables(), + columns: map[string]bool{ + "leases.granted_node": true, + }, + want: 21, + }, + { + // Taking the MAXIMUM floor instead of the minimum in + // cursorRealityFloor would read 21 here (the journal floor); the + // minimum reads 11 (the leases floor). + name: "leases and journal actor both absent", + raw: latest, + present: allSentinelTables(), + columns: map[string]bool{ + "bd_events_journal.comment_json": true, + }, + want: 11, }, } { t.Run(tc.name, func(t *testing.T) { @@ -583,7 +665,10 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { AddRow("content_hash", "char(64)", "YES", "", nil, "")) // The guarded read: cursor claims at-latest, both table sentinels - // corroborate it, and only the leases column does not. + // corroborate it, and only the leases column does not — the + // bd_events_journal sentinels (PR A2) are probed too, in the same + // sentinelColumns order, and corroborate here so the leases contradiction + // stays the only thing driving the floor. expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) for range ignoredSource.sentinelTables { @@ -592,6 +677,10 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { } mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) + for range 2 { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } boom := errors.New("stop here: the applier resumed above the floor") mock.ExpectExec(".*").WillReturnError(boom) @@ -653,3 +742,105 @@ func TestPendingVersionsUnderFloorDoNotReArmAuxMarkers(t *testing.T) { t.Error("ignored/0007 is pending on a cursor-11 store; its unguarded UPDATE would restamp wisps.updated_at") } } + +// journalSentinelReplayDML matches an (?i)-insensitive statement that +// mutates data: UPDATE ... SET, DELETE FROM, INSERT [IGNORE] INTO, +// REPLACE INTO, or TRUNCATE TABLE. It is deliberately loose about what +// follows — this test's job is to flag every candidate and then check it +// against an allowlist, not to parse SQL. +var journalSentinelReplayDML = regexp.MustCompile(`(?i)\b(update\s+\S+\s+set|delete\s+from|insert\s+(ignore\s+)?into|replace\s+into|truncate\s+table)\b`) + +var ( + journalSentinelCommentLineRE = regexp.MustCompile(`(?m)^\s*--.*$`) + journalSentinelOnUpdateRE = regexp.MustCompile(`(?i)on\s+update\s+current_timestamp(\s*\(\s*\))?`) + journalSentinelOnDupKeyRE = regexp.MustCompile(`(?i)on\s+duplicate\s+key\s+update`) + journalSentinelWhitespaceRE = regexp.MustCompile(`\s+`) +) + +// journalSentinelReplayAllowlist is every statement, by migration version, +// that the bd_events_journal sentinels' replay range (today: above floor 21) +// is permitted to carry, keyed by its whitespace-collapsed lowercase form. +// Each entry needs a justification here before it is added. +var journalSentinelReplayAllowlist = map[int][]string{ + 22: { + // Idempotent: INSERT IGNORE no-ops once bd_events_seq's single row + // (id=0) exists, on every replay after the first. + "insert ignore into bd_events_seq (id, next_seq) values (0, 0)", + // Monotone: identical in effect to issueops.healEventSeqCounter — a + // no-op when the counter is already at or ahead of the journal's + // high-water mark, and a raise to exactly MAX(seq) when it is behind. + // See TestSentinelReplayCounterConverges (embeddeddolt package) for + // the behavioral proof. + "update bd_events_seq set next_seq = greatest(next_seq, coalesce((select max(seq) from bd_events_journal), 0)) where id = 0", + }, +} + +// TestJournalSentinelReplayRangeDMLIsAllowlisted is T2.14: every ignored +// migration above the bd_events_journal sentinels' floor is a migration a +// sentinel contradiction can force to replay against a store that already +// has real data in it (section 3 of the A2 conflict decision). A DML +// statement in that range is therefore not merely "guarded" by an IF — it +// must also be safe to re-run against live rows, which an IF alone does not +// guarantee (an unconditional UPDATE inside a guarded ALTER branch would +// still re-fire every time the branch's condition holds). This pins the +// range to exactly the DML this decision reviewed, so a future migration +// added above the floor cannot silently introduce an unreviewed one. +// +// The floor is read from the registry, not hard-coded, so a future change to +// the sentinel values updates this test's scope automatically. +// +// Kills: any future ignored migration above the floor that adds DML which +// every journal-sentinel replay would silently re-run — for example a +// jv/txn backfill `UPDATE bd_events_journal ...` or an `UPDATE wisps ...` +// restamp (A4-A6 add ignored/0029-0031 inside this range). +func TestJournalSentinelReplayRangeDMLIsAllowlisted(t *testing.T) { + floor := 0 + haveFloor := false + for _, sc := range ignoredSource.sentinelColumns { + if sc.table != "bd_events_journal" { + continue + } + if !haveFloor || sc.replayFloor < floor { + floor = sc.replayFloor + } + haveFloor = true + } + if !haveFloor { + t.Fatal("no bd_events_journal sentinel column is registered in ignoredSource; this test has nothing to pin") + } + + for _, mf := range ignoredSource.list() { + if mf.version <= floor { + continue + } + blob, err := ignoredSource.files.ReadFile(ignoredSource.dir + "/" + mf.name) + if err != nil { + t.Fatalf("read %s: %v", mf.name, err) + } + + body := journalSentinelCommentLineRE.ReplaceAllString(string(blob), "") + body = journalSentinelOnUpdateRE.ReplaceAllString(body, " ") + body = journalSentinelOnDupKeyRE.ReplaceAllString(body, " ") + + for _, stmt := range strings.Split(body, ";") { + if !journalSentinelReplayDML.MatchString(stmt) { + continue + } + normalized := strings.ToLower(strings.TrimSpace(journalSentinelWhitespaceRE.ReplaceAllString(stmt, " "))) + if normalized == "" { + continue + } + allowed := false + for _, want := range journalSentinelReplayAllowlist[mf.version] { + if normalized == want { + allowed = true + break + } + } + if !allowed { + t.Errorf("ignored/%04d (above the bd_events_journal replay floor %d) carries DML not on the replay-safety allowlist:\n %s\nEvery journal-sentinel-triggered replay re-runs this statement against a live store; add it to journalSentinelReplayAllowlist with a justification, or make it a no-op above the floor.", + mf.version, floor, normalized) + } + } + } +} diff --git a/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql new file mode 100644 index 000000000..be319e19a --- /dev/null +++ b/internal/storage/schema/migrations/ignored/0028_converge_events_journal_shape.up.sql @@ -0,0 +1,172 @@ +-- Converge every observed shape of bd_events_journal on the canonical one +-- (BEADS-JOURNAL-PLAN.md §4.2b, PR A2). +-- +-- ignored/0023 repairs this table too, but only by probing DATA_TYPE: a +-- column that is entirely ABSENT returns no row, DATA_TYPE reads NULL, and +-- `NULL = 1` takes the no-op branch. That gap is exactly what let three +-- distinct field shapes survive 0023 unrepaired: +-- +-- - gas-city-inc (gci): seq, ts, op, issue_id, issue_json, dep_json — +-- comment_json and actor both absent outright (plan §1.1). 0023 widened +-- dep_json and added the ts index (both already correct, i.e. both were +-- present with some type), so neither of the two missing-column probes +-- ever fired. +-- - the events-journal-3666e5026 release shape: issue_json LONGTEXT, +-- dep_json TEXT, no comment_json, no ts index (plan §1.1, "where that +-- shape came from"). 0023's dep_json widen DOES fire here, but the +-- missing comment_json still does not, for the same reason as gci. +-- - the enterprise fork's ignored/0017 shape (0023's own header, +-- bd-t9ovd): dep_json and comment_json both TEXT (present, so 0023's +-- widen fires for both), no ts index, no idx_wisps_defer_until. This one +-- IS fully healed by 0023 already; it is included here as a fixture +-- (TestIgnored0028ConvergesAllShapes) precisely to pin that this file +-- stays a no-op on a shape 0023 already fixed, not to add new coverage +-- for it. +-- +-- This file closes the gap left open above by probing EXISTENCE first for +-- the two columns that can be missing outright, then falling through to +-- 0023's widen-if-TEXT check for both payload columns and both indexes — +-- each guarded step probes its own drift independently and does nothing when +-- it is absent, so a healthy workspace through either door (fresh init, +-- fresh clone) runs every probe below and issues no DDL. See 0023's header +-- for why that independence matters (crash-replay of an interrupted pass) +-- and for the no-DROP, no-rename invariant this file also upholds. +-- +-- It never reads or writes bd_events_seq or a single journal row: the +-- counter and every row must be byte-identical before and after +-- (TestIgnored0028ConvergesAllShapes). +-- +-- Two of these columns are registered as sentinels in schema.go's +-- ignoredSource (`{bd_events_journal, comment_json, replayFloor: 21}` and +-- `{bd_events_journal, actor, replayFloor: 21}`): a store whose cursor already +-- reads past 28 but is missing either column — e.g. restored from a backup +-- taken between the two doors, or damaged out of band — heals itself on the +-- next writable open instead of staying silently broken forever +-- (TestSentinelJournalColumnsReplay). The actor sentinel alone also fires on +-- every store sitting at ignored cursor 22-27 without it, which is every +-- production Dolt store measured at cursor 24 (BEADS-JOURNAL-PLAN.md §4.1): +-- those stores replay 22-28 once, on their first open by a binary carrying +-- this sentinel. Floor 21 sits below this file (28) and below the table's +-- creator (ignored/0022), per schema.go's two replay-floor constraints, and +-- the only DML above it is 0022's INSERT IGNORE seed and GREATEST raise +-- (TestJournalSentinelReplayRangeDMLIsAllowlisted; +-- TestReplay22To28NoOpOnHealthyStore pins that nothing else in the range +-- touches any other table or row). +-- +-- Journal tables change only on this (ignored) plane: scripts/check- +-- migration-hygiene.sh fails a main-plane migration that carries DDL against +-- any bd_events_* table, unconditionally, so this file — not a main-plane +-- twin — is the only place bd_events_journal's shape can converge. + +-- comment_json: add as LONGTEXT if missing; widen if TEXT. An absent TABLE +-- takes the no-op branch explicitly (a COLUMNS probe alone cannot tell "no +-- column" from "no table"), so this step never depends on 0022 having run +-- earlier in the same pass. +SET @journal_exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'); +SET @comment_json_type = ( + SELECT DATA_TYPE + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'comment_json' +); +SET @sql = IF(@journal_exists = 0, 'SELECT 1', + IF(@comment_json_type IS NULL, + 'ALTER TABLE bd_events_journal ADD COLUMN comment_json LONGTEXT', + IF(@comment_json_type = 'text', + 'ALTER TABLE bd_events_journal MODIFY COLUMN comment_json LONGTEXT', + 'SELECT 1'))); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- actor: add VARCHAR(255) NOT NULL DEFAULT '' if missing. ignored/0025 +-- already does exactly this on the lineage that carries it; this is a second, +-- idempotent guard — identical definition, explicit table-exists probe as in +-- 0023/0025 — for any lineage that reaches 0028 without having run 0025 (the +-- sentinel above is what makes that heal rather than strand). +SET @needs_actor = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'actor') = 0, + 1, 0 +); +SET @sql = IF(@needs_actor = 1, + 'ALTER TABLE bd_events_journal ADD COLUMN actor VARCHAR(255) NOT NULL DEFAULT ''''', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- dep_json: widen if TEXT. 0023's own step, repeated here (not removed there: +-- shipped migrations are frozen, check C) for any lineage that reaches 0028 +-- without ever having run it. Every creator of this table this repo or its +-- fork lineage has ever shipped declares dep_json with SOME type, so a +-- missing table yields NULL here too, which takes the no-op branch (the +-- comment_json step above makes the same table-absence case explicit). +SET @dep_json_is_text = ( + SELECT IF(DATA_TYPE = 'text', 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'dep_json' +); +SET @sql = IF(@dep_json_is_text = 1, + 'ALTER TABLE bd_events_journal MODIFY COLUMN dep_json LONGTEXT', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- issue_json: widen if TEXT, same reasoning as dep_json. No observed shape +-- (gci, 3666e5026, the enterprise fork) has ever shipped this column as TEXT, +-- so this is a no-op everywhere it has been checked; included for the same +-- completeness reason as idx_bd_events_journal_issue below. +SET @issue_json_is_text = ( + SELECT IF(DATA_TYPE = 'text', 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND COLUMN_NAME = 'issue_json' +); +SET @sql = IF(@issue_json_is_text = 1, + 'ALTER TABLE bd_events_journal MODIFY COLUMN issue_json LONGTEXT', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- idx_bd_events_journal_ts: create if missing. 0023's own step, repeated for +-- the same reason as actor above. +SET @needs_ts_index = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND INDEX_NAME = 'idx_bd_events_journal_ts') = 0, + 1, 0 +); +SET @sql = IF(@needs_ts_index = 1, + 'CREATE INDEX idx_bd_events_journal_ts ON bd_events_journal(ts)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- idx_bd_events_journal_issue: create if missing. Every observed shape +-- (gci, 3666e5026, the enterprise fork) already carries this index — it is +-- declared inline in every CREATE this repo or its fork lineage has ever +-- shipped — but an unconditional guard costs one more INFORMATION_SCHEMA +-- round trip and closes the same class of gap 0023 left open for +-- idx_bd_events_journal_ts, against a shape nobody has observed yet. +SET @needs_issue_index = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'bd_events_journal' + AND INDEX_NAME = 'idx_bd_events_journal_issue') = 0, + 1, 0 +); +SET @sql = IF(@needs_issue_index = 1, + 'CREATE INDEX idx_bd_events_journal_issue ON bd_events_journal(issue_id)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/old_ignored_ceiling_test.go b/internal/storage/schema/old_ignored_ceiling_test.go new file mode 100644 index 000000000..00b587ab8 --- /dev/null +++ b/internal/storage/schema/old_ignored_ceiling_test.go @@ -0,0 +1,124 @@ +package schema + +import ( + "context" + "embed" + "regexp" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" +) + +// oldIgnoredCeiling24Files stands in for the ignored-migration set a binary +// built before BEADS-JOURNAL-PLAN.md's PR A2 would have embedded: 0025 +// (actor), 0026 (dep rekey marker), 0027 (wisps.current_revision) and 0028 +// (this plan's journal-shape convergence) all postdate it, so its ceiling is +// 24 — the d3ab CLI's measured ignored ceiling in the plan's evidence base +// (§4.1). Only the filename's version prefix is ever read +// (migrationSource.list/.latest), so one placeholder file is enough; it need +// not match the real ignored/0024 migration's SQL. +// +//go:embed testdata/old_ignored_ceiling_24/*.up.sql +var oldIgnoredCeiling24Files embed.FS + +// TestOldIgnoredCeilingOpensAfter0028 is T2.9 (BEADS-JOURNAL-PLAN.md §4.3, PR +// A2): a binary that only knows the ignored series through 24 opens a store +// whose ignored cursor is already at 28 without refusing, because nothing on +// the open path compares a binary's ignored ceiling against the stored +// cursor — CurrentIgnoredVersion/LatestIgnoredVersion feed no skew guard the +// way CurrentVersion/LatestVersion do for the main plane (checkSchemaSkew). +// +// Kills: +// - adding an ignored-plane skew check: oldIgnored (ceiling 24) reporting +// atLatest()==true and migrate() applying nothing against a mocked cursor +// of 28 pins that an old-ceiling source does not refuse or try to "catch +// up" just because the stored cursor is ahead of what it knows; and the +// checkSchemaSkew/CheckForwardDrift/CheckBehindDrift sub-tests below use a +// strict sqlmock (ExpectationsWereMet) that would fail if any of them +// issued a query against ignored_schema_migrations at all. +// - adding a main-plane twin for the journal migrations: that would move +// mainSource's own cursor, which the same checkSchemaSkew mock (driven +// only by schema_migrations, unaffected by the ignored plane) would catch +// as a changed main version the moment the twin's migration shipped. +func TestOldIgnoredCeilingOpensAfter0028(t *testing.T) { + oldIgnored := migrationSource{ + files: oldIgnoredCeiling24Files, + dir: "testdata/old_ignored_ceiling_24", + cursorTable: ignoredSource.cursorTable, + } + if got := oldIgnored.latest(); got != 24 { + t.Fatalf("oldIgnored.latest() = %d, want 24", got) + } + + t.Run("old-ceiling source sees the store as at-latest, not behind", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + expectCursorProbe(mock, "ignored_schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", 28) + + if !oldIgnored.atLatest(context.Background(), db) { + t.Error("oldIgnored.atLatest() = false for a cursor (28) past its own ceiling (24); an old binary must not see this store as behind") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } + }) + + t.Run("old-ceiling source applies nothing against a cursor past its ceiling", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CREATE TABLE IF NOT EXISTS ignored_schema_migrations")). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(regexp.QuoteMeta("SHOW COLUMNS FROM ignored_schema_migrations LIKE 'content_hash'")). + WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}). + AddRow("content_hash", "char(64)", "YES", "", nil, "")) + expectCursorProbe(mock, "ignored_schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", 28) + // No sentinelTables/sentinelColumns on oldIgnored (zero value), so + // cursorRealityFloor has nothing further to probe, and no + // ExecContext should run at all: migrate must see current(28) >= + // target(24) and return immediately. + + applied, _, err := oldIgnored.migrate(context.Background(), db, 0) + if err != nil { + t.Fatalf("oldIgnored.migrate: %v", err) + } + if applied != 0 { + t.Errorf("oldIgnored.migrate applied = %d, want 0 (a cursor past this source's ceiling has nothing pending)", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations (an old-ceiling binary must not touch the database at all here): %v", err) + } + }) + + t.Run("the real open-path guards never consult the ignored cursor", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + // Main plane only: at the binary's own latest, unaffected by the + // ignored cursor being 28. If checkSchemaSkew/CheckForwardDrift ever + // queried ignored_schema_migrations, the strict mock below (no such + // expectation registered) would fail the call. + expectCursorProbe(mock, "schema_migrations", true) + mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(version), 0) FROM schema_migrations")). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(LatestVersion())) + + if err := CheckForwardDrift(context.Background(), db); err != nil { + t.Errorf("CheckForwardDrift = %v, want nil (ignored=28 must not drive a main-plane forward-drift refusal)", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } + }) +} diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index b9dae90b6..a06a0e6ad 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -363,7 +363,9 @@ type schemaSentinelColumn struct { // restamping every wisp on a plane with no history to restore from // (#5981 harm class). ignored/0015 is the guarded successor and is // genuinely pending on those stores, so the recompute still happens - // — just without the timestamp damage. + // — just without the timestamp damage. Statements above the floor + // must be replay-safe, not merely guarded; see + // TestJournalSentinelReplayRangeDMLIsAllowlisted. // (ii) The migration that creates the column AND the migration that // creates the table carrying it must both be ABOVE the floor, so the // replay can still heal both shapes the single INFORMATION_SCHEMA @@ -405,7 +407,31 @@ var ( // 0001–0025 series replays, dragging in ignored/0007 and restamping // wisps.updated_at across the plane. Believing the cursor up to 11 // applies exactly the pending tail 0012–0025 instead. - sentinelColumns: []schemaSentinelColumn{{table: "leases", column: "granted_node", replayFloor: 11}}, + // + // bd_events_journal carries two more, registered together: a store + // whose cursor already reads past 28 but is missing comment_json or + // actor (restored from a backup taken between doors, or damaged out + // of band) heals itself on the next writable open instead of staying + // silently broken forever (BEADS-JOURNAL-PLAN.md §4.2b, PR A2). Floor + // 21 satisfies the same two constraints as the leases floor above: + // 0022 (the table's creator) and 0028 (comment_json's adder; actor's + // adder is 0025, also above 21) both sit strictly above it; + // ignored/0007's unguarded restamp sits below it, and the only DML + // above it is 0022's INSERT IGNORE seed and GREATEST raise + // (TestJournalSentinelReplayRangeDMLIsAllowlisted). + // + // The actor sentinel alone also fires on every store sitting at + // ignored cursor 22-27 that predates 0025, which is every production + // Dolt store measured at cursor 24 (BEADS-JOURNAL-PLAN.md §4.1): + // those stores replay 22-28 once, on their first open by a binary + // carrying this sentinel. 0022's tail is the healEventSeqCounter + // statement: a no-op on a counter already at or ahead of the + // journal, and a raise to exactly MAX(seq) on one that is behind. + sentinelColumns: []schemaSentinelColumn{ + {table: "leases", column: "granted_node", replayFloor: 11}, + {table: "bd_events_journal", column: "comment_json", replayFloor: 21}, + {table: "bd_events_journal", column: "actor", replayFloor: 21}, + }, } ) diff --git a/internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql b/internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql new file mode 100644 index 000000000..6f9e51c14 --- /dev/null +++ b/internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql @@ -0,0 +1,8 @@ +-- Placeholder for TestOldIgnoredCeilingOpensAfter0028 (T2.9, +-- BEADS-JOURNAL-PLAN.md PR A2): stands in for the newest ignored migration a +-- binary built before this plan's 0025-0028 would have embedded (the d3ab +-- CLI's measured ignored ceiling in the plan's evidence base, §4.1). Its +-- content is never executed by the test -- migrationSource.latest() and +-- .list() only parse the filename for a version number -- so this file need +-- not match the real ignored/0024 migration's SQL. +SELECT 1; diff --git a/internal/storage/storage.go b/internal/storage/storage.go index ab27581d8..6ac8539cb 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -915,6 +915,32 @@ type EventsJournalConfigurer interface { SetEventsJournalEnabled(enabled bool) } +// EventsJournalShapeChecker reports whether the LAST SetEventsJournalEnabled +// call on this instance was able to activate the journal. +// +// It exists because SetEventsJournalEnabled itself stays void: dozens of +// tests across three backends already call it as a bare statement, and +// adaptive I/O (PR A1) must not force every one of them to start checking an +// error it never had before. Instead, enabling the journal probes the table's +// shape internally and caches any failure; this is the seam a factory's open +// path checks immediately afterward to decide whether activation actually +// succeeded. A missing or unsupported table makes SetEventsJournalEnabled(true) +// a no-op for WRITES, from that store's perspective: nothing panics, no +// INSERT is attempted, and no write is lost or corrupted — but +// eventsjournal.Apply uses this checker to fail the overall open with a typed +// error (issueops.JournalShapeError) instead of letting every subsequent +// write fail one at a time, which is the gci failure mode this interface +// exists to avoid. A caller that enables the journal directly, bypassing +// Apply, and then reads it anyway gets a loud SQL error from the read itself +// rather than a silently wrong answer — reads are never gated by the enabled +// flag, so there is no "no-op" to fall back to there. +// +// A store that does not implement it is assumed never to fail activation +// (today, every implementation does). +type EventsJournalShapeChecker interface { + EventsJournalActivationError() error +} + // VersionedHistoryConfigurer controls dual-write issue-version history // activation on ONE storage instance. Implementations must never use // process-global state, for the same reason as EventsJournalConfigurer: diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index e7681d800..608601488 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -57,6 +57,15 @@ type doltSQLProvider struct { // eventsJournalEnabled activates the durable events journal for THIS // provider instance only. See SetEventsJournalEnabled. eventsJournalEnabled atomic.Bool + // journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled + // runs on activation (issueops.ProbeJournalShape, PR A1). Nil means either + // the journal is disabled or the probe failed — see journalActivationErr. + journalShape atomic.Pointer[issueops.JournalShape] + // journalActivationErr holds the probe's error when SetEventsJournalEnabled(true) + // found a table shape the journal cannot run against (a missing required + // column, or a missing table). Nil whenever journalShape is non-nil or the + // journal is disabled. See storage.EventsJournalShapeChecker. + journalActivationErr atomic.Pointer[error] // versionedHistoryEnabled activates dual-write issue-version history for // THIS provider instance only. See SetVersionedHistoryEnabled. versionedHistoryEnabled atomic.Bool @@ -72,8 +81,38 @@ type doltSQLProvider struct { // BeginTx). Without that binding the uow plumbing writes mutations while // journaling nothing — the failure is invisible, because the code runs and the // write lands and the journal is simply empty. +// +// Activating also probes bd_events_journal's actual shape ONCE +// (issueops.ProbeJournalShape, PR A1) and caches the result for this +// instance's lifetime; EventsJournalActivationError reports whether that +// probe found a shape the journal can actually run against. Deactivating +// clears both and skips the probe entirely — a disabled provider accepts any +// shape. func (p *doltSQLProvider) SetEventsJournalEnabled(enabled bool) { p.eventsJournalEnabled.Store(enabled) + if !enabled { + p.journalShape.Store(nil) + p.journalActivationErr.Store(nil) + return + } + shape, err := issueops.ProbeJournalShape(context.Background(), p.db) + if err != nil { + p.journalShape.Store(nil) + p.journalActivationErr.Store(&err) + return + } + p.journalShape.Store(shape) + p.journalActivationErr.Store(nil) +} + +// EventsJournalActivationError implements storage.EventsJournalShapeChecker: +// nil unless the last SetEventsJournalEnabled(true) call found a table shape +// the journal cannot run against (PR A1). +func (p *doltSQLProvider) EventsJournalActivationError() error { + if ptr := p.journalActivationErr.Load(); ptr != nil { + return *ptr + } + return nil } // SetVersionedHistoryEnabled activates dual-write issue-version history for @@ -160,6 +199,7 @@ var ( _ UnitOfWorkProvider = (*doltSQLProvider)(nil) _ TxProvider = (*doltSQLProvider)(nil) _ storage.EventsJournalConfigurer = (*doltSQLProvider)(nil) + _ storage.EventsJournalShapeChecker = (*doltSQLProvider)(nil) _ storage.VersionedHistoryConfigurer = (*doltSQLProvider)(nil) ) @@ -195,9 +235,20 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { // (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its // transaction. The blocked-recheck scope is bound and released the same // way; Commit takes what it recorded once the transaction has committed. + // The activation switch folds in the shape probe's success: a probe that + // found an unsupported table (journalShape nil) scopes this transaction as + // DISABLED regardless of what SetEventsJournalEnabled was called with, so a + // caller that enables the journal directly and ignores the returned + // EventsJournalActivationError still never attempts a write against a shape + // that cannot support it (storage.EventsJournalShapeChecker). Reads share + // this same BeginTx, and are never gated by the activation switch — only by + // the shape — since EventsJournalCursor has no enabled/disabled concept of + // its own. + journalShape := p.journalShape.Load() return &doltServerTx{ conn: conn, - clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), + clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load() && journalShape != nil), + clearJournalShape: issueops.ScopeEventsJournalShape(conn, journalShape), clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()), clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn), }, nil diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index 4fd396e7a..6c6f81702 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -18,6 +18,10 @@ type doltServerTx struct { // releaseConn and poisonConn — so the activation entry cannot outlive the // transaction it describes, whichever way the transaction ends. clearJournalScope func() + // clearJournalShape releases the probed *issueops.JournalShape BeginTx bound + // to conn (PR A1), the shape counterpart of clearJournalScope's enabled/disabled + // switch. Same binding at BeginTx, same release at releaseConn/poisonConn. + clearJournalShape func() // clearVersionScope is clearJournalScope's counterpart for dual-write // issue-version history: same binding at BeginTx, same release at // releaseConn/poisonConn. @@ -197,6 +201,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() + t.releaseJournalShapeScope() t.releaseVersionScope() t.releaseRecheckScope() if t.conn != nil { @@ -215,6 +220,15 @@ func (t *doltServerTx) releaseJournalScope() { } } +// releaseJournalShapeScope is releaseJournalScope's counterpart for the probed +// *issueops.JournalShape (PR A1). Idempotent for the same reason. +func (t *doltServerTx) releaseJournalShapeScope() { + if t.clearJournalShape != nil { + t.clearJournalShape() + t.clearJournalShape = nil + } +} + // releaseVersionScope is releaseJournalScope's counterpart for dual-write // issue-version history. Idempotent for the same reason. func (t *doltServerTx) releaseVersionScope() { @@ -241,6 +255,7 @@ func (t *doltServerTx) releaseRecheckScope() { // database/sql close the connection and drop it from the pool. func (t *doltServerTx) poisonConn() { t.releaseJournalScope() + t.releaseJournalShapeScope() t.releaseVersionScope() t.releaseRecheckScope() if t.conn == nil { diff --git a/internal/storage/uow/journal_scope_test.go b/internal/storage/uow/journal_scope_test.go index e41504494..05ccc37bc 100644 --- a/internal/storage/uow/journal_scope_test.go +++ b/internal/storage/uow/journal_scope_test.go @@ -22,6 +22,19 @@ import ( // commits, and the journal is simply empty — so it must be pinned from both // directions: enabled emits, disabled does not. +// expectCanonicalJournalShapeProbe arms the one INFORMATION_SCHEMA.COLUMNS +// query SetEventsJournalEnabled(true) now issues to probe bd_events_journal's +// shape (issueops.ProbeJournalShape, PR A1), returning every column a +// canonical table carries so activation succeeds with the shape every test in +// this file was written against before shape probing existed. +func expectCanonicalJournalShapeProbe(mock sqlmock.Sqlmock) { + mock.ExpectQuery("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS"). + WithArgs("bd_events_journal"). + WillReturnRows(sqlmock.NewRows([]string{"COLUMN_NAME"}). + AddRow("seq").AddRow("ts").AddRow("op").AddRow("issue_id"). + AddRow("actor").AddRow("issue_json").AddRow("dep_json").AddRow("comment_json")) +} + func TestProviderImplementsEventsJournalConfigurer(t *testing.T) { p, _ := newMockTxProvider(t) var configurer storage.EventsJournalConfigurer = p @@ -37,7 +50,10 @@ func TestProviderImplementsEventsJournalConfigurer(t *testing.T) { // internal bookkeeping. func TestBeginTxScopesJournalActivationToThePinnedConn(t *testing.T) { p, mock := newMockTxProvider(t) + expectCanonicalJournalShapeProbe(mock) p.SetEventsJournalEnabled(true) + require.NoError(t, p.EventsJournalActivationError(), + "activation against the canonical shape the probe expectation below describes must succeed") mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("UPDATE bd_events_seq SET next_seq").WillReturnResult(sqlmock.NewResult(0, 1)) diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index d61b68529..29bb5ee8c 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -352,6 +352,18 @@ func (p *notifyingProvider) SetEventsJournalEnabled(enabled bool) { } } +// EventsJournalActivationError forwards the inner provider's shape-probe +// result (storage.EventsJournalShapeChecker, PR A1), for the same reason +// SetEventsJournalEnabled forwards activation itself: the wrapper holds no +// journal state of its own, and eventsjournal.Apply must see the inner +// provider's real probe result through whichever wrapper sits on top of it. +func (p *notifyingProvider) EventsJournalActivationError() error { + if checker, ok := p.inner.(storage.EventsJournalShapeChecker); ok { + return checker.EventsJournalActivationError() + } + return nil +} + // SetVersionedHistoryEnabled forwards dual-write issue-version history // activation to the provider that actually binds it to a transaction // (doltSQLProvider.BeginTx), for the same reason and in the same shape as diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 5bccc063e..3ccc611cc 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -299,9 +299,13 @@ func TestNotifyingProviderBuildsRolesOnItself(t *testing.T) { // for both binds on the inner provider's transactions, and a retention // pass writes no bead. None of these build a role, so none of them can // build one "on itself". - "SetEventsJournalEnabled": true, - "RunEventsMaintenanceTx": true, - "SetVersionedHistoryEnabled": true, + "SetEventsJournalEnabled": true, + // EventsJournalActivationError (PR A1) forwards the inner provider's + // shape-probe result the same way SetEventsJournalEnabled forwards + // activation: a query, not a role constructor. + "EventsJournalActivationError": true, + "RunEventsMaintenanceTx": true, + "SetVersionedHistoryEnabled": true, } var checked int diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index f7aa0b8cb..e7ae732b8 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -26,6 +26,7 @@ go_test( "bazel_skip_allowlist_test.go", "check_build_tags_test.go", "check_doc_freshness_test.go", + "check_migration_hygiene_journal_plane_test.go", "check_release_cross_compile_test.go", "check_testing_short_test.go", "check_winget_portable_alias_test.go", diff --git a/scripts/check-migration-hygiene.sh b/scripts/check-migration-hygiene.sh index 7a1db5af4..50250abea 100755 --- a/scripts/check-migration-hygiene.sh +++ b/scripts/check-migration-hygiene.sh @@ -86,7 +86,23 @@ # point of the check is that the idiom stops spreading into migrations # where the write does matter. # -# Checks C, D, and E compare against $BASE_SHA if set (CI passes the PR base), +# F. A new main-plane migration must not contain DDL (CREATE/ALTER/RENAME/ +# DROP TABLE) against a journal table (`bd_events_journal`, +# `bd_events_seq`). Unlike check D, there is no twin exemption: journal +# tables change only on the ignored plane (Dolt) or in Postgres DDL +# (BEADS-JOURNAL-PLAN.md §4.2c, PR A2) — an ignored-series twin does not +# excuse a main-plane DDL statement here the way it does for the other +# clone-local tables check D covers, because a main-plane change to +# these specific tables bumps the skew-checked main cursor (schema.go's +# forward-drift guard) for a table whose whole point is to stay +# invisible to binaries that predate it. BEADS-JOURNAL-PLAN.md names +# this rule "hygiene check E" and its test `TestHygieneCheckE`; it is +# implemented here as check F and the test is named for what it checks +# rather than for the plan's letter, because this repo's own check E +# already means something else (no PREPARE'd DML, above) and reusing the +# letter would misdescribe both. +# +# Checks C, D, E, and F compare against $BASE_SHA if set (CI passes the PR base), # else origin/main, else main; they are skipped with a warning when no base # is resolvable (e.g. shallow clone without the base commit). @@ -439,6 +455,37 @@ EOF done fi +# --- Check F: no main-plane DDL on journal tables --------------------------- +# bd_events_journal/bd_events_seq change only on the ignored plane (Dolt) or +# in Postgres DDL (BEADS-JOURNAL-PLAN.md §4.2c, PR A2). Scoped to added_main +# the same as checks D and E; unlike D, there is no twin exemption here — +# see this check's doc comment at the top of the file for why. +if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then + echo "WARN (journal plane) no usable base ref; skipping check F." >&2 +else + journal_tables='bd_events_journal|bd_events_seq' + ddl_re_f="(alter|create|rename|drop)[[:space:]]+table([[:space:]]+if[[:space:]]+(not[[:space:]]+)?exists)?[[:space:]]+[\`']?(${journal_tables})\b" + for f in $added_main; do + [ -f "$f" ] || continue + stripped=$(sed 's/--.*$//' "$f" | tr '[:upper:]' '[:lower:]') + hits=$(grep -n -E "$ddl_re_f" <<<"$stripped" || true) + if [ -n "$hits" ]; then + fail=1 + echo "FAIL (journal plane) $f contains main-plane DDL on a journal table:" + echo "$hits" | sed 's/^/ line /' + cat <<'EOF' + bd_events_journal and bd_events_seq change only on the ignored plane: a + main-plane migration touching either one bumps the skew-checked main + cursor for a table whose whole point is to stay invisible to binaries that + predate it (BEADS-JOURNAL-PLAN.md §4.2c). Move this DDL to a new file under + internal/storage/schema/migrations/ignored/ instead; there is no twin + exemption for these two tables the way check D allows for other + clone-local tables. +EOF + fi + done +fi + if [ "$fail" -ne 0 ]; then echo echo "Migration hygiene check failed. See docs in scripts/check-migration-hygiene.sh." diff --git a/scripts/check_migration_hygiene_journal_plane_test.go b/scripts/check_migration_hygiene_journal_plane_test.go new file mode 100644 index 000000000..0d99258aa --- /dev/null +++ b/scripts/check_migration_hygiene_journal_plane_test.go @@ -0,0 +1,139 @@ +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// This file is T2.8 (BEADS-JOURNAL-PLAN.md §4.3, PR A2): a main-plane +// migration with DDL on bd_events_* fails check-migration-hygiene.sh. The +// plan names the rule "hygiene check E" and the test TestHygieneCheckE, but +// this repo's own check E already means something else (no PREPARE'd DML in +// new main-plane migrations — see that check's doc comment in +// check-migration-hygiene.sh). The rule is implemented here as a new check F +// instead, and these tests are named for what they check rather than for the +// plan's letter, to avoid misdescribing either check. + +// runCheckMigrationHygiene runs check-migration-hygiene.sh in a throwaway git +// repository: baseFiles are committed on main (standing in for the PR's +// base); newFiles are then added to the working tree uncommitted (standing in +// for the PR's diff — checks C, D, E and F all diff the working tree against +// this base commit). +func runCheckMigrationHygiene(t *testing.T, baseFiles, newFiles map[string]string) (string, error) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("checker is a Bash boundary") + } + requireHostTool(t, "git") + requireHostTool(t, "bash") + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "check-migration-hygiene.sh")) + if err != nil { + t.Fatalf("read check-migration-hygiene.sh: %v", err) + } + dir := t.TempDir() + write := func(files map[string]string) { + for rel, content := range files { + full := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(full), err) + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + } + all := map[string]string{"scripts/check-migration-hygiene.sh": string(script)} + for rel, content := range baseFiles { + all[rel] = content + } + write(all) + env := append(os.Environ(), + "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"add", "-A"}, {"commit", "-q", "-m", "base"}} { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = env + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + write(newFiles) + cmd := exec.Command("bash", filepath.Join(dir, "scripts", "check-migration-hygiene.sh")) + cmd.Dir = dir + cmd.Env = env + out, err := cmd.CombinedOutput() + return string(out), err +} + +// TestHygieneCheckFRejectsMainPlaneJournalDDL is T2.8: a new main-plane +// migration altering bd_events_journal fails the script even though it ships +// an ignored-series twin in the same change — proving this is check F (no +// twin exemption, ever) and not merely check D (twin required) passing +// through by accident. Kills: removing check F, or weakening it to accept a +// twin the way check D does. +func TestHygieneCheckFRejectsMainPlaneJournalDDL(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/0100_add_journal_widget.up.sql": "ALTER TABLE bd_events_journal ADD COLUMN widget VARCHAR(32);\n", + "internal/storage/schema/migrations/ignored/0100_add_journal_widget.up.sql": "-- twin: not what check F looks at; its presence must not save the main-plane file.\nSELECT 1;\n", + }) + if err == nil { + t.Fatalf("main-plane DDL on bd_events_journal was accepted even with an ignored twin:\n%s", out) + } + if !strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("expected a \"FAIL (journal plane)\" report, got:\n%s", out) + } + if !strings.Contains(out, "0100_add_journal_widget.up.sql") { + t.Errorf("expected the offending file named in output, got:\n%s", out) + } +} + +// TestHygieneCheckFRejectsMainPlaneEventsSeqDDL pins that the rule covers +// bd_events_seq too, not only bd_events_journal. +func TestHygieneCheckFRejectsMainPlaneEventsSeqDDL(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/0100_rename_events_seq.up.sql": "ALTER TABLE bd_events_seq ADD COLUMN note VARCHAR(8);\n", + }) + if err == nil { + t.Fatalf("main-plane DDL on bd_events_seq was accepted:\n%s", out) + } + if !strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("expected a \"FAIL (journal plane)\" report, got:\n%s", out) + } +} + +// TestHygieneCheckFAllowsMainPlaneDDLOnNonJournalTables is the negative +// control: a main-plane migration that never mentions a journal table passes +// cleanly, so check F is not a blanket rejection of main-plane DDL. +func TestHygieneCheckFAllowsMainPlaneDDLOnNonJournalTables(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/0100_add_issues_widget.up.sql": "ALTER TABLE issues ADD COLUMN widget VARCHAR(32);\n", + }) + if err != nil { + t.Fatalf("main-plane DDL on a non-journal table was rejected: %v\n%s", err, out) + } + if strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("did not expect a journal-plane failure, got:\n%s", out) + } +} + +// TestHygieneCheckFIgnoresIgnoredPlaneJournalDDL confirms the rule is +// main-plane-only: the same DDL shipped as an ignored-series migration (the +// only place journal tables are allowed to change, PR A2) passes. +func TestHygieneCheckFIgnoresIgnoredPlaneJournalDDL(t *testing.T) { + out, err := runCheckMigrationHygiene(t, nil, map[string]string{ + "internal/storage/schema/migrations/ignored/0100_add_journal_widget.up.sql": "ALTER TABLE bd_events_journal ADD COLUMN widget VARCHAR(32);\n", + }) + if err != nil { + t.Fatalf("ignored-plane DDL on bd_events_journal was rejected: %v\n%s", err, out) + } + if strings.Contains(out, "FAIL (journal plane)") { + t.Errorf("did not expect a journal-plane failure for an ignored-series migration, got:\n%s", out) + } +}