Compare commits
6 commits
main
...
a2-journal
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e81d02c195 | ||
|
|
9895023d85 | ||
|
|
851db307dd | ||
|
|
28ea285662 | ||
|
|
b2ad36a797 | ||
|
|
5a808e3eef |
37 changed files with 2975 additions and 57 deletions
12
.github/scripts/embedded-storage-test-shards.txt
vendored
12
.github/scripts/embedded-storage-test-shards.txt
vendored
|
|
@ -154,8 +154,10 @@
|
|||
15 1 TestEmbeddedRoleFixtureKitHooksAreUsable
|
||||
15 1 TestLifecycleCreateContract
|
||||
15 1 TestMergeWithStrategy_OursKeepsLocal
|
||||
15 1 TestStoreAtIgnored28WritesAndJournals
|
||||
|
||||
15 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase
|
||||
15 2 TestAdaptiveInsert_GciShape
|
||||
15 2 TestCleanGitRemoteCacheGarbage
|
||||
15 2 TestCreateBatchFastPathsMatchPerRow_Embedded
|
||||
15 2 TestDepRekeyMarkerStillInIgnoredSeries
|
||||
|
|
@ -229,6 +231,7 @@
|
|||
15 4 TestIgnoredCursorHealNeverResurrectsRolledBackVersions
|
||||
15 4 TestMain
|
||||
15 4 TestMemoriesContract
|
||||
15 4 TestMissingRequiredColumnRefusesAtOpen
|
||||
15 4 TestPromoteClearsNoHistory
|
||||
15 4 TestRemoteVerbPlainRemoteKeepsAmbientEnv
|
||||
15 4 TestReopenIssue
|
||||
|
|
@ -236,6 +239,7 @@
|
|||
15 4 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials
|
||||
|
||||
15 5 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase
|
||||
15 5 TestAdaptiveRead_GciShape
|
||||
15 5 TestBlockingAnnotatorContract
|
||||
15 5 TestBootstrapperContract
|
||||
15 5 TestBuildDSN_WithDatabase
|
||||
|
|
@ -290,6 +294,7 @@
|
|||
15 6 TestRenameLabel
|
||||
15 6 TestRenameLabelMintsOneVersionPerTouchedBead
|
||||
15 6 TestRunInTransactionPostCommitStageFailureIsIndeterminate
|
||||
15 6 TestSentinelReplayCounterConverges
|
||||
15 6 TestStatsReporterContract
|
||||
15 6 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials
|
||||
15 6 TestWithPeerAuth_RestoresEnvWhenCallbackFails
|
||||
|
|
@ -304,6 +309,7 @@
|
|||
15 7 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType
|
||||
15 7 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane
|
||||
15 7 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp
|
||||
15 7 TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched
|
||||
15 7 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender
|
||||
15 7 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext
|
||||
15 7 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet
|
||||
|
|
@ -346,6 +352,7 @@
|
|||
15 8 TestEmbeddedSQLCommitResponseLossIsIndeterminate
|
||||
15 8 TestGetReadyWorkIncludeEphemeralAppliesWispFilters
|
||||
15 8 TestHasRepository
|
||||
15 8 TestIgnored0028ConvergesAllShapes
|
||||
15 8 TestLenientGateWarning_ComposedOutput
|
||||
15 8 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle
|
||||
15 8 TestReclaimExpiredLeaseSurvivesRestartEmbedded
|
||||
|
|
@ -375,6 +382,7 @@
|
|||
15 9 TestMetadataCASContract
|
||||
15 9 TestPromoteStillWorksWithCollisionGuard
|
||||
15 9 TestReadyWorkWithMissingDeferredParentTables
|
||||
15 9 TestReplay22To28NoOpOnHealthyStore
|
||||
15 9 TestUpdateMergeOps_DoltEngine
|
||||
|
||||
15 10 TestBuildDSN_SpacesInPath
|
||||
|
|
@ -403,6 +411,7 @@
|
|||
15 10 TestGetReadyWorkMoleculeFilter
|
||||
15 10 TestHelperProcess
|
||||
15 10 TestIgnoredCursorHealResumeRespectsOperatorOverride
|
||||
15 10 TestJournalShapeProbeOncePerOpen
|
||||
15 10 TestReclaimRefusesForeignReplicaLease
|
||||
15 10 TestRemoteEntryPointsUseVersionControlOps
|
||||
15 10 TestTransactionCleanupBeforeSQLCommitRemainsDefinite
|
||||
|
|
@ -452,6 +461,7 @@
|
|||
15 12 TestEmbeddedRecomputeAllBlockedWiring
|
||||
15 12 TestEmbeddedRelationsResolvesAWispAnchor
|
||||
15 12 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges
|
||||
15 12 TestIgnored0028AloneLeavesCounterAndRowsUntouched
|
||||
15 12 TestLargeBatchApplyStatementCounts712_Embedded
|
||||
15 12 TestLifecycleUpdateContract
|
||||
15 12 TestPreviewStoreRefusesLocalMetadataWrite
|
||||
|
|
@ -506,6 +516,7 @@
|
|||
15 14 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits
|
||||
15 14 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor
|
||||
15 14 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff
|
||||
15 14 TestIgnored0028NoopsWithoutItsTables
|
||||
15 14 TestMergeWithStrategy_TheirsResolvesAndCommits
|
||||
15 14 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint
|
||||
15 14 TestOpenAfterCloseCreatesNewStore
|
||||
|
|
@ -539,3 +550,4 @@
|
|||
15 15 TestIgnoredCursorHealIsIdempotent
|
||||
15 15 TestReadyWorkStatementBudget
|
||||
15 15 TestReleaserContract
|
||||
15 15 TestSentinelJournalColumnsReplay
|
||||
|
|
|
|||
|
|
@ -683,6 +683,7 @@ go_test(
|
|||
"doctor_health_smoke_test.go",
|
||||
"doctor_health_test.go",
|
||||
"doctor_init_diagnostics_cgo_test.go",
|
||||
"doctor_journal_shape_test.go",
|
||||
"doctor_max_rows_test.go",
|
||||
"doctor_migrate_fix_test.go",
|
||||
"doctor_mutation_gate_test.go",
|
||||
|
|
@ -1250,6 +1251,7 @@ go_test(
|
|||
deps = [
|
||||
"//:beads",
|
||||
"//cmd/bd/doctor",
|
||||
"//cmd/bd/doctor/fix",
|
||||
"//internal/ado",
|
||||
"//internal/beads",
|
||||
"//internal/ceiling",
|
||||
|
|
|
|||
|
|
@ -748,6 +748,13 @@ func runDiagnostics(path string) doctorResult {
|
|||
skewCheck := convertWithCategory(doctor.CheckMigrationContentSkew(sharedStore), doctor.CategoryData)
|
||||
result.Checks = append(result.Checks, skewCheck)
|
||||
|
||||
// Check 7g: Events journal shape (BEADS-JOURNAL-PLAN.md §4.2f, PR A2, T2.12).
|
||||
journalShapeCheck := convertDoctorCheck(doctor.CheckEventsJournalShape(path))
|
||||
result.Checks = append(result.Checks, journalShapeCheck)
|
||||
if journalShapeCheck.Status == statusError {
|
||||
result.OverallOK = false
|
||||
}
|
||||
|
||||
// Dolt health checks (connection, schema, issue count, status).
|
||||
for _, dc := range doctor.RunDoltHealthChecks(path) {
|
||||
result.Checks = append(result.Checks, convertDoctorCheck(dc))
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ go_library(
|
|||
"hooks_migration.go",
|
||||
"installation.go",
|
||||
"integrity.go",
|
||||
"journal_shape.go",
|
||||
"kv.go",
|
||||
"label_whitespace.go",
|
||||
"legacy.go",
|
||||
|
|
|
|||
|
|
@ -136,6 +136,16 @@ func SchemaCompatibility(path string) error {
|
|||
return DatabaseVersion(path)
|
||||
}
|
||||
|
||||
// EventsJournalShape fixes the "Events Journal Shape" doctor finding
|
||||
// (BEADS-JOURNAL-PLAN.md §4.2f, PR A2: missing optional bd_events_journal
|
||||
// columns, or a bd_events_seq counter behind bd_events_journal's MAX(seq)).
|
||||
// The converging ignored-plane migration (ignored/0028) is applied as a side
|
||||
// effect of DatabaseVersion's writable store open, the same as every other
|
||||
// pending-migration fix in this file.
|
||||
func EventsJournalShape(path string) error {
|
||||
return DatabaseVersion(path)
|
||||
}
|
||||
|
||||
// FreshCloneImport handles the "Fresh Clone" fix: imports JSONL issues into an
|
||||
// existing (possibly empty) Dolt store. This covers the case where the Database
|
||||
// fix already created the store but a prior version didn't import.
|
||||
|
|
|
|||
|
|
@ -243,6 +243,8 @@ func SanitizeFixRecommendation(fix string, gate FixGate) string {
|
|||
// audited against that rule. The ones deliberately left out open a store, so
|
||||
// being withheld under a blocked gate is correct rather than an omission:
|
||||
// "Database", "Database Integrity", "Fresh Clone", "Schema Compatibility",
|
||||
// "Events Journal Shape" (opens the store, applying the converging
|
||||
// ignored-plane migration as a side effect, same as Schema Compatibility),
|
||||
// "Repo Fingerprint" (fix.RepoFingerprint and fix.FixMissingMetadata both write
|
||||
// through the store), "Dolt Schema", "Pending Migrations", and the row-level
|
||||
// data repairs. "Sync Divergence", "JSONL Config" and "Untracked Files" are
|
||||
|
|
|
|||
240
cmd/bd/doctor/journal_shape.go
Normal file
240
cmd/bd/doctor/journal_shape.go
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
package doctor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// journalOptionalColumns mirrors the optional-column list
|
||||
// internal/storage/issueops/journal_shape.go's ProbeJournalShape treats as
|
||||
// optional (BEADS-JOURNAL-PLAN.md §4.2a, PR A1). It is duplicated here
|
||||
// rather than imported: this check reports over doctor's own plain
|
||||
// *sql.DB (openDoltConn), and ProbeJournalShape prints its own
|
||||
// once-per-probe STDERR warning as a side effect, which would duplicate
|
||||
// this check's finding on every `bd doctor` run.
|
||||
var journalOptionalColumns = []string{"actor", "issue_json", "dep_json", "comment_json"}
|
||||
|
||||
// CheckEventsJournalShape is the standalone entry point for the "Events
|
||||
// Journal Shape" doctor check (T2.12, BEADS-JOURNAL-PLAN.md §4.2f, PR A2).
|
||||
// It opens its own connection rather than joining RunDoltHealthChecks'
|
||||
// shared one, the same standalone shape as CheckDoltRemoteGitOrigin and
|
||||
// CheckMigrationContentSkew, so it can be registered in doctor.go beside
|
||||
// them without touching runDoltHealthChecksInternal's fixed-shape branches.
|
||||
func CheckEventsJournalShape(path string) DoctorCheck {
|
||||
beadsDir := ResolveBeadsDirForRepo(path)
|
||||
|
||||
if !IsDoltBackend(beadsDir) {
|
||||
return DoctorCheck{
|
||||
Name: "Events Journal Shape",
|
||||
Status: StatusOK,
|
||||
Message: "N/A (non-Dolt backend)",
|
||||
Category: CategoryDolt,
|
||||
}
|
||||
}
|
||||
|
||||
conn, err := openDoltConn(beadsDir)
|
||||
if err != nil {
|
||||
// Matches runDoltHealthChecksInternal's own no-server-running
|
||||
// tolerance: a store that hasn't been opened yet (server not started)
|
||||
// is not a journal-shape finding.
|
||||
return DoctorCheck{
|
||||
Name: "Events Journal Shape",
|
||||
Status: StatusOK,
|
||||
Message: "Skipped (no server running; will auto-start on next bd command)",
|
||||
Category: CategoryDolt,
|
||||
}
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
return checkEventsJournalShapeWithDB(conn)
|
||||
}
|
||||
|
||||
// checkEventsJournalShapeWithDB is T2.12 (BEADS-JOURNAL-PLAN.md §4.2f, PR
|
||||
// A2): "bd doctor 'events journal shape'" reports:
|
||||
// - optional columns bd_events_journal is still missing. Adaptive I/O (PR
|
||||
// A1) keeps a table in this state writable and readable, dropping only
|
||||
// the affected payloads, but a dropped payload is still a finding — the
|
||||
// plan's own risk note says to "keep the doctor finding and a
|
||||
// once-per-store warning, never silence".
|
||||
// - the bd_events_seq counter sitting behind bd_events_journal's own
|
||||
// MAX(seq): the counter must never trail what has already been durably
|
||||
// written (nextEventSeq increments it inside the same transaction as the
|
||||
// row that carries its new value), so this can only mean the counter was
|
||||
// lost and reseeded (e.g. restored from a stale backup) without the rows
|
||||
// that came after it.
|
||||
//
|
||||
// The plan's design note (f) also lists "outbox depth and the age of its
|
||||
// oldest row" and "a NULL epoch". Those refer to bd_events_outbox and
|
||||
// bd_events_seq.epoch, introduced by PRs A5/A6 (§4.3's own table scopes
|
||||
// T2.10/T2.11 to Postgres/PR B1 and T2.13 to PR A12; A5/A6 are likewise a
|
||||
// separate, later plan item, §5) — neither object exists in this schema yet,
|
||||
// so there is nothing to probe. Once they land, this is the function to
|
||||
// extend.
|
||||
func checkEventsJournalShapeWithDB(conn *doltConn) DoctorCheck {
|
||||
ctx := context.Background()
|
||||
|
||||
exists, err := tableExists(ctx, conn.db, "bd_events_journal")
|
||||
if err != nil {
|
||||
return DoctorCheck{
|
||||
Name: "Events Journal Shape",
|
||||
Status: StatusWarning,
|
||||
Message: "Could not probe bd_events_journal",
|
||||
Detail: err.Error(),
|
||||
Category: CategoryDolt,
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
return DoctorCheck{
|
||||
Name: "Events Journal Shape",
|
||||
Status: StatusOK,
|
||||
Message: "N/A (bd_events_journal not created yet)",
|
||||
Category: CategoryDolt,
|
||||
}
|
||||
}
|
||||
|
||||
var findings []string
|
||||
var hasMissingColumns, hasUnfixableFinding bool
|
||||
|
||||
missingOptional, err := missingJournalOptionalColumns(ctx, conn.db)
|
||||
if err != nil {
|
||||
findings = append(findings, fmt.Sprintf("could not read bd_events_journal's columns: %v", err))
|
||||
hasUnfixableFinding = true
|
||||
} else if len(missingOptional) > 0 {
|
||||
findings = append(findings, fmt.Sprintf(
|
||||
"bd_events_journal is missing optional column(s): %s (affected payloads are dropped on write)",
|
||||
strings.Join(missingOptional, ", ")))
|
||||
hasMissingColumns = true
|
||||
}
|
||||
|
||||
// The counter-behind finding is deliberately NOT claimed as --fix-able,
|
||||
// but that is a scoping choice, not a capability gap: `--fix` (what
|
||||
// fix.EventsJournalShape/DatabaseVersion does) does not target this
|
||||
// finding directly, yet it is repaired as a side effect whenever the same
|
||||
// open replays ignored/0022 (any sentinel contradiction — see
|
||||
// schema.go's bd_events_journal sentinels), by the same GREATEST raise
|
||||
// the writer's own heal uses. A finding about the counter alone, with no
|
||||
// column also missing, triggers no such replay. The plan's design note
|
||||
// (f) only promises "--fix applies the converging DDL" — it says nothing
|
||||
// about reseeding a counter on its own, so this stays a report-only
|
||||
// finding in A2. Making it fixable on its own is an optional follow-up.
|
||||
counterBehind, counterDetail, err := journalCounterBehindMaxSeq(ctx, conn.db)
|
||||
if err != nil {
|
||||
findings = append(findings, fmt.Sprintf("could not compare bd_events_seq to MAX(seq): %v", err))
|
||||
hasUnfixableFinding = true
|
||||
} else if counterBehind {
|
||||
findings = append(findings, counterDetail+" (not repaired by 'bd doctor --fix'; needs manual investigation)")
|
||||
hasUnfixableFinding = true
|
||||
}
|
||||
|
||||
if len(findings) == 0 {
|
||||
return DoctorCheck{
|
||||
Name: "Events Journal Shape",
|
||||
Status: StatusOK,
|
||||
Message: "bd_events_journal shape converged",
|
||||
Category: CategoryDolt,
|
||||
}
|
||||
}
|
||||
|
||||
check := DoctorCheck{
|
||||
Name: "Events Journal Shape",
|
||||
Status: StatusWarning,
|
||||
Message: fmt.Sprintf("%d finding(s)", len(findings)),
|
||||
Detail: strings.Join(findings, "\n"),
|
||||
Category: CategoryDolt,
|
||||
}
|
||||
switch {
|
||||
case hasMissingColumns && hasUnfixableFinding:
|
||||
check.Fix = "Run 'bd doctor --fix' to add the missing column(s); the remaining finding(s) above need manual investigation"
|
||||
case hasMissingColumns:
|
||||
check.Fix = "Run 'bd doctor --fix' to apply the converging ignored-plane migration (adds comment_json/actor, widens columns, creates indexes)"
|
||||
}
|
||||
return check
|
||||
}
|
||||
|
||||
// tableExists reports whether table exists in the connected database. A
|
||||
// read-only probe (COUNT with LIMIT 1), matching checkSchemaWithDB's idiom
|
||||
// elsewhere in this package.
|
||||
func tableExists(ctx context.Context, db *sql.DB, table string) (bool, error) {
|
||||
var count int
|
||||
err := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s LIMIT 1", table)).Scan(&count) //nolint:gosec // table is one of this file's own constant strings, never user input
|
||||
if err != nil {
|
||||
// A missing table is not this function's error to report: every other
|
||||
// SQL failure is.
|
||||
if strings.Contains(strings.ToLower(err.Error()), "doesn't exist") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "not found") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "unknown table") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// missingJournalOptionalColumns runs the same INFORMATION_SCHEMA.COLUMNS
|
||||
// probe internal/storage/issueops.ProbeJournalShape uses (over a plain
|
||||
// connection, with no side-effecting warning), and returns which of
|
||||
// journalOptionalColumns bd_events_journal does not carry.
|
||||
func missingJournalOptionalColumns(ctx context.Context, db *sql.DB) ([]string, error) {
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
present := make(map[string]bool, len(journalOptionalColumns))
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
present[strings.ToLower(name)] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var missing []string
|
||||
for _, col := range journalOptionalColumns {
|
||||
if !present[col] {
|
||||
missing = append(missing, col)
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
// journalCounterBehindMaxSeq reports whether bd_events_seq's counter trails
|
||||
// the highest seq actually recorded in bd_events_journal. bd_events_seq may
|
||||
// not exist on a database that predates ignored/0019 (the events table) or
|
||||
// has never journaled; that reads as "not behind" (nothing to compare),
|
||||
// never as a finding.
|
||||
func journalCounterBehindMaxSeq(ctx context.Context, db *sql.DB) (bool, string, error) {
|
||||
counterExists, err := tableExists(ctx, db, "bd_events_seq")
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if !counterExists {
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
var counter, maxSeq int64
|
||||
err = db.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
(SELECT COALESCE(next_seq, 0) FROM bd_events_seq WHERE id = 0),
|
||||
(SELECT COALESCE(MAX(seq), 0) FROM bd_events_journal)
|
||||
`).Scan(&counter, &maxSeq)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
if counter < maxSeq {
|
||||
return true, fmt.Sprintf(
|
||||
"bd_events_seq counter (%d) is behind bd_events_journal's MAX(seq) (%d): "+
|
||||
"the counter was likely lost and reseeded without the rows written after it",
|
||||
counter, maxSeq), nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
|
|
@ -403,6 +403,8 @@ func applyFixList(path string, fixes []doctorCheck) {
|
|||
err = fix.DatabaseIntegrity(path)
|
||||
case "Schema Compatibility":
|
||||
err = fix.SchemaCompatibility(path)
|
||||
case "Events Journal Shape":
|
||||
err = fix.EventsJournalShape(path)
|
||||
case "Repo Fingerprint":
|
||||
err = fix.RepoFingerprint(path, doctorYes)
|
||||
// Also repair any other missing metadata fields (bd_version, repo_id, clone_id)
|
||||
|
|
|
|||
177
cmd/bd/doctor_journal_shape_test.go
Normal file
177
cmd/bd/doctor_journal_shape_test.go
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
//go:build cgo
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/cmd/bd/doctor"
|
||||
"github.com/steveyegge/beads/cmd/bd/doctor/fix"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// findDoctorCheck returns the check named name from checks, or nil.
|
||||
func findDoctorCheck(checks []doctorCheck, name string) *doctorCheck {
|
||||
for i := range checks {
|
||||
if checks[i].Name == name {
|
||||
return &checks[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestDoctorJournalShape is T2.12 (BEADS-JOURNAL-PLAN.md §4.2f, PR A2):
|
||||
// "bd doctor" raises an "Events Journal Shape" finding for a bd_events_journal
|
||||
// missing an optional column, and `--fix` (fix.EventsJournalShape) clears it
|
||||
// by applying the converging ignored-plane migration (ignored/0028). A
|
||||
// counter behind bd_events_journal's own MAX(seq) is also raised, but is
|
||||
// deliberately NOT claimed as --fix-able: ignored/0028's own header states it
|
||||
// "never reads or writes bd_events_seq or a single journal row", and when no
|
||||
// column is also missing there is no sentinel contradiction to float the
|
||||
// ignored cursor's floor and force the replay that would otherwise touch it
|
||||
// (ignored/0022's own idempotent high-water-mark raise, re-run only as part
|
||||
// of that replay).
|
||||
//
|
||||
// Scope: this test covers only what A1/A2 actually built and what exists in
|
||||
// this schema today. The plan's design note (f) also lists "outbox depth and
|
||||
// the age of its oldest row" and "a NULL epoch"; those reference
|
||||
// bd_events_outbox and bd_events_seq.epoch, introduced by PRs A5/A6 (§5, a
|
||||
// separate, later plan item) — neither object exists yet, so there is
|
||||
// nothing for this test (or the check itself) to exercise for them.
|
||||
//
|
||||
// The two sub-findings are tested from independently damaged stores (not one
|
||||
// store carrying both faults at once): dropping comment_json ALSO floors the
|
||||
// ignored cursor via its sentinel and forces a 22..28 replay on the next
|
||||
// writable open, and that replay's pass through ignored/0022 re-raises
|
||||
// bd_events_seq to GREATEST(current, MAX(seq)) as an unrelated side effect —
|
||||
// entangling the two findings' fixability would make the counter case look
|
||||
// fixable only because a *different* finding's repair happened to touch it.
|
||||
//
|
||||
// Kills: the check not wired into doctor (runDiagnostics never calling
|
||||
// doctor.CheckEventsJournalShape) — confirmed via the clean-store sub-test
|
||||
// below, which goes through runDiagnostics itself rather than calling the
|
||||
// check function directly, and manually by temporarily removing doctor.go's
|
||||
// wiring call and observing it fail with "check not found" before restoring
|
||||
// it; the check function itself going unimplemented (a compile failure);
|
||||
// and --fix not actually converging the column shape (the post-fix re-probe
|
||||
// would still see missing comment_json).
|
||||
func TestDoctorJournalShape(t *testing.T) {
|
||||
t.Run("wired into runDiagnostics on a healthy store", func(t *testing.T) {
|
||||
tmpDir, store := setupValidateTestDB(t, "jnlok")
|
||||
ctx := context.Background()
|
||||
store.SetEventsJournalEnabled(true)
|
||||
if err := store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation: %v", err)
|
||||
}
|
||||
issue := &types.Issue{Title: "t2.12 wiring seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := store.CreateIssue(ctx, issue, "jnlok"); err != nil {
|
||||
t.Fatalf("CreateIssue: %v", err)
|
||||
}
|
||||
store.Close()
|
||||
|
||||
result := runDiagnostics(tmpDir)
|
||||
check := findDoctorCheck(result.Checks, "Events Journal Shape")
|
||||
if check == nil {
|
||||
t.Fatal("Events Journal Shape check not found in runDiagnostics' output")
|
||||
}
|
||||
if check.Status != statusOK {
|
||||
t.Errorf("status = %q, want %q (message: %s, detail: %s)", check.Status, statusOK, check.Message, check.Detail)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing optional column is raised and fixed", func(t *testing.T) {
|
||||
tmpDir, store := setupValidateTestDB(t, "jnlcol")
|
||||
ctx := context.Background()
|
||||
store.SetEventsJournalEnabled(true)
|
||||
if err := store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation: %v", err)
|
||||
}
|
||||
issue := &types.Issue{Title: "t2.12 column seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := store.CreateIssue(ctx, issue, "jnlcol"); err != nil {
|
||||
t.Fatalf("CreateIssue: %v", err)
|
||||
}
|
||||
|
||||
// Damage exactly as the plan's evidence base found it (gas-city-inc:
|
||||
// comment_json absent outright).
|
||||
if _, err := store.DB().ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil {
|
||||
t.Fatalf("drop comment_json: %v", err)
|
||||
}
|
||||
store.Close()
|
||||
|
||||
check := doctor.CheckEventsJournalShape(tmpDir)
|
||||
if check.Status != statusWarning {
|
||||
t.Fatalf("after damage: status = %q, want %q (message: %s)", check.Status, statusWarning, check.Message)
|
||||
}
|
||||
if !strings.Contains(check.Detail, "comment_json") {
|
||||
t.Errorf("after damage: Detail = %q, want it to mention comment_json", check.Detail)
|
||||
}
|
||||
if strings.Contains(check.Detail, "bd_events_seq counter") {
|
||||
t.Errorf("after damage: Detail = %q, unexpectedly mentions the counter (this store never touched it)", check.Detail)
|
||||
}
|
||||
if check.Fix == "" || !strings.Contains(check.Fix, "bd doctor --fix") {
|
||||
t.Errorf("after damage: Fix = %q, want guidance to run 'bd doctor --fix'", check.Fix)
|
||||
}
|
||||
|
||||
if err := fix.EventsJournalShape(tmpDir); err != nil {
|
||||
t.Fatalf("fix.EventsJournalShape: %v", err)
|
||||
}
|
||||
|
||||
check = doctor.CheckEventsJournalShape(tmpDir)
|
||||
if check.Status != statusOK {
|
||||
t.Fatalf("after fix: status = %q, want %q (message: %s, detail: %s)", check.Status, statusOK, check.Message, check.Detail)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("counter behind MAX(seq) is raised and not claimed fixable", func(t *testing.T) {
|
||||
tmpDir, store := setupValidateTestDB(t, "jnlseq")
|
||||
ctx := context.Background()
|
||||
store.SetEventsJournalEnabled(true)
|
||||
if err := store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation: %v", err)
|
||||
}
|
||||
issue := &types.Issue{Title: "t2.12 counter seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := store.CreateIssue(ctx, issue, "jnlseq"); err != nil {
|
||||
t.Fatalf("CreateIssue: %v", err)
|
||||
}
|
||||
|
||||
// Damage ONLY the counter (no column touched, so no sentinel
|
||||
// contradiction floors the ignored cursor and nothing replays
|
||||
// ignored/0022 on the next writable open).
|
||||
if _, err := store.DB().ExecContext(ctx, "UPDATE bd_events_seq SET next_seq = 0 WHERE id = 0"); err != nil {
|
||||
t.Fatalf("reseed next_seq behind MAX(seq): %v", err)
|
||||
}
|
||||
store.Close()
|
||||
|
||||
check := doctor.CheckEventsJournalShape(tmpDir)
|
||||
if check.Status != statusWarning {
|
||||
t.Fatalf("after damage: status = %q, want %q (message: %s)", check.Status, statusWarning, check.Message)
|
||||
}
|
||||
if !strings.Contains(check.Detail, "bd_events_seq counter") {
|
||||
t.Errorf("after damage: Detail = %q, want it to mention the counter behind MAX(seq)", check.Detail)
|
||||
}
|
||||
if strings.Contains(check.Detail, "comment_json") {
|
||||
t.Errorf("after damage: Detail = %q, unexpectedly mentions comment_json (this store never touched it)", check.Detail)
|
||||
}
|
||||
if !strings.Contains(check.Detail, "not repaired by 'bd doctor --fix'") {
|
||||
t.Errorf("after damage: Detail = %q, want it to disclose this finding is not --fix-able", check.Detail)
|
||||
}
|
||||
if strings.Contains(check.Fix, "bd doctor --fix") {
|
||||
t.Errorf("after damage: Fix = %q, must not claim 'bd doctor --fix' resolves a counter-behind finding", check.Fix)
|
||||
}
|
||||
|
||||
if err := fix.EventsJournalShape(tmpDir); err != nil {
|
||||
t.Fatalf("fix.EventsJournalShape: %v", err)
|
||||
}
|
||||
|
||||
check = doctor.CheckEventsJournalShape(tmpDir)
|
||||
if check.Status != statusWarning {
|
||||
t.Fatalf("after fix: status = %q, want %q (no pending migration exists to repair a counter-only fault, so --fix is a no-op here): message=%s detail=%s",
|
||||
check.Status, statusWarning, check.Message, check.Detail)
|
||||
}
|
||||
if !strings.Contains(check.Detail, "bd_events_seq counter") {
|
||||
t.Errorf("after fix: Detail = %q, want the still-unrepaired counter finding to remain", check.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -146,6 +146,12 @@ func ActivateStore(beadsDir string, s storage.DoltStorage, err error) (storage.D
|
|||
// the journal is that a consumer can trust its cursor, and a plumbing that
|
||||
// records nothing while reporting success is the one outcome that breaks that
|
||||
// trust invisibly. A disabled workspace accepts any plumbing.
|
||||
//
|
||||
// It also fails there when the configurer DOES support the journal but its
|
||||
// table's shape does not (storage.EventsJournalShapeChecker, PR A1): a missing
|
||||
// required column refuses at open, the same way a nil configurer does, rather
|
||||
// than letting SetEventsJournalEnabled succeed and every write fail
|
||||
// afterward — the gci failure mode this check replaces.
|
||||
func Apply(configurer storage.EventsJournalConfigurer, enabled bool) error {
|
||||
if configurer == nil {
|
||||
if enabled {
|
||||
|
|
@ -154,5 +160,10 @@ func Apply(configurer storage.EventsJournalConfigurer, enabled bool) error {
|
|||
return nil
|
||||
}
|
||||
configurer.SetEventsJournalEnabled(enabled)
|
||||
if checker, ok := configurer.(storage.EventsJournalShapeChecker); ok {
|
||||
if err := checker.EventsJournalActivationError(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -308,6 +308,7 @@ var _ storage.Compactor = (*DoltStore)(nil)
|
|||
var _ storage.SchemaMigrator = (*DoltStore)(nil)
|
||||
var _ storage.ExternalRefHistoryQuerier = (*DoltStore)(nil)
|
||||
var _ storage.EventsJournalConfigurer = (*DoltStore)(nil)
|
||||
var _ storage.EventsJournalShapeChecker = (*DoltStore)(nil)
|
||||
var _ storage.VersionedHistoryConfigurer = (*DoltStore)(nil)
|
||||
|
||||
// DoltStore implements the Storage interface using Dolt
|
||||
|
|
@ -320,6 +321,17 @@ type DoltStore struct {
|
|||
// eventsJournalEnabled activates the durable events journal for THIS store
|
||||
// instance only (storage.EventsJournalConfigurer); never process-global.
|
||||
eventsJournalEnabled atomic.Bool
|
||||
// journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled
|
||||
// runs on activation (issueops.ProbeJournalShape, PR A1). Nil means either
|
||||
// the journal is disabled or the probe failed; scopeEventsJournalTransaction
|
||||
// treats nil the same as "do not journal this transaction" — see
|
||||
// journalActivationErr.
|
||||
journalShape atomic.Pointer[issueops.JournalShape]
|
||||
// journalActivationErr holds the probe's error when SetEventsJournalEnabled(true)
|
||||
// found a table shape the journal cannot run against (a missing required
|
||||
// column, or a missing table). Nil whenever journalShape is non-nil or the
|
||||
// journal is disabled. See storage.EventsJournalShapeChecker.
|
||||
journalActivationErr atomic.Pointer[error]
|
||||
// versionedHistoryEnabled activates dual-write issue-version history for
|
||||
// THIS store instance only (storage.VersionedHistoryConfigurer); never
|
||||
// process-global.
|
||||
|
|
@ -1083,6 +1095,13 @@ func (s *DoltStore) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) e
|
|||
return fmt.Errorf("begin read tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
// ReadEventsJournal/ReadEventsJournalPage read through this helper, and
|
||||
// the column list they SELECT depends on the probed shape, not on
|
||||
// whether the journal is enabled for writes — scope the shape only, so
|
||||
// readEventsRowsInTx sees this store's actual table instead of
|
||||
// defaulting to canonicalJournalShape (PR A1).
|
||||
clearShape := issueops.ScopeEventsJournalShape(tx, s.journalShape.Load())
|
||||
defer clearShape()
|
||||
return fn(tx)
|
||||
})
|
||||
}
|
||||
|
|
@ -1379,12 +1398,56 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche
|
|||
}
|
||||
|
||||
// SetEventsJournalEnabled activates the journal for this store instance only.
|
||||
//
|
||||
// Activating probes bd_events_journal's actual shape ONCE
|
||||
// (issueops.ProbeJournalShape, PR A1) and caches the result for this
|
||||
// instance's lifetime; EventsJournalActivationError reports whether that
|
||||
// probe found a shape the journal can actually run against. Deactivating
|
||||
// clears both and skips the probe entirely — a disabled store accepts any
|
||||
// shape.
|
||||
func (s *DoltStore) SetEventsJournalEnabled(enabled bool) {
|
||||
s.eventsJournalEnabled.Store(enabled)
|
||||
if !enabled {
|
||||
s.journalShape.Store(nil)
|
||||
s.journalActivationErr.Store(nil)
|
||||
return
|
||||
}
|
||||
shape, err := issueops.ProbeJournalShape(context.Background(), s.db)
|
||||
if err != nil {
|
||||
s.journalShape.Store(nil)
|
||||
s.journalActivationErr.Store(&err)
|
||||
return
|
||||
}
|
||||
s.journalShape.Store(shape)
|
||||
s.journalActivationErr.Store(nil)
|
||||
}
|
||||
|
||||
// EventsJournalActivationError implements storage.EventsJournalShapeChecker:
|
||||
// nil unless the last SetEventsJournalEnabled(true) call found a table shape
|
||||
// the journal cannot run against.
|
||||
func (s *DoltStore) EventsJournalActivationError() error {
|
||||
if p := s.journalActivationErr.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scopeEventsJournalTransaction binds this store's cached activation state and
|
||||
// probed shape to tx. The activation switch folds in the shape probe's
|
||||
// success: a probe that found an unsupported table (journalShape nil) scopes
|
||||
// the transaction as DISABLED regardless of what SetEventsJournalEnabled was
|
||||
// called with, so a caller that enables the journal directly and ignores the
|
||||
// returned EventsJournalActivationError still never attempts a write against
|
||||
// a shape that cannot support it (storage.EventsJournalShapeChecker).
|
||||
func (s *DoltStore) scopeEventsJournalTransaction(tx *sql.Tx) func() {
|
||||
return issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load())
|
||||
shape := s.journalShape.Load()
|
||||
enabled := s.eventsJournalEnabled.Load() && shape != nil
|
||||
clearActivation := issueops.ScopeEventsJournalTransaction(tx, enabled)
|
||||
clearShape := issueops.ScopeEventsJournalShape(tx, shape)
|
||||
return func() {
|
||||
clearShape()
|
||||
clearActivation()
|
||||
}
|
||||
}
|
||||
|
||||
// SetVersionedHistoryEnabled activates dual-write issue-version history for
|
||||
|
|
|
|||
|
|
@ -222,7 +222,14 @@ func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg s
|
|||
// exists to make impossible. In journal mode both planes therefore share the
|
||||
// pinned regular transaction. The default journal-off path keeps the
|
||||
// established split transactions untouched.
|
||||
journalEnabled := s.eventsJournalEnabled.Load()
|
||||
// journalShape is folded into journalEnabled the same way
|
||||
// scopeEventsJournalTransaction folds it in for every other raw-tx mutator
|
||||
// (store.go): a probe that found an unsupported shape (journalShape nil)
|
||||
// must pin neither plane together nor attempt a write here, even if a
|
||||
// caller enabled the journal directly and ignored the returned
|
||||
// EventsJournalActivationError (storage.EventsJournalShapeChecker, PR A1).
|
||||
journalShape := s.journalShape.Load()
|
||||
journalEnabled := s.eventsJournalEnabled.Load() && journalShape != nil
|
||||
ignoredTx := regularTx
|
||||
if !journalEnabled {
|
||||
// NOTE (GH#3140 metrics skew): the pool-wait bracket above measures only
|
||||
|
|
@ -241,6 +248,16 @@ func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg s
|
|||
}
|
||||
clearJournalScope := issueops.ScopeEventsJournalTransaction(regularTx, journalEnabled)
|
||||
defer clearJournalScope()
|
||||
// Shape counterpart of the activation switch just above (PR A1):
|
||||
// insertEventRow/readEventsRowsInTx reach this store's probed shape only
|
||||
// through a tx-scoped value, never through the store itself, so without
|
||||
// this call every write on this path — CreateIssue, UpdateIssue, comments,
|
||||
// the whole doltTransaction mutator surface reached from
|
||||
// runDoltTransactionRecording — would silently fall back to
|
||||
// canonicalJournalShape and defeat adaptive I/O for this store's main
|
||||
// write path.
|
||||
clearJournalShape := issueops.ScopeEventsJournalShape(regularTx, journalShape)
|
||||
defer clearJournalShape()
|
||||
// Versioned history binds to the SAME regular transaction the mutation runs
|
||||
// in, for the same reason the journal does: RecordVersionInTx no-ops unless
|
||||
// this scope is set. Without it every issues-plane mutation routed through
|
||||
|
|
|
|||
|
|
@ -156,6 +156,8 @@ go_test(
|
|||
"journal_concurrency_test.go",
|
||||
"journal_contract_test.go",
|
||||
"journal_retention_test.go",
|
||||
"journal_shape_adaptive_test.go",
|
||||
"journal_shape_converge_test.go",
|
||||
"journal_truncation_test.go",
|
||||
"label_list_all_benchmark_test.go",
|
||||
"label_rename_test.go",
|
||||
|
|
|
|||
218
internal/storage/embeddeddolt/journal_shape_adaptive_test.go
Normal file
218
internal/storage/embeddeddolt/journal_shape_adaptive_test.go
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
//go:build cgo
|
||||
|
||||
package embeddeddolt_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage/issueops"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// Adaptive I/O for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2a/§4.3, PR
|
||||
// A1). These tests reproduce gas-city-inc's exact shape — a table built
|
||||
// before `comment_json` and `actor` ever existed — directly on an
|
||||
// otherwise-ordinary embedded store, and prove the writer and reader adapt to
|
||||
// it instead of failing every journaled write (T2.1/T2.2), that adaptation
|
||||
// probes the shape once per activation rather than once per write (T2.3),
|
||||
// and that a table missing a REQUIRED column refuses at open rather than
|
||||
// silently disabling the journal or failing per write (T2.4).
|
||||
|
||||
// newGciShapeStore returns a store whose bd_events_journal has had
|
||||
// comment_json and actor dropped, reproducing gas-city-inc's shape
|
||||
// (BEADS-JOURNAL-PLAN.md §1.1: "gci.bd_events_journal has the columns seq,
|
||||
// ts, op, issue_id, issue_json, dep_json" — no actor, no comment_json). The
|
||||
// journal is NOT activated; callers that want to observe activation itself
|
||||
// (the probe, its warning, its cached result) call SetEventsJournalEnabled
|
||||
// themselves.
|
||||
func newGciShapeStore(t *testing.T, prefix string) *testEnv {
|
||||
t.Helper()
|
||||
te := newTestEnv(t, prefix)
|
||||
ctx := context.Background()
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json")
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor")
|
||||
return te
|
||||
}
|
||||
|
||||
// TestAdaptiveInsert_GciShape is T2.1: on the gci shape, create, update and
|
||||
// comment all succeed, the rows land, and the comment payload is dropped with
|
||||
// exactly one warning. A fixed INSERT column list (the pre-A1 behavior) fails
|
||||
// every one of these three writes, because each one names either actor or
|
||||
// comment_json.
|
||||
func TestAdaptiveInsert_GciShape(t *testing.T) {
|
||||
te := newGciShapeStore(t, "ai")
|
||||
ctx := context.Background()
|
||||
|
||||
stderr := captureStderr(t, func() {
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
})
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation on the gci shape (missing only OPTIONAL columns) must succeed, got: %v", err)
|
||||
}
|
||||
if n := strings.Count(stderr, "events journal:"); n != 1 {
|
||||
t.Fatalf("activation warning count = %d, want exactly 1 (one warning per store, not per write):\n%s", n, stderr)
|
||||
}
|
||||
for _, want := range []string{"actor", "comment_json"} {
|
||||
if !strings.Contains(stderr, want) {
|
||||
t.Errorf("activation warning does not name missing column %q:\n%s", want, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
issue := &types.Issue{Title: "gci create", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("CreateIssue on gci shape: %v", err)
|
||||
}
|
||||
if err := te.store.UpdateIssue(ctx, issue.ID, map[string]interface{}{"title": "gci updated"}, "tester"); err != nil {
|
||||
t.Fatalf("UpdateIssue on gci shape: %v", err)
|
||||
}
|
||||
if _, err := te.store.AddIssueComment(ctx, issue.ID, "tester", "gci comment"); err != nil {
|
||||
t.Fatalf("AddIssueComment on gci shape: %v", err)
|
||||
}
|
||||
|
||||
rows, err := te.store.ReadEventsJournal(ctx, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadEventsJournal on gci shape: %v", err)
|
||||
}
|
||||
var ops []string
|
||||
for _, r := range rows {
|
||||
if r.IssueID != issue.ID {
|
||||
continue
|
||||
}
|
||||
ops = append(ops, r.Op)
|
||||
// Dropped, not merely blank: the column does not exist on this shape,
|
||||
// so the adaptive reader must never have tried to select it.
|
||||
if r.Actor != "" {
|
||||
t.Errorf("row op=%s: Actor = %q, want \"\" (column does not exist on this shape)", r.Op, r.Actor)
|
||||
}
|
||||
if r.CommentJSON != "" {
|
||||
t.Errorf("row op=%s: CommentJSON = %q, want \"\" (column does not exist on this shape)", r.Op, r.CommentJSON)
|
||||
}
|
||||
}
|
||||
wantOps := []string{"create", "update", "comment"}
|
||||
if len(ops) != len(wantOps) {
|
||||
t.Fatalf("journaled ops for %s = %v, want %v (a fixed INSERT column list fails every one of these on the gci shape)", issue.ID, ops, wantOps)
|
||||
}
|
||||
for i, want := range wantOps {
|
||||
if ops[i] != want {
|
||||
t.Errorf("journaled op[%d] = %q, want %q", i, ops[i], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdaptiveRead_GciShape is T2.2: a page read on the gci shape succeeds,
|
||||
// and actor/comment come back empty rather than failing the SELECT. A fixed
|
||||
// SELECT list (the pre-A1 behavior) fails the read outright, because it names
|
||||
// both actor and comment_json.
|
||||
func TestAdaptiveRead_GciShape(t *testing.T) {
|
||||
te := newGciShapeStore(t, "ar")
|
||||
ctx := context.Background()
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation on the gci shape: %v", err)
|
||||
}
|
||||
|
||||
issue := &types.Issue{ID: "ar-1", Title: "gci read", Status: types.StatusOpen, IssueType: types.TypeTask}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("CreateIssue on gci shape: %v", err)
|
||||
}
|
||||
|
||||
page, err := te.store.ReadEventsJournalPage(ctx, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadEventsJournalPage on gci shape: %v", err)
|
||||
}
|
||||
if len(page.Rows) == 0 {
|
||||
t.Fatal("ReadEventsJournalPage on gci shape returned no rows")
|
||||
}
|
||||
for _, r := range page.Rows {
|
||||
if r.Actor != "" {
|
||||
t.Errorf("row op=%s: Actor = %q, want \"\" on a shape with no actor column", r.Op, r.Actor)
|
||||
}
|
||||
if r.CommentJSON != "" {
|
||||
t.Errorf("row op=%s: CommentJSON = %q, want \"\" on a shape with no comment_json column", r.Op, r.CommentJSON)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestJournalShapeProbeOncePerOpen is T2.3: 100 writes against one already-
|
||||
// activated store issue no INFORMATION_SCHEMA probes of their own — only the
|
||||
// ONE SetEventsJournalEnabled ran. A per-write probe would show up here as a
|
||||
// nonzero delta across the 100 writes.
|
||||
func TestJournalShapeProbeOncePerOpen(t *testing.T) {
|
||||
te := newTestEnv(t, "pr")
|
||||
ctx := context.Background()
|
||||
|
||||
before := issueops.JournalShapeProbeCountForTest()
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation on canonical shape: %v", err)
|
||||
}
|
||||
afterActivate := issueops.JournalShapeProbeCountForTest()
|
||||
if delta := afterActivate - before; delta != 1 {
|
||||
t.Fatalf("probes issued by SetEventsJournalEnabled = %d, want exactly 1", delta)
|
||||
}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
issue := &types.Issue{
|
||||
Title: fmt.Sprintf("probe-once %d", i), Status: types.StatusOpen,
|
||||
Priority: 2, IssueType: types.TypeTask,
|
||||
}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("CreateIssue %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
after := issueops.JournalShapeProbeCountForTest()
|
||||
if delta := after - afterActivate; delta != 0 {
|
||||
t.Fatalf("probes issued by 100 writes against an already-activated store = %d, want 0 (a per-write probe, not a per-open probe)", delta)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMissingRequiredColumnRefusesAtOpen is T2.4: a table missing a REQUIRED
|
||||
// column (op) makes activation fail with a typed error when the journal is
|
||||
// enabled, but leaves it succeeding when disabled — a disabled workspace
|
||||
// accepts any shape. It also pins that a caller who enables the journal
|
||||
// anyway (ignoring the returned activation error) still never attempts the
|
||||
// INSERT: the mutation itself must not fail, which is the whole point of
|
||||
// moving this failure to open time instead of per write.
|
||||
func TestMissingRequiredColumnRefusesAtOpen(t *testing.T) {
|
||||
te := newTestEnv(t, "mr")
|
||||
ctx := context.Background()
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN op")
|
||||
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
err := te.store.EventsJournalActivationError()
|
||||
if err == nil {
|
||||
t.Fatal("activation with the journal enabled against a table missing a required column (op) succeeded; want a typed error")
|
||||
}
|
||||
if !errors.Is(err, issueops.ErrJournalShapeUnsupported) {
|
||||
t.Errorf("activation error = %v, does not match issueops.ErrJournalShapeUnsupported", err)
|
||||
}
|
||||
var shapeErr *issueops.JournalShapeError
|
||||
if !errors.As(err, &shapeErr) {
|
||||
t.Fatalf("activation error is not a *issueops.JournalShapeError: %v (%T)", err, err)
|
||||
}
|
||||
if len(shapeErr.Missing) != 1 || shapeErr.Missing[0] != "op" {
|
||||
t.Errorf("JournalShapeError.Missing = %v, want [op]", shapeErr.Missing)
|
||||
}
|
||||
|
||||
// Disabling must still succeed: a disabled workspace accepts any shape, and
|
||||
// never runs the probe that would otherwise fail it.
|
||||
te.store.SetEventsJournalEnabled(false)
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Errorf("activation with the journal disabled against the same broken shape failed: %v", err)
|
||||
}
|
||||
|
||||
// Re-enable (probe fails again, same cached nil shape) and prove the
|
||||
// mutation itself still lands: this is the gci failure mode A1 exists to
|
||||
// close, where a per-write INSERT against a missing column rolled back the
|
||||
// user's own write.
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
issue := &types.Issue{Title: "no op column", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("CreateIssue must still succeed even though the journal cannot activate on this shape: %v", err)
|
||||
}
|
||||
}
|
||||
848
internal/storage/embeddeddolt/journal_shape_converge_test.go
Normal file
848
internal/storage/embeddeddolt/journal_shape_converge_test.go
Normal file
|
|
@ -0,0 +1,848 @@
|
|||
//go:build cgo
|
||||
|
||||
package embeddeddolt_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage/embeddeddolt"
|
||||
"github.com/steveyegge/beads/internal/storage/schema"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// Shape convergence for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2b/§4.3,
|
||||
// PR A2). ignored/0028 closes the exact gap ignored/0023 left open (probing
|
||||
// DATA_TYPE alone cannot see a column that is entirely absent): these tests
|
||||
// reproduce every historical shape the plan's evidence base found, prove each
|
||||
// one converges on canonical without perturbing a single row or the seq
|
||||
// counter (T2.5), pin the sentinel-triggered replay's floor at exactly 21
|
||||
// (T2.6), and pin that the same replay is a true no-op everywhere outside
|
||||
// bd_events_journal itself when nothing else is damaged (T2.7).
|
||||
|
||||
// newShape3666e5026Store reproduces the events-journal-3666e5026 release
|
||||
// shape (BEADS-JOURNAL-PLAN.md §1.1, "where that shape came from"):
|
||||
// issue_json LONGTEXT, dep_json TEXT, no comment_json, no actor, no ts index.
|
||||
// This predates both comment_json and actor outright.
|
||||
func newShape3666e5026Store(t *testing.T, prefix string) *testEnv {
|
||||
t.Helper()
|
||||
te := newTestEnv(t, prefix)
|
||||
ctx := context.Background()
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json")
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor")
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN dep_json TEXT")
|
||||
te.exec(t, ctx, "DROP INDEX idx_bd_events_journal_ts ON bd_events_journal")
|
||||
return te
|
||||
}
|
||||
|
||||
// newForkIgnored0017ShapeStore reproduces the enterprise fork's
|
||||
// ignored/0017_create_events_journal shape (0023's own header, bd-t9ovd):
|
||||
// both payload columns TEXT, no actor (actor postdates this shape; it is
|
||||
// bd OSS migration 0025), no ts index. ignored/0023 already fully heals this
|
||||
// one — dropping actor is what forces the sentinel-triggered replay that
|
||||
// exercises 0023's widen-if-TEXT step here, the same mechanism a lineage
|
||||
// that genuinely never ran 0023/0025 before reaching this shape would hit.
|
||||
func newForkIgnored0017ShapeStore(t *testing.T, prefix string) *testEnv {
|
||||
t.Helper()
|
||||
te := newTestEnv(t, prefix)
|
||||
ctx := context.Background()
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal DROP COLUMN actor")
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN dep_json TEXT")
|
||||
te.exec(t, ctx, "ALTER TABLE bd_events_journal MODIFY COLUMN comment_json TEXT")
|
||||
te.exec(t, ctx, "DROP INDEX idx_bd_events_journal_ts ON bd_events_journal")
|
||||
return te
|
||||
}
|
||||
|
||||
// openDB opens a short-lived raw SQL connection to te's store and registers
|
||||
// its cleanup with t.Cleanup, for tests that need to hold the connection
|
||||
// across several calls (schema.CurrentIgnoredVersion, schema.MigrateUp,
|
||||
// direct probes) rather than te.exec's one-shot open/close.
|
||||
func (te *testEnv) openDB(t *testing.T, ctx context.Context) *sql.DB {
|
||||
t.Helper()
|
||||
db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQL: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cleanup() })
|
||||
return db
|
||||
}
|
||||
|
||||
// journalCoreRow is the subset of bd_events_journal's columns every observed
|
||||
// shape (canonical or historical) carries. comment_json and actor are
|
||||
// deliberately excluded: they are exactly what this migration adds back, so
|
||||
// a before/after comparison that is scoped to the columns every shape shares
|
||||
// is what proves the healing touched only the columns it should.
|
||||
type journalCoreRow struct {
|
||||
Seq int64
|
||||
Ts string
|
||||
Op string
|
||||
IssueID string
|
||||
IssueJSON sql.NullString
|
||||
DepJSON sql.NullString
|
||||
}
|
||||
|
||||
// snapshotJournalCore captures every bd_events_journal row (core columns
|
||||
// only, see journalCoreRow) in seq order, plus bd_events_seq's counter.
|
||||
func snapshotJournalCore(t *testing.T, ctx context.Context, db *sql.DB) ([]journalCoreRow, int64) {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, "SELECT seq, ts, op, issue_id, issue_json, dep_json FROM bd_events_journal ORDER BY seq")
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot bd_events_journal: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []journalCoreRow
|
||||
for rows.Next() {
|
||||
var r journalCoreRow
|
||||
if err := rows.Scan(&r.Seq, &r.Ts, &r.Op, &r.IssueID, &r.IssueJSON, &r.DepJSON); err != nil {
|
||||
t.Fatalf("scan bd_events_journal row: %v", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate bd_events_journal: %v", err)
|
||||
}
|
||||
var nextSeq int64
|
||||
if err := db.QueryRowContext(ctx, "SELECT next_seq FROM bd_events_seq WHERE id = 0").Scan(&nextSeq); err != nil {
|
||||
t.Fatalf("snapshot bd_events_seq: %v", err)
|
||||
}
|
||||
return out, nextSeq
|
||||
}
|
||||
|
||||
// assertIndexExists fails the test if the named index is missing.
|
||||
func assertIndexExists(t *testing.T, ctx context.Context, db *sql.DB, table, index string) {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?`,
|
||||
table, index).Scan(&count); err != nil {
|
||||
t.Fatalf("probe index %s on %s: %v", index, table, err)
|
||||
}
|
||||
if count == 0 {
|
||||
t.Errorf("%s: index %s missing after convergence", table, index)
|
||||
}
|
||||
}
|
||||
|
||||
// assertColumnShapeConverged diffs got against want (both from
|
||||
// clonePlaneTableShape, migrate_ignored_plane_shape_test.go) with messaging
|
||||
// specific to this file's "did this fixture converge on canonical" question,
|
||||
// rather than that helper's own fresh-init-vs-fresh-clone framing.
|
||||
func assertColumnShapeConverged(t *testing.T, fixtureName, table string, want, got map[string]clonePlaneColumn) {
|
||||
t.Helper()
|
||||
names := map[string]bool{}
|
||||
for name := range want {
|
||||
names[name] = true
|
||||
}
|
||||
for name := range got {
|
||||
names[name] = true
|
||||
}
|
||||
for name := range names {
|
||||
w, inWant := want[name]
|
||||
g, inGot := got[name]
|
||||
switch {
|
||||
case !inGot:
|
||||
t.Errorf("%s: %s.%s present on the canonical reference but missing after convergence", fixtureName, table, name)
|
||||
case !inWant:
|
||||
t.Errorf("%s: %s.%s present after convergence but not on the canonical reference", fixtureName, table, name)
|
||||
case w != g:
|
||||
t.Errorf("%s: %s.%s shape differs from canonical: want %s, got %s",
|
||||
fixtureName, table, name, formatClonePlaneColumn(w), formatClonePlaneColumn(g))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIgnored0028ConvergesAllShapes is T2.5: every historical shape the
|
||||
// plan's evidence base found ends up byte-identical to canonical after
|
||||
// MigrateUp, and not a single row or the seq counter moved getting there.
|
||||
// Kills: probing only DATA_TYPE (0023's gap — a dropped fixture here would
|
||||
// stay missing comment_json/actor and fail the shape diff); any non-monotone
|
||||
// write to the counter or rows (caught by the before/after snapshot; monotone
|
||||
// writes are TestIgnored0028AloneLeavesCounterAndRowsUntouched's job — this
|
||||
// test's forced replay always re-runs ignored/0022 too, whose own GREATEST
|
||||
// raise would mask one).
|
||||
func TestIgnored0028ConvergesAllShapes(t *testing.T) {
|
||||
// dbPrefix is separate from name (rather than derived from it): embedded
|
||||
// mode database names reject hyphens, but "fork-ignored-0017" is the
|
||||
// plan's own name for that fixture and reads better in -v output and
|
||||
// failure messages than a mangled version of it would.
|
||||
fixtures := []struct {
|
||||
name string
|
||||
dbPrefix string
|
||||
build func(t *testing.T, prefix string) *testEnv
|
||||
damaged bool
|
||||
}{
|
||||
{name: "3666e5026", dbPrefix: "cv3666e5026", build: newShape3666e5026Store, damaged: true},
|
||||
{name: "gci", dbPrefix: "cvgci", build: newGciShapeStore, damaged: true},
|
||||
{name: "fork-ignored-0017", dbPrefix: "cvfork0017", build: newForkIgnored0017ShapeStore, damaged: true},
|
||||
{name: "canonical", dbPrefix: "cvcanon", build: func(t *testing.T, prefix string) *testEnv { return newTestEnv(t, prefix) }, damaged: false},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
reference := newTestEnv(t, "cvref")
|
||||
refDB := reference.openDB(t, ctx)
|
||||
refConn, err := refDB.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reference Conn: %v", err)
|
||||
}
|
||||
defer refConn.Close()
|
||||
wantShape := clonePlaneTableShape(t, ctx, refConn, "bd_events_journal")
|
||||
|
||||
for _, fx := range fixtures {
|
||||
t.Run(fx.name, func(t *testing.T) {
|
||||
te := fx.build(t, fx.dbPrefix)
|
||||
|
||||
// One real journaled event while the shape is still canonical (every
|
||||
// builder above damages bd_events_journal only AFTER newTestEnv
|
||||
// returns), so the snapshot below compares real data, not an empty
|
||||
// table.
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation before damaging the shape: %v", err)
|
||||
}
|
||||
issue := &types.Issue{Title: fx.name + " seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("seed CreateIssue: %v", err)
|
||||
}
|
||||
|
||||
db := te.openDB(t, ctx)
|
||||
beforeRows, beforeSeq := snapshotJournalCore(t, ctx, db)
|
||||
|
||||
// MigrateUp's own return value cannot prove convergence here: it
|
||||
// reports only the main-plane applied count (plus a reconcile
|
||||
// flag), never the ignored-plane count, so the sentinel
|
||||
// contradiction and its heal are both checked through
|
||||
// CurrentIgnoredVersion instead (see TestSentinelJournalColumnsReplay).
|
||||
if fx.damaged {
|
||||
before, err := schema.CurrentIgnoredVersion(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (before MigrateUp): %v", err)
|
||||
}
|
||||
if want := schema.LatestIgnoredVersion(); before >= want {
|
||||
t.Fatalf("CurrentIgnoredVersion before MigrateUp = %d, want below latest (%d): this fixture's damage did not float a sentinel contradiction", before, want)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := schema.MigrateUp(ctx, db); err != nil {
|
||||
t.Fatalf("MigrateUp: %v", err)
|
||||
}
|
||||
|
||||
if after, err := schema.CurrentIgnoredVersion(ctx, db); err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after MigrateUp): %v", err)
|
||||
} else if want := schema.LatestIgnoredVersion(); after != want {
|
||||
t.Fatalf("CurrentIgnoredVersion after MigrateUp = %d, want latest (%d): this fixture did not converge", after, want)
|
||||
}
|
||||
|
||||
afterRows, afterSeq := snapshotJournalCore(t, ctx, db)
|
||||
if beforeSeq != afterSeq {
|
||||
t.Errorf("bd_events_seq.next_seq changed: before=%d after=%d", beforeSeq, afterSeq)
|
||||
}
|
||||
if !reflect.DeepEqual(beforeRows, afterRows) {
|
||||
t.Errorf("bd_events_journal rows changed across MigrateUp:\nbefore=%+v\nafter=%+v", beforeRows, afterRows)
|
||||
}
|
||||
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Conn: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
gotShape := clonePlaneTableShape(t, ctx, conn, "bd_events_journal")
|
||||
assertColumnShapeConverged(t, fx.name, "bd_events_journal", wantShape, gotShape)
|
||||
assertIndexExists(t, ctx, db, "bd_events_journal", "idx_bd_events_journal_ts")
|
||||
assertIndexExists(t, ctx, db, "bd_events_journal", "idx_bd_events_journal_issue")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// isolationRow is the row shape TestIgnored0028AloneLeavesCounterAndRowsUntouched
|
||||
// compares before and after running ignored/0028 directly: the four columns
|
||||
// every damaged fixture in that test shares, plus comment_json only on the
|
||||
// one fixture (fork-ignored-0017) that still has it when 0028 starts.
|
||||
type isolationRow struct {
|
||||
Seq int64
|
||||
Ts, Op, IssueID string
|
||||
IssueJSON, DepJSON, CommentJSON sql.NullString
|
||||
}
|
||||
|
||||
// readIsolationRows reads isolationRows in seq order. comment_json is only
|
||||
// selected when hasCommentJSON is true: on two of the three fixtures the
|
||||
// column does not exist yet when this runs, so selecting it unconditionally
|
||||
// would fail with "column not found" rather than compare anything.
|
||||
func readIsolationRows(t *testing.T, ctx context.Context, conn *sql.Conn, hasCommentJSON bool) []isolationRow {
|
||||
t.Helper()
|
||||
query := "SELECT seq, ts, op, issue_id, issue_json, dep_json FROM bd_events_journal ORDER BY seq"
|
||||
if hasCommentJSON {
|
||||
query = "SELECT seq, ts, op, issue_id, issue_json, dep_json, comment_json FROM bd_events_journal ORDER BY seq"
|
||||
}
|
||||
rows, err := conn.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
t.Fatalf("read bd_events_journal: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []isolationRow
|
||||
for rows.Next() {
|
||||
var r isolationRow
|
||||
dest := []any{&r.Seq, &r.Ts, &r.Op, &r.IssueID, &r.IssueJSON, &r.DepJSON}
|
||||
if hasCommentJSON {
|
||||
dest = append(dest, &r.CommentJSON)
|
||||
}
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
t.Fatalf("scan bd_events_journal row: %v", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate bd_events_journal: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertIsolationRowsEqual(t *testing.T, before, after []isolationRow) {
|
||||
t.Helper()
|
||||
if len(before) != len(after) {
|
||||
t.Fatalf("journal row count = %d, want %d (0028 alone must not touch data)", len(after), len(before))
|
||||
}
|
||||
for i := range before {
|
||||
if before[i] != after[i] {
|
||||
t.Errorf("journal row seq %d changed: before %+v, after %+v", before[i].Seq, before[i], after[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIgnored0028AloneLeavesCounterAndRowsUntouched is T2.5b. T2.5 forces its
|
||||
// replay through a sentinel contradiction, which always re-runs ignored/0022
|
||||
// too (every fixture there drops actor), so a monotone GREATEST write to the
|
||||
// counter by 0028 itself would be invisible: 0022's own identical GREATEST
|
||||
// raise on the same pass would mask it. Running 0028's frozen bytes directly,
|
||||
// with the counter seeded below MAX(seq), closes that gap: any counter write
|
||||
// at all, monotone or not, shows. Kills: any counter or row write in 0028,
|
||||
// including a monotone GREATEST.
|
||||
func TestIgnored0028AloneLeavesCounterAndRowsUntouched(t *testing.T) {
|
||||
fixtures := []struct {
|
||||
name string
|
||||
dbPrefix string
|
||||
build func(t *testing.T, prefix string) *testEnv
|
||||
hasCommentJSON bool
|
||||
}{
|
||||
{name: "3666e5026", dbPrefix: "i28a3666e5026", build: newShape3666e5026Store, hasCommentJSON: false},
|
||||
{name: "gci", dbPrefix: "i28agci", build: newGciShapeStore, hasCommentJSON: false},
|
||||
{name: "fork-ignored-0017", dbPrefix: "i28afork0017", build: newForkIgnored0017ShapeStore, hasCommentJSON: true},
|
||||
}
|
||||
|
||||
for _, fx := range fixtures {
|
||||
t.Run(fx.name, func(t *testing.T) {
|
||||
te := fx.build(t, fx.dbPrefix)
|
||||
ctx := context.Background()
|
||||
|
||||
db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQL: %v", err)
|
||||
}
|
||||
defer func() { _ = cleanup() }()
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Conn: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
seed := []int64{1, 2, 3}
|
||||
for _, seq := range seed {
|
||||
ts := tsForSeq(seq)
|
||||
query := "INSERT INTO bd_events_journal (seq, ts, op, issue_id, issue_json, dep_json) VALUES (?, ?, 'create', 'bd-i28a', ?, ?)"
|
||||
args := []any{seq, ts, `{"id":"bd-i28a"}`, `[]`}
|
||||
if fx.hasCommentJSON {
|
||||
query = "INSERT INTO bd_events_journal (seq, ts, op, issue_id, issue_json, dep_json, comment_json) VALUES (?, ?, 'create', 'bd-i28a', ?, ?, ?)"
|
||||
args = append(args, `{"text":"hi"}`)
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, query, args...); err != nil {
|
||||
t.Fatalf("seed journal row %d: %v", seq, err)
|
||||
}
|
||||
}
|
||||
execFrozenGuard(t, ctx, conn, "UPDATE bd_events_seq SET next_seq = 1 WHERE id = 0")
|
||||
rowsBefore := readIsolationRows(t, ctx, conn, fx.hasCommentJSON)
|
||||
if len(rowsBefore) != len(seed) {
|
||||
t.Fatalf("seeded journal rows = %d, want %d", len(rowsBefore), len(seed))
|
||||
}
|
||||
|
||||
sqlText, err := schema.IgnoredMigrationSQL("0028_converge_events_journal_shape.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read ignored 0028 migration: %v", err)
|
||||
}
|
||||
execFrozenGuard(t, ctx, conn, sqlText)
|
||||
|
||||
if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 {
|
||||
t.Errorf("next_seq = %d after applying 0028 alone, want 1: 0028 must never write the counter, not even a monotone raise", got)
|
||||
}
|
||||
assertIsolationRowsEqual(t, rowsBefore, readIsolationRows(t, ctx, conn, fx.hasCommentJSON))
|
||||
|
||||
for _, column := range []string{"comment_json", "dep_json"} {
|
||||
if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" {
|
||||
t.Errorf("bd_events_journal.%s DATA_TYPE = %q after 0028, want %q", column, got, "longtext")
|
||||
}
|
||||
}
|
||||
requireColumnCount(t, ctx, conn, "bd_events_journal", "actor", 1)
|
||||
requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1)
|
||||
requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIgnored0028NoopsWithoutItsTables is the 0028 analogue of 0023's
|
||||
// TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables
|
||||
// (migrate_journal_shape_repair_test.go): 0028 must survive running against a
|
||||
// workspace with no bd_events_journal table at all, the same intermediate
|
||||
// state 0023 already has to survive. Kills: removing or omitting the
|
||||
// table-exists conjunct from 0028's comment_json step.
|
||||
func TestIgnored0028NoopsWithoutItsTables(t *testing.T) {
|
||||
te := newTestEnv(t, "i28notab")
|
||||
ctx := context.Background()
|
||||
|
||||
db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQL: %v", err)
|
||||
}
|
||||
defer func() { _ = cleanup() }()
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Conn: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execFrozenGuard(t, ctx, conn, "DROP TABLE IF EXISTS bd_events_journal")
|
||||
|
||||
sqlText, err := schema.IgnoredMigrationSQL("0028_converge_events_journal_shape.up.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read ignored 0028 migration: %v", err)
|
||||
}
|
||||
execFrozenGuard(t, ctx, conn, sqlText)
|
||||
|
||||
var got int
|
||||
if err := conn.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal'`).Scan(&got); err != nil {
|
||||
t.Fatalf("probe bd_events_journal presence: %v", err)
|
||||
}
|
||||
if got != 0 {
|
||||
t.Errorf("0028 created bd_events_journal out of nothing (count = %d); it must only ever repair an existing table", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSentinelJournalColumnsReplay is T2.6: a store already at the latest
|
||||
// ignored cursor that loses comment_json out of band (a backup restored
|
||||
// between doors, or damage out of band) floors at exactly 21 and heals on the
|
||||
// next writable open. MigrateUp's own return value cannot prove this: it
|
||||
// reports only the main-plane applied count (plus a reconcile flag), never
|
||||
// the ignored-plane count, so every assertion here goes through
|
||||
// CurrentIgnoredVersion instead. Kills: no sentinel registered (the floor
|
||||
// read would stay at latest instead of dropping to 21, caught by the
|
||||
// exact-21 assertion below); a floor of 22 or more (same assertion — 22
|
||||
// would mean 0022, bd_events_journal's own creator, never replays, which
|
||||
// cannot heal a shape where the table already exists but a column does
|
||||
// not); a replay that runs but does not actually add comment_json back
|
||||
// (caught by the final post-heal probe).
|
||||
func TestSentinelJournalColumnsReplay(t *testing.T) {
|
||||
te := newTestEnv(t, "sj")
|
||||
ctx := context.Background()
|
||||
db := te.openDB(t, ctx)
|
||||
|
||||
before, err := schema.CurrentIgnoredVersion(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (before damage): %v", err)
|
||||
}
|
||||
if want := schema.LatestIgnoredVersion(); before != want {
|
||||
t.Fatalf("CurrentIgnoredVersion before damage = %d, want latest (%d)", before, want)
|
||||
}
|
||||
|
||||
// Damage through the SAME connection db already holds open, not a second
|
||||
// embeddeddolt.OpenSQL handle (te.exec's own short-lived one): embedded
|
||||
// mode's engine-open path serializes on the directory, and a second
|
||||
// concurrent handle to it deadlocks against the first instead of simply
|
||||
// queuing.
|
||||
if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil {
|
||||
t.Fatalf("drop comment_json: %v", err)
|
||||
}
|
||||
|
||||
floored, err := schema.CurrentIgnoredVersion(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after dropping comment_json): %v", err)
|
||||
}
|
||||
if floored != 21 {
|
||||
t.Fatalf("CurrentIgnoredVersion after dropping comment_json = %d, want exactly 21", floored)
|
||||
}
|
||||
|
||||
if _, err := schema.MigrateUp(ctx, db); err != nil {
|
||||
t.Fatalf("MigrateUp (forced replay): %v", err)
|
||||
}
|
||||
|
||||
healed, err := schema.CurrentIgnoredVersion(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after heal): %v", err)
|
||||
}
|
||||
if want := schema.LatestIgnoredVersion(); healed != want {
|
||||
t.Fatalf("CurrentIgnoredVersion after heal = %d, want latest (%d)", healed, want)
|
||||
}
|
||||
|
||||
var commentType sql.NullString
|
||||
if err := db.QueryRowContext(ctx, `
|
||||
SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = 'comment_json'`,
|
||||
).Scan(&commentType); err != nil {
|
||||
t.Fatalf("probe comment_json after heal: %v", err)
|
||||
}
|
||||
if !commentType.Valid {
|
||||
t.Fatal("comment_json is still missing after the forced replay")
|
||||
}
|
||||
}
|
||||
|
||||
// tsForSeq gives TestSentinelReplayCounterConverges's and
|
||||
// TestIgnored0028AloneLeavesCounterAndRowsUntouched's seeded rows distinct,
|
||||
// ordered timestamps without threading one more field through every fixture
|
||||
// for a value no assertion in either test reads.
|
||||
func tsForSeq(seq int64) string {
|
||||
return fmt.Sprintf("2026-08-01 00:00:%02d", seq)
|
||||
}
|
||||
|
||||
// counterConvergeState is one bd_events_journal/bd_events_seq relationship
|
||||
// TestSentinelReplayCounterConverges seeds: the rows to leave in the journal,
|
||||
// the counter value to seed, and the counter the sentinel-triggered replay
|
||||
// must converge to (GREATEST(before, MAX(seq))).
|
||||
type counterConvergeState struct {
|
||||
name string
|
||||
code string // short, hyphen/underscore-free token for database name prefixes
|
||||
seqs []int64
|
||||
before int64
|
||||
want int64
|
||||
}
|
||||
|
||||
var counterConvergeStates = []counterConvergeState{
|
||||
{name: "behind", code: "behind", seqs: []int64{1, 2, 3}, before: 1, want: 3},
|
||||
{name: "equal", code: "equal", seqs: []int64{1, 2, 3}, before: 3, want: 3},
|
||||
{name: "ahead_journal_restored_older", code: "ajro", seqs: []int64{1, 2, 3}, before: 10, want: 10},
|
||||
{name: "empty_journal_counter_ahead", code: "ejca", seqs: nil, before: 7, want: 7},
|
||||
{name: "hole_above_counter", code: "hac", seqs: []int64{1, 3}, before: 1, want: 3},
|
||||
}
|
||||
|
||||
// TestSentinelReplayCounterConverges is T2.6b: across every counter/journal
|
||||
// relationship #5496's evidence base found, a sentinel-triggered replay of
|
||||
// ignored 22..28 must leave bd_events_seq at exactly
|
||||
// GREATEST(seeded, MAX(seq)) — never lower, never past the high-water mark —
|
||||
// and then the very next real write must land contiguously right after it.
|
||||
// Run against both sentinel columns: dropping actor across all five states,
|
||||
// plus dropping comment_json on "behind" alone to prove that sentinel also
|
||||
// reaches the counter (it is registered on the same table, same floor).
|
||||
//
|
||||
// Kills: the sentinel floor sitting at 22 or above (behind stays at 1, and
|
||||
// hole_above_counter's next write lands at the stale seq 2, a late arrival);
|
||||
// a replayed statement that lowers the counter, e.g. a future
|
||||
// `SET next_seq = COALESCE(MAX(seq), 0)` with no GREATEST (the ahead_* and
|
||||
// empty_* cases fail); an over-raise such as MAX(seq)+1 (behind and equal
|
||||
// read 4, and the first new row is 5, a gap); dropping the actor sentinel
|
||||
// (actor/behind stays at 1 and the column is never restored).
|
||||
func TestSentinelReplayCounterConverges(t *testing.T) {
|
||||
type subtest struct {
|
||||
name string
|
||||
dbPrefix string
|
||||
state counterConvergeState
|
||||
column string
|
||||
}
|
||||
var subtests []subtest
|
||||
for _, state := range counterConvergeStates {
|
||||
subtests = append(subtests, subtest{
|
||||
name: state.name + "/actor",
|
||||
dbPrefix: "scc" + state.code + "a",
|
||||
state: state,
|
||||
column: "actor",
|
||||
})
|
||||
}
|
||||
subtests = append(subtests, subtest{
|
||||
name: "behind/comment_json",
|
||||
dbPrefix: "sccbehindc",
|
||||
state: counterConvergeStates[0],
|
||||
column: "comment_json",
|
||||
})
|
||||
|
||||
for _, st := range subtests {
|
||||
t.Run(st.name, func(t *testing.T) {
|
||||
te := newTestEnv(t, st.dbPrefix)
|
||||
ctx := context.Background()
|
||||
|
||||
// A raw handle with an explicit cleanup, not te.openDB: this test
|
||||
// closes it early (below) and reopens afterward, which te.openDB's
|
||||
// t.Cleanup-deferred-to-test-end handle cannot do.
|
||||
db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQL: %v", err)
|
||||
}
|
||||
closed := false
|
||||
closeDB := func() {
|
||||
if !closed {
|
||||
_ = cleanup()
|
||||
closed = true
|
||||
}
|
||||
}
|
||||
defer closeDB()
|
||||
|
||||
if _, err := db.ExecContext(ctx, "DELETE FROM bd_events_journal"); err != nil {
|
||||
t.Fatalf("clear bd_events_journal: %v", err)
|
||||
}
|
||||
for _, seq := range st.state.seqs {
|
||||
if _, err := db.ExecContext(ctx,
|
||||
"INSERT INTO bd_events_journal (seq, ts, op, issue_id) VALUES (?, ?, 'create', 'bd-scc')",
|
||||
seq, tsForSeq(seq)); err != nil {
|
||||
t.Fatalf("seed journal row %d: %v", seq, err)
|
||||
}
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "UPDATE bd_events_seq SET next_seq = ? WHERE id = 0", st.state.before); err != nil {
|
||||
t.Fatalf("seed next_seq: %v", err)
|
||||
}
|
||||
//nolint:gosec // G201: st.column is one of two test-local literals ("actor", "comment_json").
|
||||
if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN "+st.column); err != nil {
|
||||
t.Fatalf("drop %s: %v", st.column, err)
|
||||
}
|
||||
|
||||
if floored, err := schema.CurrentIgnoredVersion(ctx, db); err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after dropping %s): %v", st.column, err)
|
||||
} else if floored != 21 {
|
||||
t.Fatalf("CurrentIgnoredVersion after dropping %s = %d, want exactly 21", st.column, floored)
|
||||
}
|
||||
rowsBefore, _ := snapshotJournalCore(t, ctx, db)
|
||||
|
||||
if _, err := schema.MigrateUp(ctx, db); err != nil {
|
||||
t.Fatalf("MigrateUp (forced replay): %v", err)
|
||||
}
|
||||
|
||||
if after, err := schema.CurrentIgnoredVersion(ctx, db); err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after replay): %v", err)
|
||||
} else if want := schema.LatestIgnoredVersion(); after != want {
|
||||
t.Fatalf("CurrentIgnoredVersion after replay = %d, want latest (%d)", after, want)
|
||||
}
|
||||
var colCount int
|
||||
if err := db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal' AND COLUMN_NAME = ?`, st.column).Scan(&colCount); err != nil {
|
||||
t.Fatalf("probe %s after replay: %v", st.column, err)
|
||||
}
|
||||
if colCount == 0 {
|
||||
t.Errorf("bd_events_journal.%s is still missing after the forced replay", st.column)
|
||||
}
|
||||
|
||||
rowsAfter, nextSeq := snapshotJournalCore(t, ctx, db)
|
||||
if nextSeq != st.state.want {
|
||||
t.Errorf("next_seq = %d after replay, want %d (GREATEST(seeded %d, MAX(seq)))", nextSeq, st.state.want, st.state.before)
|
||||
}
|
||||
if !reflect.DeepEqual(rowsBefore, rowsAfter) {
|
||||
t.Errorf("bd_events_journal rows changed across the replay:\nbefore=%+v\nafter=%+v", rowsBefore, rowsAfter)
|
||||
}
|
||||
|
||||
// Close the raw handle before any te.store call: embedded opens
|
||||
// serialize on the directory and a second concurrent handle
|
||||
// deadlocks against this one instead of queuing (see
|
||||
// TestStoreAtIgnored28WritesAndJournals).
|
||||
closeDB()
|
||||
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation after convergence: %v", err)
|
||||
}
|
||||
issue := &types.Issue{Title: st.name + " seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("CreateIssue after convergence: %v", err)
|
||||
}
|
||||
|
||||
reopened := te.openDB(t, ctx)
|
||||
var minSeq, maxSeq, count int64
|
||||
if err := reopened.QueryRowContext(ctx,
|
||||
"SELECT COALESCE(MIN(seq),0), COALESCE(MAX(seq),0), COUNT(*) FROM bd_events_journal WHERE seq > ?",
|
||||
st.state.want).Scan(&minSeq, &maxSeq, &count); err != nil {
|
||||
t.Fatalf("probe new journal rows: %v", err)
|
||||
}
|
||||
if count == 0 {
|
||||
t.Fatal("CreateIssue after convergence did not journal a row above the converged counter")
|
||||
}
|
||||
if minSeq != st.state.want+1 {
|
||||
t.Errorf("lowest new seq = %d, want %d (want + 1)", minSeq, st.state.want+1)
|
||||
}
|
||||
if maxSeq-minSeq+1 != count {
|
||||
t.Errorf("new seqs are not contiguous: min=%d max=%d count=%d", minSeq, maxSeq, count)
|
||||
}
|
||||
if st.state.name == "hole_above_counter" {
|
||||
var staleCount int64
|
||||
if err := reopened.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal WHERE seq = 2").Scan(&staleCount); err != nil {
|
||||
t.Fatalf("probe seq 2: %v", err)
|
||||
}
|
||||
if staleCount != 0 {
|
||||
t.Errorf("seq 2 (the hole below the converged counter) has a row; a late arrival filled it instead of the new write landing above the counter")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ignoredPlaneTablesExceptJournal lists every clone-local (dolt_ignore) table
|
||||
// other than bd_events_journal itself, matching
|
||||
// TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape's table list
|
||||
// (migrate_ignored_plane_shape_test.go). bd_events_journal's own shape
|
||||
// legitimately changes during the replay this test forces; every one of
|
||||
// these must not.
|
||||
var ignoredPlaneTablesExceptJournal = []string{
|
||||
"wisps", "wisp_labels", "wisp_dependencies", "wisp_events",
|
||||
"wisp_comments", "wisp_child_counters", "events", "leases",
|
||||
"repo_mtimes", "local_metadata", "bd_events_seq",
|
||||
}
|
||||
|
||||
// dumpTable renders every row of table as an order-stable, type-agnostic
|
||||
// snapshot: each value read as text (or "<NULL>"), ordered by the table's
|
||||
// first column. It exists so TestReplay22To28NoOpOnHealthyStore can compare
|
||||
// whole tables it otherwise knows nothing about the schema of, rather than
|
||||
// hand-maintaining a column list per table.
|
||||
func dumpTable(t *testing.T, ctx context.Context, db *sql.DB, table string) []map[string]string {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, "SELECT * FROM "+table+" ORDER BY 1")
|
||||
if err != nil {
|
||||
t.Fatalf("dump %s: %v", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
t.Fatalf("dump %s columns: %v", table, err)
|
||||
}
|
||||
var out []map[string]string
|
||||
for rows.Next() {
|
||||
vals := make([]sql.NullString, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
t.Fatalf("dump %s row: %v", table, err)
|
||||
}
|
||||
row := make(map[string]string, len(cols))
|
||||
for i, c := range cols {
|
||||
if vals[i].Valid {
|
||||
row[c] = vals[i].String
|
||||
} else {
|
||||
row[c] = "<NULL>"
|
||||
}
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate %s: %v", table, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func snapshotIgnoredPlaneTablesExceptJournal(t *testing.T, ctx context.Context, db *sql.DB) map[string][]map[string]string {
|
||||
t.Helper()
|
||||
snap := make(map[string][]map[string]string, len(ignoredPlaneTablesExceptJournal))
|
||||
for _, table := range ignoredPlaneTablesExceptJournal {
|
||||
snap[table] = dumpTable(t, ctx, db, table)
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
// TestReplay22To28NoOpOnHealthyStore is T2.7: forcing the sentinel-triggered
|
||||
// replay of ignored 22..28 must perturb nothing outside bd_events_journal
|
||||
// itself — not a single row in any other clone-local table, and critically
|
||||
// not wisps.updated_at, which an unguarded statement in this range would
|
||||
// restamp via ON UPDATE CURRENT_TIMESTAMP (the #5981 harm class the
|
||||
// replay-floor's two placement constraints exist to prevent; see schema.go's
|
||||
// schemaSentinelColumn doc comment). Kills: an unguarded statement above the
|
||||
// floor touching a table or row it should not.
|
||||
func TestReplay22To28NoOpOnHealthyStore(t *testing.T) {
|
||||
te := newTestEnv(t, "rp")
|
||||
ctx := context.Background()
|
||||
|
||||
wispID := "wisp-rp-1"
|
||||
te.exec(t, ctx,
|
||||
"INSERT INTO wisps (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type, ephemeral) VALUES (?, 'w', '', '', '', '', 'open', 2, 'task', 1)",
|
||||
wispID)
|
||||
|
||||
db := te.openDB(t, ctx)
|
||||
before := snapshotIgnoredPlaneTablesExceptJournal(t, ctx, db)
|
||||
|
||||
// Force the replay the only way it ever happens in production: contradict
|
||||
// a sentinel, which floors the cursor to 21 and makes 22..28 pending again
|
||||
// (TestSentinelJournalColumnsReplay proves the floor and the heal; this
|
||||
// test only cares what else the resulting replay does or does not touch).
|
||||
// Damage through db itself, not a second embeddeddolt.OpenSQL handle: see
|
||||
// the matching comment in TestSentinelJournalColumnsReplay.
|
||||
if _, err := db.ExecContext(ctx, "ALTER TABLE bd_events_journal DROP COLUMN comment_json"); err != nil {
|
||||
t.Fatalf("drop comment_json: %v", err)
|
||||
}
|
||||
// MigrateUp's own return value cannot prove the replay ran: it reports
|
||||
// only the main-plane applied count, never the ignored-plane count (see
|
||||
// TestSentinelJournalColumnsReplay), so the contradiction and its heal are
|
||||
// both checked through CurrentIgnoredVersion instead.
|
||||
if floored, err := schema.CurrentIgnoredVersion(ctx, db); err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after damage): %v", err)
|
||||
} else if floored != 21 {
|
||||
t.Fatalf("CurrentIgnoredVersion after dropping comment_json = %d, want exactly 21", floored)
|
||||
}
|
||||
if _, err := schema.MigrateUp(ctx, db); err != nil {
|
||||
t.Fatalf("MigrateUp (forced 22..28 replay): %v", err)
|
||||
}
|
||||
if healed, err := schema.CurrentIgnoredVersion(ctx, db); err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion (after replay): %v", err)
|
||||
} else if want := schema.LatestIgnoredVersion(); healed != want {
|
||||
t.Fatalf("CurrentIgnoredVersion after replay = %d, want latest (%d): the replay did not run", healed, want)
|
||||
}
|
||||
|
||||
after := snapshotIgnoredPlaneTablesExceptJournal(t, ctx, db)
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
t.Errorf("replaying ignored 22..28 perturbed a table it should never touch:\nbefore=%+v\nafter=%+v", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStoreAtIgnored28WritesAndJournals is the real-store half of T2.9
|
||||
// (BEADS-JOURNAL-PLAN.md §4.3, PR A2; schema.TestOldIgnoredCeilingOpensAfter0028
|
||||
// is the other half, pinning architecturally that no open-path guard compares
|
||||
// a binary's own ignored ceiling against the stored cursor). A fresh store
|
||||
// migrates to ignored 28 on its own, with no ignored-ceiling input from the
|
||||
// caller at all; this just confirms that store opens, writes, and journals
|
||||
// normally once there, matching the plan's own framing of the claim.
|
||||
func TestStoreAtIgnored28WritesAndJournals(t *testing.T) {
|
||||
te := newTestEnv(t, "oldceil")
|
||||
ctx := context.Background()
|
||||
|
||||
// Short-lived connection for the pre-check only: te.store's own calls
|
||||
// below (SetEventsJournalEnabled, CreateIssue) each open and close their
|
||||
// own connection to the same directory, and OpenSQL's backoff waits
|
||||
// until ctx cancellation (never), so holding a second connection open
|
||||
// concurrently via te.openDB here would deadlock them forever instead of
|
||||
// failing. te.openDB is only safe to call once nothing else still needs
|
||||
// to open its own connection to this store (see T2.5/T2.6/T2.7 above,
|
||||
// which all finish their te.store.* calls first).
|
||||
func() {
|
||||
db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQL: %v", err)
|
||||
}
|
||||
defer func() { _ = cleanup() }()
|
||||
if got, err := schema.CurrentIgnoredVersion(ctx, db); err != nil {
|
||||
t.Fatalf("CurrentIgnoredVersion: %v", err)
|
||||
} else if want := schema.LatestIgnoredVersion(); got != want {
|
||||
t.Fatalf("CurrentIgnoredVersion = %d, want latest (%d) on a freshly initialized store", got, want)
|
||||
}
|
||||
}()
|
||||
|
||||
te.store.SetEventsJournalEnabled(true)
|
||||
if err := te.store.EventsJournalActivationError(); err != nil {
|
||||
t.Fatalf("activation at ignored 28: %v", err)
|
||||
}
|
||||
issue := &types.Issue{Title: "t2.9 seed", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
|
||||
if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil {
|
||||
t.Fatalf("CreateIssue at ignored 28: %v", err)
|
||||
}
|
||||
|
||||
db := te.openDB(t, ctx)
|
||||
var journaled int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ?", issue.ID).Scan(&journaled); err != nil {
|
||||
t.Fatalf("probe bd_events_journal: %v", err)
|
||||
}
|
||||
if journaled == 0 {
|
||||
t.Error("CreateIssue at ignored 28 did not journal")
|
||||
}
|
||||
}
|
||||
|
|
@ -62,6 +62,72 @@ func TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape(t *testing.T
|
|||
conn, closeConn := newWorkspaceBeforeMigration0023(t, ctx)
|
||||
defer closeConn()
|
||||
|
||||
rowsBefore := seedForkLineageDrift(t, ctx, conn)
|
||||
|
||||
if _, err := schema.MigrateUp(ctx, conn); err != nil {
|
||||
t.Fatalf("repair MigrateUp: %v", err)
|
||||
}
|
||||
requireIgnoredCursorAtLatest(t, ctx, conn)
|
||||
|
||||
// Shape converged.
|
||||
for _, column := range []string{"dep_json", "comment_json"} {
|
||||
if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" {
|
||||
t.Errorf("bd_events_journal.%s DATA_TYPE = %q after repair, want %q", column, got, "longtext")
|
||||
}
|
||||
}
|
||||
requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1)
|
||||
requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1)
|
||||
requireIndex(t, ctx, conn, "wisps", "idx_wisps_defer_until", 1)
|
||||
|
||||
// Data survived byte for byte.
|
||||
assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn))
|
||||
// The pass may raise a counter that is behind the journal to its
|
||||
// high-water mark (ignored/0022's tail, re-run because this fork shape has
|
||||
// no actor column and trips the bd_events_journal.actor sentinel at floor
|
||||
// 21) and must leave it nowhere else: GREATEST(seeded, MAX(seq)).
|
||||
const seeded = 1
|
||||
high := int(rowsBefore[len(rowsBefore)-1].seq)
|
||||
if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != max(seeded, high) {
|
||||
t.Errorf("bd_events_seq.next_seq = %d after repair, want GREATEST(seeded %d, MAX(seq) %d) = %d: a pass may only raise a behind counter to the high-water mark, never lower it or move it past MAX(seq)",
|
||||
got, seeded, high, max(seeded, high))
|
||||
}
|
||||
|
||||
// The repair is what makes the write that used to roll back a user's
|
||||
// mutation land instead.
|
||||
if _, err := conn.ExecContext(ctx,
|
||||
"INSERT INTO bd_events_journal (seq, ts, op, issue_id, comment_json) VALUES (99, '2026-08-01 00:00:03', 'comment', 'bd-t9ovd', ?)",
|
||||
strings.Repeat("x", 70000)); err != nil {
|
||||
t.Fatalf("oversized comment_json insert still refused after repair: %v", err)
|
||||
}
|
||||
if got := scalarInt(t, ctx, conn, "SELECT LENGTH(comment_json) FROM bd_events_journal WHERE seq = 99"); got != 70000 {
|
||||
t.Errorf("stored oversized comment_json length = %d, want 70000", got)
|
||||
}
|
||||
execFrozenGuard(t, ctx, conn, "DELETE FROM bd_events_journal WHERE seq = 99")
|
||||
|
||||
// Crash-replay: the pass can be killed and re-run, so the frozen bytes must
|
||||
// be a clean no-op against the state they just produced. Re-arm the
|
||||
// canary first: if replaying 0023 alone ever wrote the counter, letting it
|
||||
// start already at the high-water mark would hide that.
|
||||
createBefore := showCreateTable(t, ctx, conn, "bd_events_journal")
|
||||
execFrozenGuard(t, ctx, conn, "UPDATE bd_events_seq SET next_seq = 1 WHERE id = 0")
|
||||
execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t))
|
||||
if createAfter := showCreateTable(t, ctx, conn, "bd_events_journal"); createAfter != createBefore {
|
||||
t.Errorf("replaying 0023 changed bd_events_journal:\nbefore:\n%s\nafter:\n%s", createBefore, createAfter)
|
||||
}
|
||||
if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 {
|
||||
t.Errorf("next_seq = %d after replaying 0023, want 1: replaying 0023 wrote the counter", got)
|
||||
}
|
||||
assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn))
|
||||
}
|
||||
|
||||
// seedForkLineageDrift builds the drifted fork-lineage journal shape 0023
|
||||
// repairs: the fork's TEXT-column DDL, idx_wisps_defer_until dropped, three
|
||||
// journal rows, and a seq counter deliberately set BELOW the journal's own
|
||||
// MAX(seq). It also proves the fixture really is drifted before handing it
|
||||
// back: an oversized comment_json insert must be refused by the TEXT column.
|
||||
func seedForkLineageDrift(t *testing.T, ctx context.Context, conn *sql.Conn) []journalRow {
|
||||
t.Helper()
|
||||
|
||||
// The drift. bd_events_journal and wisps are both clone-local and absent
|
||||
// from HEAD here, so replacing the table and dropping the index leaves the
|
||||
// working set exactly as clean as it was.
|
||||
|
|
@ -71,9 +137,14 @@ func TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape(t *testing.T
|
|||
// Data the repair must not touch: three journal rows whose payloads carry
|
||||
// quotes, backslashes, NULLs, and a multi-KB blob (bound as parameters, so
|
||||
// the column holds the literal Go string), plus a seq counter deliberately
|
||||
// set BELOW the journal's own MAX(seq). Only 0022's seeding raises it to
|
||||
// the high-water mark, so next_seq staying at 1 is proof that 0022 did not
|
||||
// re-run and that 0023 left the counter alone.
|
||||
// set BELOW the journal's own MAX(seq). This seeds two separate claims:
|
||||
// that 0023 alone never touches the counter or a row
|
||||
// (TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched runs
|
||||
// its frozen bytes directly to check that), and that the full MigrateUp
|
||||
// pass converges the counter to GREATEST(seeded, MAX(seq)) — this fork
|
||||
// shape has no actor column, so it trips the bd_events_journal.actor
|
||||
// sentinel at floor 21, which re-runs ignored/0022, whose tail is the same
|
||||
// statement as issueops.healEventSeqCounter.
|
||||
bigPayload := strings.Repeat(`{"k":"v'\"\\","n":0},`, 400) + `{"end":true}`
|
||||
seed := []struct {
|
||||
seq int64
|
||||
|
|
@ -107,47 +178,36 @@ VALUES (?, ?, ?, 'bd-t9ovd', ?, ?, ?)`, s.seq, s.ts, s.op, s.issueJSON, s.depJSO
|
|||
t.Fatal("oversized comment_json insert succeeded on the drifted TEXT column; the fixture is not reproducing the drift this migration repairs")
|
||||
}
|
||||
|
||||
if _, err := schema.MigrateUp(ctx, conn); err != nil {
|
||||
t.Fatalf("repair MigrateUp: %v", err)
|
||||
}
|
||||
requireIgnoredCursorAtLatest(t, ctx, conn)
|
||||
return rowsBefore
|
||||
}
|
||||
|
||||
// TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched pins
|
||||
// 0023's own contract ("neither bd_events_seq nor a single journal row is
|
||||
// read or written", its header) with the runner out of the picture: the
|
||||
// frozen bytes run directly against the drifted fork shape, so every repair
|
||||
// branch fires and no sentinel, floor, or neighbouring migration can cause or
|
||||
// mask a counter write. The counter is seeded below MAX(seq), so even a
|
||||
// monotone GREATEST-style write would show.
|
||||
func TestEmbeddedIgnoredMigration0023AloneLeavesCounterAndRowsUntouched(t *testing.T) {
|
||||
requireEmbedded(t)
|
||||
ctx := t.Context()
|
||||
conn, closeConn := newWorkspaceBeforeMigration0023(t, ctx)
|
||||
defer closeConn()
|
||||
rowsBefore := seedForkLineageDrift(t, ctx, conn)
|
||||
|
||||
execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t))
|
||||
|
||||
// Shape converged.
|
||||
for _, column := range []string{"dep_json", "comment_json"} {
|
||||
if got := journalColumnDataType(t, ctx, conn, column); got != "longtext" {
|
||||
t.Errorf("bd_events_journal.%s DATA_TYPE = %q after repair, want %q", column, got, "longtext")
|
||||
t.Fatalf("0023 did not widen %s (got %q): the fixture no longer exercises its branches", column, got)
|
||||
}
|
||||
}
|
||||
requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_ts", 1)
|
||||
requireIndex(t, ctx, conn, "bd_events_journal", "idx_bd_events_journal_issue", 1)
|
||||
requireIndex(t, ctx, conn, "wisps", "idx_wisps_defer_until", 1)
|
||||
|
||||
// Data survived byte for byte, and the counter was not touched.
|
||||
assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn))
|
||||
if got := scalarInt(t, ctx, conn, "SELECT next_seq FROM bd_events_seq WHERE id = 0"); got != 1 {
|
||||
t.Errorf("bd_events_seq.next_seq = %d after repair, want 1 (0023 must touch neither the counter nor any data)", got)
|
||||
t.Errorf("next_seq = %d after applying 0023 alone, want 1: 0023 must never write the counter", got)
|
||||
}
|
||||
|
||||
// The repair is what makes the write that used to roll back a user's
|
||||
// mutation land instead.
|
||||
if _, err := conn.ExecContext(ctx,
|
||||
"INSERT INTO bd_events_journal (seq, ts, op, issue_id, comment_json) VALUES (99, '2026-08-01 00:00:03', 'comment', 'bd-t9ovd', ?)",
|
||||
strings.Repeat("x", 70000)); err != nil {
|
||||
t.Fatalf("oversized comment_json insert still refused after repair: %v", err)
|
||||
}
|
||||
if got := scalarInt(t, ctx, conn, "SELECT LENGTH(comment_json) FROM bd_events_journal WHERE seq = 99"); got != 70000 {
|
||||
t.Errorf("stored oversized comment_json length = %d, want 70000", got)
|
||||
}
|
||||
execFrozenGuard(t, ctx, conn, "DELETE FROM bd_events_journal WHERE seq = 99")
|
||||
|
||||
// Crash-replay: the pass can be killed and re-run, so the frozen bytes must
|
||||
// be a clean no-op against the state they just produced.
|
||||
createBefore := showCreateTable(t, ctx, conn, "bd_events_journal")
|
||||
execFrozenGuard(t, ctx, conn, ignoredMigration0023SQL(t))
|
||||
if createAfter := showCreateTable(t, ctx, conn, "bd_events_journal"); createAfter != createBefore {
|
||||
t.Errorf("replaying 0023 changed bd_events_journal:\nbefore:\n%s\nafter:\n%s", createBefore, createAfter)
|
||||
}
|
||||
assertJournalRowsEqual(t, rowsBefore, readJournalRows(t, ctx, conn))
|
||||
}
|
||||
|
||||
// TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace is the insurance
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ var _ storage.Flattener = (*EmbeddedDoltStore)(nil)
|
|||
var _ storage.Compactor = (*EmbeddedDoltStore)(nil)
|
||||
var _ storage.SchemaMigrator = (*EmbeddedDoltStore)(nil)
|
||||
var _ storage.EventsJournalConfigurer = (*EmbeddedDoltStore)(nil)
|
||||
var _ storage.EventsJournalShapeChecker = (*EmbeddedDoltStore)(nil)
|
||||
var _ storage.VersionedHistoryConfigurer = (*EmbeddedDoltStore)(nil)
|
||||
var _ storage.ExternalRefHistoryQuerier = (*EmbeddedDoltStore)(nil)
|
||||
|
||||
|
|
@ -53,6 +54,16 @@ type EmbeddedDoltStore struct {
|
|||
// eventsJournalEnabled activates the durable events journal for THIS store
|
||||
// instance only (storage.EventsJournalConfigurer); never process-global.
|
||||
eventsJournalEnabled atomic.Bool
|
||||
// journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled
|
||||
// runs on activation (issueops.ProbeJournalShape, PR A1), over a short-lived
|
||||
// connection since this store has no persistent db handle. Nil means either
|
||||
// the journal is disabled or the probe failed — see journalActivationErr.
|
||||
journalShape atomic.Pointer[issueops.JournalShape]
|
||||
// journalActivationErr holds the probe's error when SetEventsJournalEnabled(true)
|
||||
// found a table shape the journal cannot run against (a missing required
|
||||
// column, or a missing table). Nil whenever journalShape is non-nil or the
|
||||
// journal is disabled. See storage.EventsJournalShapeChecker.
|
||||
journalActivationErr atomic.Pointer[error]
|
||||
// versionedHistoryEnabled activates dual-write issue-version history for
|
||||
// THIS store instance only (storage.VersionedHistoryConfigurer); never
|
||||
// process-global.
|
||||
|
|
@ -457,8 +468,20 @@ func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func
|
|||
err = fmt.Errorf("embeddeddolt: begin tx: %w", err)
|
||||
return
|
||||
}
|
||||
clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load())
|
||||
// The activation switch folds in the shape probe's success: a probe that
|
||||
// found an unsupported table (journalShape nil) scopes this transaction as
|
||||
// DISABLED regardless of what SetEventsJournalEnabled was called with, so a
|
||||
// caller that enables the journal directly and ignores the returned
|
||||
// EventsJournalActivationError still never attempts a write against a shape
|
||||
// that cannot support it (storage.EventsJournalShapeChecker). Reads
|
||||
// (commit=false callers) are never gated by this switch — only by the
|
||||
// shape — since ReadEventsJournal has no enabled/disabled concept of its
|
||||
// own.
|
||||
journalShape := s.journalShape.Load()
|
||||
clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load() && journalShape != nil)
|
||||
defer clearJournalScope()
|
||||
clearJournalShapeScope := issueops.ScopeEventsJournalShape(tx, journalShape)
|
||||
defer clearJournalShapeScope()
|
||||
clearVersionScope := issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load())
|
||||
defer clearVersionScope()
|
||||
clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx)
|
||||
|
|
@ -484,8 +507,47 @@ func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func
|
|||
}
|
||||
|
||||
// SetEventsJournalEnabled activates the journal for this store instance only.
|
||||
//
|
||||
// Activating probes bd_events_journal's actual shape ONCE
|
||||
// (issueops.ProbeJournalShape, PR A1), over a short-lived connection opened
|
||||
// and closed just for the probe since this store keeps no persistent db
|
||||
// handle, and caches the result for this instance's lifetime;
|
||||
// EventsJournalActivationError reports whether that probe found a shape the
|
||||
// journal can actually run against. Deactivating clears both and skips the
|
||||
// probe entirely — a disabled store accepts any shape.
|
||||
func (s *EmbeddedDoltStore) SetEventsJournalEnabled(enabled bool) {
|
||||
s.eventsJournalEnabled.Store(enabled)
|
||||
if !enabled {
|
||||
s.journalShape.Store(nil)
|
||||
s.journalActivationErr.Store(nil)
|
||||
return
|
||||
}
|
||||
db, cleanup, err := OpenSQL(context.Background(), s.dataDir, s.database, s.branch)
|
||||
if err != nil {
|
||||
s.journalShape.Store(nil)
|
||||
wrapped := fmt.Errorf("journal: open connection to probe shape: %w", err)
|
||||
s.journalActivationErr.Store(&wrapped)
|
||||
return
|
||||
}
|
||||
defer func() { _ = cleanup() }()
|
||||
shape, probeErr := issueops.ProbeJournalShape(context.Background(), db)
|
||||
if probeErr != nil {
|
||||
s.journalShape.Store(nil)
|
||||
s.journalActivationErr.Store(&probeErr)
|
||||
return
|
||||
}
|
||||
s.journalShape.Store(shape)
|
||||
s.journalActivationErr.Store(nil)
|
||||
}
|
||||
|
||||
// EventsJournalActivationError implements storage.EventsJournalShapeChecker:
|
||||
// nil unless the last SetEventsJournalEnabled(true) call found a table shape
|
||||
// the journal cannot run against.
|
||||
func (s *EmbeddedDoltStore) EventsJournalActivationError() error {
|
||||
if p := s.journalActivationErr.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetVersionedHistoryEnabled activates dual-write issue-version history for
|
||||
|
|
|
|||
|
|
@ -134,6 +134,20 @@ func (p *uowProvider) SetEventsJournalEnabled(enabled bool) {
|
|||
}
|
||||
}
|
||||
|
||||
// EventsJournalActivationError forwards the inner provider's shape-probe
|
||||
// result (storage.EventsJournalShapeChecker, PR A1). This wrapper is the
|
||||
// OUTERMOST provider on both real chains (see SetVersionedHistoryEnabled), so
|
||||
// without this forwarder eventsjournal.Apply's type assertion for the checker
|
||||
// would fail here even when the inner provider's own probe found an
|
||||
// unsupported table shape — the open would then succeed against a shape it
|
||||
// cannot journal, the gci failure mode this interface exists to avoid.
|
||||
func (p *uowProvider) EventsJournalActivationError() error {
|
||||
if checker, ok := p.UnitOfWorkProvider.(storage.EventsJournalShapeChecker); ok {
|
||||
return checker.EventsJournalActivationError()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetVersionedHistoryEnabled forwards dual-write issue-version activation
|
||||
// inward, for the same reason SetEventsJournalEnabled does.
|
||||
//
|
||||
|
|
@ -162,6 +176,7 @@ func (p *uowProvider) RunEventsMaintenanceTx(ctx context.Context, fn func(contex
|
|||
var (
|
||||
_ uow.PoolTuner = (*uowProvider)(nil)
|
||||
_ storage.EventsJournalConfigurer = (*uowProvider)(nil)
|
||||
_ storage.EventsJournalShapeChecker = (*uowProvider)(nil)
|
||||
_ storage.VersionedHistoryConfigurer = (*uowProvider)(nil)
|
||||
_ issueops.EventsMaintenanceRunner = (*uowProvider)(nil)
|
||||
_ uow.IssueLifecycleSource = (*uowProvider)(nil)
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ go_library(
|
|||
"journal_autoprune.go",
|
||||
"journal_dependencies.go",
|
||||
"journal_prune.go",
|
||||
"journal_shape.go",
|
||||
"labels.go",
|
||||
"lease.go",
|
||||
"metadata.go",
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -420,6 +421,15 @@ func getJournalIssueInTx(ctx context.Context, tx DBTX, issueID string) (*types.I
|
|||
// (non-dependency ops). ts is the insert time, stamped inside the committing
|
||||
// transaction. actor is stored as-is — "" for the genuinely unattributable
|
||||
// paths — in the NOT NULL DEFAULT ” actor column.
|
||||
//
|
||||
// The column list is shape-adaptive (journal_shape.go, PR A1): seq, ts, op
|
||||
// and issue_id are always named, and each optional column — actor,
|
||||
// issue_json, dep_json, comment_json — is named only when journalShapeFor(tx)
|
||||
// says this store's table actually has it. On a canonical table that is every
|
||||
// column, in the same order this INSERT has always used; on an old-lineage
|
||||
// table missing one, the write still lands and that one payload is simply not
|
||||
// recorded, rather than failing the whole mutation the journal row shares a
|
||||
// transaction with.
|
||||
func insertEventRow(ctx context.Context, tx DBTX, op EventOp, issueID string, issue *types.Issue, dep *EventDep, comment *EventComment, actor string) error {
|
||||
var issueJSON any
|
||||
if issue != nil {
|
||||
|
|
@ -445,11 +455,41 @@ func insertEventRow(ctx context.Context, tx DBTX, op EventOp, issueID string, is
|
|||
}
|
||||
commentJSON = string(b)
|
||||
}
|
||||
|
||||
shape := journalShapeFor(tx)
|
||||
type optionalJournalValue struct {
|
||||
column string
|
||||
value any
|
||||
}
|
||||
var optionalValues []optionalJournalValue
|
||||
if shape.HasActor {
|
||||
optionalValues = append(optionalValues, optionalJournalValue{"actor", actor})
|
||||
}
|
||||
if shape.HasIssueJSON {
|
||||
optionalValues = append(optionalValues, optionalJournalValue{"issue_json", issueJSON})
|
||||
}
|
||||
if shape.HasDepJSON {
|
||||
optionalValues = append(optionalValues, optionalJournalValue{"dep_json", depJSON})
|
||||
}
|
||||
if shape.HasCommentJSON {
|
||||
optionalValues = append(optionalValues, optionalJournalValue{"comment_json", commentJSON})
|
||||
}
|
||||
columns := make([]string, 0, 4+len(optionalValues))
|
||||
columns = append(columns, "seq", "ts", "op", "issue_id")
|
||||
for _, ov := range optionalValues {
|
||||
columns = append(columns, ov.column)
|
||||
}
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(columns)), ", ")
|
||||
//nolint:gosec // columns is built entirely from the fixed name literals above, never from caller input.
|
||||
query := fmt.Sprintf("INSERT INTO bd_events_journal (%s) VALUES (%s)", strings.Join(columns, ", "), placeholders)
|
||||
|
||||
insert := func(seq int64) error {
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO bd_events_journal (seq, ts, op, issue_id, actor, issue_json, dep_json, comment_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, seq, time.Now().UTC(), string(op), issueID, actor, issueJSON, depJSON, commentJSON)
|
||||
args := make([]any, 0, len(columns))
|
||||
args = append(args, seq, time.Now().UTC(), string(op), issueID)
|
||||
for _, ov := range optionalValues {
|
||||
args = append(args, ov.value)
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, query, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
|
|
@ -251,10 +252,30 @@ func readEventsHeadInTx(ctx context.Context, tx DBTX) (int64, error) {
|
|||
return head, nil
|
||||
}
|
||||
|
||||
// readEventsRowsInTx's column list is shape-adaptive (journal_shape.go, PR
|
||||
// A1): seq, ts, op and issue_id are always selected, and each optional column
|
||||
// is selected only when journalShapeFor(tx) says this store's table actually
|
||||
// has it. A column the probe did not find is simply never requested, so the
|
||||
// corresponding field on storage.EventsJournalRow reads as "" — never a scan
|
||||
// error — for a consumer on an old-lineage table.
|
||||
func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([]storage.EventsJournalRow, error) {
|
||||
shape := journalShapeFor(tx)
|
||||
// CAST(ts AS CHAR) normalizes the DATETIME to a stable string across drivers.
|
||||
q := `SELECT seq, CAST(ts AS CHAR), op, issue_id, actor, issue_json, dep_json, comment_json
|
||||
FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC`
|
||||
columns := []string{"seq", "CAST(ts AS CHAR)", "op", "issue_id"}
|
||||
if shape.HasActor {
|
||||
columns = append(columns, "actor")
|
||||
}
|
||||
if shape.HasIssueJSON {
|
||||
columns = append(columns, "issue_json")
|
||||
}
|
||||
if shape.HasDepJSON {
|
||||
columns = append(columns, "dep_json")
|
||||
}
|
||||
if shape.HasCommentJSON {
|
||||
columns = append(columns, "comment_json")
|
||||
}
|
||||
//nolint:gosec // columns is built entirely from the fixed name literals above, never from caller input.
|
||||
q := "SELECT " + strings.Join(columns, ", ") + " FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC"
|
||||
if limit > 0 {
|
||||
q += " LIMIT " + strconv.Itoa(limit)
|
||||
}
|
||||
|
|
@ -268,14 +289,29 @@ func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([
|
|||
for rows.Next() {
|
||||
var (
|
||||
r storage.EventsJournalRow
|
||||
actor sql.NullString
|
||||
issueJS sql.NullString
|
||||
depJS sql.NullString
|
||||
commentJS sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&r.Seq, &r.TS, &r.Op, &r.IssueID, &r.Actor, &issueJS, &depJS, &commentJS); err != nil {
|
||||
dest := []any{&r.Seq, &r.TS, &r.Op, &r.IssueID}
|
||||
if shape.HasActor {
|
||||
dest = append(dest, &actor)
|
||||
}
|
||||
if shape.HasIssueJSON {
|
||||
dest = append(dest, &issueJS)
|
||||
}
|
||||
if shape.HasDepJSON {
|
||||
dest = append(dest, &depJS)
|
||||
}
|
||||
if shape.HasCommentJSON {
|
||||
dest = append(dest, &commentJS)
|
||||
}
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
return nil, fmt.Errorf("journal: scan row: %w", err)
|
||||
}
|
||||
r.TS = normalizeEventsTimestamp(r.TS)
|
||||
r.Actor = actor.String
|
||||
r.IssueJSON = issueJS.String
|
||||
r.DepJSON = depJS.String
|
||||
r.CommentJSON = commentJS.String
|
||||
|
|
|
|||
257
internal/storage/issueops/journal_shape.go
Normal file
257
internal/storage/issueops/journal_shape.go
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
package issueops
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/steveyegge/beads/internal/debug"
|
||||
)
|
||||
|
||||
// Adaptive I/O for bd_events_journal (BEADS-JOURNAL-PLAN.md §4.2a, PR A1).
|
||||
//
|
||||
// The writer and reader used to name a fixed column list. That is fine on a
|
||||
// table built by the current migrations, but a table built by an earlier
|
||||
// lineage — gas-city-inc's is the live case — can lack an optional column
|
||||
// forever: the ignored series only ever creates the table when it is wholly
|
||||
// absent, and its one repair (ignored/0023) probes column TYPES, never
|
||||
// EXISTENCE. A fixed INSERT naming a missing column then fails every
|
||||
// journaled write, and because the journal row commits in the SAME
|
||||
// transaction as the mutation it records, that failure rolls back the user's
|
||||
// write along with it.
|
||||
//
|
||||
// The fix probes once, at activation, rather than guessing or failing per
|
||||
// write:
|
||||
// - REQUIRED columns {seq, ts, op, issue_id} have existed on every lineage
|
||||
// this table has ever shipped under; a table missing one of them, or
|
||||
// missing entirely, cannot journal at all. ProbeJournalShape refuses with
|
||||
// a typed error so a store's "enable the journal" step fails at open,
|
||||
// before any mutation is attempted — never as a per-write failure after
|
||||
// the work is otherwise done, which is the gci failure mode above.
|
||||
// - OPTIONAL columns {actor, issue_json, dep_json, comment_json} may be
|
||||
// missing on an old-lineage table. The writer drops whichever of them the
|
||||
// probe did not find; the reader returns "" for them. One warning is
|
||||
// logged per probe (i.e. once per store instance, not once per write),
|
||||
// and ignored/0028 (PR A2) is what actually converges the shape — this
|
||||
// file only keeps a drifted table WRITABLE and READABLE in the meantime.
|
||||
//
|
||||
// A store instance probes exactly once, when storage.EventsJournalConfigurer
|
||||
// activates the journal (see each backend's SetEventsJournalEnabled), and
|
||||
// caches the *JournalShape for its own lifetime. ScopeEventsJournalShape
|
||||
// carries that cached value onto one transaction, next to
|
||||
// ScopeEventsJournalTransaction's activation switch, so insertEventRow and
|
||||
// readEventsRowsInTx — both of which only ever see a tx, never the store —
|
||||
// can reach it without a per-write probe (TestJournalShapeProbeOncePerOpen).
|
||||
|
||||
// journalTableName is the clone-local table every probe, insert and read in
|
||||
// this file targets.
|
||||
const journalTableName = "bd_events_journal"
|
||||
|
||||
// requiredJournalColumns are the columns ProbeJournalShape treats as
|
||||
// non-negotiable: every lineage of bd_events_journal has always carried them,
|
||||
// and the seq/ts/op/issue_id contract (nextEventSeq, the reader's ordering)
|
||||
// is meaningless without them.
|
||||
var requiredJournalColumns = []string{"seq", "ts", "op", "issue_id"}
|
||||
|
||||
// optionalJournalColumns are the columns a pre-merge lineage may lack. Order
|
||||
// matches the canonical INSERT column order a healthy table has carried since
|
||||
// migration 0066/ignored 0025 added actor.
|
||||
var optionalJournalColumns = []string{"actor", "issue_json", "dep_json", "comment_json"}
|
||||
|
||||
// JournalShape records which OPTIONAL bd_events_journal columns exist on one
|
||||
// store's table. A nil *JournalShape is never handed to a caller; probing a
|
||||
// table missing a REQUIRED column returns an error instead (see
|
||||
// ProbeJournalShape).
|
||||
type JournalShape struct {
|
||||
HasActor bool
|
||||
HasIssueJSON bool
|
||||
HasDepJSON bool
|
||||
HasCommentJSON bool
|
||||
}
|
||||
|
||||
// missingOptional names the optional columns this shape does not have, in
|
||||
// optionalJournalColumns order, for the doctor finding and the one-per-probe
|
||||
// warning.
|
||||
func (s *JournalShape) missingOptional() []string {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
var missing []string
|
||||
if !s.HasActor {
|
||||
missing = append(missing, "actor")
|
||||
}
|
||||
if !s.HasIssueJSON {
|
||||
missing = append(missing, "issue_json")
|
||||
}
|
||||
if !s.HasDepJSON {
|
||||
missing = append(missing, "dep_json")
|
||||
}
|
||||
if !s.HasCommentJSON {
|
||||
missing = append(missing, "comment_json")
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
// canonicalJournalShape is every optional column present — the shape every
|
||||
// workspace created from main migration 0064 plus ignored 0022/0025 has, and
|
||||
// the default journalShapeFor returns for a transaction nobody scoped. That
|
||||
// default matters: it is what every call site written before this file
|
||||
// existed (and every test that activates the journal through the ctx-only
|
||||
// WithEventsJournal test helper rather than a real store) keeps getting, so
|
||||
// adaptive I/O is additive and changes nothing for a canonical table.
|
||||
var canonicalJournalShape = &JournalShape{
|
||||
HasActor: true,
|
||||
HasIssueJSON: true,
|
||||
HasDepJSON: true,
|
||||
HasCommentJSON: true,
|
||||
}
|
||||
|
||||
// ErrJournalShapeUnsupported is the sentinel errors.Is matches against any
|
||||
// *JournalShapeError returned by ProbeJournalShape, regardless of which
|
||||
// required columns a particular table happens to be missing.
|
||||
var ErrJournalShapeUnsupported = errors.New("events journal: table shape does not support the events journal")
|
||||
|
||||
// JournalShapeError reports that bd_events_journal (or the table itself) is
|
||||
// missing one or more REQUIRED columns. It is returned by ProbeJournalShape
|
||||
// and is the typed error storage.EventsJournalConfigurer implementations
|
||||
// surface from activation, so "enable the journal" fails at open rather than
|
||||
// the gci failure mode: succeeding at open and then failing every write thereafter.
|
||||
type JournalShapeError struct {
|
||||
Table string
|
||||
Missing []string
|
||||
}
|
||||
|
||||
func (e *JournalShapeError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"events journal: table %s is missing required column(s) %s; the events journal cannot be enabled against this shape (see `bd doctor`)",
|
||||
e.Table, strings.Join(e.Missing, ", "),
|
||||
)
|
||||
}
|
||||
|
||||
// Is makes errors.Is(err, ErrJournalShapeUnsupported) match any
|
||||
// *JournalShapeError, so a caller can test for the class of failure without
|
||||
// caring which columns a particular table lacks.
|
||||
func (e *JournalShapeError) Is(target error) bool {
|
||||
return target == ErrJournalShapeUnsupported //nolint:errorlint // sentinel identity comparison is the documented contract of Is.
|
||||
}
|
||||
|
||||
// journalShapeProbeCount counts how many times ProbeJournalShape has actually
|
||||
// queried INFORMATION_SCHEMA in this process. It exists for
|
||||
// TestJournalShapeProbeOncePerOpen, which asserts that a batch of writes
|
||||
// against one already-activated store issues none of its own.
|
||||
var journalShapeProbeCount atomic.Int64
|
||||
|
||||
// JournalShapeProbeCountForTest reports journalShapeProbeCount's current
|
||||
// value. Test-only instrumentation, read by TestJournalShapeProbeOncePerOpen;
|
||||
// production code never calls it.
|
||||
func JournalShapeProbeCountForTest() int64 { return journalShapeProbeCount.Load() }
|
||||
|
||||
// ProbeJournalShape runs the one INFORMATION_SCHEMA.COLUMNS query that
|
||||
// determines what q's bd_events_journal table actually supports. Callers run
|
||||
// it exactly once per store/provider instance, from
|
||||
// storage.EventsJournalConfigurer.SetEventsJournalEnabled, and cache the
|
||||
// result for that instance's lifetime — never per write or per transaction.
|
||||
//
|
||||
// A table missing any required column, including a wholly absent table
|
||||
// (which reads as every column missing, since a COLUMNS query against a
|
||||
// nonexistent table returns zero rows — the same shape ignored/0023's
|
||||
// existing probes rely on), returns *JournalShapeError instead of a shape.
|
||||
func ProbeJournalShape(ctx context.Context, q DBTX) (*JournalShape, error) {
|
||||
journalShapeProbeCount.Add(1)
|
||||
|
||||
rows, err := q.QueryContext(ctx, `
|
||||
SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`, journalTableName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("journal: probe %s shape: %w", journalTableName, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
present := make(map[string]bool, len(requiredJournalColumns)+len(optionalJournalColumns))
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, fmt.Errorf("journal: scan %s column: %w", journalTableName, err)
|
||||
}
|
||||
present[strings.ToLower(name)] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("journal: read %s shape: %w", journalTableName, err)
|
||||
}
|
||||
|
||||
var missing []string
|
||||
for _, col := range requiredJournalColumns {
|
||||
if !present[col] {
|
||||
missing = append(missing, col)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return nil, &JournalShapeError{Table: journalTableName, Missing: missing}
|
||||
}
|
||||
|
||||
shape := &JournalShape{
|
||||
HasActor: present["actor"],
|
||||
HasIssueJSON: present["issue_json"],
|
||||
HasDepJSON: present["dep_json"],
|
||||
HasCommentJSON: present["comment_json"],
|
||||
}
|
||||
warnJournalShapeDegraded(shape)
|
||||
return shape, nil
|
||||
}
|
||||
|
||||
// warnJournalShapeDegraded writes one audit line to STDERR, suppressed only
|
||||
// by quiet mode, the first (and every) time a store probes a table missing an
|
||||
// optional column — one call per ProbeJournalShape invocation, which is one
|
||||
// per store activation, which is "one warning per store" regardless of how
|
||||
// many writes that store goes on to make.
|
||||
func warnJournalShapeDegraded(shape *JournalShape) {
|
||||
missing := shape.missingOptional()
|
||||
if len(missing) == 0 {
|
||||
return
|
||||
}
|
||||
if debug.IsQuiet() {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"events journal: %s is missing optional column(s) %s; affected payload(s) will be dropped until it converges (see `bd doctor`)\n",
|
||||
journalTableName, strings.Join(missing, ", "))
|
||||
}
|
||||
|
||||
// journalShapes associates a probed *JournalShape with ONE concrete
|
||||
// transaction, the same per-tx lifetime journalTransactions gives the
|
||||
// activation switch. Entries live for one transaction only.
|
||||
var journalShapes sync.Map // map[DBTX]*JournalShape
|
||||
|
||||
// ScopeEventsJournalShape binds shape to tx so insertEventRow and
|
||||
// readEventsRowsInTx can reach the store's already-cached probe result
|
||||
// without taking the store itself as a parameter. Store implementations call
|
||||
// it immediately after BeginTx, alongside ScopeEventsJournalTransaction,
|
||||
// passing the *JournalShape SetEventsJournalEnabled cached at activation —
|
||||
// never a fresh probe.
|
||||
//
|
||||
// A nil tx or nil shape is a no-op; journalShapeFor then returns
|
||||
// canonicalJournalShape, today's fixed eight-column behavior.
|
||||
func ScopeEventsJournalShape(tx DBTX, shape *JournalShape) func() {
|
||||
if tx == nil || shape == nil {
|
||||
return func() {}
|
||||
}
|
||||
journalShapes.Store(tx, shape)
|
||||
return func() { journalShapes.Delete(tx) }
|
||||
}
|
||||
|
||||
// journalShapeFor returns the shape scoped to tx, or canonicalJournalShape
|
||||
// when nobody scoped one — which is every call site that predates this file,
|
||||
// and every test driving the journal through the ctx-only WithEventsJournal
|
||||
// helper rather than a real store's activation.
|
||||
func journalShapeFor(tx DBTX) *JournalShape {
|
||||
if v, ok := journalShapes.Load(tx); ok {
|
||||
if shape, ok := v.(*JournalShape); ok && shape != nil {
|
||||
return shape
|
||||
}
|
||||
}
|
||||
return canonicalJournalShape
|
||||
}
|
||||
|
|
@ -117,6 +117,7 @@ go_library(
|
|||
"migrations/ignored/0026_dep_rekey_dedup_marker.up.sql",
|
||||
"migrations/ignored/0027_add_wisps_current_revision.up.sql",
|
||||
"migrations/0069_widen_issue_versions_datetime_precision.up.sql",
|
||||
"migrations/ignored/0028_converge_events_journal_shape.up.sql",
|
||||
],
|
||||
importpath = "github.com/steveyegge/beads/internal/storage/schema",
|
||||
visibility = ["//:__subpackages__"],
|
||||
|
|
@ -151,6 +152,7 @@ go_test(
|
|||
"migration_0069_datetime_precision_test.go",
|
||||
"migration_body_drain_test.go",
|
||||
"migration_watchdog_test.go",
|
||||
"old_ignored_ceiling_test.go",
|
||||
"progress_test.go",
|
||||
"remote_migrate_gate_test.go",
|
||||
"schema_skew_test.go",
|
||||
|
|
@ -160,6 +162,7 @@ go_test(
|
|||
],
|
||||
data = [":migrations"],
|
||||
embed = [":schema"],
|
||||
embedsrcs = ["testdata/old_ignored_ceiling_24/0024_placeholder.up.sql"],
|
||||
deps = [
|
||||
"//internal/debug",
|
||||
"//internal/storage/depid",
|
||||
|
|
|
|||
|
|
@ -133,12 +133,40 @@ func TestCursorRealityFloor(t *testing.T) {
|
|||
wantLimited: true,
|
||||
},
|
||||
{
|
||||
name: "schema corroborates the cursor",
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{"leases.granted_node": true},
|
||||
name: "schema corroborates the cursor",
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
"bd_events_journal.comment_json": true,
|
||||
"bd_events_journal.actor": true,
|
||||
},
|
||||
wantFloor: 0,
|
||||
wantLimited: false,
|
||||
},
|
||||
{
|
||||
// T2.6c: the journal floor limits on its own, with leases
|
||||
// corroborated and the OTHER journal column present. Moving actor
|
||||
// off floor 21 would read 24 here instead (see
|
||||
// TestCurrentVersionClampsToSentinelFloor's matching case).
|
||||
name: "journal actor absent alone",
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
"bd_events_journal.comment_json": true,
|
||||
},
|
||||
wantFloor: 21,
|
||||
wantLimited: true,
|
||||
},
|
||||
{
|
||||
name: "journal comment_json absent alone",
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
"bd_events_journal.actor": true,
|
||||
},
|
||||
wantFloor: 21,
|
||||
wantLimited: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
stubSentinelTables(t, tc.present, nil)
|
||||
|
|
@ -252,8 +280,62 @@ func TestCurrentVersionClampsToSentinelFloor(t *testing.T) {
|
|||
name: "fully corroborated cursor is believed as read",
|
||||
raw: latest,
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{"leases.granted_node": true},
|
||||
want: latest,
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
"bd_events_journal.comment_json": true,
|
||||
"bd_events_journal.actor": true,
|
||||
},
|
||||
want: latest,
|
||||
},
|
||||
{
|
||||
// T2.6c: the fleet's shape at the first A2 open. Every production
|
||||
// Dolt store sits at ignored cursor 24 (BEADS-JOURNAL-PLAN.md §4.1)
|
||||
// and predates actor outright, so this clamps to 21 and 22..28
|
||||
// replay once.
|
||||
name: "fleet first open: cursor 24, journal actor absent",
|
||||
raw: 24,
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
"bd_events_journal.comment_json": true,
|
||||
},
|
||||
want: 21,
|
||||
},
|
||||
{
|
||||
// #5496's fork-lineage fixture: cursor 22, actor absent (that shape
|
||||
// predates ignored/0025, which adds actor, outright).
|
||||
name: "fork-lineage fixture: cursor 22, actor absent",
|
||||
raw: 22,
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
"bd_events_journal.comment_json": true,
|
||||
},
|
||||
want: 21,
|
||||
},
|
||||
{
|
||||
// A raw cursor already sitting exactly at the journal floor: the
|
||||
// clamp is a ceiling, not a promotion, so a contradiction at or
|
||||
// below it changes nothing.
|
||||
name: "cursor at the journal floor is untouched",
|
||||
raw: 21,
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"leases.granted_node": true,
|
||||
},
|
||||
want: 21,
|
||||
},
|
||||
{
|
||||
// Taking the MAXIMUM floor instead of the minimum in
|
||||
// cursorRealityFloor would read 21 here (the journal floor); the
|
||||
// minimum reads 11 (the leases floor).
|
||||
name: "leases and journal actor both absent",
|
||||
raw: latest,
|
||||
present: allSentinelTables(),
|
||||
columns: map[string]bool{
|
||||
"bd_events_journal.comment_json": true,
|
||||
},
|
||||
want: 11,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
|
@ -583,7 +665,10 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) {
|
|||
AddRow("content_hash", "char(64)", "YES", "", nil, ""))
|
||||
|
||||
// The guarded read: cursor claims at-latest, both table sentinels
|
||||
// corroborate it, and only the leases column does not.
|
||||
// corroborate it, and only the leases column does not — the
|
||||
// bd_events_journal sentinels (PR A2) are probed too, in the same
|
||||
// sentinelColumns order, and corroborate here so the leases contradiction
|
||||
// stays the only thing driving the floor.
|
||||
expectCursorProbe(mock, "ignored_schema_migrations", true)
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion())
|
||||
for range ignoredSource.sentinelTables {
|
||||
|
|
@ -592,6 +677,10 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) {
|
|||
}
|
||||
mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
|
||||
for range 2 {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
}
|
||||
|
||||
boom := errors.New("stop here: the applier resumed above the floor")
|
||||
mock.ExpectExec(".*").WillReturnError(boom)
|
||||
|
|
@ -653,3 +742,105 @@ func TestPendingVersionsUnderFloorDoNotReArmAuxMarkers(t *testing.T) {
|
|||
t.Error("ignored/0007 is pending on a cursor-11 store; its unguarded UPDATE would restamp wisps.updated_at")
|
||||
}
|
||||
}
|
||||
|
||||
// journalSentinelReplayDML matches an (?i)-insensitive statement that
|
||||
// mutates data: UPDATE ... SET, DELETE FROM, INSERT [IGNORE] INTO,
|
||||
// REPLACE INTO, or TRUNCATE TABLE. It is deliberately loose about what
|
||||
// follows — this test's job is to flag every candidate and then check it
|
||||
// against an allowlist, not to parse SQL.
|
||||
var journalSentinelReplayDML = regexp.MustCompile(`(?i)\b(update\s+\S+\s+set|delete\s+from|insert\s+(ignore\s+)?into|replace\s+into|truncate\s+table)\b`)
|
||||
|
||||
var (
|
||||
journalSentinelCommentLineRE = regexp.MustCompile(`(?m)^\s*--.*$`)
|
||||
journalSentinelOnUpdateRE = regexp.MustCompile(`(?i)on\s+update\s+current_timestamp(\s*\(\s*\))?`)
|
||||
journalSentinelOnDupKeyRE = regexp.MustCompile(`(?i)on\s+duplicate\s+key\s+update`)
|
||||
journalSentinelWhitespaceRE = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// journalSentinelReplayAllowlist is every statement, by migration version,
|
||||
// that the bd_events_journal sentinels' replay range (today: above floor 21)
|
||||
// is permitted to carry, keyed by its whitespace-collapsed lowercase form.
|
||||
// Each entry needs a justification here before it is added.
|
||||
var journalSentinelReplayAllowlist = map[int][]string{
|
||||
22: {
|
||||
// Idempotent: INSERT IGNORE no-ops once bd_events_seq's single row
|
||||
// (id=0) exists, on every replay after the first.
|
||||
"insert ignore into bd_events_seq (id, next_seq) values (0, 0)",
|
||||
// Monotone: identical in effect to issueops.healEventSeqCounter — a
|
||||
// no-op when the counter is already at or ahead of the journal's
|
||||
// high-water mark, and a raise to exactly MAX(seq) when it is behind.
|
||||
// See TestSentinelReplayCounterConverges (embeddeddolt package) for
|
||||
// the behavioral proof.
|
||||
"update bd_events_seq set next_seq = greatest(next_seq, coalesce((select max(seq) from bd_events_journal), 0)) where id = 0",
|
||||
},
|
||||
}
|
||||
|
||||
// TestJournalSentinelReplayRangeDMLIsAllowlisted is T2.14: every ignored
|
||||
// migration above the bd_events_journal sentinels' floor is a migration a
|
||||
// sentinel contradiction can force to replay against a store that already
|
||||
// has real data in it (section 3 of the A2 conflict decision). A DML
|
||||
// statement in that range is therefore not merely "guarded" by an IF — it
|
||||
// must also be safe to re-run against live rows, which an IF alone does not
|
||||
// guarantee (an unconditional UPDATE inside a guarded ALTER branch would
|
||||
// still re-fire every time the branch's condition holds). This pins the
|
||||
// range to exactly the DML this decision reviewed, so a future migration
|
||||
// added above the floor cannot silently introduce an unreviewed one.
|
||||
//
|
||||
// The floor is read from the registry, not hard-coded, so a future change to
|
||||
// the sentinel values updates this test's scope automatically.
|
||||
//
|
||||
// Kills: any future ignored migration above the floor that adds DML which
|
||||
// every journal-sentinel replay would silently re-run — for example a
|
||||
// jv/txn backfill `UPDATE bd_events_journal ...` or an `UPDATE wisps ...`
|
||||
// restamp (A4-A6 add ignored/0029-0031 inside this range).
|
||||
func TestJournalSentinelReplayRangeDMLIsAllowlisted(t *testing.T) {
|
||||
floor := 0
|
||||
haveFloor := false
|
||||
for _, sc := range ignoredSource.sentinelColumns {
|
||||
if sc.table != "bd_events_journal" {
|
||||
continue
|
||||
}
|
||||
if !haveFloor || sc.replayFloor < floor {
|
||||
floor = sc.replayFloor
|
||||
}
|
||||
haveFloor = true
|
||||
}
|
||||
if !haveFloor {
|
||||
t.Fatal("no bd_events_journal sentinel column is registered in ignoredSource; this test has nothing to pin")
|
||||
}
|
||||
|
||||
for _, mf := range ignoredSource.list() {
|
||||
if mf.version <= floor {
|
||||
continue
|
||||
}
|
||||
blob, err := ignoredSource.files.ReadFile(ignoredSource.dir + "/" + mf.name)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", mf.name, err)
|
||||
}
|
||||
|
||||
body := journalSentinelCommentLineRE.ReplaceAllString(string(blob), "")
|
||||
body = journalSentinelOnUpdateRE.ReplaceAllString(body, " ")
|
||||
body = journalSentinelOnDupKeyRE.ReplaceAllString(body, " ")
|
||||
|
||||
for _, stmt := range strings.Split(body, ";") {
|
||||
if !journalSentinelReplayDML.MatchString(stmt) {
|
||||
continue
|
||||
}
|
||||
normalized := strings.ToLower(strings.TrimSpace(journalSentinelWhitespaceRE.ReplaceAllString(stmt, " ")))
|
||||
if normalized == "" {
|
||||
continue
|
||||
}
|
||||
allowed := false
|
||||
for _, want := range journalSentinelReplayAllowlist[mf.version] {
|
||||
if normalized == want {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
t.Errorf("ignored/%04d (above the bd_events_journal replay floor %d) carries DML not on the replay-safety allowlist:\n %s\nEvery journal-sentinel-triggered replay re-runs this statement against a live store; add it to journalSentinelReplayAllowlist with a justification, or make it a no-op above the floor.",
|
||||
mf.version, floor, normalized)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
-- Converge every observed shape of bd_events_journal on the canonical one
|
||||
-- (BEADS-JOURNAL-PLAN.md §4.2b, PR A2).
|
||||
--
|
||||
-- ignored/0023 repairs this table too, but only by probing DATA_TYPE: a
|
||||
-- column that is entirely ABSENT returns no row, DATA_TYPE reads NULL, and
|
||||
-- `NULL = 1` takes the no-op branch. That gap is exactly what let three
|
||||
-- distinct field shapes survive 0023 unrepaired:
|
||||
--
|
||||
-- - gas-city-inc (gci): seq, ts, op, issue_id, issue_json, dep_json —
|
||||
-- comment_json and actor both absent outright (plan §1.1). 0023 widened
|
||||
-- dep_json and added the ts index (both already correct, i.e. both were
|
||||
-- present with some type), so neither of the two missing-column probes
|
||||
-- ever fired.
|
||||
-- - the events-journal-3666e5026 release shape: issue_json LONGTEXT,
|
||||
-- dep_json TEXT, no comment_json, no ts index (plan §1.1, "where that
|
||||
-- shape came from"). 0023's dep_json widen DOES fire here, but the
|
||||
-- missing comment_json still does not, for the same reason as gci.
|
||||
-- - the enterprise fork's ignored/0017 shape (0023's own header,
|
||||
-- bd-t9ovd): dep_json and comment_json both TEXT (present, so 0023's
|
||||
-- widen fires for both), no ts index, no idx_wisps_defer_until. This one
|
||||
-- IS fully healed by 0023 already; it is included here as a fixture
|
||||
-- (TestIgnored0028ConvergesAllShapes) precisely to pin that this file
|
||||
-- stays a no-op on a shape 0023 already fixed, not to add new coverage
|
||||
-- for it.
|
||||
--
|
||||
-- This file closes the gap left open above by probing EXISTENCE first for
|
||||
-- the two columns that can be missing outright, then falling through to
|
||||
-- 0023's widen-if-TEXT check for both payload columns and both indexes —
|
||||
-- each guarded step probes its own drift independently and does nothing when
|
||||
-- it is absent, so a healthy workspace through either door (fresh init,
|
||||
-- fresh clone) runs every probe below and issues no DDL. See 0023's header
|
||||
-- for why that independence matters (crash-replay of an interrupted pass)
|
||||
-- and for the no-DROP, no-rename invariant this file also upholds.
|
||||
--
|
||||
-- It never reads or writes bd_events_seq or a single journal row: the
|
||||
-- counter and every row must be byte-identical before and after
|
||||
-- (TestIgnored0028ConvergesAllShapes).
|
||||
--
|
||||
-- Two of these columns are registered as sentinels in schema.go's
|
||||
-- ignoredSource (`{bd_events_journal, comment_json, replayFloor: 21}` and
|
||||
-- `{bd_events_journal, actor, replayFloor: 21}`): a store whose cursor already
|
||||
-- reads past 28 but is missing either column — e.g. restored from a backup
|
||||
-- taken between the two doors, or damaged out of band — heals itself on the
|
||||
-- next writable open instead of staying silently broken forever
|
||||
-- (TestSentinelJournalColumnsReplay). The actor sentinel alone also fires on
|
||||
-- every store sitting at ignored cursor 22-27 without it, which is every
|
||||
-- production Dolt store measured at cursor 24 (BEADS-JOURNAL-PLAN.md §4.1):
|
||||
-- those stores replay 22-28 once, on their first open by a binary carrying
|
||||
-- this sentinel. Floor 21 sits below this file (28) and below the table's
|
||||
-- creator (ignored/0022), per schema.go's two replay-floor constraints, and
|
||||
-- the only DML above it is 0022's INSERT IGNORE seed and GREATEST raise
|
||||
-- (TestJournalSentinelReplayRangeDMLIsAllowlisted;
|
||||
-- TestReplay22To28NoOpOnHealthyStore pins that nothing else in the range
|
||||
-- touches any other table or row).
|
||||
--
|
||||
-- Journal tables change only on this (ignored) plane: scripts/check-
|
||||
-- migration-hygiene.sh fails a main-plane migration that carries DDL against
|
||||
-- any bd_events_* table, unconditionally, so this file — not a main-plane
|
||||
-- twin — is the only place bd_events_journal's shape can converge.
|
||||
|
||||
-- comment_json: add as LONGTEXT if missing; widen if TEXT. An absent TABLE
|
||||
-- takes the no-op branch explicitly (a COLUMNS probe alone cannot tell "no
|
||||
-- column" from "no table"), so this step never depends on 0022 having run
|
||||
-- earlier in the same pass.
|
||||
SET @journal_exists = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal');
|
||||
SET @comment_json_type = (
|
||||
SELECT DATA_TYPE
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'bd_events_journal'
|
||||
AND COLUMN_NAME = 'comment_json'
|
||||
);
|
||||
SET @sql = IF(@journal_exists = 0, 'SELECT 1',
|
||||
IF(@comment_json_type IS NULL,
|
||||
'ALTER TABLE bd_events_journal ADD COLUMN comment_json LONGTEXT',
|
||||
IF(@comment_json_type = 'text',
|
||||
'ALTER TABLE bd_events_journal MODIFY COLUMN comment_json LONGTEXT',
|
||||
'SELECT 1')));
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- actor: add VARCHAR(255) NOT NULL DEFAULT '' if missing. ignored/0025
|
||||
-- already does exactly this on the lineage that carries it; this is a second,
|
||||
-- idempotent guard — identical definition, explicit table-exists probe as in
|
||||
-- 0023/0025 — for any lineage that reaches 0028 without having run 0025 (the
|
||||
-- sentinel above is what makes that heal rather than strand).
|
||||
SET @needs_actor = IF(
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0
|
||||
AND
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'bd_events_journal'
|
||||
AND COLUMN_NAME = 'actor') = 0,
|
||||
1, 0
|
||||
);
|
||||
SET @sql = IF(@needs_actor = 1,
|
||||
'ALTER TABLE bd_events_journal ADD COLUMN actor VARCHAR(255) NOT NULL DEFAULT ''''',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- dep_json: widen if TEXT. 0023's own step, repeated here (not removed there:
|
||||
-- shipped migrations are frozen, check C) for any lineage that reaches 0028
|
||||
-- without ever having run it. Every creator of this table this repo or its
|
||||
-- fork lineage has ever shipped declares dep_json with SOME type, so a
|
||||
-- missing table yields NULL here too, which takes the no-op branch (the
|
||||
-- comment_json step above makes the same table-absence case explicit).
|
||||
SET @dep_json_is_text = (
|
||||
SELECT IF(DATA_TYPE = 'text', 1, 0)
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'bd_events_journal'
|
||||
AND COLUMN_NAME = 'dep_json'
|
||||
);
|
||||
SET @sql = IF(@dep_json_is_text = 1,
|
||||
'ALTER TABLE bd_events_journal MODIFY COLUMN dep_json LONGTEXT',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- issue_json: widen if TEXT, same reasoning as dep_json. No observed shape
|
||||
-- (gci, 3666e5026, the enterprise fork) has ever shipped this column as TEXT,
|
||||
-- so this is a no-op everywhere it has been checked; included for the same
|
||||
-- completeness reason as idx_bd_events_journal_issue below.
|
||||
SET @issue_json_is_text = (
|
||||
SELECT IF(DATA_TYPE = 'text', 1, 0)
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'bd_events_journal'
|
||||
AND COLUMN_NAME = 'issue_json'
|
||||
);
|
||||
SET @sql = IF(@issue_json_is_text = 1,
|
||||
'ALTER TABLE bd_events_journal MODIFY COLUMN issue_json LONGTEXT',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- idx_bd_events_journal_ts: create if missing. 0023's own step, repeated for
|
||||
-- the same reason as actor above.
|
||||
SET @needs_ts_index = IF(
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0
|
||||
AND
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'bd_events_journal'
|
||||
AND INDEX_NAME = 'idx_bd_events_journal_ts') = 0,
|
||||
1, 0
|
||||
);
|
||||
SET @sql = IF(@needs_ts_index = 1,
|
||||
'CREATE INDEX idx_bd_events_journal_ts ON bd_events_journal(ts)',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- idx_bd_events_journal_issue: create if missing. Every observed shape
|
||||
-- (gci, 3666e5026, the enterprise fork) already carries this index — it is
|
||||
-- declared inline in every CREATE this repo or its fork lineage has ever
|
||||
-- shipped — but an unconditional guard costs one more INFORMATION_SCHEMA
|
||||
-- round trip and closes the same class of gap 0023 left open for
|
||||
-- idx_bd_events_journal_ts, against a shape nobody has observed yet.
|
||||
SET @needs_issue_index = IF(
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'bd_events_journal') > 0
|
||||
AND
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'bd_events_journal'
|
||||
AND INDEX_NAME = 'idx_bd_events_journal_issue') = 0,
|
||||
1, 0
|
||||
);
|
||||
SET @sql = IF(@needs_issue_index = 1,
|
||||
'CREATE INDEX idx_bd_events_journal_issue ON bd_events_journal(issue_id)',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
124
internal/storage/schema/old_ignored_ceiling_test.go
Normal file
124
internal/storage/schema/old_ignored_ceiling_test.go
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
package schema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
sqlmock "github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
// oldIgnoredCeiling24Files stands in for the ignored-migration set a binary
|
||||
// built before BEADS-JOURNAL-PLAN.md's PR A2 would have embedded: 0025
|
||||
// (actor), 0026 (dep rekey marker), 0027 (wisps.current_revision) and 0028
|
||||
// (this plan's journal-shape convergence) all postdate it, so its ceiling is
|
||||
// 24 — the d3ab CLI's measured ignored ceiling in the plan's evidence base
|
||||
// (§4.1). Only the filename's version prefix is ever read
|
||||
// (migrationSource.list/.latest), so one placeholder file is enough; it need
|
||||
// not match the real ignored/0024 migration's SQL.
|
||||
//
|
||||
//go:embed testdata/old_ignored_ceiling_24/*.up.sql
|
||||
var oldIgnoredCeiling24Files embed.FS
|
||||
|
||||
// TestOldIgnoredCeilingOpensAfter0028 is T2.9 (BEADS-JOURNAL-PLAN.md §4.3, PR
|
||||
// A2): a binary that only knows the ignored series through 24 opens a store
|
||||
// whose ignored cursor is already at 28 without refusing, because nothing on
|
||||
// the open path compares a binary's ignored ceiling against the stored
|
||||
// cursor — CurrentIgnoredVersion/LatestIgnoredVersion feed no skew guard the
|
||||
// way CurrentVersion/LatestVersion do for the main plane (checkSchemaSkew).
|
||||
//
|
||||
// Kills:
|
||||
// - adding an ignored-plane skew check: oldIgnored (ceiling 24) reporting
|
||||
// atLatest()==true and migrate() applying nothing against a mocked cursor
|
||||
// of 28 pins that an old-ceiling source does not refuse or try to "catch
|
||||
// up" just because the stored cursor is ahead of what it knows; and the
|
||||
// checkSchemaSkew/CheckForwardDrift/CheckBehindDrift sub-tests below use a
|
||||
// strict sqlmock (ExpectationsWereMet) that would fail if any of them
|
||||
// issued a query against ignored_schema_migrations at all.
|
||||
// - adding a main-plane twin for the journal migrations: that would move
|
||||
// mainSource's own cursor, which the same checkSchemaSkew mock (driven
|
||||
// only by schema_migrations, unaffected by the ignored plane) would catch
|
||||
// as a changed main version the moment the twin's migration shipped.
|
||||
func TestOldIgnoredCeilingOpensAfter0028(t *testing.T) {
|
||||
oldIgnored := migrationSource{
|
||||
files: oldIgnoredCeiling24Files,
|
||||
dir: "testdata/old_ignored_ceiling_24",
|
||||
cursorTable: ignoredSource.cursorTable,
|
||||
}
|
||||
if got := oldIgnored.latest(); got != 24 {
|
||||
t.Fatalf("oldIgnored.latest() = %d, want 24", got)
|
||||
}
|
||||
|
||||
t.Run("old-ceiling source sees the store as at-latest, not behind", func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
expectCursorProbe(mock, "ignored_schema_migrations", true)
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", 28)
|
||||
|
||||
if !oldIgnored.atLatest(context.Background(), db) {
|
||||
t.Error("oldIgnored.atLatest() = false for a cursor (28) past its own ceiling (24); an old binary must not see this store as behind")
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet sql expectations: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("old-ceiling source applies nothing against a cursor past its ceiling", func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
mock.ExpectExec(regexp.QuoteMeta("CREATE TABLE IF NOT EXISTS ignored_schema_migrations")).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SHOW COLUMNS FROM ignored_schema_migrations LIKE 'content_hash'")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}).
|
||||
AddRow("content_hash", "char(64)", "YES", "", nil, ""))
|
||||
expectCursorProbe(mock, "ignored_schema_migrations", true)
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", 28)
|
||||
// No sentinelTables/sentinelColumns on oldIgnored (zero value), so
|
||||
// cursorRealityFloor has nothing further to probe, and no
|
||||
// ExecContext should run at all: migrate must see current(28) >=
|
||||
// target(24) and return immediately.
|
||||
|
||||
applied, _, err := oldIgnored.migrate(context.Background(), db, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("oldIgnored.migrate: %v", err)
|
||||
}
|
||||
if applied != 0 {
|
||||
t.Errorf("oldIgnored.migrate applied = %d, want 0 (a cursor past this source's ceiling has nothing pending)", applied)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet sql expectations (an old-ceiling binary must not touch the database at all here): %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the real open-path guards never consult the ignored cursor", func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Main plane only: at the binary's own latest, unaffected by the
|
||||
// ignored cursor being 28. If checkSchemaSkew/CheckForwardDrift ever
|
||||
// queried ignored_schema_migrations, the strict mock below (no such
|
||||
// expectation registered) would fail the call.
|
||||
expectCursorProbe(mock, "schema_migrations", true)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(version), 0) FROM schema_migrations")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(LatestVersion()))
|
||||
|
||||
if err := CheckForwardDrift(context.Background(), db); err != nil {
|
||||
t.Errorf("CheckForwardDrift = %v, want nil (ignored=28 must not drive a main-plane forward-drift refusal)", err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet sql expectations: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -363,7 +363,9 @@ type schemaSentinelColumn struct {
|
|||
// restamping every wisp on a plane with no history to restore from
|
||||
// (#5981 harm class). ignored/0015 is the guarded successor and is
|
||||
// genuinely pending on those stores, so the recompute still happens
|
||||
// — just without the timestamp damage.
|
||||
// — just without the timestamp damage. Statements above the floor
|
||||
// must be replay-safe, not merely guarded; see
|
||||
// TestJournalSentinelReplayRangeDMLIsAllowlisted.
|
||||
// (ii) The migration that creates the column AND the migration that
|
||||
// creates the table carrying it must both be ABOVE the floor, so the
|
||||
// replay can still heal both shapes the single INFORMATION_SCHEMA
|
||||
|
|
@ -405,7 +407,31 @@ var (
|
|||
// 0001–0025 series replays, dragging in ignored/0007 and restamping
|
||||
// wisps.updated_at across the plane. Believing the cursor up to 11
|
||||
// applies exactly the pending tail 0012–0025 instead.
|
||||
sentinelColumns: []schemaSentinelColumn{{table: "leases", column: "granted_node", replayFloor: 11}},
|
||||
//
|
||||
// bd_events_journal carries two more, registered together: a store
|
||||
// whose cursor already reads past 28 but is missing comment_json or
|
||||
// actor (restored from a backup taken between doors, or damaged out
|
||||
// of band) heals itself on the next writable open instead of staying
|
||||
// silently broken forever (BEADS-JOURNAL-PLAN.md §4.2b, PR A2). Floor
|
||||
// 21 satisfies the same two constraints as the leases floor above:
|
||||
// 0022 (the table's creator) and 0028 (comment_json's adder; actor's
|
||||
// adder is 0025, also above 21) both sit strictly above it;
|
||||
// ignored/0007's unguarded restamp sits below it, and the only DML
|
||||
// above it is 0022's INSERT IGNORE seed and GREATEST raise
|
||||
// (TestJournalSentinelReplayRangeDMLIsAllowlisted).
|
||||
//
|
||||
// The actor sentinel alone also fires on every store sitting at
|
||||
// ignored cursor 22-27 that predates 0025, which is every production
|
||||
// Dolt store measured at cursor 24 (BEADS-JOURNAL-PLAN.md §4.1):
|
||||
// those stores replay 22-28 once, on their first open by a binary
|
||||
// carrying this sentinel. 0022's tail is the healEventSeqCounter
|
||||
// statement: a no-op on a counter already at or ahead of the
|
||||
// journal, and a raise to exactly MAX(seq) on one that is behind.
|
||||
sentinelColumns: []schemaSentinelColumn{
|
||||
{table: "leases", column: "granted_node", replayFloor: 11},
|
||||
{table: "bd_events_journal", column: "comment_json", replayFloor: 21},
|
||||
{table: "bd_events_journal", column: "actor", replayFloor: 21},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
8
internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql
vendored
Normal file
8
internal/storage/schema/testdata/old_ignored_ceiling_24/0024_placeholder.up.sql
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
-- Placeholder for TestOldIgnoredCeilingOpensAfter0028 (T2.9,
|
||||
-- BEADS-JOURNAL-PLAN.md PR A2): stands in for the newest ignored migration a
|
||||
-- binary built before this plan's 0025-0028 would have embedded (the d3ab
|
||||
-- CLI's measured ignored ceiling in the plan's evidence base, §4.1). Its
|
||||
-- content is never executed by the test -- migrationSource.latest() and
|
||||
-- .list() only parse the filename for a version number -- so this file need
|
||||
-- not match the real ignored/0024 migration's SQL.
|
||||
SELECT 1;
|
||||
|
|
@ -915,6 +915,32 @@ type EventsJournalConfigurer interface {
|
|||
SetEventsJournalEnabled(enabled bool)
|
||||
}
|
||||
|
||||
// EventsJournalShapeChecker reports whether the LAST SetEventsJournalEnabled
|
||||
// call on this instance was able to activate the journal.
|
||||
//
|
||||
// It exists because SetEventsJournalEnabled itself stays void: dozens of
|
||||
// tests across three backends already call it as a bare statement, and
|
||||
// adaptive I/O (PR A1) must not force every one of them to start checking an
|
||||
// error it never had before. Instead, enabling the journal probes the table's
|
||||
// shape internally and caches any failure; this is the seam a factory's open
|
||||
// path checks immediately afterward to decide whether activation actually
|
||||
// succeeded. A missing or unsupported table makes SetEventsJournalEnabled(true)
|
||||
// a no-op for WRITES, from that store's perspective: nothing panics, no
|
||||
// INSERT is attempted, and no write is lost or corrupted — but
|
||||
// eventsjournal.Apply uses this checker to fail the overall open with a typed
|
||||
// error (issueops.JournalShapeError) instead of letting every subsequent
|
||||
// write fail one at a time, which is the gci failure mode this interface
|
||||
// exists to avoid. A caller that enables the journal directly, bypassing
|
||||
// Apply, and then reads it anyway gets a loud SQL error from the read itself
|
||||
// rather than a silently wrong answer — reads are never gated by the enabled
|
||||
// flag, so there is no "no-op" to fall back to there.
|
||||
//
|
||||
// A store that does not implement it is assumed never to fail activation
|
||||
// (today, every implementation does).
|
||||
type EventsJournalShapeChecker interface {
|
||||
EventsJournalActivationError() error
|
||||
}
|
||||
|
||||
// VersionedHistoryConfigurer controls dual-write issue-version history
|
||||
// activation on ONE storage instance. Implementations must never use
|
||||
// process-global state, for the same reason as EventsJournalConfigurer:
|
||||
|
|
|
|||
|
|
@ -57,6 +57,15 @@ type doltSQLProvider struct {
|
|||
// eventsJournalEnabled activates the durable events journal for THIS
|
||||
// provider instance only. See SetEventsJournalEnabled.
|
||||
eventsJournalEnabled atomic.Bool
|
||||
// journalShape caches the ONE INFORMATION_SCHEMA probe SetEventsJournalEnabled
|
||||
// runs on activation (issueops.ProbeJournalShape, PR A1). Nil means either
|
||||
// the journal is disabled or the probe failed — see journalActivationErr.
|
||||
journalShape atomic.Pointer[issueops.JournalShape]
|
||||
// journalActivationErr holds the probe's error when SetEventsJournalEnabled(true)
|
||||
// found a table shape the journal cannot run against (a missing required
|
||||
// column, or a missing table). Nil whenever journalShape is non-nil or the
|
||||
// journal is disabled. See storage.EventsJournalShapeChecker.
|
||||
journalActivationErr atomic.Pointer[error]
|
||||
// versionedHistoryEnabled activates dual-write issue-version history for
|
||||
// THIS provider instance only. See SetVersionedHistoryEnabled.
|
||||
versionedHistoryEnabled atomic.Bool
|
||||
|
|
@ -72,8 +81,38 @@ type doltSQLProvider struct {
|
|||
// BeginTx). Without that binding the uow plumbing writes mutations while
|
||||
// journaling nothing — the failure is invisible, because the code runs and the
|
||||
// write lands and the journal is simply empty.
|
||||
//
|
||||
// Activating also probes bd_events_journal's actual shape ONCE
|
||||
// (issueops.ProbeJournalShape, PR A1) and caches the result for this
|
||||
// instance's lifetime; EventsJournalActivationError reports whether that
|
||||
// probe found a shape the journal can actually run against. Deactivating
|
||||
// clears both and skips the probe entirely — a disabled provider accepts any
|
||||
// shape.
|
||||
func (p *doltSQLProvider) SetEventsJournalEnabled(enabled bool) {
|
||||
p.eventsJournalEnabled.Store(enabled)
|
||||
if !enabled {
|
||||
p.journalShape.Store(nil)
|
||||
p.journalActivationErr.Store(nil)
|
||||
return
|
||||
}
|
||||
shape, err := issueops.ProbeJournalShape(context.Background(), p.db)
|
||||
if err != nil {
|
||||
p.journalShape.Store(nil)
|
||||
p.journalActivationErr.Store(&err)
|
||||
return
|
||||
}
|
||||
p.journalShape.Store(shape)
|
||||
p.journalActivationErr.Store(nil)
|
||||
}
|
||||
|
||||
// EventsJournalActivationError implements storage.EventsJournalShapeChecker:
|
||||
// nil unless the last SetEventsJournalEnabled(true) call found a table shape
|
||||
// the journal cannot run against (PR A1).
|
||||
func (p *doltSQLProvider) EventsJournalActivationError() error {
|
||||
if ptr := p.journalActivationErr.Load(); ptr != nil {
|
||||
return *ptr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetVersionedHistoryEnabled activates dual-write issue-version history for
|
||||
|
|
@ -160,6 +199,7 @@ var (
|
|||
_ UnitOfWorkProvider = (*doltSQLProvider)(nil)
|
||||
_ TxProvider = (*doltSQLProvider)(nil)
|
||||
_ storage.EventsJournalConfigurer = (*doltSQLProvider)(nil)
|
||||
_ storage.EventsJournalShapeChecker = (*doltSQLProvider)(nil)
|
||||
_ storage.VersionedHistoryConfigurer = (*doltSQLProvider)(nil)
|
||||
)
|
||||
|
||||
|
|
@ -195,9 +235,20 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) {
|
|||
// (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its
|
||||
// transaction. The blocked-recheck scope is bound and released the same
|
||||
// way; Commit takes what it recorded once the transaction has committed.
|
||||
// The activation switch folds in the shape probe's success: a probe that
|
||||
// found an unsupported table (journalShape nil) scopes this transaction as
|
||||
// DISABLED regardless of what SetEventsJournalEnabled was called with, so a
|
||||
// caller that enables the journal directly and ignores the returned
|
||||
// EventsJournalActivationError still never attempts a write against a shape
|
||||
// that cannot support it (storage.EventsJournalShapeChecker). Reads share
|
||||
// this same BeginTx, and are never gated by the activation switch — only by
|
||||
// the shape — since EventsJournalCursor has no enabled/disabled concept of
|
||||
// its own.
|
||||
journalShape := p.journalShape.Load()
|
||||
return &doltServerTx{
|
||||
conn: conn,
|
||||
clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()),
|
||||
clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load() && journalShape != nil),
|
||||
clearJournalShape: issueops.ScopeEventsJournalShape(conn, journalShape),
|
||||
clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()),
|
||||
clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn),
|
||||
}, nil
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ type doltServerTx struct {
|
|||
// releaseConn and poisonConn — so the activation entry cannot outlive the
|
||||
// transaction it describes, whichever way the transaction ends.
|
||||
clearJournalScope func()
|
||||
// clearJournalShape releases the probed *issueops.JournalShape BeginTx bound
|
||||
// to conn (PR A1), the shape counterpart of clearJournalScope's enabled/disabled
|
||||
// switch. Same binding at BeginTx, same release at releaseConn/poisonConn.
|
||||
clearJournalShape func()
|
||||
// clearVersionScope is clearJournalScope's counterpart for dual-write
|
||||
// issue-version history: same binding at BeginTx, same release at
|
||||
// releaseConn/poisonConn.
|
||||
|
|
@ -197,6 +201,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error {
|
|||
|
||||
func (t *doltServerTx) releaseConn() {
|
||||
t.releaseJournalScope()
|
||||
t.releaseJournalShapeScope()
|
||||
t.releaseVersionScope()
|
||||
t.releaseRecheckScope()
|
||||
if t.conn != nil {
|
||||
|
|
@ -215,6 +220,15 @@ func (t *doltServerTx) releaseJournalScope() {
|
|||
}
|
||||
}
|
||||
|
||||
// releaseJournalShapeScope is releaseJournalScope's counterpart for the probed
|
||||
// *issueops.JournalShape (PR A1). Idempotent for the same reason.
|
||||
func (t *doltServerTx) releaseJournalShapeScope() {
|
||||
if t.clearJournalShape != nil {
|
||||
t.clearJournalShape()
|
||||
t.clearJournalShape = nil
|
||||
}
|
||||
}
|
||||
|
||||
// releaseVersionScope is releaseJournalScope's counterpart for dual-write
|
||||
// issue-version history. Idempotent for the same reason.
|
||||
func (t *doltServerTx) releaseVersionScope() {
|
||||
|
|
@ -241,6 +255,7 @@ func (t *doltServerTx) releaseRecheckScope() {
|
|||
// database/sql close the connection and drop it from the pool.
|
||||
func (t *doltServerTx) poisonConn() {
|
||||
t.releaseJournalScope()
|
||||
t.releaseJournalShapeScope()
|
||||
t.releaseVersionScope()
|
||||
t.releaseRecheckScope()
|
||||
if t.conn == nil {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,19 @@ import (
|
|||
// commits, and the journal is simply empty — so it must be pinned from both
|
||||
// directions: enabled emits, disabled does not.
|
||||
|
||||
// expectCanonicalJournalShapeProbe arms the one INFORMATION_SCHEMA.COLUMNS
|
||||
// query SetEventsJournalEnabled(true) now issues to probe bd_events_journal's
|
||||
// shape (issueops.ProbeJournalShape, PR A1), returning every column a
|
||||
// canonical table carries so activation succeeds with the shape every test in
|
||||
// this file was written against before shape probing existed.
|
||||
func expectCanonicalJournalShapeProbe(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS").
|
||||
WithArgs("bd_events_journal").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"COLUMN_NAME"}).
|
||||
AddRow("seq").AddRow("ts").AddRow("op").AddRow("issue_id").
|
||||
AddRow("actor").AddRow("issue_json").AddRow("dep_json").AddRow("comment_json"))
|
||||
}
|
||||
|
||||
func TestProviderImplementsEventsJournalConfigurer(t *testing.T) {
|
||||
p, _ := newMockTxProvider(t)
|
||||
var configurer storage.EventsJournalConfigurer = p
|
||||
|
|
@ -37,7 +50,10 @@ func TestProviderImplementsEventsJournalConfigurer(t *testing.T) {
|
|||
// internal bookkeeping.
|
||||
func TestBeginTxScopesJournalActivationToThePinnedConn(t *testing.T) {
|
||||
p, mock := newMockTxProvider(t)
|
||||
expectCanonicalJournalShapeProbe(mock)
|
||||
p.SetEventsJournalEnabled(true)
|
||||
require.NoError(t, p.EventsJournalActivationError(),
|
||||
"activation against the canonical shape the probe expectation below describes must succeed")
|
||||
|
||||
mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec("UPDATE bd_events_seq SET next_seq").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
|
|
|||
|
|
@ -352,6 +352,18 @@ func (p *notifyingProvider) SetEventsJournalEnabled(enabled bool) {
|
|||
}
|
||||
}
|
||||
|
||||
// EventsJournalActivationError forwards the inner provider's shape-probe
|
||||
// result (storage.EventsJournalShapeChecker, PR A1), for the same reason
|
||||
// SetEventsJournalEnabled forwards activation itself: the wrapper holds no
|
||||
// journal state of its own, and eventsjournal.Apply must see the inner
|
||||
// provider's real probe result through whichever wrapper sits on top of it.
|
||||
func (p *notifyingProvider) EventsJournalActivationError() error {
|
||||
if checker, ok := p.inner.(storage.EventsJournalShapeChecker); ok {
|
||||
return checker.EventsJournalActivationError()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetVersionedHistoryEnabled forwards dual-write issue-version history
|
||||
// activation to the provider that actually binds it to a transaction
|
||||
// (doltSQLProvider.BeginTx), for the same reason and in the same shape as
|
||||
|
|
|
|||
|
|
@ -299,9 +299,13 @@ func TestNotifyingProviderBuildsRolesOnItself(t *testing.T) {
|
|||
// for both binds on the inner provider's transactions, and a retention
|
||||
// pass writes no bead. None of these build a role, so none of them can
|
||||
// build one "on itself".
|
||||
"SetEventsJournalEnabled": true,
|
||||
"RunEventsMaintenanceTx": true,
|
||||
"SetVersionedHistoryEnabled": true,
|
||||
"SetEventsJournalEnabled": true,
|
||||
// EventsJournalActivationError (PR A1) forwards the inner provider's
|
||||
// shape-probe result the same way SetEventsJournalEnabled forwards
|
||||
// activation: a query, not a role constructor.
|
||||
"EventsJournalActivationError": true,
|
||||
"RunEventsMaintenanceTx": true,
|
||||
"SetVersionedHistoryEnabled": true,
|
||||
}
|
||||
|
||||
var checked int
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ go_test(
|
|||
"bazel_skip_allowlist_test.go",
|
||||
"check_build_tags_test.go",
|
||||
"check_doc_freshness_test.go",
|
||||
"check_migration_hygiene_journal_plane_test.go",
|
||||
"check_release_cross_compile_test.go",
|
||||
"check_testing_short_test.go",
|
||||
"check_winget_portable_alias_test.go",
|
||||
|
|
|
|||
|
|
@ -86,7 +86,23 @@
|
|||
# point of the check is that the idiom stops spreading into migrations
|
||||
# where the write does matter.
|
||||
#
|
||||
# Checks C, D, and E compare against $BASE_SHA if set (CI passes the PR base),
|
||||
# F. A new main-plane migration must not contain DDL (CREATE/ALTER/RENAME/
|
||||
# DROP TABLE) against a journal table (`bd_events_journal`,
|
||||
# `bd_events_seq`). Unlike check D, there is no twin exemption: journal
|
||||
# tables change only on the ignored plane (Dolt) or in Postgres DDL
|
||||
# (BEADS-JOURNAL-PLAN.md §4.2c, PR A2) — an ignored-series twin does not
|
||||
# excuse a main-plane DDL statement here the way it does for the other
|
||||
# clone-local tables check D covers, because a main-plane change to
|
||||
# these specific tables bumps the skew-checked main cursor (schema.go's
|
||||
# forward-drift guard) for a table whose whole point is to stay
|
||||
# invisible to binaries that predate it. BEADS-JOURNAL-PLAN.md names
|
||||
# this rule "hygiene check E" and its test `TestHygieneCheckE`; it is
|
||||
# implemented here as check F and the test is named for what it checks
|
||||
# rather than for the plan's letter, because this repo's own check E
|
||||
# already means something else (no PREPARE'd DML, above) and reusing the
|
||||
# letter would misdescribe both.
|
||||
#
|
||||
# Checks C, D, E, and F compare against $BASE_SHA if set (CI passes the PR base),
|
||||
# else origin/main, else main; they are skipped with a warning when no base
|
||||
# is resolvable (e.g. shallow clone without the base commit).
|
||||
|
||||
|
|
@ -439,6 +455,37 @@ EOF
|
|||
done
|
||||
fi
|
||||
|
||||
# --- Check F: no main-plane DDL on journal tables ---------------------------
|
||||
# bd_events_journal/bd_events_seq change only on the ignored plane (Dolt) or
|
||||
# in Postgres DDL (BEADS-JOURNAL-PLAN.md §4.2c, PR A2). Scoped to added_main
|
||||
# the same as checks D and E; unlike D, there is no twin exemption here —
|
||||
# see this check's doc comment at the top of the file for why.
|
||||
if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then
|
||||
echo "WARN (journal plane) no usable base ref; skipping check F." >&2
|
||||
else
|
||||
journal_tables='bd_events_journal|bd_events_seq'
|
||||
ddl_re_f="(alter|create|rename|drop)[[:space:]]+table([[:space:]]+if[[:space:]]+(not[[:space:]]+)?exists)?[[:space:]]+[\`']?(${journal_tables})\b"
|
||||
for f in $added_main; do
|
||||
[ -f "$f" ] || continue
|
||||
stripped=$(sed 's/--.*$//' "$f" | tr '[:upper:]' '[:lower:]')
|
||||
hits=$(grep -n -E "$ddl_re_f" <<<"$stripped" || true)
|
||||
if [ -n "$hits" ]; then
|
||||
fail=1
|
||||
echo "FAIL (journal plane) $f contains main-plane DDL on a journal table:"
|
||||
echo "$hits" | sed 's/^/ line /'
|
||||
cat <<'EOF'
|
||||
bd_events_journal and bd_events_seq change only on the ignored plane: a
|
||||
main-plane migration touching either one bumps the skew-checked main
|
||||
cursor for a table whose whole point is to stay invisible to binaries that
|
||||
predate it (BEADS-JOURNAL-PLAN.md §4.2c). Move this DDL to a new file under
|
||||
internal/storage/schema/migrations/ignored/ instead; there is no twin
|
||||
exemption for these two tables the way check D allows for other
|
||||
clone-local tables.
|
||||
EOF
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo
|
||||
echo "Migration hygiene check failed. See docs in scripts/check-migration-hygiene.sh."
|
||||
|
|
|
|||
139
scripts/check_migration_hygiene_journal_plane_test.go
Normal file
139
scripts/check_migration_hygiene_journal_plane_test.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package scripts_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This file is T2.8 (BEADS-JOURNAL-PLAN.md §4.3, PR A2): a main-plane
|
||||
// migration with DDL on bd_events_* fails check-migration-hygiene.sh. The
|
||||
// plan names the rule "hygiene check E" and the test TestHygieneCheckE, but
|
||||
// this repo's own check E already means something else (no PREPARE'd DML in
|
||||
// new main-plane migrations — see that check's doc comment in
|
||||
// check-migration-hygiene.sh). The rule is implemented here as a new check F
|
||||
// instead, and these tests are named for what they check rather than for the
|
||||
// plan's letter, to avoid misdescribing either check.
|
||||
|
||||
// runCheckMigrationHygiene runs check-migration-hygiene.sh in a throwaway git
|
||||
// repository: baseFiles are committed on main (standing in for the PR's
|
||||
// base); newFiles are then added to the working tree uncommitted (standing in
|
||||
// for the PR's diff — checks C, D, E and F all diff the working tree against
|
||||
// this base commit).
|
||||
func runCheckMigrationHygiene(t *testing.T, baseFiles, newFiles map[string]string) (string, error) {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("checker is a Bash boundary")
|
||||
}
|
||||
requireHostTool(t, "git")
|
||||
requireHostTool(t, "bash")
|
||||
script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "check-migration-hygiene.sh"))
|
||||
if err != nil {
|
||||
t.Fatalf("read check-migration-hygiene.sh: %v", err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
write := func(files map[string]string) {
|
||||
for rel, content := range files {
|
||||
full := filepath.Join(dir, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", filepath.Dir(full), err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("write %s: %v", rel, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
all := map[string]string{"scripts/check-migration-hygiene.sh": string(script)}
|
||||
for rel, content := range baseFiles {
|
||||
all[rel] = content
|
||||
}
|
||||
write(all)
|
||||
env := append(os.Environ(),
|
||||
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1",
|
||||
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
|
||||
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
|
||||
)
|
||||
for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"add", "-A"}, {"commit", "-q", "-m", "base"}} {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
}
|
||||
write(newFiles)
|
||||
cmd := exec.Command("bash", filepath.Join(dir, "scripts", "check-migration-hygiene.sh"))
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// TestHygieneCheckFRejectsMainPlaneJournalDDL is T2.8: a new main-plane
|
||||
// migration altering bd_events_journal fails the script even though it ships
|
||||
// an ignored-series twin in the same change — proving this is check F (no
|
||||
// twin exemption, ever) and not merely check D (twin required) passing
|
||||
// through by accident. Kills: removing check F, or weakening it to accept a
|
||||
// twin the way check D does.
|
||||
func TestHygieneCheckFRejectsMainPlaneJournalDDL(t *testing.T) {
|
||||
out, err := runCheckMigrationHygiene(t, nil, map[string]string{
|
||||
"internal/storage/schema/migrations/0100_add_journal_widget.up.sql": "ALTER TABLE bd_events_journal ADD COLUMN widget VARCHAR(32);\n",
|
||||
"internal/storage/schema/migrations/ignored/0100_add_journal_widget.up.sql": "-- twin: not what check F looks at; its presence must not save the main-plane file.\nSELECT 1;\n",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("main-plane DDL on bd_events_journal was accepted even with an ignored twin:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "FAIL (journal plane)") {
|
||||
t.Errorf("expected a \"FAIL (journal plane)\" report, got:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "0100_add_journal_widget.up.sql") {
|
||||
t.Errorf("expected the offending file named in output, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHygieneCheckFRejectsMainPlaneEventsSeqDDL pins that the rule covers
|
||||
// bd_events_seq too, not only bd_events_journal.
|
||||
func TestHygieneCheckFRejectsMainPlaneEventsSeqDDL(t *testing.T) {
|
||||
out, err := runCheckMigrationHygiene(t, nil, map[string]string{
|
||||
"internal/storage/schema/migrations/0100_rename_events_seq.up.sql": "ALTER TABLE bd_events_seq ADD COLUMN note VARCHAR(8);\n",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("main-plane DDL on bd_events_seq was accepted:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "FAIL (journal plane)") {
|
||||
t.Errorf("expected a \"FAIL (journal plane)\" report, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHygieneCheckFAllowsMainPlaneDDLOnNonJournalTables is the negative
|
||||
// control: a main-plane migration that never mentions a journal table passes
|
||||
// cleanly, so check F is not a blanket rejection of main-plane DDL.
|
||||
func TestHygieneCheckFAllowsMainPlaneDDLOnNonJournalTables(t *testing.T) {
|
||||
out, err := runCheckMigrationHygiene(t, nil, map[string]string{
|
||||
"internal/storage/schema/migrations/0100_add_issues_widget.up.sql": "ALTER TABLE issues ADD COLUMN widget VARCHAR(32);\n",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("main-plane DDL on a non-journal table was rejected: %v\n%s", err, out)
|
||||
}
|
||||
if strings.Contains(out, "FAIL (journal plane)") {
|
||||
t.Errorf("did not expect a journal-plane failure, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHygieneCheckFIgnoresIgnoredPlaneJournalDDL confirms the rule is
|
||||
// main-plane-only: the same DDL shipped as an ignored-series migration (the
|
||||
// only place journal tables are allowed to change, PR A2) passes.
|
||||
func TestHygieneCheckFIgnoresIgnoredPlaneJournalDDL(t *testing.T) {
|
||||
out, err := runCheckMigrationHygiene(t, nil, map[string]string{
|
||||
"internal/storage/schema/migrations/ignored/0100_add_journal_widget.up.sql": "ALTER TABLE bd_events_journal ADD COLUMN widget VARCHAR(32);\n",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ignored-plane DDL on bd_events_journal was rejected: %v\n%s", err, out)
|
||||
}
|
||||
if strings.Contains(out, "FAIL (journal plane)") {
|
||||
t.Errorf("did not expect a journal-plane failure for an ignored-series migration, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue