fix(schema): crash mid-aux-rekey no longer strands rows behind the 0009 marker (bd-578h9.16)

A pass that died mid-rekey had already advanced the main cursor past
the bd-578h9.4 watershed, so the next MigrateUp read the lineage as a
converged fresh clone: it skipped the rewrite, recorded marker 0009
over partially re-keyed rows, and the rekey never completed - the
un-rekeyed rows kept their per-clone-random 0037 ids and the merge
duplicates the pass exists to fix came back.

The rekey now records a clone-local in-progress sentinel in
local_metadata (dolt-ignored; created by main migration 0029, which is
always applied before the rekey runs) before its first UPDATE and
clears it only after the last table completes. A pending sentinel
overrides the fresh-clone skip, so the next pass resumes the idempotent
rewrite. MigrateUp also treats the aux tables as migration-owned when
resuming: their partial-rekey dirt is exempted from the
changed-signature guard (the resume is about to change them) and staged
into the pass's commit instead of being left to the autocommit net.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Steve Yegge 2026-06-11 18:22:50 -04:00
commit 81b3308c30
4 changed files with 187 additions and 2 deletions

View file

@ -27,6 +27,62 @@ const auxRowRekeyMarkerVersion = 9
// (bd-578h9.4). The marker is still recorded.
const auxRowRekeyShippedMainVersion = 51
// auxRowRekeyInProgressKey is the clone-local sentinel (local_metadata,
// dolt-ignored) set before the re-key's first UPDATE and cleared after the
// last table completes. A crashed pass leaves it set, and the next MigrateUp
// resumes the re-key even though the main cursor already advanced past
// auxRowRekeyShippedMainVersion in the crashed pass — without it the
// fresh-clone skip above reads the crashed lineage as already converged and
// strands the partially re-keyed rows forever (bd-578h9.16).
const auxRowRekeyInProgressKey = "aux_row_rekey_in_progress"
// auxRekeyResumePending reports whether a previous re-key pass on this clone
// recorded the in-progress sentinel and never cleared it. A missing
// local_metadata table means no sentinel: the table is dolt-ignored and
// therefore clone-local, so a fresh clone lacks it until something recreates
// it — setAuxRekeyInProgress creates it on demand.
func auxRekeyResumePending(ctx context.Context, db DBConn) (bool, error) {
var tableCount int
if err := db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'local_metadata'`,
).Scan(&tableCount); err != nil {
return false, err
}
if tableCount == 0 {
return false, nil
}
var n int
if err := db.QueryRowContext(ctx,
"SELECT COUNT(*) FROM local_metadata WHERE `key` = ?",
auxRowRekeyInProgressKey).Scan(&n); err != nil {
return false, err
}
return n > 0, nil
}
func setAuxRekeyInProgress(ctx context.Context, db DBConn) error {
// local_metadata is dolt-ignored, hence clone-local: a fresh clone whose
// main cursor is already past 0029 does not have the table (the migration
// will not re-run) until EnsureIgnoredTables recreates it. Create it here
// with 0029's DDL — its dolt_ignore pattern is committed history, so the
// sentinel stays clone-local.
if _, err := db.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS local_metadata (`key` VARCHAR(255) PRIMARY KEY, value TEXT NOT NULL DEFAULT '')"); err != nil {
return fmt.Errorf("ensuring local_metadata: %w", err)
}
_, err := db.ExecContext(ctx,
"REPLACE INTO local_metadata (`key`, value) VALUES (?, '1')",
auxRowRekeyInProgressKey)
return err
}
func clearAuxRekeyInProgress(ctx context.Context, db DBConn) error {
_, err := db.ExecContext(ctx,
"DELETE FROM local_metadata WHERE `key` = ?",
auxRowRekeyInProgressKey)
return err
}
// auxRekeyTable describes one table covered by the re-key. columns is the
// frozen SELECT list of every non-id column, in creation order, with datetime
// columns CAST to CHAR server-side so the scanned text is identical across
@ -98,10 +154,25 @@ func rekeyAuxRowIDs(ctx context.Context, db DBConn, mainVersionBefore int) (bool
if !markerPending {
return false, nil
}
if mainVersionBefore >= auxRowRekeyShippedMainVersion {
resume, err := auxRekeyResumePending(ctx, db)
if err != nil {
return false, fmt.Errorf("reading aux rekey sentinel: %w", err)
}
// The fresh-clone skip must not fire on a lineage whose previous pass
// crashed mid-rekey: that pass already advanced the main cursor past the
// shipped version, but its sentinel proves the rewrite never finished
// (bd-578h9.16).
if mainVersionBefore >= auxRowRekeyShippedMainVersion && !resume {
return false, nil
}
// Sentinel before the first UPDATE: a crash anywhere in the rewrite
// leaves it set, so the next pass resumes (the rewrite is idempotent)
// instead of recording the marker over partially re-keyed rows.
if err := setAuxRekeyInProgress(ctx, db); err != nil {
return false, fmt.Errorf("recording aux rekey sentinel: %w", err)
}
wrote := false
for _, t := range auxRekeyTables {
w, err := rekeyAuxRowTable(ctx, db, t)
@ -110,6 +181,9 @@ func rekeyAuxRowIDs(ctx context.Context, db DBConn, mainVersionBefore int) (bool
return wrote, fmt.Errorf("%s: %w", t.name, err)
}
}
if err := clearAuxRekeyInProgress(ctx, db); err != nil {
return wrote, fmt.Errorf("clearing aux rekey sentinel: %w", err)
}
return wrote, nil
}

View file

@ -3,6 +3,7 @@ package schema
import (
"context"
"database/sql"
"errors"
"regexp"
"testing"
@ -225,11 +226,14 @@ func TestRekeyAuxRowIDsRunsAllTablesWhenMarkerPending(t *testing.T) {
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
"version", auxRowRekeyMarkerVersion-1)
expectAuxRekeySentinel(mock, false)
expectSetAuxRekeySentinel(mock)
// Each of the four tables is probed; this mocked world has none of them,
// so each probe returns 0 and the loop completes without scanning.
for range auxRekeyTables {
expectColumnExists(mock, false)
}
expectClearAuxRekeySentinel(mock)
wrote, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion-1)
if err != nil {
@ -243,6 +247,36 @@ func TestRekeyAuxRowIDsRunsAllTablesWhenMarkerPending(t *testing.T) {
}
}
// expectAuxRekeySentinel mocks auxRekeyResumePending: the local_metadata
// table-existence probe, then (when the table exists) the sentinel-row count.
func expectAuxRekeySentinel(mock sqlmock.Sqlmock, pending bool) {
mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
n := 0
if pending {
n = 1
}
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM local_metadata WHERE `key` = ?")).
WithArgs(auxRowRekeyInProgressKey).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(n))
}
func expectSetAuxRekeySentinel(mock sqlmock.Sqlmock) {
// Fresh clones lack the dolt-ignored local_metadata table, so the set
// path ensures it exists first.
mock.ExpectExec(`CREATE TABLE IF NOT EXISTS local_metadata`).
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec(regexp.QuoteMeta("REPLACE INTO local_metadata (`key`, value) VALUES (?, '1')")).
WithArgs(auxRowRekeyInProgressKey).
WillReturnResult(sqlmock.NewResult(0, 1))
}
func expectClearAuxRekeySentinel(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta("DELETE FROM local_metadata WHERE `key` = ?")).
WithArgs(auxRowRekeyInProgressKey).
WillReturnResult(sqlmock.NewResult(0, 1))
}
// TestRekeyAuxRowIDsSkipsConvergedLineage verifies the bd-578h9.4 gate: when
// the main cursor already reached the version that shipped with the re-key
// BEFORE this pass ran, the lineage was migrated by a rekey-aware binary and
@ -257,8 +291,10 @@ func TestRekeyAuxRowIDsSkipsConvergedLineage(t *testing.T) {
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
"version", auxRowRekeyMarkerVersion-1)
expectAuxRekeySentinel(mock, false)
// No further expectations: marker is pending, but the pre-pass main
// cursor at the watershed means no table may be probed or scanned.
// cursor at the watershed and no crash sentinel means no table may be
// probed or scanned.
wrote, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion)
if err != nil {
@ -271,3 +307,61 @@ func TestRekeyAuxRowIDsSkipsConvergedLineage(t *testing.T) {
t.Fatalf("unmet sql expectations: %v", err)
}
}
// TestRekeyAuxRowIDsResumesAfterCrash covers bd-578h9.16: a pass that died
// mid-rekey already advanced the main cursor past the watershed, so without
// the sentinel the next pass would read the lineage as a converged fresh
// clone, record the marker, and strand the partially re-keyed rows forever.
// A pending sentinel must override the fresh-clone skip and resume the
// rewrite.
func TestRekeyAuxRowIDsResumesAfterCrash(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New: %v", err)
}
defer db.Close()
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
"version", auxRowRekeyMarkerVersion-1)
expectAuxRekeySentinel(mock, true)
expectSetAuxRekeySentinel(mock)
for range auxRekeyTables {
expectColumnExists(mock, false)
}
expectClearAuxRekeySentinel(mock)
// Pre-pass cursor at the watershed — the crashed pass already migrated
// main — yet the rewrite must run because the sentinel is pending.
if _, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion); err != nil {
t.Fatalf("rekeyAuxRowIDs: %v", err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet sql expectations: %v", err)
}
}
// TestRekeyAuxRowIDsKeepsSentinelOnFailure: when a table rewrite fails, the
// sentinel must NOT be cleared — it is the only record that the rewrite is
// incomplete once the main cursor has advanced.
func TestRekeyAuxRowIDsKeepsSentinelOnFailure(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New: %v", err)
}
defer db.Close()
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
"version", auxRowRekeyMarkerVersion-1)
expectAuxRekeySentinel(mock, false)
expectSetAuxRekeySentinel(mock)
// First table probe fails; no DELETE of the sentinel may follow.
mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`).
WillReturnError(errors.New("boom"))
if _, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion-1); err == nil {
t.Fatal("expected the table failure to propagate")
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet sql expectations: %v", err)
}
}

View file

@ -104,6 +104,10 @@ func expectOnePendingMigration(t *testing.T, mock sqlmock.Sqlmock) {
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1)
expectDoltStatusRows(mock)
expectDoltStatusRows(mock)
// MigrateUp probes the aux-rekey crash sentinel (bd-578h9.16); this
// mocked world has no local_metadata table, so no crashed pass.
mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
// MigrateUp captures the pre-pass main cursor for the aux re-key
// watershed (bd-578h9.4) before the main migrations run.
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1)

View file

@ -274,6 +274,19 @@ func MigrateUp(ctx context.Context, db DBConn) (int, error) {
if err != nil {
return 0, fmt.Errorf("reading pre-migration status: %w", err)
}
// A previous pass that crashed mid-aux-rekey left its partial UPDATEs
// dirty in the working set with the in-progress sentinel still recorded
// (bd-578h9.16). Those tables are this pass's own migration state, not
// pre-existing user writes: dropping them from dirtyBefore exempts them
// from the changed-signature guard (the resumed rekey is about to change
// them) and lets stageSchemaTables commit them with the rest of the pass.
if resuming, err := auxRekeyResumePending(ctx, db); err != nil {
return 0, fmt.Errorf("reading aux rekey sentinel: %w", err)
} else if resuming {
for _, t := range auxRekeyTables {
delete(dirtyBefore, t.name)
}
}
touchedDirtyTables, err := mainSource.pendingMigrationDirtyTables(ctx, db, dirtyBefore)
if err != nil {
return 0, fmt.Errorf("checking dirty tables against pending migrations: %w", err)