fix(schema): crash mid-aux-rekey no longer strands rows behind the 0009 marker (bd-578h9.16)
A pass that died mid-rekey had already advanced the main cursor past the bd-578h9.4 watershed, so the next MigrateUp read the lineage as a converged fresh clone: it skipped the rewrite, recorded marker 0009 over partially re-keyed rows, and the rekey never completed - the un-rekeyed rows kept their per-clone-random 0037 ids and the merge duplicates the pass exists to fix came back. The rekey now records a clone-local in-progress sentinel in local_metadata (dolt-ignored; created by main migration 0029, which is always applied before the rekey runs) before its first UPDATE and clears it only after the last table completes. A pending sentinel overrides the fresh-clone skip, so the next pass resumes the idempotent rewrite. MigrateUp also treats the aux tables as migration-owned when resuming: their partial-rekey dirt is exempted from the changed-signature guard (the resume is about to change them) and staged into the pass's commit instead of being left to the autocommit net. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
da59f139e0
commit
81b3308c30
4 changed files with 187 additions and 2 deletions
|
|
@ -27,6 +27,62 @@ const auxRowRekeyMarkerVersion = 9
|
|||
// (bd-578h9.4). The marker is still recorded.
|
||||
const auxRowRekeyShippedMainVersion = 51
|
||||
|
||||
// auxRowRekeyInProgressKey is the clone-local sentinel (local_metadata,
|
||||
// dolt-ignored) set before the re-key's first UPDATE and cleared after the
|
||||
// last table completes. A crashed pass leaves it set, and the next MigrateUp
|
||||
// resumes the re-key even though the main cursor already advanced past
|
||||
// auxRowRekeyShippedMainVersion in the crashed pass — without it the
|
||||
// fresh-clone skip above reads the crashed lineage as already converged and
|
||||
// strands the partially re-keyed rows forever (bd-578h9.16).
|
||||
const auxRowRekeyInProgressKey = "aux_row_rekey_in_progress"
|
||||
|
||||
// auxRekeyResumePending reports whether a previous re-key pass on this clone
|
||||
// recorded the in-progress sentinel and never cleared it. A missing
|
||||
// local_metadata table means no sentinel: the table is dolt-ignored and
|
||||
// therefore clone-local, so a fresh clone lacks it until something recreates
|
||||
// it — setAuxRekeyInProgress creates it on demand.
|
||||
func auxRekeyResumePending(ctx context.Context, db DBConn) (bool, error) {
|
||||
var tableCount int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'local_metadata'`,
|
||||
).Scan(&tableCount); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if tableCount == 0 {
|
||||
return false, nil
|
||||
}
|
||||
var n int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM local_metadata WHERE `key` = ?",
|
||||
auxRowRekeyInProgressKey).Scan(&n); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func setAuxRekeyInProgress(ctx context.Context, db DBConn) error {
|
||||
// local_metadata is dolt-ignored, hence clone-local: a fresh clone whose
|
||||
// main cursor is already past 0029 does not have the table (the migration
|
||||
// will not re-run) until EnsureIgnoredTables recreates it. Create it here
|
||||
// with 0029's DDL — its dolt_ignore pattern is committed history, so the
|
||||
// sentinel stays clone-local.
|
||||
if _, err := db.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS local_metadata (`key` VARCHAR(255) PRIMARY KEY, value TEXT NOT NULL DEFAULT '')"); err != nil {
|
||||
return fmt.Errorf("ensuring local_metadata: %w", err)
|
||||
}
|
||||
_, err := db.ExecContext(ctx,
|
||||
"REPLACE INTO local_metadata (`key`, value) VALUES (?, '1')",
|
||||
auxRowRekeyInProgressKey)
|
||||
return err
|
||||
}
|
||||
|
||||
func clearAuxRekeyInProgress(ctx context.Context, db DBConn) error {
|
||||
_, err := db.ExecContext(ctx,
|
||||
"DELETE FROM local_metadata WHERE `key` = ?",
|
||||
auxRowRekeyInProgressKey)
|
||||
return err
|
||||
}
|
||||
|
||||
// auxRekeyTable describes one table covered by the re-key. columns is the
|
||||
// frozen SELECT list of every non-id column, in creation order, with datetime
|
||||
// columns CAST to CHAR server-side so the scanned text is identical across
|
||||
|
|
@ -98,10 +154,25 @@ func rekeyAuxRowIDs(ctx context.Context, db DBConn, mainVersionBefore int) (bool
|
|||
if !markerPending {
|
||||
return false, nil
|
||||
}
|
||||
if mainVersionBefore >= auxRowRekeyShippedMainVersion {
|
||||
resume, err := auxRekeyResumePending(ctx, db)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading aux rekey sentinel: %w", err)
|
||||
}
|
||||
// The fresh-clone skip must not fire on a lineage whose previous pass
|
||||
// crashed mid-rekey: that pass already advanced the main cursor past the
|
||||
// shipped version, but its sentinel proves the rewrite never finished
|
||||
// (bd-578h9.16).
|
||||
if mainVersionBefore >= auxRowRekeyShippedMainVersion && !resume {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Sentinel before the first UPDATE: a crash anywhere in the rewrite
|
||||
// leaves it set, so the next pass resumes (the rewrite is idempotent)
|
||||
// instead of recording the marker over partially re-keyed rows.
|
||||
if err := setAuxRekeyInProgress(ctx, db); err != nil {
|
||||
return false, fmt.Errorf("recording aux rekey sentinel: %w", err)
|
||||
}
|
||||
|
||||
wrote := false
|
||||
for _, t := range auxRekeyTables {
|
||||
w, err := rekeyAuxRowTable(ctx, db, t)
|
||||
|
|
@ -110,6 +181,9 @@ func rekeyAuxRowIDs(ctx context.Context, db DBConn, mainVersionBefore int) (bool
|
|||
return wrote, fmt.Errorf("%s: %w", t.name, err)
|
||||
}
|
||||
}
|
||||
if err := clearAuxRekeyInProgress(ctx, db); err != nil {
|
||||
return wrote, fmt.Errorf("clearing aux rekey sentinel: %w", err)
|
||||
}
|
||||
return wrote, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package schema
|
|||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
|
|
@ -225,11 +226,14 @@ func TestRekeyAuxRowIDsRunsAllTablesWhenMarkerPending(t *testing.T) {
|
|||
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
|
||||
"version", auxRowRekeyMarkerVersion-1)
|
||||
expectAuxRekeySentinel(mock, false)
|
||||
expectSetAuxRekeySentinel(mock)
|
||||
// Each of the four tables is probed; this mocked world has none of them,
|
||||
// so each probe returns 0 and the loop completes without scanning.
|
||||
for range auxRekeyTables {
|
||||
expectColumnExists(mock, false)
|
||||
}
|
||||
expectClearAuxRekeySentinel(mock)
|
||||
|
||||
wrote, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion-1)
|
||||
if err != nil {
|
||||
|
|
@ -243,6 +247,36 @@ func TestRekeyAuxRowIDsRunsAllTablesWhenMarkerPending(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// expectAuxRekeySentinel mocks auxRekeyResumePending: the local_metadata
|
||||
// table-existence probe, then (when the table exists) the sentinel-row count.
|
||||
func expectAuxRekeySentinel(mock sqlmock.Sqlmock, pending bool) {
|
||||
mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
n := 0
|
||||
if pending {
|
||||
n = 1
|
||||
}
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM local_metadata WHERE `key` = ?")).
|
||||
WithArgs(auxRowRekeyInProgressKey).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(n))
|
||||
}
|
||||
|
||||
func expectSetAuxRekeySentinel(mock sqlmock.Sqlmock) {
|
||||
// Fresh clones lack the dolt-ignored local_metadata table, so the set
|
||||
// path ensures it exists first.
|
||||
mock.ExpectExec(`CREATE TABLE IF NOT EXISTS local_metadata`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec(regexp.QuoteMeta("REPLACE INTO local_metadata (`key`, value) VALUES (?, '1')")).
|
||||
WithArgs(auxRowRekeyInProgressKey).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
}
|
||||
|
||||
func expectClearAuxRekeySentinel(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectExec(regexp.QuoteMeta("DELETE FROM local_metadata WHERE `key` = ?")).
|
||||
WithArgs(auxRowRekeyInProgressKey).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
}
|
||||
|
||||
// TestRekeyAuxRowIDsSkipsConvergedLineage verifies the bd-578h9.4 gate: when
|
||||
// the main cursor already reached the version that shipped with the re-key
|
||||
// BEFORE this pass ran, the lineage was migrated by a rekey-aware binary and
|
||||
|
|
@ -257,8 +291,10 @@ func TestRekeyAuxRowIDsSkipsConvergedLineage(t *testing.T) {
|
|||
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
|
||||
"version", auxRowRekeyMarkerVersion-1)
|
||||
expectAuxRekeySentinel(mock, false)
|
||||
// No further expectations: marker is pending, but the pre-pass main
|
||||
// cursor at the watershed means no table may be probed or scanned.
|
||||
// cursor at the watershed and no crash sentinel means no table may be
|
||||
// probed or scanned.
|
||||
|
||||
wrote, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion)
|
||||
if err != nil {
|
||||
|
|
@ -271,3 +307,61 @@ func TestRekeyAuxRowIDsSkipsConvergedLineage(t *testing.T) {
|
|||
t.Fatalf("unmet sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRekeyAuxRowIDsResumesAfterCrash covers bd-578h9.16: a pass that died
|
||||
// mid-rekey already advanced the main cursor past the watershed, so without
|
||||
// the sentinel the next pass would read the lineage as a converged fresh
|
||||
// clone, record the marker, and strand the partially re-keyed rows forever.
|
||||
// A pending sentinel must override the fresh-clone skip and resume the
|
||||
// rewrite.
|
||||
func TestRekeyAuxRowIDsResumesAfterCrash(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
|
||||
"version", auxRowRekeyMarkerVersion-1)
|
||||
expectAuxRekeySentinel(mock, true)
|
||||
expectSetAuxRekeySentinel(mock)
|
||||
for range auxRekeyTables {
|
||||
expectColumnExists(mock, false)
|
||||
}
|
||||
expectClearAuxRekeySentinel(mock)
|
||||
|
||||
// Pre-pass cursor at the watershed — the crashed pass already migrated
|
||||
// main — yet the rewrite must run because the sentinel is pending.
|
||||
if _, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion); err != nil {
|
||||
t.Fatalf("rekeyAuxRowIDs: %v", err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRekeyAuxRowIDsKeepsSentinelOnFailure: when a table rewrite fails, the
|
||||
// sentinel must NOT be cleared — it is the only record that the rewrite is
|
||||
// incomplete once the main cursor has advanced.
|
||||
func TestRekeyAuxRowIDsKeepsSentinelOnFailure(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations",
|
||||
"version", auxRowRekeyMarkerVersion-1)
|
||||
expectAuxRekeySentinel(mock, false)
|
||||
expectSetAuxRekeySentinel(mock)
|
||||
// First table probe fails; no DELETE of the sentinel may follow.
|
||||
mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`).
|
||||
WillReturnError(errors.New("boom"))
|
||||
|
||||
if _, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion-1); err == nil {
|
||||
t.Fatal("expected the table failure to propagate")
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,6 +104,10 @@ func expectOnePendingMigration(t *testing.T, mock sqlmock.Sqlmock) {
|
|||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1)
|
||||
expectDoltStatusRows(mock)
|
||||
expectDoltStatusRows(mock)
|
||||
// MigrateUp probes the aux-rekey crash sentinel (bd-578h9.16); this
|
||||
// mocked world has no local_metadata table, so no crashed pass.
|
||||
mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
|
||||
// MigrateUp captures the pre-pass main cursor for the aux re-key
|
||||
// watershed (bd-578h9.4) before the main migrations run.
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1)
|
||||
|
|
|
|||
|
|
@ -274,6 +274,19 @@ func MigrateUp(ctx context.Context, db DBConn) (int, error) {
|
|||
if err != nil {
|
||||
return 0, fmt.Errorf("reading pre-migration status: %w", err)
|
||||
}
|
||||
// A previous pass that crashed mid-aux-rekey left its partial UPDATEs
|
||||
// dirty in the working set with the in-progress sentinel still recorded
|
||||
// (bd-578h9.16). Those tables are this pass's own migration state, not
|
||||
// pre-existing user writes: dropping them from dirtyBefore exempts them
|
||||
// from the changed-signature guard (the resumed rekey is about to change
|
||||
// them) and lets stageSchemaTables commit them with the rest of the pass.
|
||||
if resuming, err := auxRekeyResumePending(ctx, db); err != nil {
|
||||
return 0, fmt.Errorf("reading aux rekey sentinel: %w", err)
|
||||
} else if resuming {
|
||||
for _, t := range auxRekeyTables {
|
||||
delete(dirtyBefore, t.name)
|
||||
}
|
||||
}
|
||||
touchedDirtyTables, err := mainSource.pendingMigrationDirtyTables(ctx, db, dirtyBefore)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("checking dirty tables against pending migrations: %w", err)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue