From 81b3308c30bc0fa4bb9e5bbd154dfd3f0059aa44 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Thu, 11 Jun 2026 18:22:50 -0400 Subject: [PATCH] fix(schema): crash mid-aux-rekey no longer strands rows behind the 0009 marker (bd-578h9.16) A pass that died mid-rekey had already advanced the main cursor past the bd-578h9.4 watershed, so the next MigrateUp read the lineage as a converged fresh clone: it skipped the rewrite, recorded marker 0009 over partially re-keyed rows, and the rekey never completed - the un-rekeyed rows kept their per-clone-random 0037 ids and the merge duplicates the pass exists to fix came back. The rekey now records a clone-local in-progress sentinel in local_metadata (dolt-ignored; created by main migration 0029, which is always applied before the rekey runs) before its first UPDATE and clears it only after the last table completes. A pending sentinel overrides the fresh-clone skip, so the next pass resumes the idempotent rewrite. MigrateUp also treats the aux tables as migration-owned when resuming: their partial-rekey dirt is exempted from the changed-signature guard (the resume is about to change them) and staged into the pass's commit instead of being left to the autocommit net. Co-Authored-By: Claude Fable 5 --- .../storage/schema/aux_row_id_backfill.go | 76 ++++++++++++++- .../schema/aux_row_id_backfill_test.go | 96 ++++++++++++++++++- internal/storage/schema/lock_test.go | 4 + internal/storage/schema/schema.go | 13 +++ 4 files changed, 187 insertions(+), 2 deletions(-) diff --git a/internal/storage/schema/aux_row_id_backfill.go b/internal/storage/schema/aux_row_id_backfill.go index a047d6f2f..3315b1e68 100644 --- a/internal/storage/schema/aux_row_id_backfill.go +++ b/internal/storage/schema/aux_row_id_backfill.go @@ -27,6 +27,62 @@ const auxRowRekeyMarkerVersion = 9 // (bd-578h9.4). The marker is still recorded. const auxRowRekeyShippedMainVersion = 51 +// auxRowRekeyInProgressKey is the clone-local sentinel (local_metadata, +// dolt-ignored) set before the re-key's first UPDATE and cleared after the +// last table completes. A crashed pass leaves it set, and the next MigrateUp +// resumes the re-key even though the main cursor already advanced past +// auxRowRekeyShippedMainVersion in the crashed pass — without it the +// fresh-clone skip above reads the crashed lineage as already converged and +// strands the partially re-keyed rows forever (bd-578h9.16). +const auxRowRekeyInProgressKey = "aux_row_rekey_in_progress" + +// auxRekeyResumePending reports whether a previous re-key pass on this clone +// recorded the in-progress sentinel and never cleared it. A missing +// local_metadata table means no sentinel: the table is dolt-ignored and +// therefore clone-local, so a fresh clone lacks it until something recreates +// it — setAuxRekeyInProgress creates it on demand. +func auxRekeyResumePending(ctx context.Context, db DBConn) (bool, error) { + var tableCount int + if err := db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'local_metadata'`, + ).Scan(&tableCount); err != nil { + return false, err + } + if tableCount == 0 { + return false, nil + } + var n int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM local_metadata WHERE `key` = ?", + auxRowRekeyInProgressKey).Scan(&n); err != nil { + return false, err + } + return n > 0, nil +} + +func setAuxRekeyInProgress(ctx context.Context, db DBConn) error { + // local_metadata is dolt-ignored, hence clone-local: a fresh clone whose + // main cursor is already past 0029 does not have the table (the migration + // will not re-run) until EnsureIgnoredTables recreates it. Create it here + // with 0029's DDL — its dolt_ignore pattern is committed history, so the + // sentinel stays clone-local. + if _, err := db.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS local_metadata (`key` VARCHAR(255) PRIMARY KEY, value TEXT NOT NULL DEFAULT '')"); err != nil { + return fmt.Errorf("ensuring local_metadata: %w", err) + } + _, err := db.ExecContext(ctx, + "REPLACE INTO local_metadata (`key`, value) VALUES (?, '1')", + auxRowRekeyInProgressKey) + return err +} + +func clearAuxRekeyInProgress(ctx context.Context, db DBConn) error { + _, err := db.ExecContext(ctx, + "DELETE FROM local_metadata WHERE `key` = ?", + auxRowRekeyInProgressKey) + return err +} + // auxRekeyTable describes one table covered by the re-key. columns is the // frozen SELECT list of every non-id column, in creation order, with datetime // columns CAST to CHAR server-side so the scanned text is identical across @@ -98,10 +154,25 @@ func rekeyAuxRowIDs(ctx context.Context, db DBConn, mainVersionBefore int) (bool if !markerPending { return false, nil } - if mainVersionBefore >= auxRowRekeyShippedMainVersion { + resume, err := auxRekeyResumePending(ctx, db) + if err != nil { + return false, fmt.Errorf("reading aux rekey sentinel: %w", err) + } + // The fresh-clone skip must not fire on a lineage whose previous pass + // crashed mid-rekey: that pass already advanced the main cursor past the + // shipped version, but its sentinel proves the rewrite never finished + // (bd-578h9.16). + if mainVersionBefore >= auxRowRekeyShippedMainVersion && !resume { return false, nil } + // Sentinel before the first UPDATE: a crash anywhere in the rewrite + // leaves it set, so the next pass resumes (the rewrite is idempotent) + // instead of recording the marker over partially re-keyed rows. + if err := setAuxRekeyInProgress(ctx, db); err != nil { + return false, fmt.Errorf("recording aux rekey sentinel: %w", err) + } + wrote := false for _, t := range auxRekeyTables { w, err := rekeyAuxRowTable(ctx, db, t) @@ -110,6 +181,9 @@ func rekeyAuxRowIDs(ctx context.Context, db DBConn, mainVersionBefore int) (bool return wrote, fmt.Errorf("%s: %w", t.name, err) } } + if err := clearAuxRekeyInProgress(ctx, db); err != nil { + return wrote, fmt.Errorf("clearing aux rekey sentinel: %w", err) + } return wrote, nil } diff --git a/internal/storage/schema/aux_row_id_backfill_test.go b/internal/storage/schema/aux_row_id_backfill_test.go index 202b349a5..4f204f8b4 100644 --- a/internal/storage/schema/aux_row_id_backfill_test.go +++ b/internal/storage/schema/aux_row_id_backfill_test.go @@ -3,6 +3,7 @@ package schema import ( "context" "database/sql" + "errors" "regexp" "testing" @@ -225,11 +226,14 @@ func TestRekeyAuxRowIDsRunsAllTablesWhenMarkerPending(t *testing.T) { expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", auxRowRekeyMarkerVersion-1) + expectAuxRekeySentinel(mock, false) + expectSetAuxRekeySentinel(mock) // Each of the four tables is probed; this mocked world has none of them, // so each probe returns 0 and the loop completes without scanning. for range auxRekeyTables { expectColumnExists(mock, false) } + expectClearAuxRekeySentinel(mock) wrote, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion-1) if err != nil { @@ -243,6 +247,36 @@ func TestRekeyAuxRowIDsRunsAllTablesWhenMarkerPending(t *testing.T) { } } +// expectAuxRekeySentinel mocks auxRekeyResumePending: the local_metadata +// table-existence probe, then (when the table exists) the sentinel-row count. +func expectAuxRekeySentinel(mock sqlmock.Sqlmock, pending bool) { + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + n := 0 + if pending { + n = 1 + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM local_metadata WHERE `key` = ?")). + WithArgs(auxRowRekeyInProgressKey). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(n)) +} + +func expectSetAuxRekeySentinel(mock sqlmock.Sqlmock) { + // Fresh clones lack the dolt-ignored local_metadata table, so the set + // path ensures it exists first. + mock.ExpectExec(`CREATE TABLE IF NOT EXISTS local_metadata`). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("REPLACE INTO local_metadata (`key`, value) VALUES (?, '1')")). + WithArgs(auxRowRekeyInProgressKey). + WillReturnResult(sqlmock.NewResult(0, 1)) +} + +func expectClearAuxRekeySentinel(mock sqlmock.Sqlmock) { + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM local_metadata WHERE `key` = ?")). + WithArgs(auxRowRekeyInProgressKey). + WillReturnResult(sqlmock.NewResult(0, 1)) +} + // TestRekeyAuxRowIDsSkipsConvergedLineage verifies the bd-578h9.4 gate: when // the main cursor already reached the version that shipped with the re-key // BEFORE this pass ran, the lineage was migrated by a rekey-aware binary and @@ -257,8 +291,10 @@ func TestRekeyAuxRowIDsSkipsConvergedLineage(t *testing.T) { expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", auxRowRekeyMarkerVersion-1) + expectAuxRekeySentinel(mock, false) // No further expectations: marker is pending, but the pre-pass main - // cursor at the watershed means no table may be probed or scanned. + // cursor at the watershed and no crash sentinel means no table may be + // probed or scanned. wrote, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion) if err != nil { @@ -271,3 +307,61 @@ func TestRekeyAuxRowIDsSkipsConvergedLineage(t *testing.T) { t.Fatalf("unmet sql expectations: %v", err) } } + +// TestRekeyAuxRowIDsResumesAfterCrash covers bd-578h9.16: a pass that died +// mid-rekey already advanced the main cursor past the watershed, so without +// the sentinel the next pass would read the lineage as a converged fresh +// clone, record the marker, and strand the partially re-keyed rows forever. +// A pending sentinel must override the fresh-clone skip and resume the +// rewrite. +func TestRekeyAuxRowIDsResumesAfterCrash(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", + "version", auxRowRekeyMarkerVersion-1) + expectAuxRekeySentinel(mock, true) + expectSetAuxRekeySentinel(mock) + for range auxRekeyTables { + expectColumnExists(mock, false) + } + expectClearAuxRekeySentinel(mock) + + // Pre-pass cursor at the watershed — the crashed pass already migrated + // main — yet the rewrite must run because the sentinel is pending. + if _, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion); err != nil { + t.Fatalf("rekeyAuxRowIDs: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +// TestRekeyAuxRowIDsKeepsSentinelOnFailure: when a table rewrite fails, the +// sentinel must NOT be cleared — it is the only record that the rewrite is +// incomplete once the main cursor has advanced. +func TestRekeyAuxRowIDsKeepsSentinelOnFailure(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", + "version", auxRowRekeyMarkerVersion-1) + expectAuxRekeySentinel(mock, false) + expectSetAuxRekeySentinel(mock) + // First table probe fails; no DELETE of the sentinel may follow. + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`). + WillReturnError(errors.New("boom")) + + if _, err := rekeyAuxRowIDs(context.Background(), db, auxRowRekeyShippedMainVersion-1); err == nil { + t.Fatal("expected the table failure to propagate") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 7adbdb45e..c3df755bf 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -104,6 +104,10 @@ func expectOnePendingMigration(t *testing.T, mock sqlmock.Sqlmock) { expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1) expectDoltStatusRows(mock) expectDoltStatusRows(mock) + // MigrateUp probes the aux-rekey crash sentinel (bd-578h9.16); this + // mocked world has no local_metadata table, so no crashed pass. + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) // MigrateUp captures the pre-pass main cursor for the aux re-key // watershed (bd-578h9.4) before the main migrations run. expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1) diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 7cd1c9401..f05e5d324 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -274,6 +274,19 @@ func MigrateUp(ctx context.Context, db DBConn) (int, error) { if err != nil { return 0, fmt.Errorf("reading pre-migration status: %w", err) } + // A previous pass that crashed mid-aux-rekey left its partial UPDATEs + // dirty in the working set with the in-progress sentinel still recorded + // (bd-578h9.16). Those tables are this pass's own migration state, not + // pre-existing user writes: dropping them from dirtyBefore exempts them + // from the changed-signature guard (the resumed rekey is about to change + // them) and lets stageSchemaTables commit them with the rest of the pass. + if resuming, err := auxRekeyResumePending(ctx, db); err != nil { + return 0, fmt.Errorf("reading aux rekey sentinel: %w", err) + } else if resuming { + for _, t := range auxRekeyTables { + delete(dirtyBefore, t.name) + } + } touchedDirtyTables, err := mainSource.pendingMigrationDirtyTables(ctx, db, dirtyBefore) if err != nil { return 0, fmt.Errorf("checking dirty tables against pending migrations: %w", err)