diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 703661fe0..5f5b336b2 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -116,13 +116,17 @@ func expectOnePendingMigration(t *testing.T, mock sqlmock.Sqlmock) { expectContentHashColumnExists(mock) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", latest-1) if latest == 53 { - // The v53 pre-repair (#4502) probes the six rig/agent columns on - // issues; this mocked world has them all, so no ALTERs follow. + // The v53 pre-repair probes the six rig/agent columns on issues and + // then the local wisp_dependencies table; this mocked world has all + // issue columns and no local wisp_dependencies table, so no ALTERs follow. for _, col := range []string{"hook_bead", "role_bead", "agent_state", "last_activity", "role_type", "rig"} { mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`). - WithArgs(col). + WithArgs("issues", col). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) } + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES`). + WithArgs("wisp_dependencies"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) } mock.ExpectExec("(?s).*"). WillReturnResult(sqlmock.NewResult(0, 0)) diff --git a/internal/storage/schema/migration_repairs.go b/internal/storage/schema/migration_repairs.go index b42c9f2ad..245dcf955 100644 --- a/internal/storage/schema/migration_repairs.go +++ b/internal/storage/schema/migration_repairs.go @@ -18,7 +18,10 @@ import ( // preMigrationRepair dispatches any repair registered for (source, version). func (m migrationSource) preMigrationRepair(ctx context.Context, db DBConn, version int) error { if m.cursorTable == "schema_migrations" && version == 53 { - return ensureIssuesRigColumns(ctx, db) + if err := ensureIssuesRigColumns(ctx, db); err != nil { + return err + } + return ensureWispDependenciesSplitTargets(ctx, db) } return nil } @@ -40,14 +43,11 @@ func ensureIssuesRigColumns(ctx context.Context, db DBConn) error { {"rig", "VARCHAR(255) DEFAULT ''"}, } for _, col := range columns { - var present int - if err := db.QueryRowContext(ctx, ` - SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'issues' AND COLUMN_NAME = ? - `, col.name).Scan(&present); err != nil { + present, err := schemaColumnExists(ctx, db, "issues", col.name) + if err != nil { return fmt.Errorf("checking issues.%s: %w", col.name, err) } - if present > 0 { + if present { continue } if _, err := db.ExecContext(ctx, "ALTER TABLE issues ADD COLUMN "+col.name+" "+col.definition); err != nil { @@ -56,3 +56,93 @@ func ensureIssuesRigColumns(ctx context.Context, db DBConn) error { } return nil } + +// ensureWispDependenciesSplitTargets repairs #4555: a mixed-vintage local +// wisp_dependencies table can have the post-0005 id column while still lacking +// one or more split target columns. Migration 0053 reads those columns when it +// repairs rig wisps, so add the missing columns and backfill them from the +// legacy depends_on_id column when that source column is still available. +func ensureWispDependenciesSplitTargets(ctx context.Context, db DBConn) error { + table, err := schemaTableExists(ctx, db, "wisp_dependencies") + if err != nil { + return fmt.Errorf("checking wisp_dependencies table: %w", err) + } + if !table { + return nil + } + + columns := wispDependenciesSplitTargetColumns() + missing := make([]struct{ name, definition string }, 0, len(columns)) + for _, col := range columns { + present, err := schemaColumnExists(ctx, db, "wisp_dependencies", col.name) + if err != nil { + return fmt.Errorf("checking wisp_dependencies.%s: %w", col.name, err) + } + if !present { + missing = append(missing, col) + } + } + if len(missing) == 0 { + return nil + } + + for _, col := range missing { + if _, err := db.ExecContext(ctx, "ALTER TABLE wisp_dependencies ADD COLUMN "+col.name+" "+col.definition); err != nil { + return fmt.Errorf("adding wisp_dependencies.%s for migration 0053: %w", col.name, err) + } + } + + legacyTarget, err := schemaColumnExists(ctx, db, "wisp_dependencies", "depends_on_id") + if err != nil { + return fmt.Errorf("checking wisp_dependencies.depends_on_id: %w", err) + } + if !legacyTarget { + return nil + } + + for _, repair := range wispDependenciesSplitTargetBackfillSQL() { + if _, err := db.ExecContext(ctx, repair); err != nil { + return fmt.Errorf("backfilling wisp_dependencies split targets for migration 0053: %w", err) + } + } + return nil +} + +func wispDependenciesSplitTargetColumns() []struct{ name, definition string } { + return []struct{ name, definition string }{ + {"depends_on_issue_id", "VARCHAR(255) NULL"}, + {"depends_on_wisp_id", "VARCHAR(255) NULL"}, + {"depends_on_external", "VARCHAR(255) NULL"}, + } +} + +func wispDependenciesSplitTargetBackfillSQL() []string { + return []string{ + "UPDATE wisp_dependencies SET depends_on_external = depends_on_id WHERE depends_on_external IS NULL AND depends_on_id LIKE 'external:%'", + "UPDATE wisp_dependencies wd JOIN wisps w ON w.id = wd.depends_on_id SET wd.depends_on_wisp_id = wd.depends_on_id WHERE wd.depends_on_wisp_id IS NULL AND wd.depends_on_external IS NULL", + "UPDATE wisp_dependencies wd JOIN issues i ON i.id = wd.depends_on_id SET wd.depends_on_issue_id = wd.depends_on_id WHERE wd.depends_on_issue_id IS NULL AND wd.depends_on_external IS NULL AND wd.depends_on_wisp_id IS NULL", + "UPDATE wisp_dependencies SET depends_on_external = depends_on_id WHERE depends_on_external IS NULL AND depends_on_wisp_id IS NULL AND depends_on_issue_id IS NULL", + } +} + +func schemaTableExists(ctx context.Context, db DBConn, table string) (bool, error) { + var count int + if err := db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? + `, table).Scan(&count); err != nil { + return false, err + } + return count > 0, nil +} + +func schemaColumnExists(ctx context.Context, db DBConn, table, column string) (bool, error) { + var count int + if err := db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? + `, table, column).Scan(&count); err != nil { + return false, err + } + return count > 0, nil +} diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 500f758ed..d7526606c 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -294,6 +294,14 @@ func MigrateUp(ctx context.Context, db DBConn) (int, error) { delete(dirtyBefore, t.name) } } + if recoverable, err := failed0053DirtyTablesAreRecoverable(ctx, db, dirtyBefore); err != nil { + return 0, fmt.Errorf("checking failed v53 migration recovery: %w", err) + } else if recoverable { + log.Printf("schema migration recovering known failed v53 dirty tables: %s", strings.Join(sortedDirtyTableNames(dirtyBefore), ", ")) + for table := range dirtyBefore { + delete(dirtyBefore, table) + } + } touchedDirtyTables, err := mainSource.pendingMigrationDirtyTables(ctx, db, dirtyBefore) if err != nil { return 0, fmt.Errorf("checking dirty tables against pending migrations: %w", err) @@ -395,6 +403,67 @@ func MigrateUp(ctx context.Context, db DBConn) (int, error) { return applied, nil } +func failed0053DirtyTablesAreRecoverable(ctx context.Context, db DBConn, dirtyBefore map[string]dirtyTableState) (bool, error) { + if len(dirtyBefore) == 0 { + return false, nil + } + current, err := mainSource.currentVersion(ctx, db) + if err != nil { + return false, err + } + if current != 52 { + return false, nil + } + + allowed := map[string]struct{}{ + "child_counters": {}, + "comments": {}, + "dependencies": {}, + "events": {}, + "issues": {}, + "labels": {}, + // 0051 drops the legacy DEFAULT (UUID()) on these aux tables. In a + // single-pass v49->v53 batch (MigrateUp commits once at the end), + // that DROP DEFAULT is still uncommitted when 0053 fails, so a + // legacy-default DB trips this gate with these tables dirty too. + // The change is an idempotent, schema-only DROP DEFAULT, so it is + // safe to fold into the recovery commit (#4555). + "issue_snapshots": {}, + "compaction_snapshots": {}, + } + for table := range dirtyBefore { + if _, ok := allowed[table]; !ok { + return false, nil + } + } + + needsRepair, err := wispDependenciesNeed0053Repair(ctx, db) + if err != nil { + return false, err + } + return needsRepair, nil +} + +func wispDependenciesNeed0053Repair(ctx context.Context, db DBConn) (bool, error) { + table, err := schemaTableExists(ctx, db, "wisp_dependencies") + if err != nil { + return false, err + } + if !table { + return false, nil + } + for _, column := range []string{"depends_on_issue_id", "depends_on_wisp_id", "depends_on_external"} { + present, err := schemaColumnExists(ctx, db, "wisp_dependencies", column) + if err != nil { + return false, err + } + if !present { + return true, nil + } + } + return false, nil +} + func migrationWorkNeeded(ctx context.Context, db DBConn) (bool, error) { if !mainSource.atLatest(ctx, db) || !ignoredSource.atLatest(ctx, db) { return true, nil diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index dea203985..4315cfbf9 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -227,7 +227,7 @@ func TestEnsureIssuesRigColumnsAddsOnlyMissing(t *testing.T) { // in 0001 reach v52 without them; the 0053 pre-repair must add exactly // the missing ones. Simulate hook_bead present, the other five absent. countQuery := `SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS` - mock.ExpectQuery(countQuery).WithArgs("hook_bead"). + mock.ExpectQuery(countQuery).WithArgs("issues", "hook_bead"). WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) for _, col := range []struct{ name, ddl string }{ {"role_bead", "ALTER TABLE issues ADD COLUMN role_bead VARCHAR\\(255\\) DEFAULT ''"}, @@ -236,7 +236,7 @@ func TestEnsureIssuesRigColumnsAddsOnlyMissing(t *testing.T) { {"role_type", "ALTER TABLE issues ADD COLUMN role_type VARCHAR\\(32\\) DEFAULT ''"}, {"rig", "ALTER TABLE issues ADD COLUMN rig VARCHAR\\(255\\) DEFAULT ''"}, } { - mock.ExpectQuery(countQuery).WithArgs(col.name). + mock.ExpectQuery(countQuery).WithArgs("issues", col.name). WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0)) mock.ExpectExec(col.ddl).WillReturnResult(sqlmock.NewResult(0, 0)) } @@ -249,6 +249,205 @@ func TestEnsureIssuesRigColumnsAddsOnlyMissing(t *testing.T) { } } +func TestEnsureWispDependenciesSplitTargetsAddsMissingAndBackfills(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + tableQuery := `(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES.*TABLE_NAME = \?` + columnQuery := `(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS.*TABLE_NAME = \? AND COLUMN_NAME = \?` + mock.ExpectQuery(tableQuery).WithArgs("wisp_dependencies"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + for _, col := range []string{"depends_on_issue_id", "depends_on_wisp_id", "depends_on_external"} { + mock.ExpectQuery(columnQuery).WithArgs("wisp_dependencies", col). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0)) + } + for _, ddl := range []string{ + "ALTER TABLE wisp_dependencies ADD COLUMN depends_on_issue_id VARCHAR\\(255\\) NULL", + "ALTER TABLE wisp_dependencies ADD COLUMN depends_on_wisp_id VARCHAR\\(255\\) NULL", + "ALTER TABLE wisp_dependencies ADD COLUMN depends_on_external VARCHAR\\(255\\) NULL", + } { + mock.ExpectExec(ddl).WillReturnResult(sqlmock.NewResult(0, 0)) + } + mock.ExpectQuery(columnQuery).WithArgs("wisp_dependencies", "depends_on_id"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + for _, update := range []string{ + `UPDATE wisp_dependencies SET depends_on_external = depends_on_id`, + `UPDATE wisp_dependencies wd JOIN wisps w ON w\.id = wd\.depends_on_id`, + `UPDATE wisp_dependencies wd JOIN issues i ON i\.id = wd\.depends_on_id`, + `UPDATE wisp_dependencies SET depends_on_external = depends_on_id WHERE depends_on_external IS NULL`, + } { + mock.ExpectExec(update).WillReturnResult(sqlmock.NewResult(0, 1)) + } + + if err := ensureWispDependenciesSplitTargets(context.Background(), db); err != nil { + t.Fatalf("ensureWispDependenciesSplitTargets: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestFailed0053DirtyTablesAreRecoverable(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(52)) + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES.*TABLE_NAME = \?`). + WithArgs("wisp_dependencies"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS.*TABLE_NAME = \? AND COLUMN_NAME = \?`). + WithArgs("wisp_dependencies", "depends_on_issue_id"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0)) + + recoverable, err := failed0053DirtyTablesAreRecoverable(context.Background(), db, map[string]dirtyTableState{ + "comments": {}, + "dependencies": {}, + "events": {}, + "issues": {}, + }) + if err != nil { + t.Fatalf("failed0053DirtyTablesAreRecoverable: %v", err) + } + if !recoverable { + t.Fatal("recoverable = false, want true") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestFailed0053DirtyTablesAreRecoverableWithDirtySnapshotAuxTables(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(52)) + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES.*TABLE_NAME = \?`). + WithArgs("wisp_dependencies"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS.*TABLE_NAME = \? AND COLUMN_NAME = \?`). + WithArgs("wisp_dependencies", "depends_on_issue_id"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0)) + + // 0051's DROP DEFAULT on the legacy UUID() default leaves these aux + // snapshot tables dirty too when a v49->v53 batch trips over 0053 + // (#4555); the gate must still recover. + recoverable, err := failed0053DirtyTablesAreRecoverable(context.Background(), db, map[string]dirtyTableState{ + "comments": {}, + "dependencies": {}, + "events": {}, + "issues": {}, + "issue_snapshots": {}, + "compaction_snapshots": {}, + }) + if err != nil { + t.Fatalf("failed0053DirtyTablesAreRecoverable: %v", err) + } + if !recoverable { + t.Fatal("recoverable = false, want true") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestFailed0053DirtyTablesRejectsUnrelatedDirtyTable(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(52)) + + recoverable, err := failed0053DirtyTablesAreRecoverable(context.Background(), db, map[string]dirtyTableState{ + "comments": {}, + "settings": {}, + }) + if err != nil { + t.Fatalf("failed0053DirtyTablesAreRecoverable: %v", err) + } + if recoverable { + t.Fatal("recoverable = true, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestFailed0053DirtyTablesRejectsWhenWispDependenciesAlreadyRepaired(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(52)) + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.TABLES.*TABLE_NAME = \?`). + WithArgs("wisp_dependencies"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + for _, col := range []string{"depends_on_issue_id", "depends_on_wisp_id", "depends_on_external"} { + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS.*TABLE_NAME = \? AND COLUMN_NAME = \?`). + WithArgs("wisp_dependencies", col). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + } + + recoverable, err := failed0053DirtyTablesAreRecoverable(context.Background(), db, map[string]dirtyTableState{ + "comments": {}, + "dependencies": {}, + "events": {}, + "issues": {}, + }) + if err != nil { + t.Fatalf("failed0053DirtyTablesAreRecoverable: %v", err) + } + if recoverable { + t.Fatal("recoverable = true, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestFailed0053DirtyTablesRejectsWrongVersion(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(51)) + + recoverable, err := failed0053DirtyTablesAreRecoverable(context.Background(), db, map[string]dirtyTableState{ + "comments": {}, + "dependencies": {}, + "events": {}, + "issues": {}, + }) + if err != nil { + t.Fatalf("failed0053DirtyTablesAreRecoverable: %v", err) + } + if recoverable { + t.Fatal("recoverable = true, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + func TestPreMigrationRepairScopedToMain0053(t *testing.T) { // The repair must not fire for other versions or for the ignored source // (whose cursor table differs); nil DB proves no queries are attempted. @@ -376,6 +575,52 @@ INSERT INTO wisp_child_counters (parent_id, last_child) VALUES (%s, 7); `SELECT COUNT(*) AS c FROM wisp_child_counters WHERE parent_id = 'schema-cli-rig'`, "0") } +func TestWispDependenciesSplitTargetBackfillPrefersWispOverIssueThroughDoltCLI(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := filepath.Join(t.TempDir(), "wisp-dependency-split") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create wisp dependency split dir: %v", err) + } + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + + var repairSQL strings.Builder + for _, col := range wispDependenciesSplitTargetColumns() { + fmt.Fprintf(&repairSQL, "ALTER TABLE wisp_dependencies ADD COLUMN %s %s;\n", col.name, col.definition) + } + for _, stmt := range wispDependenciesSplitTargetBackfillSQL() { + repairSQL.WriteString(stmt) + repairSQL.WriteString(";\n") + } + + const sourceID = "source-wisp" + const ambiguousID = "ambiguous-target" + seedSQL := fmt.Sprintf(` +CREATE TABLE issues ( + id VARCHAR(255) PRIMARY KEY +); +CREATE TABLE wisps ( + id VARCHAR(255) PRIMARY KEY +); +CREATE TABLE wisp_dependencies ( + issue_id VARCHAR(255) NOT NULL, + depends_on_id VARCHAR(255) NOT NULL, + PRIMARY KEY (issue_id, depends_on_id) +); +INSERT INTO issues (id) VALUES (%s); +INSERT INTO wisps (id) VALUES (%s), (%s); +INSERT INTO wisp_dependencies (issue_id, depends_on_id) VALUES (%s, %s); +`, doltSQLString(ambiguousID), + doltSQLString(sourceID), doltSQLString(ambiguousID), + doltSQLString(sourceID), doltSQLString(ambiguousID)) + runDoltSQL(t, dir, seedSQL+repairSQL.String()) + + requireDoltCount(t, dir, + `SELECT COUNT(*) AS c FROM wisp_dependencies WHERE issue_id = 'source-wisp' AND depends_on_wisp_id = 'ambiguous-target' AND depends_on_issue_id IS NULL AND depends_on_external IS NULL`, "1") + requireDoltCount(t, dir, + `SELECT COUNT(*) AS c FROM wisp_dependencies WHERE issue_id = 'source-wisp' AND depends_on_issue_id = 'ambiguous-target'`, "0") +} + func TestMigration0047HandlesLegacyWispDependenciesShape(t *testing.T) { sql, err := os.ReadFile("migrations/0047_recompute_mixed_is_blocked.up.sql") if err != nil {