diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a3e409ea..5f6c18930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -185,6 +185,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 proxied-server route share the banner. A workspace-less directory, and a healthy store with no memories, stay silent as before. +- **`bd` no longer serializes every invocation behind the schema-init advisory + lock.** The schema-init probe ran entirely inside the database-scoped + `GET_LOCK('bd_schema_init:', 5)`, so on a shared Dolt server every `bd` + process queued behind every other one to discover that it had nothing to + migrate. Measured on an 18-seat rig: 1500 `is_free_lock` samples over 14s + found the lock held 96.7% of the time, held runs had a 673ms median and a + 4.2s max, and free gaps had a 0ms median — the lock was handed straight from + holder to holder. That queue was 0.4-2.4s of pure waiting on every claim, + heartbeat, list, comment and mail check. `MigrateUpWithLock` now answers the + steady-state question first, without the lock — it puts the session on the + target database (proving the database exists before issuing any statement + that could fail, so a fresh bootstrap still falls through to the locked + `CREATE DATABASE`; callers whose pool already names the database inject + nothing and the probe simply declines a session that is elsewhere), then + asks whether both migration cursors are at this + binary's latest with the content-hash column and custom-status/type backfills + done, whether all the canonical `dolt_ignore` patterns are present, and + finally whether the migration lock is free — and takes `GET_LOCK` only on the + path that can actually migrate. That last term is what keeps the probe honest + while a peer is mid-pass: the cursors land before the backfills and rekeys + finish, so a held lock, not the cursors alone, is the proof that nobody is + rewriting the tables underneath this caller. The probe issues no writes — + its one session mutation is the `USE` that pins the target database — and fails + closed: any error, any unreadable state, and anything short of provably + converged falls through to the locked path unchanged, as does any caller + carrying fresh-bootstrap heal authority. + - **Incremental auto-export now actually takes the incremental path** ([#5806](https://github.com/gastownhall/beads/pull/5806)). Change detection compared `GetStateHash()` values — a hash of the entire database plus diff --git a/internal/storage/domain/db/ddl.go b/internal/storage/domain/db/ddl.go index 3f5f7f42e..babdeb608 100644 --- a/internal/storage/domain/db/ddl.go +++ b/internal/storage/domain/db/ddl.go @@ -76,9 +76,18 @@ func (r *ddlSQLRepository) UseDatabase(ctx context.Context, database string) err return nil } -func quoteIdentifier(name string) (string, error) { +// QuoteIdentifier validates name and returns it back-quoted, ready to embed in +// a statement. It is the one place this repository's identifiers are quoted; +// callers outside the package that must build a schema-qualified name (see +// uow's convergence-probe database selector) go through it rather than +// re-deriving the rule. +func QuoteIdentifier(name string) (string, error) { if err := ValidateIdentifier(name); err != nil { return "", err } return "`" + name + "`", nil } + +func quoteIdentifier(name string) (string, error) { + return QuoteIdentifier(name) +} diff --git a/internal/storage/schema/converged.go b/internal/storage/schema/converged.go new file mode 100644 index 000000000..d3c923750 --- /dev/null +++ b/internal/storage/schema/converged.go @@ -0,0 +1,244 @@ +package schema + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage/dberrors" +) + +// alreadyConverged reports whether databaseName is already at this binary's +// target schema in every respect MigrateUp checks before its no-work +// short-circuit: no migration work is pending, the canonical dolt_ignore +// patterns are all present, and nobody is holding the migration lock. When it +// says yes, a MigrateUp pass would acquire the migration lock, do nothing, +// and return (0, nil). +// +// Why it exists: the whole schema-init probe ran INSIDE the database-scoped +// GET_LOCK, so every bd invocation against a shared Dolt server serialized on +// it even though the probe is a no-op in the steady state. Measured on an +// 18-seat rig, the lock was held 96.7% of a 14s sampling window, held runs had +// a 673ms median and a 4.2s max, and free gaps had a 0ms median — handed +// straight from holder to holder. That queue was 0.4-2.4s of pure waiting on +// every claim, heartbeat, list and comment. The statements under the lock are +// individually cheap; the SERIALIZATION is the cost, so batching them does not +// help and answering the steady-state question WITHOUT the lock does. +// +// It is deliberately advisory and fails closed onto the locked path. Any +// error, any unreadable state, and anything short of provably converged +// returns false, and the caller then behaves exactly as it did before this +// check existed. A false negative costs one ordinary locked pass; false +// positives are avoided by evaluating the same predicates MigrateUp itself +// evaluates, on the same pinned session, with no writes of its own. +func alreadyConverged(ctx context.Context, db DBConn, databaseName string, selector DatabaseSelector) (bool, error) { + if databaseName == "" { + return false, nil + } + + // Put the session on the target database first. The hot path — the proxied + // CLI open in uow.openAndInitSchema — pins its schema-init pool with an + // EMPTY DSN database and only USEs the database after GET_LOCK, so a probe + // that merely ASKED whether the session was already on databaseName read + // NULL from DATABASE() and declined on every single invocation: the fast + // path never fired where it was needed. + onTarget, qualifier, err := selectTargetDatabase(ctx, db, databaseName, selector) + if err != nil { + return false, err + } + if !onTarget { + return false, nil + } + + // Exactly MigrateUp's own gate, and it runs first: on a fresh or + // mid-upgrade database it reports work needed from the cursor probe alone, + // before any statement that could fail against a missing table. + needed, err := migrationWorkNeeded(ctx, db) + if err != nil { + return false, fmt.Errorf("checking schema migration work: %w", err) + } + if needed { + return false, nil + } + + // MigrateUp re-asserts the canonical dolt_ignore patterns ahead of that + // gate precisely because an out-of-band-materialized database can arrive + // with its cursors at-latest and the patterns missing. Read the same + // question instead of writing it: an under-seeded database is not + // converged and must take the locked path, which heals and commits it. + // + // migrationWorkNeeded has just proved mainSource.atLatest, i.e. the main + // cursor is at or past mainSource.latest(); every version-gated pattern's + // flip migration is part of that embedded set, so the gate can be + // evaluated against latest() with no second cursor read. + seeded, err := doltIgnoreSeeded(ctx, db, qualifier, mainSource.latest()) + if err != nil { + return false, err + } + if !seeded { + return false, nil + } + + // LAST, and deliberately so: everything above is the cheap steady-state + // question and short-circuits first. This term closes the window where a + // peer is midway through a real migration pass. migrationWorkNeeded only + // covers the FRONT half of MigrateUp — once the version cursors and + // content_hash columns have landed it reports "nothing pending" while the + // pass is still running backfills, dependency/aux PK rekeys, the ignored + // series, and the schema commit. A lock-free prober in that window would + // otherwise conclude "converged" and hand its caller a database whose + // tables are being rewritten underneath it. If anyone holds the migration + // lock we are not entitled to that conclusion: fail closed into the locked + // path and wait our turn, exactly as every caller did before the fast path + // existed. + return migrationLockFree(ctx, db, MigrationLockName(databaseName)) +} + +// selectTargetDatabase puts the pinned session on databaseName, reporting +// whether it is now provably there and — when a selector had to be used to get +// there — the identifier-quoted name to schema-qualify later reads with. +// +// A session already on databaseName needs nothing: it is provably on the +// target because DATABASE() was just read and matched, and later reads may +// therefore run unqualified against it. An empty qualifier means exactly that. +// +// Without a selector the probe may not issue USE at all, so a session on any +// other database (or on none) is declined. That is the whole behavior for +// callers whose pool DSN already names the database (internal/storage/dolt), +// and it keeps this package free of the DDL repository it would otherwise need +// (see DatabaseSelector for why that import cannot exist). +// +// A database that does not exist yet is a fresh bootstrap: report false so the +// caller falls through to the locked path, whose CREATE DATABASE arbitrates +// creation and issues the #5012 fresh-bootstrap heal capability. +// +// Selecting is also what makes skipping a caller's locked bootstrap +// preparation safe: preparation creates the database and USEs it, and both +// have provably happened here — the database existed before we touched it (so +// preparation's bare CREATE DATABASE could only have failed with "database +// exists" and captured no heal authority), and the selector issues the same +// USE preparation would. +// +// The existence probe is not decoration. A Dolt session that issues a FAILING +// statement stays pinned to its pre-statement catalog snapshot, so a USE of a +// not-yet-created database would poison this pooled connection for the rest of +// its life (be-bv7x). Probe with a query that always succeeds, then act. +func selectTargetDatabase(ctx context.Context, db DBConn, databaseName string, selector DatabaseSelector) (bool, string, error) { + var current sql.NullString + if err := db.QueryRowContext(ctx, "SELECT DATABASE()").Scan(¤t); err != nil { + return false, "", fmt.Errorf("reading current database: %w", err) + } + if current.Valid && current.String == databaseName { + return true, "", nil + } + if selector == nil { + return false, "", nil + } + + var exists int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?", + databaseName, + ).Scan(&exists); err != nil { + return false, "", fmt.Errorf("probing database %q existence: %w", databaseName, err) + } + if exists == 0 { + return false, "", nil + } + + quoted, err := selector(ctx, db, databaseName) + if err != nil { + return false, "", fmt.Errorf("selecting database %q: %w", databaseName, err) + } + if quoted == "" { + return false, "", fmt.Errorf("selecting database %q: selector returned no quoted name", databaseName) + } + return true, quoted, nil +} + +// migrationLockFree reports whether the database-scoped migration lock is +// currently unheld. IS_FREE_LOCK is a read: it never queues, never acquires, +// and costs one round trip, which is the entire point — the fast path exists +// to stop paying GET_LOCK's queue. +// +// A NULL answer means the server would not tell us, which is not the same as +// "free": fail closed. +func migrationLockFree(ctx context.Context, db DBConn, lockName string) (bool, error) { + var free sql.NullInt64 + if err := db.QueryRowContext(ctx, "SELECT IS_FREE_LOCK(?)", lockName).Scan(&free); err != nil { + return false, fmt.Errorf("probing migration lock %q: %w", lockName, err) + } + if !free.Valid { + return false, nil + } + return free.Int64 == 1, nil +} + +// doltIgnoreSeeded reports whether every canonical dolt_ignore pattern +// seedDoltIgnorePatterns would assert is already present. It is the read-only +// counterpart of that seed and shares its version gate: a pattern whose flip +// migration has not been reached yet is not expected, exactly as the seed +// would not insert it. mainVersionAtLeast is a lower bound on the main cursor +// that the caller has already established, so this read costs one round trip +// rather than three. +// +// Presence is judged on the pattern alone, never on its ignored value, because +// INSERT IGNORE would leave an explicit operator override (a pattern recorded +// with ignored=false) untouched. Reporting such a row as missing would send +// every invocation down the locked path forever. +// +// qualifier is the identifier-quoted database name selectTargetDatabase +// returned, or empty when the session was already on the target database and +// nothing needed selecting. dolt_ignore is a Dolt system table: it is NOT +// listed in information_schema.tables (verified against a live dolt sql-server +// — the cursor-table existence probe pattern from migrationSource.currentVersion +// would report it absent and disable the fast path permanently), so this read +// cannot get the full be-bv7x probe-before-act treatment at table granularity. +// What it has instead: selectTargetDatabase has already proved this session is +// on the target database (by reading DATABASE(), or by probing +// information_schema.schemata and selecting it), Dolt materializes dolt_ignore +// on every real database (live-verified), the name is stated explicitly +// whenever one was quoted for us rather than re-derived here, and +// IsTableNotExist below fails closed onto the locked path should that +// materialization assumption break. +func doltIgnoreSeeded(ctx context.Context, db DBConn, qualifier string, mainVersionAtLeast int) (bool, error) { + table := "dolt_ignore" + if qualifier != "" { + table = qualifier + ".dolt_ignore" + } + rows, err := db.QueryContext(ctx, "SELECT pattern FROM "+table) + if err != nil { + if dberrors.IsTableNotExist(err) { + return false, nil + } + return false, fmt.Errorf("reading dolt_ignore: %w", err) + } + defer rows.Close() + + present := make(map[string]struct{}) + for rows.Next() { + var pattern string + if err := rows.Scan(&pattern); err != nil { + return false, fmt.Errorf("reading dolt_ignore: %w", err) + } + present[pattern] = struct{}{} + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("reading dolt_ignore: %w", err) + } + + for _, pattern := range doltIgnorePatterns { + if _, ok := present[pattern]; !ok { + return false, nil + } + } + for _, gated := range versionGatedDoltIgnorePatterns { + if mainVersionAtLeast < gated.minMainVersion { + continue + } + if _, ok := present[gated.pattern]; !ok { + return false, nil + } + } + return true, nil +} diff --git a/internal/storage/schema/converged_test.go b/internal/storage/schema/converged_test.go new file mode 100644 index 000000000..68d3d16d2 --- /dev/null +++ b/internal/storage/schema/converged_test.go @@ -0,0 +1,826 @@ +package schema + +import ( + "bytes" + "context" + "database/sql" + "errors" + "io" + "os" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/debug" +) + +// expectConvergedFastPathMiss walks the REAL pre-lock probe on the shape the +// proxied CLI actually presents — a pinned session with no database selected — +// and declines at the first schema predicate: the database exists, the session +// is put on it, and the main cursor turns out to be one migration behind, so +// the locked path runs exactly as it did before the probe existed. +// +// It deliberately does NOT decline at statement 1. An earlier version of this +// helper primed DATABASE() to NULL and stopped there, which meant every +// retrofitted locked-path test in lock_test.go skipped the probe entirely — +// and the probe's inability to fire on that very shape (uow's schema-init pool +// opens with an EMPTY DSN database) went unnoticed through review. +func expectConvergedFastPathMiss(mock sqlmock.Sqlmock, database string) { + expectSessionPutOnDatabase(mock, database) + expectCursorProbe(mock, "schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()-1) +} + +// testDatabaseSelector stands in for the selector uow injects: it issues the +// same USE through the same connection and returns the same quoted name, so +// the mocked statement stream matches production byte for byte. +var testDatabaseSelector DatabaseSelector = func(ctx context.Context, conn DBConn, database string) (string, error) { + quoted := "`" + database + "`" + if _, err := conn.ExecContext(ctx, "USE "+quoted); err != nil { + return "", err + } + return quoted, nil +} + +// qualifiedDoltIgnore is the table expression the probe reads after a selector +// put the session on database; unqualifiedDoltIgnore is what it reads when the +// session was already there and nothing had to be selected. +func qualifiedDoltIgnore(database string) string { + return "`" + database + "`.dolt_ignore" +} + +const unqualifiedDoltIgnore = "dolt_ignore" + +// expectCurrentDatabase mocks the fast path's opening question: which database +// is this pinned session actually on? +func expectCurrentDatabase(mock sqlmock.Sqlmock, name any) { + mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE()")). + WillReturnRows(sqlmock.NewRows([]string{"DATABASE()"}).AddRow(name)) +} + +// expectDatabaseExistsProbe mocks the always-succeeding existence probe that +// must precede any USE (be-bv7x: a failing statement pins a Dolt session to +// its pre-statement catalog snapshot for the rest of its pooled life). +func expectDatabaseExistsProbe(mock sqlmock.Sqlmock, name string, exists bool) { + n := 0 + if exists { + n = 1 + } + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.schemata`). + WithArgs(name). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(n)) +} + +// expectSessionPutOnDatabase mocks the whole database-selection preamble on +// the hot-path shape: nothing selected, the database exists, USE it. +func expectSessionPutOnDatabase(mock sqlmock.Sqlmock, name string) { + expectCurrentDatabase(mock, nil) + expectDatabaseExistsProbe(mock, name, true) + mock.ExpectExec(regexp.QuoteMeta("USE `" + name + "`")). + WillReturnResult(sqlmock.NewResult(0, 0)) +} + +// expectNoMigrationWorkNeeded mocks migrationWorkNeeded on a fully upgraded +// database: both cursors at latest, both content_hash columns present, no +// custom-status/type backfill pending. +func expectNoMigrationWorkNeeded(mock sqlmock.Sqlmock) { + expectNoMigrationWorkNeededAtVersion(mock, LatestVersion()) +} + +// expectNoMigrationWorkNeededAtVersion is the same, with the main cursor +// reporting mainVersion (which may be ahead of this binary's latest). +func expectNoMigrationWorkNeededAtVersion(mock sqlmock.Sqlmock, mainVersion int) { + expectCursorProbe(mock, "schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", mainVersion) + expectCursorProbe(mock, "ignored_schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) + expectIgnoredSentinelProbes(mock, true) + expectContentHashColumnExists(mock) + expectContentHashColumnExists(mock) + expectScalar(mock, "SELECT COUNT(*) FROM custom_types", "count", 1) + expectScalar(mock, "SELECT COUNT(*) FROM custom_statuses", "count", 1) +} + +// expectDoltIgnoreRead mocks the read-only dolt_ignore probe, returning +// exactly the patterns given. The read is schema-qualified (dolt_ignore is a +// Dolt system table and is not listed in information_schema.tables, so it +// cannot carry the cursor tables' existence probe) and takes no cursor read of +// its own: migrationWorkNeeded has already proved the main cursor is at or +// past this binary's latest, which settles every version gate. +func expectDoltIgnoreRead(mock sqlmock.Sqlmock, table string, patterns []string) { + rows := sqlmock.NewRows([]string{"pattern"}) + for _, pattern := range patterns { + rows.AddRow(pattern) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT pattern FROM " + table)). + WillReturnRows(rows) +} + +// expectMigrationLockProbe mocks the fast path's LAST predicate: is anyone +// holding the database-scoped migration lock right now? +func expectMigrationLockProbe(mock sqlmock.Sqlmock, database string, free any) { + mock.ExpectQuery(regexp.QuoteMeta("SELECT IS_FREE_LOCK(?)")). + WithArgs(MigrationLockName(database)). + WillReturnRows(sqlmock.NewRows([]string{"free"}).AddRow(free)) +} + +// seededIgnorePatterns is every pattern seedDoltIgnorePatterns would assert on +// a database whose main cursor is at mainVersion. +func seededIgnorePatterns(mainVersion int) []string { + patterns := append([]string(nil), doltIgnorePatterns...) + for _, gated := range versionGatedDoltIgnorePatterns { + if mainVersion >= gated.minMainVersion { + patterns = append(patterns, gated.pattern) + } + } + return patterns +} + +// expectConvergedProbe mocks the whole fast path on a converged database whose +// session is already on it: nothing is selected, so the dolt_ignore read runs +// unqualified against the session database DATABASE() just confirmed. +func expectConvergedProbe(mock sqlmock.Sqlmock, database string) { + expectCurrentDatabase(mock, database) + expectNoMigrationWorkNeeded(mock) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) + expectMigrationLockProbe(mock, database, 1) +} + +func newMockConn(t *testing.T) (*sql.Conn, sqlmock.Sqlmock, func()) { + t.Helper() + + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + conn, err := db.Conn(context.Background()) + if err != nil { + db.Close() + t.Fatalf("pin mock connection: %v", err) + } + return conn, mock, func() { + conn.Close() + db.Close() + } +} + +func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) { + t.Helper() + + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db, mock +} + +// TestMigrateUpWithLockSkipsLockWhenAlreadyConverged is the point of the whole +// change: on a database that needs nothing, no GET_LOCK is issued at all. The +// mock is primed with the probe and nothing else, so a GET_LOCK — or the +// caller's locked preparation, whose CREATE DATABASE the fast path also skips +// — would surface as an unexpected statement and fail the call. +func TestMigrateUpWithLockSkipsLockWhenAlreadyConverged(t *testing.T) { + conn, mock, cleanup := newMockConn(t) + defer cleanup() + + expectConvergedProbe(mock, "testdb") + + prepared := 0 + applied, err := MigrateUpWithLock(context.Background(), conn, "testdb", + WithLockedPreparation("tcp:test", func(context.Context, *sql.Conn) (*FreshBootstrapHealCapability, error) { + prepared++ + return nil, nil + })) + if err != nil { + t.Fatalf("MigrateUpWithLock() error = %v, want nil (converged database must short-circuit before GET_LOCK)", err) + } + if applied != 0 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied) + } + if prepared != 0 { + t.Fatalf("locked preparation ran %d times on a converged database, want 0", prepared) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestMigrateUpWithLockSkipsLockOnAnUnselectedSession is the regression test +// for the reason this fast path did not work: uow.openAndInitSchema opens the +// schema-init pool with an EMPTY DSN database and only issues USE after +// GET_LOCK, so on every production seat's path DATABASE() was NULL when the +// probe ran. A probe that merely ASKS whether the session is on the target +// database therefore declined every single time and the lock was taken exactly +// as before — the whole change was a no-op on the only path that mattered. +// +// The probe must instead establish the database through the injected +// selector: prove it exists with a query that cannot fail, USE it, and only +// then evaluate the schema predicates — reading dolt_ignore under the name the +// selector quoted. +func TestMigrateUpWithLockSkipsLockOnAnUnselectedSession(t *testing.T) { + conn, mock, cleanup := newMockConn(t) + defer cleanup() + + expectSessionPutOnDatabase(mock, "testdb") + expectNoMigrationWorkNeeded(mock) + expectDoltIgnoreRead(mock, qualifiedDoltIgnore("testdb"), seededIgnorePatterns(LatestVersion())) + expectMigrationLockProbe(mock, "testdb", 1) + + prepared := 0 + applied, err := MigrateUpWithLock(context.Background(), conn, "testdb", + WithDatabaseSelector(testDatabaseSelector), + WithLockedPreparation("tcp:test", func(context.Context, *sql.Conn) (*FreshBootstrapHealCapability, error) { + prepared++ + return nil, nil + })) + if err != nil { + t.Fatalf("MigrateUpWithLock() error = %v, want nil", err) + } + if applied != 0 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied) + } + if prepared != 0 { + t.Fatalf("locked preparation ran %d times, want 0", prepared) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (an unselected session must reach the probe, not decline at DATABASE()): %v", err) + } +} + +// TestMigrateUpWithLockTakesLockWhenBehind pins the other half of the +// contract: a database one migration behind still goes through GET_LOCK and +// the full pass. Without this, a fast path that answered "converged" +// unconditionally would still pass the test above. +func TestMigrateUpWithLockTakesLockWhenBehind(t *testing.T) { + conn, mock, cleanup := newMockConn(t) + defer cleanup() + + // Fast path: right database, but the main cursor is behind -> work needed. + expectCurrentDatabase(mock, "testdb") + expectCursorProbe(mock, "schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()-1) + + lockName := MigrationLockName("testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + expectOnePendingMigration(t, mock) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + applied, err := MigrateUpWithLock(context.Background(), conn, "testdb") + if err != nil { + t.Fatalf("MigrateUpWithLock() error = %v", err) + } + if applied != 1 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 1", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestMigrateUpWithLockKeepsLockWithBootstrapHeal pins that a caller carrying +// fresh-bootstrap reset authority never enters the fast path: the very first +// statement it issues is GET_LOCK, so the #5012 bootstrap sequence is +// byte-identical to what it was before the probe existed. +func TestMigrateUpWithLockKeepsLockWithBootstrapHeal(t *testing.T) { + conn, mock, cleanup := newMockConn(t) + defer cleanup() + + lockName := MigrationLockName("testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + expectDirtyGuardRefusal(t, mock) + expectFreshBootstrapIdentityMatch(mock) + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")). + WillReturnRows(sqlmock.NewRows([]string{"status"})) + expectOnePendingMigration(t, mock) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + applied, err := MigrateUpWithLock(context.Background(), conn, "testdb", + WithFreshBootstrapHeal(testFreshBootstrapHealCapability(), testBootstrapEndpoint)) + if err != nil { + t.Fatalf("MigrateUpWithLock() error = %v", err) + } + if applied != 1 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 1", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestAlreadyConvergedFallsThroughOnAMissingDatabase is the fresh-bootstrap +// case. The database does not exist yet, so the probe must decline WITHOUT +// issuing the USE that would fail — a failing statement pins a Dolt session to +// its pre-statement catalog snapshot for the rest of its pooled life (be-bv7x) +// — and hand the open to the locked path, whose bare CREATE DATABASE +// arbitrates creation and issues the #5012 heal capability. +func TestAlreadyConvergedFallsThroughOnAMissingDatabase(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, nil) + expectDatabaseExistsProbe(mock, "testdb", false) + + converged, err := alreadyConverged(context.Background(), db, "testdb", testDatabaseSelector) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatal("alreadyConverged() = true on a database that does not exist, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (a missing database must decline before USE, not after): %v", err) + } +} + +// TestAlreadyConvergedWithoutASelectorNeverSelects pins the default for +// callers that inject nothing. The probe has no legal way to issue USE on its +// own — selecting a database and quoting an identifier belong to the DDL +// repository, which this package cannot import without breaking that package's +// test build — so a session that is not already on the target is simply +// declined, exactly as it was before the selector existed. That is the whole +// behavior for internal/storage/dolt, whose pool DSN already names the +// database; the mock is primed with nothing after DATABASE(), so an existence +// probe or a USE would surface as an unexpected statement. +func TestAlreadyConvergedWithoutASelectorNeverSelects(t *testing.T) { + for _, tt := range []struct { + name string + current any + }{ + {name: "no database selected", current: nil}, + {name: "different database", current: "otherdb"}, + } { + t.Run(tt.name, func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, tt.current) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatal("alreadyConverged() = true without a selector and off the target database, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (an uninjected probe must not probe or select): %v", err) + } + }) + } +} + +// TestAlreadyConvergedFailsClosedOnDatabaseSelection covers the ways putting +// the session on the target database can go wrong. All must surface as "not +// converged" so MigrateUpWithLock takes the lock. +func TestAlreadyConvergedFailsClosedOnDatabaseSelection(t *testing.T) { + t.Run("empty database name", func(t *testing.T) { + db, mock := newMockDB(t) + + converged, err := alreadyConverged(context.Background(), db, "", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatal("alreadyConverged() = true with no database name, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (an empty name must issue no statements at all): %v", err) + } + }) + + t.Run("current database unreadable", func(t *testing.T) { + db, mock := newMockDB(t) + + mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE()")). + WillReturnError(sql.ErrConnDone) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err == nil { + t.Fatal("alreadyConverged() error = nil, want the DATABASE() read failure") + } + if converged { + t.Fatal("alreadyConverged() = true on an unreadable session, want false") + } + }) + + t.Run("selector's use fails", func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, nil) + expectDatabaseExistsProbe(mock, "testdb", true) + mock.ExpectExec(regexp.QuoteMeta("USE `testdb`")). + WillReturnError(errors.New("database dropped underneath us")) + + converged, err := alreadyConverged(context.Background(), db, "testdb", testDatabaseSelector) + if err == nil { + t.Fatal("alreadyConverged() error = nil, want the USE failure") + } + if converged { + t.Fatal("alreadyConverged() = true after a failed USE, want false") + } + }) + + t.Run("selector rejects the name", func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, nil) + expectDatabaseExistsProbe(mock, "bad`name", true) + rejecting := DatabaseSelector(func(context.Context, DBConn, string) (string, error) { + return "", errors.New("invalid identifier") + }) + + converged, err := alreadyConverged(context.Background(), db, "bad`name", rejecting) + if err == nil { + t.Fatal("alreadyConverged() error = nil, want the identifier rejection") + } + if converged { + t.Fatal("alreadyConverged() = true on a name the selector refused, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (a refused name must issue no USE): %v", err) + } + }) + + // A selector that reports success without handing back a quoted name would + // leave the dolt_ignore read to guess at qualification. Nothing downstream + // may run on that: refuse it here. + t.Run("selector returns no quoted name", func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, nil) + expectDatabaseExistsProbe(mock, "testdb", true) + silent := DatabaseSelector(func(context.Context, DBConn, string) (string, error) { + return "", nil + }) + + converged, err := alreadyConverged(context.Background(), db, "testdb", silent) + if err == nil { + t.Fatal("alreadyConverged() error = nil, want the empty-name refusal") + } + if converged { + t.Fatal("alreadyConverged() = true on a selector that quoted nothing, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) +} + +// TestAlreadyConvergedDeclinesWhileTheMigrationLockIsHeld is the containment +// for the mutual-exclusion hole. migrationWorkNeeded only covers the FRONT +// half of MigrateUp: once the version cursors and content_hash columns have +// landed it reports "nothing pending" while the pass is still running +// backfills, dependency/aux PK rekeys, the ignored series, and the schema +// commit. Every predicate above can therefore be satisfied by a database that +// a peer is actively rewriting — a fleet-wide binary roll is exactly that +// window — so a held migration lock must veto the fast path. +func TestAlreadyConvergedDeclinesWhileTheMigrationLockIsHeld(t *testing.T) { + for _, tt := range []struct { + name string + free any + }{ + {name: "lock held by a peer", free: 0}, + {name: "server will not say", free: nil}, + } { + t.Run(tt.name, func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, "testdb") + expectNoMigrationWorkNeeded(mock) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) + expectMigrationLockProbe(mock, "testdb", tt.free) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatal("alreadyConverged() = true while the migration lock is not provably free, want false: a peer may be mid-pass rewriting tables this caller is about to read") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + } +} + +// TestAlreadyConvergedProbesTheLockLast pins the ordering the containment is +// cheap because of: the lock probe is the LAST term, so the ordinary +// not-converged answers cost nothing extra. A database one migration behind +// must never reach IS_FREE_LOCK at all — the mock is not primed for it, so a +// reordered probe surfaces as an unexpected statement. +func TestAlreadyConvergedProbesTheLockLast(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, "testdb") + expectCursorProbe(mock, "schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()-1) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatal("alreadyConverged() = true with the main cursor behind, want false") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestAlreadyConvergedRejectsUnderSeededDoltIgnore is the case +// seedDoltIgnorePatterns exists for: an out-of-band-materialized database +// arrives with its cursors at-latest and the ignore patterns missing. The fast +// path must refuse it so the locked pass can heal and commit the seed. +func TestAlreadyConvergedRejectsUnderSeededDoltIgnore(t *testing.T) { + for _, missing := range seededIgnorePatterns(LatestVersion()) { + t.Run(missing, func(t *testing.T) { + db, mock := newMockDB(t) + + var present []string + for _, pattern := range seededIgnorePatterns(LatestVersion()) { + if pattern != missing { + present = append(present, pattern) + } + } + expectCurrentDatabase(mock, "testdb") + expectNoMigrationWorkNeeded(mock) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, present) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatalf("alreadyConverged() = true with dolt_ignore pattern %q missing, want false", missing) + } + }) + } +} + +// TestDoltIgnoreSeededQualifiesOnlyWhatWasSelected pins where the read's +// database name comes from. When a selector put the session on the database it +// hands back the quoted name and the read states it; when the session was +// already there, DATABASE() itself was the proof and the read is +// session-scoped. The probe never re-derives or re-quotes a name of its own — +// that rule is what let the domain/db import go. +func TestDoltIgnoreSeededQualifiesOnlyWhatWasSelected(t *testing.T) { + for _, tt := range []struct { + name string + qualifier string + table string + }{ + {name: "selected", qualifier: "`testdb`", table: qualifiedDoltIgnore("testdb")}, + {name: "already on the database", qualifier: "", table: unqualifiedDoltIgnore}, + } { + t.Run(tt.name, func(t *testing.T) { + db, mock := newMockDB(t) + expectDoltIgnoreRead(mock, tt.table, seededIgnorePatterns(LatestVersion())) + + seeded, err := doltIgnoreSeeded(context.Background(), db, tt.qualifier, mainSource.latest()) + if err != nil { + t.Fatalf("doltIgnoreSeeded() error = %v", err) + } + if !seeded { + t.Fatal("doltIgnoreSeeded() = false on a fully seeded database, want true") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + } +} + +// TestDoltIgnoreSeededHonorsTheVersionGate pins both sides of the gate a +// version-gated pattern carries. Below its flip migration the pattern is not +// expected at all (seedDoltIgnorePatterns would not insert it either, and +// asserting it on a pre-flip database would strand a still-tracked table +// behind a suppressing pattern); at or past it, its absence is a miss. +func TestDoltIgnoreSeededHonorsTheVersionGate(t *testing.T) { + for _, gated := range versionGatedDoltIgnorePatterns { + t.Run(gated.pattern, func(t *testing.T) { + ungated := append([]string(nil), doltIgnorePatterns...) + + t.Run("below the gate", func(t *testing.T) { + db, mock := newMockDB(t) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, ungated) + + seeded, err := doltIgnoreSeeded(context.Background(), db, "", gated.minMainVersion-1) + if err != nil { + t.Fatalf("doltIgnoreSeeded() error = %v", err) + } + if !seeded { + t.Fatalf("doltIgnoreSeeded() = false below the %q gate (main version %d < %d), want true: the flip migration has not run, so the pattern is not expected yet", + gated.pattern, gated.minMainVersion-1, gated.minMainVersion) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (the gate must be settled without a cursor re-read): %v", err) + } + }) + + t.Run("at the gate", func(t *testing.T) { + db, mock := newMockDB(t) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, ungated) + + seeded, err := doltIgnoreSeeded(context.Background(), db, "", gated.minMainVersion) + if err != nil { + t.Fatalf("doltIgnoreSeeded() error = %v", err) + } + if seeded { + t.Fatalf("doltIgnoreSeeded() = true with %q missing at main version %d, want false", gated.pattern, gated.minMainVersion) + } + }) + }) + } +} + +// TestAlreadyConvergedAcceptsOverriddenIgnorePattern pins that presence is +// judged on the pattern alone. An operator override (the row exists with +// ignored=false) is what INSERT IGNORE would leave untouched, so treating it +// as missing would send every invocation down the locked path forever — the +// saturation this change removes. +func TestAlreadyConvergedAcceptsOverriddenIgnorePattern(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, "testdb") + expectNoMigrationWorkNeeded(mock) + // The probe selects patterns only; the mock never offers the ignored + // column, so a query that filtered on it would not match this expectation. + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) + expectMigrationLockProbe(mock, "testdb", 1) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if !converged { + t.Fatal("alreadyConverged() = false on a fully seeded database, want true") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestAlreadyConvergedOnForwardSkew documents — rather than changes — +// what the fast path does when the database is AHEAD of this binary. MigrateUp +// itself treats "cursor >= my latest" as nothing to do and returns (0, nil) +// without ever consulting the lock, so the fast path returning converged is +// the same answer for the same reason. Forward drift is a separate question, +// asked by CheckForwardDrift at the caller (cmd/bd/main.go), and this probe +// must not start answering it: doing so here would turn a diagnosable skew +// error into an unexplained lock acquisition on every invocation. +func TestAlreadyConvergedOnForwardSkew(t *testing.T) { + db, mock := newMockDB(t) + + ahead := LatestVersion() + 5 + expectCurrentDatabase(mock, "testdb") + expectNoMigrationWorkNeededAtVersion(mock, ahead) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(ahead)) + expectMigrationLockProbe(mock, "testdb", 1) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if !converged { + t.Fatal("alreadyConverged() = false on a forward-skewed database, want true: MigrateUp's own gate reports no work at or past latest, so the locked pass would return (0, nil) too") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestAlreadyConvergedFailsClosedOnUnreadableState pins the fail-closed +// contract from both sides. A cursor probe that errors is swallowed by +// atLatest into "work needed", so the answer is a plain not-converged; a +// dolt_ignore read that errors surfaces the error. MigrateUpWithLock treats +// anything but (true, nil) as "take the lock", so both end at the locked path. +func TestAlreadyConvergedFailsClosedOnUnreadableState(t *testing.T) { + t.Run("cursor probe fails", func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, "testdb") + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). + WillReturnError(sql.ErrConnDone) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err != nil { + t.Fatalf("alreadyConverged() error = %v", err) + } + if converged { + t.Fatal("alreadyConverged() = true on an unreadable cursor, want false") + } + }) + + t.Run("dolt_ignore read fails", func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, "testdb") + expectNoMigrationWorkNeeded(mock) + mock.ExpectQuery(regexp.QuoteMeta("SELECT pattern FROM dolt_ignore")). + WillReturnError(sql.ErrConnDone) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err == nil { + t.Fatal("alreadyConverged() error = nil, want the dolt_ignore read failure") + } + if converged { + t.Fatal("alreadyConverged() = true on a failed dolt_ignore read, want false") + } + }) + + t.Run("lock probe fails", func(t *testing.T) { + db, mock := newMockDB(t) + + expectCurrentDatabase(mock, "testdb") + expectNoMigrationWorkNeeded(mock) + expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) + mock.ExpectQuery(regexp.QuoteMeta("SELECT IS_FREE_LOCK(?)")). + WillReturnError(sql.ErrConnDone) + + converged, err := alreadyConverged(context.Background(), db, "testdb", nil) + if err == nil { + t.Fatal("alreadyConverged() error = nil, want the lock probe failure") + } + if converged { + t.Fatal("alreadyConverged() = true on a failed lock probe, want false") + } + }) + +} + +// TestMigrateUpWithLockLogsAnUnavailableFastPath pins the diagnostic on the +// one branch that is otherwise invisible. The probe is advisory and fails +// closed, so a probe that has silently stopped working looks exactly like a +// probe that correctly declined — and the symptom (every seat back on the +// saturated GET_LOCK) shows up nowhere near the cause. Discarding the error +// without a word is what made the unreachable-fast-path defect survive +// review; say so where BD_DEBUG and -v can see it. +func TestMigrateUpWithLockLogsAnUnavailableFastPath(t *testing.T) { + conn, mock, cleanup := newMockConn(t) + defer cleanup() + + mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE()")). + WillReturnError(errors.New("probe transport broke")) + lockName := MigrationLockName("testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + mock.ExpectExec(regexp.QuoteMeta("INSERT IGNORE INTO dolt_ignore VALUES (?, true)")). + WillReturnError(errors.New("stop here")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + logged := captureStderr(t, func() { + debug.SetVerbose(true) + defer debug.SetVerbose(false) + if _, err := MigrateUpWithLock(context.Background(), conn, "testdb"); err == nil { + t.Fatal("MigrateUpWithLock() error = nil, want the locked path's failure") + } + }) + + if !strings.Contains(logged, "convergence fast path unavailable") || + !strings.Contains(logged, "probe transport broke") { + t.Fatalf("debug output = %q, want the discarded probe error reported", logged) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (a broken probe must still fail closed onto the lock): %v", err) + } +} + +// captureStderr redirects os.Stderr — where debug.Logf writes — for the +// duration of fn and returns what was written to it. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + saved := os.Stderr + os.Stderr = w + done := make(chan string, 1) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + done <- buf.String() + }() + + func() { + defer func() { + os.Stderr = saved + w.Close() + }() + fn() + }() + return <-done +} diff --git a/internal/storage/schema/lock.go b/internal/storage/schema/lock.go index aa29b6635..a3b89205e 100644 --- a/internal/storage/schema/lock.go +++ b/internal/storage/schema/lock.go @@ -10,6 +10,8 @@ import ( "strings" "sync/atomic" "time" + + "github.com/steveyegge/beads/internal/debug" ) const ( @@ -55,6 +57,7 @@ type MigrateLockOption func(*migrateLockOptions) type migrateLockOptions struct { freshBootstrapHeal *freshBootstrapHealRequest lockedPreparation *lockedPreparationRequest + databaseSelector DatabaseSelector } type freshBootstrapHealRequest struct { @@ -169,6 +172,38 @@ func WithLockedPreparation(endpoint string, fn LockedPreparation) MigrateLockOpt } } +// DatabaseSelector puts a pinned session on databaseName for the pre-lock +// convergence probe and returns the identifier-quoted name the probe uses to +// schema-qualify its reads. Returning an error, like everything else in the +// probe, fails closed onto the locked path. +// +// It is injected rather than implemented here on purpose. Selecting a database +// and quoting an identifier belong to the DDL repository in +// internal/storage/domain/db — but that package's own test suite migrates a +// scratch database with this package, so a schema -> domain/db import compiles +// as a library and then fails the domain/db TEST build with "import cycle not +// allowed in test". Inverting the dependency lets the one caller that needs +// the probe to reach an unselected session hand in the repository's real +// implementation, so the fast path still issues preparation's exact statement +// without this package depending on it. +type DatabaseSelector func(ctx context.Context, conn DBConn, databaseName string) (quotedName string, err error) + +// WithDatabaseSelector lets the convergence probe put the pinned session on +// the target database itself. +// +// Without it the probe cannot issue USE, so it declines unless the session is +// already on databaseName — correct for callers whose pool DSN already names +// the database (internal/storage/dolt). The proxied CLI open is the caller +// that needs it: it pins its schema-init pool with an EMPTY DSN database and +// USEs only inside the locked preparation, so an uninjected probe would read +// NULL from DATABASE() and decline on every single invocation, which is +// exactly the defect this option exists to close. +func WithDatabaseSelector(fn DatabaseSelector) MigrateLockOption { + return func(o *migrateLockOptions) { + o.databaseSelector = fn + } +} + // MigrateUpWithLock serializes schema migrations for a single Dolt sql-server // database. conn must be a pinned *sql.Conn because MySQL/Dolt named locks are // session-scoped; GET_LOCK, migrations, and RELEASE_LOCK must run on the same @@ -181,6 +216,38 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string, opt(&o) } + // Steady-state fast path. In the overwhelmingly common case the database + // is already at this binary's schema and the whole locked pass is a probe + // that finds nothing to do — but running that probe under the + // database-scoped GET_LOCK serialized every bd invocation on a shared Dolt + // server behind every other one (96.7% lock saturation on an 18-seat rig; + // see alreadyConverged). Answer the same question first, without the lock, + // and take the lock only on the path that can actually migrate. + // + // Skipped when the caller carries fresh-bootstrap heal authority: that + // capability is issued only to the init that just created the database, so + // there is nothing converged to detect and the bootstrap path stays + // exactly as it was. A caller's locked preparation is skipped only once + // alreadyConverged has proved the database already existed and put the + // session on it — preparation's CREATE DATABASE would then have failed + // with "database exists" and captured no authority, and its USE is the + // statement alreadyConverged itself issued. + if o.freshBootstrapHeal == nil { + converged, convergedErr := alreadyConverged(ctx, conn, databaseName, o.databaseSelector) + switch { + case convergedErr != nil: + // Advisory only: the probe fails closed onto the locked path + // below, so this is never fatal. But a silently discarded error is + // a fast path that has quietly stopped firing — the exact failure + // mode that leaves the GET_LOCK saturation this exists to remove + // looking like a mystery. Say so where BD_DEBUG/-v can see it. + debug.Logf("schema: convergence fast path unavailable for %q, taking the migration lock: %v\n", + databaseName, convergedErr) + case converged: + return 0, nil + } + } + lockName := MigrationLockName(databaseName) if err := AcquireMigrationLock(ctx, conn, lockName); err != nil { return 0, err diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index f77d5a8cb..5209578cb 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -77,6 +77,7 @@ func TestMigrateUpWithLockUsesDatabaseScopedLockOnly(t *testing.T) { defer conn.Close() lockName := MigrationLockName("testdb") + expectConvergedFastPathMiss(mock, "testdb") mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). WithArgs(lockName, migrationLockAcquireTimeoutSeconds). WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) @@ -85,7 +86,7 @@ func TestMigrateUpWithLockUsesDatabaseScopedLockOnly(t *testing.T) { WithArgs(lockName). WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) - applied, err := MigrateUpWithLock(ctx, conn, "testdb") + applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithDatabaseSelector(testDatabaseSelector)) if err != nil { t.Fatalf("MigrateUpWithLock() error = %v", err) } @@ -112,6 +113,7 @@ func TestMigrateUpWithLockPreparationErrorReleasesAndJoinsReleaseFailure(t *test defer conn.Close() lockName := MigrationLockName("testdb") + expectConvergedFastPathMiss(mock, "testdb") preparationErr := errors.New("bootstrap preparation failed") mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). WithArgs(lockName, migrationLockAcquireTimeoutSeconds). @@ -121,7 +123,7 @@ func TestMigrateUpWithLockPreparationErrorReleasesAndJoinsReleaseFailure(t *test WillReturnError(errors.New("release failed")) called := 0 - applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithLockedPreparation("tcp:test", func(context.Context, *sql.Conn) (*FreshBootstrapHealCapability, error) { + applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithDatabaseSelector(testDatabaseSelector), WithLockedPreparation("tcp:test", func(context.Context, *sql.Conn) (*FreshBootstrapHealCapability, error) { called++ return nil, preparationErr })) @@ -165,6 +167,7 @@ func TestMigrateUpWithLockMigrationErrorNotMaskedByReleaseFailure(t *testing.T) defer conn.Close() lockName := MigrationLockName("testdb") + expectConvergedFastPathMiss(mock, "testdb") mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). WithArgs(lockName, migrationLockAcquireTimeoutSeconds). WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) @@ -178,7 +181,7 @@ func TestMigrateUpWithLockMigrationErrorNotMaskedByReleaseFailure(t *testing.T) WithArgs(lockName). WillReturnError(releaseCause) - applied, err := MigrateUpWithLock(ctx, conn, "testdb") + applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithDatabaseSelector(testDatabaseSelector)) if applied != 0 { t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied) } @@ -579,6 +582,7 @@ func TestMigrateUpWithLockDirtyGuardStaysFatalWithoutHeal(t *testing.T) { defer conn.Close() lockName := MigrationLockName("testdb") + expectConvergedFastPathMiss(mock, "testdb") mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). WithArgs(lockName, migrationLockAcquireTimeoutSeconds). WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) @@ -587,7 +591,7 @@ func TestMigrateUpWithLockDirtyGuardStaysFatalWithoutHeal(t *testing.T) { WithArgs(lockName). WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) - applied, err := MigrateUpWithLock(ctx, conn, "testdb") + applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithDatabaseSelector(testDatabaseSelector)) if applied != 0 { t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied) } diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 8c901ac7c..c1170031d 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -163,6 +163,36 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { }, nil } +// selectProbeDatabase lets schema's pre-lock convergence probe reach the +// database on a session that is not yet on one. openAndInitSchema pins its +// schema-init pool with an EMPTY DSN database, so without this the probe reads +// NULL from DATABASE(), declines, and every invocation queues on the +// server-wide migration lock it exists to skip. +// +// The USE MUST remain the DDL repository's own UseDatabase — the exact +// statement prepareBootstrap issues below. That identity is the reason the +// probe may skip locked preparation when it reports converged: preparation's +// contribution on an existing database is precisely this USE (its bare CREATE +// DATABASE can only have failed with "database exists" and captured no heal +// authority). Reimplementing the statement here, or letting the two quote +// identifiers differently, would silently break that argument. +// +// It is injected rather than reached for from schema/: domain/db's own test +// suite migrates a scratch database with schema, so a schema -> domain/db +// import compiles as a library and then fails the domain/db TEST build with +// "import cycle not allowed in test". uow already depends on both, so the +// dependency is legal exactly here. +func selectProbeDatabase(ctx context.Context, conn schema.DBConn, database string) (string, error) { + quoted, err := db.QuoteIdentifier(database) + if err != nil { + return "", err + } + if err := db.NewDDLSQLRepository(conn).UseDatabase(ctx, database); err != nil { + return "", err + } + return quoted, nil +} + func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error { bo := backoff.NewExponentialBackOff() bo.InitialInterval = 25 * time.Millisecond @@ -170,135 +200,172 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error // full cold-start migration pass (every migration + a Dolt commit each), // not just a transient blip — it grows as migrations accumulate. bo.MaxElapsedTime = 60 * time.Second - // Fresh-bootstrap ownership proof for the #4566 guard self-heal - // (gastownhall/beads#5012): the first attempt issues a bare CREATE - // DATABASE (no IF NOT EXISTS), so the server arbitrates creation - // atomically — success proves THIS init created the database, and an - // already-exists refusal (1007) proves it did not. Only the proven - // creator captures and passes a one-shot FreshBootstrapHealCapability: on - // a database this init - // created, a retry attempt that finds dirty tables can only be seeing a - // previous attempt's own half-applied migration step (a session that - // died between a step's SQL and its per-step Dolt commit — the "busy - // buffer" shape on a loaded shared server), never pre-existing user - // data, so the migrate call may discard that debris and converge instead - // of failing the init permanently. The capability also binds that proof to - // the endpoint, server UUID, database, and initial HEAD, preventing a stale - // creator from resetting a drop/recreated name. A concurrent initializer - // that loses the create race keeps the guard's refusal unchanged. `created` - // is sticky across retry attempts for availability re-creation; reset - // authority is captured only in the exact CREATE-winning attempt and is - // never inferred again from probing or CREATE IF NOT EXISTS. - created := false - var bootstrapHeal *schema.FreshBootstrapHealCapability - prepareBootstrap := func(ctx context.Context, conn *sql.Conn) (*schema.FreshBootstrapHealCapability, error) { - ddl := db.NewDDLSQLRepository(conn) - justCreated := false - if created { - // Re-assert on retries so a database dropped between attempts - // (e.g. a concurrent clean-databases) is recreated rather than - // failing the USE below. - if err := ddl.CreateDatabaseIfNotExists(ctx, database); err != nil { - return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: creating database: %w", err)} - } - } else { - switch err := ddl.CreateDatabase(ctx, database); { - case err == nil: - created = true - justCreated = true - case isDatabaseExistsError(err): - // Pre-existing (or a concurrent initializer won the create - // race): not ours, heal stays off. - case isSerializationError(err): - // Only the initial bare CREATE preserves its historical - // serialization retry classification. The later sticky CREATE, - // USE, and identity capture remain permanent regardless of - // their nested driver error. - return nil, &bootstrapPreparationError{ - err: fmt.Errorf("uow: creating database: %w", err), - retryable: true, - } - default: - return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: creating database: %w", err)} - } - } - if err := ddl.UseDatabase(ctx, database); err != nil { - return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: switching to database: %w", err)} - } - if justCreated { - // Capture authority only in the same attempt that won the exact bare - // CREATE. A later retry may re-create a missing database for - // availability, but it must never infer ownership of a replacement - // incarnation from CREATE IF NOT EXISTS. - var err error - bootstrapHeal, err = schema.CaptureFreshBootstrapHealCapability( - ctx, conn, p.serverEndpoint, database, - ) - if err != nil { - return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: capture fresh database identity: %w", err)} - } - } - return bootstrapHeal, nil - } + // One preparer per initSchema call carries the sticky fresh-bootstrap state + // (created/heal) across every backoff attempt; see bootstrapPreparer. + preparer := &bootstrapPreparer{provider: p, database: database} return backoff.Retry(func() error { - conn, err := p.db.Conn(ctx) - if err != nil { - if isSerializationError(err) { - return fmt.Errorf("uow: pin connection: %w", err) - } - return backoff.Permanent(fmt.Errorf("uow: pin connection: %w", err)) - } - defer conn.Close() - - ddl := db.NewDDLSQLRepository(conn) - if p.teamServer { - if err := ddl.UseDatabase(ctx, database); err != nil { - if isSerializationError(err) { - return fmt.Errorf("uow: switching to database: %w", err) - } - return backoff.Permanent(fmt.Errorf( - "uow: database %q not found — the schema is managed by beads-team-server; ask your operator to run 'bts init' first: %w", - database, err)) - } - if err := checkTeamServerSchema(ctx, conn, database); err != nil { - if isSerializationError(err) { - return fmt.Errorf("uow: team-server schema check: %w", err) - } - return backoff.Permanent(err) - } - // Identity is checked only after the schema check proves the - // metadata table exists at this binary's version. - if err := checkTeamServerIdentity(ctx, conn, database, p.expectedProjectID); err != nil { - if isSerializationError(err) { - return fmt.Errorf("uow: team-server identity check: %w", err) - } - return backoff.Permanent(err) - } - return nil - } - if p.preview { - // Preview open: attach to the database that is already there and - // stop. No CreateDatabase, no MigrateUpWithLock — a --dry-run or - // --inspect that migrated the workspace before rendering its plan - // would be the exact side effect the flag exists to prevent. - if err := ddl.UseDatabase(ctx, database); err != nil { - if isSerializationError(err) { - return fmt.Errorf("uow: switching to database: %w", err) - } - return backoff.Permanent(fmt.Errorf( - "uow: database %q not found — preview commands (--dry-run, --inspect) never create or migrate a database; run the command without the preview flag first: %w", - database, err)) - } - return nil - } - if _, err := schema.MigrateUpWithLock(ctx, conn, database, - schema.WithLockedPreparation(p.serverEndpoint, prepareBootstrap)); err != nil { - return classifyInitSchemaError(err) - } - return nil + return p.initSchemaAttempt(ctx, database, preparer) }, backoff.WithContext(bo, ctx)) } +// initSchemaAttempt runs a single backoff attempt of initSchema. It pins a +// connection and fans out to the team-server, preview, or migrating open path. +// Each path returns a bare error for a serialization retry and backoff.Permanent +// for a terminal failure. +func (p *doltSQLProvider) initSchemaAttempt(ctx context.Context, database string, preparer *bootstrapPreparer) error { + conn, err := p.db.Conn(ctx) + if err != nil { + if isSerializationError(err) { + return fmt.Errorf("uow: pin connection: %w", err) + } + return backoff.Permanent(fmt.Errorf("uow: pin connection: %w", err)) + } + defer conn.Close() + + if p.teamServer { + return p.verifyTeamServerSchema(ctx, conn, database) + } + if p.preview { + return p.attachPreviewDatabase(ctx, conn, database) + } + if _, err := schema.MigrateUpWithLock(ctx, conn, database, + schema.WithDatabaseSelector(selectProbeDatabase), + schema.WithLockedPreparation(p.serverEndpoint, preparer.prepare)); err != nil { + return classifyInitSchemaError(err) + } + return nil +} + +// verifyTeamServerSchema is the team-server open path: the schema is owned by +// beads-team-server (bts), so bd never creates the database or migrates. It +// attaches to the existing database and verifies the schema version, then the +// project identity — identity is checked only after the schema check proves the +// metadata table exists at this binary's version. +func (p *doltSQLProvider) verifyTeamServerSchema(ctx context.Context, conn *sql.Conn, database string) error { + ddl := db.NewDDLSQLRepository(conn) + if err := ddl.UseDatabase(ctx, database); err != nil { + if isSerializationError(err) { + return fmt.Errorf("uow: switching to database: %w", err) + } + return backoff.Permanent(fmt.Errorf( + "uow: database %q not found — the schema is managed by beads-team-server; ask your operator to run 'bts init' first: %w", + database, err)) + } + if err := checkTeamServerSchema(ctx, conn, database); err != nil { + if isSerializationError(err) { + return fmt.Errorf("uow: team-server schema check: %w", err) + } + return backoff.Permanent(err) + } + if err := checkTeamServerIdentity(ctx, conn, database, p.expectedProjectID); err != nil { + if isSerializationError(err) { + return fmt.Errorf("uow: team-server identity check: %w", err) + } + return backoff.Permanent(err) + } + return nil +} + +// attachPreviewDatabase is the preview open path: attach to the database that is +// already there and stop. No CreateDatabase, no MigrateUpWithLock — a --dry-run +// or --inspect that migrated the workspace before rendering its plan would be the +// exact side effect the flag exists to prevent. +func (p *doltSQLProvider) attachPreviewDatabase(ctx context.Context, conn *sql.Conn, database string) error { + ddl := db.NewDDLSQLRepository(conn) + if err := ddl.UseDatabase(ctx, database); err != nil { + if isSerializationError(err) { + return fmt.Errorf("uow: switching to database: %w", err) + } + return backoff.Permanent(fmt.Errorf( + "uow: database %q not found — preview commands (--dry-run, --inspect) never create or migrate a database; run the command without the preview flag first: %w", + database, err)) + } + return nil +} + +// bootstrapPreparer carries the sticky fresh-bootstrap state across the backoff +// retry attempts of a single initSchema call and runs as MigrateUpWithLock's +// locked-preparation callback (see prepare). +// +// Fresh-bootstrap ownership proof for the #4566 guard self-heal +// (gastownhall/beads#5012): the first attempt issues a bare CREATE DATABASE (no +// IF NOT EXISTS), so the server arbitrates creation atomically — success proves +// THIS init created the database, and an already-exists refusal (1007) proves it +// did not. Only the proven creator captures and passes a one-shot +// FreshBootstrapHealCapability: on a database this init created, a retry attempt +// that finds dirty tables can only be seeing a previous attempt's own +// half-applied migration step (a session that died between a step's SQL and its +// per-step Dolt commit — the "busy buffer" shape on a loaded shared server), +// never pre-existing user data, so the migrate call may discard that debris and +// converge instead of failing the init permanently. The capability also binds +// that proof to the endpoint, server UUID, database, and initial HEAD, preventing +// a stale creator from resetting a drop/recreated name. A concurrent initializer +// that loses the create race keeps the guard's refusal unchanged. +type bootstrapPreparer struct { + provider *doltSQLProvider + database string + // created is sticky across retry attempts for availability re-creation; + // reset authority is captured only in the exact CREATE-winning attempt and is + // never inferred again from probing or CREATE IF NOT EXISTS. + created bool + // heal is the one-shot capability captured in the CREATE-winning attempt and + // re-returned unchanged on later attempts. + heal *schema.FreshBootstrapHealCapability +} + +// prepare is MigrateUpWithLock's locked-preparation callback. It (re)creates and +// selects the target database and, only in the attempt that won the bare CREATE, +// captures the fresh-bootstrap heal capability. See bootstrapPreparer. +func (b *bootstrapPreparer) prepare(ctx context.Context, conn *sql.Conn) (*schema.FreshBootstrapHealCapability, error) { + ddl := db.NewDDLSQLRepository(conn) + justCreated := false + if b.created { + // Re-assert on retries so a database dropped between attempts + // (e.g. a concurrent clean-databases) is recreated rather than + // failing the USE below. + if err := ddl.CreateDatabaseIfNotExists(ctx, b.database); err != nil { + return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: creating database: %w", err)} + } + } else { + switch err := ddl.CreateDatabase(ctx, b.database); { + case err == nil: + b.created = true + justCreated = true + case isDatabaseExistsError(err): + // Pre-existing (or a concurrent initializer won the create + // race): not ours, heal stays off. + case isSerializationError(err): + // Only the initial bare CREATE preserves its historical + // serialization retry classification. The later sticky CREATE, + // USE, and identity capture remain permanent regardless of + // their nested driver error. + return nil, &bootstrapPreparationError{ + err: fmt.Errorf("uow: creating database: %w", err), + retryable: true, + } + default: + return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: creating database: %w", err)} + } + } + if err := ddl.UseDatabase(ctx, b.database); err != nil { + return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: switching to database: %w", err)} + } + if justCreated { + // Capture authority only in the same attempt that won the exact bare + // CREATE. A later retry may re-create a missing database for + // availability, but it must never infer ownership of a replacement + // incarnation from CREATE IF NOT EXISTS. + var err error + b.heal, err = schema.CaptureFreshBootstrapHealCapability( + ctx, conn, b.provider.serverEndpoint, b.database, + ) + if err != nil { + return nil, &bootstrapPreparationError{err: fmt.Errorf("uow: capture fresh database identity: %w", err)} + } + } + return b.heal, nil +} + func buildDSN(ep proxy.Endpoint, database, user, password, tlsConfigName string) string { return util.DoltServerDSN{ Host: ep.Host, diff --git a/internal/storage/uow/dolt_sql_provider_lock_test.go b/internal/storage/uow/dolt_sql_provider_lock_test.go index fc3b72916..aca6a198e 100644 --- a/internal/storage/uow/dolt_sql_provider_lock_test.go +++ b/internal/storage/uow/dolt_sql_provider_lock_test.go @@ -60,6 +60,39 @@ func TestClassifyInitSchemaErrorKeepsBootstrapPreparationErrorsDistinct(t *testi } } +// expectNoSessionDatabase mocks the opening question of the pre-lock +// convergence probe on the shape every production seat presents: initSchema +// pins its connection from a pool opened with an EMPTY DSN database (see +// openAndInitSchema), so DATABASE() is NULL until something issues USE. +func expectNoSessionDatabase(mock sqlmock.Sqlmock) { + mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE()")). + WillReturnRows(sqlmock.NewRows([]string{"DATABASE()"}).AddRow(nil)) +} + +// expectDatabaseExistsProbe mocks the always-succeeding existence probe the +// convergence probe must issue before any USE. +func expectDatabaseExistsProbe(mock sqlmock.Sqlmock, database string, exists bool) { + n := 0 + if exists { + n = 1 + } + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.schemata`). + WithArgs(database). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(n)) +} + +// TestInitSchemaAcquiresMigrationLockBeforeBootstrapDDL is the fresh-bootstrap +// ordering guard: nothing that creates or touches schema may run before +// GET_LOCK. +// +// The probe expectations at the top are load-bearing, not scenery. Without +// them the probe's statements were simply unexpected — sqlmock refuses an +// unexpected query WITHOUT consuming the next expectation, so the probe erred, +// failed closed, and GET_LOCK matched expectation #1 anyway. The test passed +// no matter what the probe did or did not do on this path, which is exactly +// how a fast path that could never fire on the proxied CLI open reached +// review. Now the probe is walked explicitly: it finds no database to converge +// on (this init is about to create it), declines, and the lock is taken first. func TestInitSchemaAcquiresMigrationLockBeforeBootstrapDDL(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -68,6 +101,8 @@ func TestInitSchemaAcquiresMigrationLockBeforeBootstrapDDL(t *testing.T) { defer db.Close() lockName := schema.MigrationLockName("beads") + expectNoSessionDatabase(mock) + expectDatabaseExistsProbe(mock, "beads", false) mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). WithArgs(lockName, 5). WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) @@ -101,3 +136,117 @@ func TestInitSchemaAcquiresMigrationLockBeforeBootstrapDDL(t *testing.T) { t.Fatalf("ordered bootstrap SQL expectations: %v", err) } } + +// TestInitSchemaConvergenceProbeRunsWithNoSessionDatabase is the caller-side +// regression for the fast path that could not fire. openAndInitSchema opens +// its schema-init pool with an EMPTY DSN database and the USE happens only +// inside the locked bootstrap preparation, so when the pre-lock convergence +// probe asked DATABASE() it read NULL on every production seat's open, gave +// up on its first statement, and took the server-wide GET_LOCK exactly as +// before — the change measured on the shared rig was a no-op on the only path +// that mattered. +// +// From this caller, with the database already present, the probe must reach +// the schema predicates: prove the database exists with a query that cannot +// fail, USE it, and read the cursor. Here that cursor is far behind, so the +// probe correctly declines and the ordinary locked pass follows — GET_LOCK +// first, bootstrap DDL after. +func TestInitSchemaConvergenceProbeRunsWithNoSessionDatabase(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + lockName := schema.MigrationLockName("beads") + expectNoSessionDatabase(mock) + expectDatabaseExistsProbe(mock, "beads", true) + mock.ExpectExec(regexp.QuoteMeta("USE `beads`")). + WillReturnResult(sqlmock.NewResult(0, 0)) + // migrationWorkNeeded's first question, reached only because the probe put + // the session on the database itself. The cursor is far behind this + // binary, so there is real work and the fast path declines. + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). + WithArgs("schema_migrations"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(version), 0) FROM schema_migrations")). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) + + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, 5). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + // The bare CREATE DATABASE loses to the database the probe just proved + // exists, so this init captures no fresh-bootstrap heal authority. + mock.ExpectExec(regexp.QuoteMeta("CREATE DATABASE `beads`")). + WillReturnError(&mysql.MySQLError{Number: 1007, Message: "database exists"}) + mock.ExpectExec(regexp.QuoteMeta("USE `beads`")). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("INSERT IGNORE INTO dolt_ignore VALUES (?, true)")). + WithArgs(sqlmock.AnyArg()). + WillReturnError(errors.New("first migration statement failed")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + p := &doltSQLProvider{ + defaultBranch: defaultBranch, + db: db, + serverEndpoint: "tcp:127.0.0.1:3306", + } + err = p.initSchema(context.Background(), "beads") + if err == nil || !strings.Contains(err.Error(), "first migration statement failed") { + t.Fatalf("initSchema() error = %v, want first migration sentinel", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("the convergence probe must reach the schema predicates on a session with no database selected: %v", err) + } +} + +// TestSelectProbeDatabaseIssuesPreparationsUse pins the two properties the +// convergence probe borrows from this injection site and cannot check for +// itself: the statement is the DDL repository's own UseDatabase (identical to +// what prepareBootstrap issues, which is why a converged probe may skip locked +// preparation), and the name is validated and quoted by that same repository +// BEFORE any statement goes out. +func TestSelectProbeDatabaseIssuesPreparationsUse(t *testing.T) { + t.Run("valid name", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("USE `beads`")). + WillReturnResult(sqlmock.NewResult(0, 0)) + + quoted, err := selectProbeDatabase(context.Background(), db, "beads") + if err != nil { + t.Fatalf("selectProbeDatabase() error = %v", err) + } + if quoted != "`beads`" { + t.Fatalf("selectProbeDatabase() quoted = %q, want %q", quoted, "`beads`") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + t.Run("invalid name", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + quoted, err := selectProbeDatabase(context.Background(), db, "bad`name") + if err == nil { + t.Fatal("selectProbeDatabase() error = nil, want the identifier rejection") + } + if quoted != "" { + t.Fatalf("selectProbeDatabase() quoted = %q, want empty", quoted) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations (an invalid name must be refused before any statement): %v", err) + } + }) +} diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index b6c23385e..bf1066d58 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -255,11 +255,22 @@ func TestNotifyingProviderAnswersTheWholeProviderSurface(t *testing.T) { if len(surface) < 25 { t.Fatalf("parsed only %d methods off doltSQLProvider — the scan is broken, not the provider", len(surface)) } + // initSchemaAttempt is the single-attempt body initSchema's backoff loop + // calls, and verifyTeamServerSchema and attachPreviewDatabase are the two open + // paths it fans out to (dolt_sql_provider.go). Like initSchema above them, each + // is an unexported startup helper reached only as p. on the concrete + // provider — never through a UnitOfWorkProvider type assertion — so the wrapper + // has no capability to answer and no bead write to fire a hook for. + initBranch := "an unexported branch of initSchema's startup path, reached only on the concrete " + + "provider, not a capability any caller can reach through the wrapper" assertPartition(t, "UnitOfWorkProvider", surface, reflect.TypeOf((*notifyingProvider)(nil)), map[string]string{ "BeginTx": "TxProvider hands out a bare transaction with no unit of work around it, so " + "there is nothing to buffer and nothing to drain: the wrapper deliberately does not " + "offer it, and only this package's own NewUOW asks a provider for one", - "initSchema": "unexported provider setup, not a capability any caller can reach", + "initSchema": "unexported provider setup, not a capability any caller can reach", + "initSchemaAttempt": initBranch, + "verifyTeamServerSchema": initBranch, + "attachPreviewDatabase": initBranch, }) }