diff --git a/cmd/bd/migrate_consent_test.go b/cmd/bd/migrate_consent_test.go index 02a21cf01..644f86a14 100644 --- a/cmd/bd/migrate_consent_test.go +++ b/cmd/bd/migrate_consent_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "regexp" "strings" "testing" @@ -279,9 +280,10 @@ func runLocalConsentGate(t *testing.T, cmdline string, consents bool) error { defer db.Close() if !consents { for range 2 { - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). - WithArgs("schema_migrations"). + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery("^SHOW TABLES$"). + WillReturnRows(sqlmock.NewRows([]string{"Tables_in_beads"}).AddRow("schema_migrations")) mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(schema.LatestVersion() - 1)) } diff --git a/cmd/bd/shared_migrate_consent_test.go b/cmd/bd/shared_migrate_consent_test.go index 4181bd02e..c6d9a7187 100644 --- a/cmd/bd/shared_migrate_consent_test.go +++ b/cmd/bd/shared_migrate_consent_test.go @@ -5,6 +5,7 @@ import ( "context" "io" "os" + "regexp" "strings" "testing" @@ -233,10 +234,11 @@ func TestPrintGlobalDatabaseConsentHint(t *testing.T) { // either outcome, so an unmet expectation means the gate took a path this test // does not describe. func expectRemoteBackedSharedProbe(mock sqlmock.Sqlmock) { - for i := 0; i < 2; i++ { - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). - WithArgs("schema_migrations"). + for range 2 { + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery("^SHOW TABLES$"). + WillReturnRows(sqlmock.NewRows([]string{"Tables_in_beads"}).AddRow("schema_migrations")) mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) } diff --git a/internal/storage/dolt/initschema_pool_poison_test.go b/internal/storage/dolt/initschema_pool_poison_test.go index c8f7a3ccd..677964213 100644 --- a/internal/storage/dolt/initschema_pool_poison_test.go +++ b/internal/storage/dolt/initschema_pool_poison_test.go @@ -137,3 +137,211 @@ func TestSentinelColumnProbe_PoolSeesLaterTables(t *testing.T) { t.Fatalf("the pooled session that ran the sentinel probe cannot see a table created after it (be-bv7x pin): %v", err) } } + +// TestCursorProbe_NoDatabaseSessionStaysUsable pins the cursor-existence probe +// to the be-bv7x rule on a real sql-server: it must succeed on any session. On +// a session with no database selected, SHOW TABLES fails ("no database +// selected"), and a failing statement pins the pooled session to its +// pre-statement catalog. The probe has to read the cursor as absent there, +// exactly as the INFORMATION_SCHEMA.TABLES count it replaced did, and leave the +// session able to see a database and table created afterwards elsewhere. +func TestCursorProbe_NoDatabaseSessionStaysUsable(t *testing.T) { + skipIfNoDolt(t) + + ctx, cancel := testContext(t) + defer cancel() + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: testServerPort, User: "root"}.String() + pool, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open no-database pool: %v", err) + } + defer pool.Close() + pool.SetMaxOpenConns(1) + pool.SetMaxIdleConns(1) + + version, err := schema.CurrentVersion(ctx, pool) + if err != nil { + t.Fatalf("CurrentVersion on a session with no database selected: %v", err) + } + if version != 0 { + t.Fatalf("CurrentVersion on a session with no database selected = %d, want 0", version) + } + + admin, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open admin connection: %v", err) + } + defer admin.Close() + dbName := uniqueTestDBName(t) + if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+dbName+"`"); err != nil { + t.Fatalf("create database: %v", err) + } + defer func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = admin.ExecContext(dropCtx, "DROP DATABASE IF EXISTS `"+dbName+"`") + }() + if _, err := admin.ExecContext(ctx, "CREATE TABLE `"+dbName+"`.probe_after (id INT PRIMARY KEY)"); err != nil { + t.Fatalf("create table: %v", err) + } + + conn, err := pool.Conn(ctx) + if err != nil { + t.Fatalf("pin the probed session: %v", err) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "USE `"+dbName+"`"); err != nil { + t.Fatalf("the session the cursor probe ran on cannot select a database created after it: %v", err) + } + var count int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM probe_after").Scan(&count); err != nil { + t.Fatalf("the session the cursor probe ran on cannot see a table created after it (be-bv7x pin): %v", err) + } +} + +// TestCursorProbe_DroppedDatabaseSessionStaysUsable is the 1049 half of +// TestCursorProbe_NoDatabaseSessionStaysUsable: on a session whose database +// was dropped under it, SHOW TABLES fails with "database not found" rather +// than "no database selected". The probe must read the cursor as absent there +// too, and leave the session able to see a database and table created +// afterwards. +func TestCursorProbe_DroppedDatabaseSessionStaysUsable(t *testing.T) { + skipIfNoDolt(t) + + ctx, cancel := testContext(t) + defer cancel() + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: testServerPort, User: "root"}.String() + admin, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open admin connection: %v", err) + } + defer admin.Close() + dropLater := func(name string) { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = admin.ExecContext(dropCtx, "DROP DATABASE IF EXISTS `"+name+"`") + } + + dropped := uniqueTestDBName(t) + if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+dropped+"`"); err != nil { + t.Fatalf("create database: %v", err) + } + defer dropLater(dropped) + + pool, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open probe pool: %v", err) + } + defer pool.Close() + pool.SetMaxOpenConns(1) + pool.SetMaxIdleConns(1) + conn, err := pool.Conn(ctx) + if err != nil { + t.Fatalf("pin the probed session: %v", err) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "USE `"+dropped+"`"); err != nil { + t.Fatalf("select the database to drop: %v", err) + } + if _, err := admin.ExecContext(ctx, "DROP DATABASE `"+dropped+"`"); err != nil { + t.Fatalf("drop the probed session's database: %v", err) + } + + version, err := schema.CurrentVersion(ctx, conn) + if err != nil { + t.Fatalf("CurrentVersion on a session whose database was dropped: %v", err) + } + if version != 0 { + t.Fatalf("CurrentVersion on a session whose database was dropped = %d, want 0", version) + } + + dbName := uniqueTestDBName(t) + if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+dbName+"`"); err != nil { + t.Fatalf("create database: %v", err) + } + defer dropLater(dbName) + if _, err := admin.ExecContext(ctx, "CREATE TABLE `"+dbName+"`.probe_after (id INT PRIMARY KEY)"); err != nil { + t.Fatalf("create table: %v", err) + } + + if _, err := conn.ExecContext(ctx, "USE `"+dbName+"`"); err != nil { + t.Fatalf("the session the cursor probe ran on cannot select a database created after it: %v", err) + } + var count int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM probe_after").Scan(&count); err != nil { + t.Fatalf("the session the cursor probe ran on cannot see a table created after it (be-bv7x pin): %v", err) + } +} + +// TestShowTablesProbes_SeeIgnoredTablesInAnyCase checks the per-open existence +// probes against a real migrated store on a fresh session. Both cursor tables' +// rows must be believed as read: the ignored cursor and every sentinel the +// cursor reality check reads are dolt_ignored, so they live only in the +// working set, and a probe that missed them would clamp the cursor and replay +// the series on every open. Dolt resolves table names case-insensitively, as +// the INFORMATION_SCHEMA.TABLES count these probes replaced does, so a +// sentinel stored in another case is still present — a case-sensitive probe +// (SHOW TABLES LIKE) reads it as absent. +func TestShowTablesProbes_SeeIgnoredTablesInAnyCase(t *testing.T) { + skipIfNoDolt(t) + + ctx, cancel := testContext(t) + defer cancel() + + dbName := uniqueTestDBName(t) + store, err := New(ctx, &Config{ + Path: t.TempDir(), + CommitterName: "test", + CommitterEmail: "test@example.com", + Database: dbName, + CreateIfMissing: true, + }) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + defer func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = store.db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbName)) + store.Close() + }() + + pool, err := sql.Open("mysql", doltutil.ServerDSN{ + Host: "127.0.0.1", Port: testServerPort, User: "root", Database: dbName, + }.String()) + if err != nil { + t.Fatalf("open single-connection pool: %v", err) + } + defer pool.Close() + pool.SetMaxOpenConns(1) + pool.SetMaxIdleConns(1) + + requireCursors := func(state string) { + t.Helper() + main, err := schema.CurrentVersion(ctx, pool) + if err != nil { + t.Fatalf("%s: CurrentVersion: %v", state, err) + } + if main != schema.LatestVersion() { + t.Fatalf("%s: CurrentVersion = %d, want %d", state, main, schema.LatestVersion()) + } + ignored, err := schema.CurrentIgnoredVersion(ctx, pool) + if err != nil { + t.Fatalf("%s: CurrentIgnoredVersion: %v", state, err) + } + if ignored != schema.LatestIgnoredVersion() { + t.Fatalf("%s: CurrentIgnoredVersion = %d, want %d: a sentinel table read as absent", state, ignored, schema.LatestIgnoredVersion()) + } + } + requireCursors("freshly migrated store") + + if _, err := store.db.ExecContext(ctx, "RENAME TABLE bd_events_seq TO bd_events_seq_case"); err != nil { + t.Fatalf("rename bd_events_seq: %v", err) + } + if _, err := store.db.ExecContext(ctx, "RENAME TABLE bd_events_seq_case TO BD_EVENTS_SEQ"); err != nil { + t.Fatalf("rename to BD_EVENTS_SEQ: %v", err) + } + requireCursors("bd_events_seq stored as BD_EVENTS_SEQ") +} diff --git a/internal/storage/schema/converged.go b/internal/storage/schema/converged.go index 587220485..3a2b606b1 100644 --- a/internal/storage/schema/converged.go +++ b/internal/storage/schema/converged.go @@ -99,7 +99,9 @@ func alreadyConverged(ctx context.Context, db DBConn, databaseName string, selec // deliberately non-ignored table that the next pull's auto-commit puts // into HEAD and push replicates fleet-wide. // - // Two always-succeeding reads on the hot path, both bounded to one row. + // Two SHOW TABLES listings on the hot path, safe to issue because + // selectTargetDatabase has put the session on the target (see + // readIgnoredCursorState). state, err := readIgnoredCursorState(ctx, db, qualifier) if err != nil { return false, err diff --git a/internal/storage/schema/converged_test.go b/internal/storage/schema/converged_test.go index 8eff9441e..67d26ead6 100644 --- a/internal/storage/schema/converged_test.go +++ b/internal/storage/schema/converged_test.go @@ -715,7 +715,7 @@ func TestAlreadyConvergedFailsClosedOnUnreadableState(t *testing.T) { db, mock := newMockDB(t) expectCurrentDatabase(mock, "testdb") - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). WillReturnError(sql.ErrConnDone) converged, err := alreadyConverged(context.Background(), db, "testdb", nil) diff --git a/internal/storage/schema/cursor_reality_test.go b/internal/storage/schema/cursor_reality_test.go index 6f722e27a..5fde1827f 100644 --- a/internal/storage/schema/cursor_reality_test.go +++ b/internal/storage/schema/cursor_reality_test.go @@ -12,7 +12,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" ) -// stubSentinelTables replaces the INFORMATION_SCHEMA probe with a fixed answer +// stubSentinelTables replaces the table-existence probe with a fixed answer // for the duration of a test, following the issueRowCounter seam convention in // this package. The DBConn is never touched, so these run with no database. func stubSentinelTables(t *testing.T, present map[string]bool, probeErr error) *[]string { @@ -47,17 +47,11 @@ func stubSentinelColumns(t *testing.T, present map[string]bool, probeErr error) } // expectSentinelColumnProbe mocks the real column-sentinel probe in order: the -// carrier table's existence through sentinelTableExists's INFORMATION_SCHEMA -// point read, then — only when the table is there — the SHOW COLUMNS read. An -// absent column is an empty SHOW COLUMNS result set, not a zero count. +// carrier table's existence through sentinelTableExists (a SHOW TABLES read), +// then — only when the table is there — the SHOW COLUMNS read. An absent column +// is an empty SHOW COLUMNS result set, not a zero count. func expectSentinelColumnProbe(mock sqlmock.Sqlmock, sc schemaSentinelColumn, tablePresent, columnPresent bool) { - count := 0 - if tablePresent { - count = 1 - } - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WithArgs(sc.table). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(count)) + expectShowTable(mock, sc.table, tablePresent) if !tablePresent { return } @@ -697,8 +691,7 @@ func TestMigrationWorkNeededWhenWispTablesAbsent(t *testing.T) { expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()) expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) + expectShowTable(mock, ignoredSource.sentinelTables[0], false) needed, err := migrationWorkNeeded(context.Background(), db) if err != nil { @@ -727,15 +720,13 @@ func TestMigrationWorkNeededWhenLeaseGrantedNodeAbsent(t *testing.T) { expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()) expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) - for range ignoredSource.sentinelTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, table := range ignoredSource.sentinelTables { + expectShowTable(mock, table, true) } // The floored-table sentinels probe the same way and must all corroborate // too, ahead of the column probe below. - for range ignoredSource.sentinelFlooredTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, st := range ignoredSource.sentinelFlooredTables { + expectShowTable(mock, st.table, true) } // leases is present; only its granted_node column is missing. for _, sc := range ignoredSource.sentinelColumns { @@ -772,13 +763,11 @@ func TestIgnoredCursorReadWithLeasesAbsentIssuesNoFailingStatement(t *testing.T) expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) - for range ignoredSource.sentinelTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, table := range ignoredSource.sentinelTables { + expectShowTable(mock, table, true) } - for range ignoredSource.sentinelFlooredTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, st := range ignoredSource.sentinelFlooredTables { + expectShowTable(mock, st.table, true) } for _, sc := range ignoredSource.sentinelColumns { expectSentinelColumnProbe(mock, sc, false, false) @@ -935,8 +924,7 @@ func TestMigrateAppliesUnderContradictedCursor(t *testing.T) { // The guarded read: cursor claims at-latest, sentinel probe contradicts it. expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) + expectShowTable(mock, ignoredSource.sentinelTables[0], false) // The proof: the applier reaches ignored/0001's SQL. Fail it distinctively // rather than mocking the whole series. @@ -987,15 +975,13 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { // corroborate it, and only the leases column does not. expectCursorProbe(mock, "ignored_schema_migrations", true) expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM ignored_schema_migrations", "version", LatestIgnoredVersion()) - for range ignoredSource.sentinelTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, table := range ignoredSource.sentinelTables { + expectShowTable(mock, table, true) } // The floored-table sentinels probe the same way and must all corroborate // too, ahead of the column probe below. - for range ignoredSource.sentinelFlooredTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, st := range ignoredSource.sentinelFlooredTables { + expectShowTable(mock, st.table, true) } for _, sc := range ignoredSource.sentinelColumns { expectSentinelColumnProbe(mock, sc, true, false) @@ -1061,3 +1047,84 @@ func TestPendingVersionsUnderFloorDoNotReArmAuxMarkers(t *testing.T) { t.Error("ignored/0007 is pending on a cursor-11 store; its unguarded UPDATE would restamp wisps.updated_at") } } + +// TestShowTableExistsAnswersLikeInformationSchema pins showTableExists to the +// INFORMATION_SCHEMA.TABLES count it replaced. Dolt's SHOW TABLES LIKE filter +// matches case-sensitively, while INFORMATION_SCHEMA — like every table +// reference — resolves names case-insensitively, so a LIKE-filtered probe +// reads a table stored as BD_EVENTS_SEQ as absent (dolt 2.1.8 and 2.2.0). The +// probe therefore reads the unfiltered listing and compares names +// case-insensitively and otherwise exactly: a '_' lookalike is not the table, +// and an empty listing is an absent table rather than an error. +func TestShowTableExistsAnswersLikeInformationSchema(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + cases := []struct { + name string + tables []string + probe string + want bool + }{ + {name: "a LIKE lookalike is not the table", tables: []string{"issues", "wispxdependencies"}, probe: "wisp_dependencies", want: false}, + {name: "a name containing the table is not the table", tables: []string{"bd_events_journal", "bd_events_seq"}, probe: "events", want: false}, + {name: "empty database", tables: nil, probe: "wisp_dependencies", want: false}, + {name: "exact name among others", tables: []string{"issues", "wisp_dependencies", "wisps"}, probe: "wisp_dependencies", want: true}, + {name: "stored in another case", tables: []string{"issues", "BD_EVENTS_SEQ"}, probe: "bd_events_seq", want: true}, + } + for _, tc := range cases { + rows := sqlmock.NewRows([]string{"Tables_in_testdb"}) + for _, table := range tc.tables { + rows.AddRow(table) + } + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) + } + for _, tc := range cases { + got, err := showTableExists(context.Background(), db, tc.probe) + if err != nil { + t.Fatalf("%s: showTableExists: %v", tc.name, err) + } + if got != tc.want { + t.Errorf("%s: showTableExists(%q) = %v, want %v", tc.name, tc.probe, got, tc.want) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} + +// TestCurrentVersionWithoutAUsableDatabaseIssuesNoFailingStatement is the +// be-bv7x rule for the cursor probe itself, which must succeed on any session. +// SHOW TABLES fails with no database selected (1105) and on a session whose +// database was dropped under it (1049), and a failing statement pins the +// pooled session to its pre-statement catalog. The schemata read answers 0 in +// both cases instead, so the probe stops there and reads the cursor as absent, +// exactly as the INFORMATION_SCHEMA.TABLES count did: the strict mock admits +// nothing after it. +func TestCurrentVersionWithoutAUsableDatabaseIssuesNoFailingStatement(t *testing.T) { + for _, src := range []migrationSource{mainSource, ignoredSource} { + t.Run(src.cursorTable, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) + + got, err := src.currentVersion(context.Background(), db) + if err != nil { + t.Fatalf("currentVersion without a usable database: %v", err) + } + if got != 0 { + t.Errorf("currentVersion without a usable database = %d, want 0", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + } +} diff --git a/internal/storage/schema/ignored_cursor_untrack.go b/internal/storage/schema/ignored_cursor_untrack.go index 014d76dde..5a7f7d4f0 100644 --- a/internal/storage/schema/ignored_cursor_untrack.go +++ b/internal/storage/schema/ignored_cursor_untrack.go @@ -91,16 +91,21 @@ func (s ignoredCursorState) reconcileNeeded() bool { return s.needsUntrack || s.strayScratch } -// readIgnoredCursorState answers both halves of the gate in two -// always-succeeding reads. That property is not incidental: a Dolt session -// that issues a FAILING statement stays pinned to its pre-statement catalog -// snapshot for the rest of its pooled life (be-bv7x), and this runs on every -// writable open and every lock-free fast-path evaluation. +// readIgnoredCursorState answers both halves of the gate. On a healthy +// database that is two SHOW TABLES listings, HEAD's and the working set's; a +// dolt_ignore read joins them only when the cursor is tracked at HEAD. None of +// them may fail, and that is not incidental: a Dolt session that issues a +// FAILING statement stays pinned to its pre-statement catalog snapshot for the +// rest of its pooled life (be-bv7x), and this runs on every writable open and +// every lock-free fast-path evaluation. A SHOW TABLES listing fails with no +// database selected (1105) and on a session whose database was dropped (1049), +// so both callers must already have the session on the target database; a +// session that may have no usable database needs tableInCurrentDatabase. // // qualifier is the identifier-quoted database name selectTargetDatabase // returned, or empty when the session is already on the target database. The -// scratch probe needs no qualifier: it reads information_schema against -// DATABASE(), which both callers have already put on the target. +// scratch probe takes no qualifier: it lists the session's current database, +// which both callers have already made the target. func readIgnoredCursorState(ctx context.Context, db DBConn, qualifier string) (ignoredCursorState, error) { var state ignoredCursorState @@ -110,7 +115,10 @@ func readIgnoredCursorState(ctx context.Context, db DBConn, qualifier string) (i } state.needsUntrack = needsUntrack - stray, err := schemaTableExists(ctx, db, ignoredCursorUntrackTempTable) + // Both callers have the session on the target database (alreadyConverged + // after selectTargetDatabase, MigrateUp after its preparation), so this + // per-open read takes the SHOW TABLES prober. + stray, err := showTableExists(ctx, db, ignoredCursorUntrackTempTable) if err != nil { return state, fmt.Errorf("probing %s: %w", ignoredCursorUntrackTempTable, err) } diff --git a/internal/storage/schema/ignored_cursor_untrack_test.go b/internal/storage/schema/ignored_cursor_untrack_test.go index c69bdfc62..466834a2e 100644 --- a/internal/storage/schema/ignored_cursor_untrack_test.go +++ b/internal/storage/schema/ignored_cursor_untrack_test.go @@ -120,7 +120,7 @@ func expectIgnoredCursorGate(mock sqlmock.Sqlmock, qualifier string, tracked boo if tracked { expectIgnoreResolution(mock, qualifier, ignoredSource.cursorTable, matches) } - expectSchemaTableExists(mock, ignoredCursorUntrackTempTable, stray) + expectShowTable(mock, ignoredCursorUntrackTempTable, stray) } // expectIgnoredCursorHealNoop mocks the whole open-time reconcile on a healthy @@ -764,7 +764,7 @@ func TestAlreadyConvergedDeclinesOnALegacyTrackedCursor(t *testing.T) { expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) expectHeadTableProbe(mock, "", ignoredSource.cursorTable, true) expectIgnoreResolution(mock, "", ignoredSource.cursorTable, exactlyIgnored(true)) - expectSchemaTableExists(mock, ignoredCursorUntrackTempTable, false) + expectShowTable(mock, ignoredCursorUntrackTempTable, false) converged, err := alreadyConverged(context.Background(), db, "testdb", nil) if err != nil { @@ -793,7 +793,7 @@ func TestAlreadyConvergedDeclinesOnASurvivingScratchTable(t *testing.T) { expectNoMigrationWorkNeeded(mock) expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) expectHeadTableProbe(mock, "", ignoredSource.cursorTable, false) - expectSchemaTableExists(mock, ignoredCursorUntrackTempTable, true) + expectShowTable(mock, ignoredCursorUntrackTempTable, true) converged, err := alreadyConverged(context.Background(), db, "testdb", nil) if err != nil { @@ -820,7 +820,7 @@ func TestAlreadyConvergedAcceptsATrackedCursorTheOperatorUnignored(t *testing.T) expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) expectHeadTableProbe(mock, "", ignoredSource.cursorTable, true) expectIgnoreResolution(mock, "", ignoredSource.cursorTable, exactlyIgnored(false)) - expectSchemaTableExists(mock, ignoredCursorUntrackTempTable, false) + expectShowTable(mock, ignoredCursorUntrackTempTable, false) expectMigrationLockProbe(mock, "testdb", 1) converged, err := alreadyConverged(context.Background(), db, "testdb", nil) @@ -864,7 +864,7 @@ func TestAlreadyConvergedKeepsTheLockProbeLast(t *testing.T) { expectDoltIgnoreRead(mock, unqualifiedDoltIgnore, seededIgnorePatterns(LatestVersion())) expectHeadTableProbe(mock, "", ignoredSource.cursorTable, true) expectIgnoreResolution(mock, "", ignoredSource.cursorTable, exactlyIgnored(true)) - expectSchemaTableExists(mock, ignoredCursorUntrackTempTable, false) + expectShowTable(mock, ignoredCursorUntrackTempTable, false) if _, err := alreadyConverged(context.Background(), db, "testdb", nil); err != nil { t.Fatalf("alreadyConverged() error = %v", err) diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index bd016ba5c..d2a1c7eaf 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -511,18 +511,15 @@ func expectScalar(mock sqlmock.Sqlmock, query, column string, value any) { WillReturnRows(sqlmock.NewRows([]string{column}).AddRow(value)) } -// expectCursorProbe mocks the information_schema existence probe that -// migrationSource.currentVersion issues before it ever reads the cursor -// table. The probe was added by be-bv7x so a not-yet-created cursor table -// never poisons the pooled Dolt session with a failing statement. +// expectCursorProbe mocks the existence probe that migrationSource.currentVersion +// issues before it ever reads the cursor table: the schemata read proving the +// session has a usable current database, then the SHOW TABLES listing. The +// probe was added by be-bv7x so a not-yet-created cursor table never poisons +// the pooled Dolt session with a failing statement. func expectCursorProbe(mock sqlmock.Sqlmock, table string, exists bool) { - present := 0 - if exists { - present = 1 - } - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). - WithArgs(table). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(present)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + expectShowTable(mock, table, exists) } func expectDoltStatusRows(mock sqlmock.Sqlmock) { @@ -807,25 +804,19 @@ func TestMigrateUpWithLockFreshBootstrapHealResetFailureStaysFatal(t *testing.T) // expectIgnoredSentinelProbes mocks the schema lookups currentVersion issues // to confirm a non-zero ignored cursor against the schema it claims (gh 5033): -// an INFORMATION_SCHEMA.TABLES count per sentinel and floored table, then per -// sentinel column its carrier table's existence and a SHOW COLUMNS read -// (expectSentinelColumnProbe — the INFORMATION_SCHEMA.COLUMNS form cost 5-6 s -// per store open on a many-database Dolt server). They fire only for a -// non-zero cursor, in ignoredSource's table then floored-table then column -// sentinel order. +// a SHOW TABLES probe per sentinel and floored table (showTableExists), then +// per sentinel column its carrier table's existence and a SHOW COLUMNS read +// (expectSentinelColumnProbe). None of them uses INFORMATION_SCHEMA, whose +// predicates Dolt does not push down, because all of them run on every store +// open. They fire only for a non-zero cursor, in ignoredSource's table then +// floored-table then column sentinel order. func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) { - count := 0 - if present { - count = 1 - } - for range ignoredSource.sentinelTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(count)) + for _, table := range ignoredSource.sentinelTables { + expectShowTable(mock, table, present) } if present { - for range ignoredSource.sentinelFlooredTables { - mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + for _, st := range ignoredSource.sentinelFlooredTables { + expectShowTable(mock, st.table, true) } for _, sc := range ignoredSource.sentinelColumns { expectSentinelColumnProbe(mock, sc, true, true) @@ -833,6 +824,18 @@ func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) { } } +// expectShowTable mocks one showTableExists probe: the database's unfiltered +// SHOW TABLES listing, which holds table only when it exists. An absent table +// is a listing without it, never a zero count. The listing always carries an +// unrelated table, so a probe that took any row as presence fails. +func expectShowTable(mock sqlmock.Sqlmock, table string, exists bool) { + rows := sqlmock.NewRows([]string{"Tables_in_testdb"}).AddRow("issues") + if exists { + rows.AddRow(table) + } + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) +} + func TestMigrateUpWithLockFreshBootstrapHealProbeFailuresStayFatal(t *testing.T) { tests := []struct { name string diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index f45e53d49..2c95d0e9f 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -1547,19 +1547,16 @@ func (m migrationSource) currentVersion(ctx context.Context, db DBConn) (int, er // not-yet-created cursor table poisons the pooled connection: tables // created afterwards on other connections stay invisible to this one for // the rest of its life in the pool (be-bv7x). - var cursorExists int - if err := db.QueryRowContext(ctx, - "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ?", - m.cursorTable, - ).Scan(&cursorExists); err != nil { + cursorExists, err := tableInCurrentDatabase(ctx, db, m.cursorTable) + if err != nil { return 0, fmt.Errorf("probing %s existence: %w", m.cursorTable, err) } - if cursorExists == 0 { + if !cursorExists { return 0, nil } var current int - err := db.QueryRowContext(ctx, "SELECT COALESCE(MAX(version), 0) FROM "+m.cursorTable).Scan(¤t) + err = db.QueryRowContext(ctx, "SELECT COALESCE(MAX(version), 0) FROM "+m.cursorTable).Scan(¤t) if err != nil && err != sql.ErrNoRows { if dberrors.IsTableNotExist(err) { return 0, nil @@ -1656,16 +1653,87 @@ func (m migrationSource) cursorRealityFloor(ctx context.Context, db DBConn) (int // sentinelTableExists is a function variable for the same reason // issueRowCounter is: it lets the cursor-reality tests exercise the real -// decision without a live database. -var sentinelTableExists = func(ctx context.Context, db DBConn, table string) (bool, error) { - var n int +// decision without a live database. It runs on EVERY store open +// (cursorRealityFloor: the sentinel and floored-sentinel tables, and the +// column sentinel's carrier), so it takes the SHOW TABLES prober for the +// reason sentinelColumnExists takes SHOW COLUMNS. +var sentinelTableExists = showTableExists + +// showTableExists reports whether the session's current database holds table, +// probing with SHOW TABLES rather than INFORMATION_SCHEMA.TABLES. Dolt does not +// push the TABLES predicate down any more than the COLUMNS one: the probe walks +// the catalog of every database on the server. On a sql-server hosting 28 +// databases the INFORMATION_SCHEMA read costs 25-30 ms against 0.6 ms for the +// listing (dolt 2.1.8), and the open path asks a dozen times per bd write. +// +// It reads the unfiltered listing and compares names case-insensitively, which +// is what the INFORMATION_SCHEMA count it replaced answers: Dolt resolves and +// counts table names case-insensitively, but its SHOW TABLES LIKE filter +// matches case-sensitively, so a filtered probe would read a table stored as +// BD_EVENTS_SEQ as absent. The comparison is otherwise exact; without a LIKE +// pattern, '_' wildcards cannot match a lookalike either. +// +// Unlike the INFORMATION_SCHEMA count, SHOW TABLES errors when no database is +// selected or the selected one was dropped, so it is only for callers already +// on the target database: the cursor-existence probe in currentVersion, which +// must succeed on any session (be-bv7x), gates it with tableInCurrentDatabase. +// table is a migration-series constant, never user input. +func showTableExists(ctx context.Context, db DBConn, table string) (bool, error) { + rows, err := db.QueryContext(ctx, "SHOW TABLES") + if err != nil { + return false, fmt.Errorf("checking table %s: %w", table, err) + } + defer func() { _ = rows.Close() }() + + cols, err := rows.Columns() + if err != nil { + return false, fmt.Errorf("checking table %s: %w", table, err) + } + // SHOW TABLES returns Tables_in_ (plus Table_type on SHOW FULL TABLES + // and on some servers); the first column is the table name. + cells := make([]sql.RawBytes, len(cols)) + dest := make([]any, len(cols)) + for i := range cells { + dest[i] = &cells[i] + } + for rows.Next() { + if err := rows.Scan(dest...); err != nil { + return false, fmt.Errorf("checking table %s: %w", table, err) + } + if len(cells) > 0 && strings.EqualFold(string(cells[0]), table) { + return true, nil + } + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("checking table %s: %w", table, err) + } + return false, nil +} + +// tableInCurrentDatabase is showTableExists for a session that may have no +// usable database: the cursor-existence probe runs on any session, so it must +// never issue a statement that can fail (be-bv7x). SHOW TABLES fails with no +// database selected (1105) and on a session whose database was dropped under +// it (1049); the schemata read answers 0 in both cases instead, which reads +// the table as absent, exactly as the INFORMATION_SCHEMA.TABLES count with +// TABLE_SCHEMA = DATABASE() did. Both reads are one round trip each. +// +// The guard is check-then-act, not atomic: a DROP DATABASE landing between the +// two reads still fails the SHOW TABLES and pins the session. On a *sql.DB the +// two reads may also land on different pooled connections, so the guard holds +// only when every connection in the pool is on the same database (a DSN that +// names it). +func tableInCurrentDatabase(ctx context.Context, db DBConn, table string) (bool, error) { + var usable int if err := db.QueryRowContext(ctx, - `SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`, - table).Scan(&n); err != nil { + "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()", + ).Scan(&usable); err != nil { return false, err } - return n > 0, nil + if usable == 0 { + return false, nil + } + return showTableExists(ctx, db, table) } // sentinelColumnExists runs on EVERY store open (cursorRealityFloor), so it diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index 2b10e591a..857df4487 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -2626,3 +2626,94 @@ DROP TABLE IF EXISTS wisps; requireDoltCount(t, dir, `SELECT COUNT(*) AS c FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps'`, "0") } + +// TestShowTableExistsMatchesInformationSchemaOnDolt proves the per-open table +// probe (the unfiltered SHOW TABLES listing, names compared case-insensitively) +// agrees with the INFORMATION_SCHEMA.TABLES count it replaced, on a real Dolt. +// It covers every table the open path probes — both cursor tables, the +// sentinel and floored-sentinel tables, the column sentinel's carrier and the +// untrack scratch table — under bd's own dolt_ignore patterns, so all but +// schema_migrations live only in the working set. It also covers a name whose +// '_' LIKE-matches a different table, and a table stored in another case: Dolt +// resolves and counts table names case-insensitively there, but its SHOW +// TABLES LIKE filter does not. +func TestShowTableExistsMatchesInformationSchemaOnDolt(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := filepath.Join(t.TempDir(), "show-tables-probe") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create probe dir: %v", err) + } + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + + infoSchemaHas := func(table string) bool { + rows := queryDoltCSV(t, dir, fmt.Sprintf(` +SELECT COUNT(*) AS cnt +FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '%s'`, table)) + return len(rows) == 1 && rows[0]["cnt"] == "1" + } + // The same listing and comparison showTableExists makes. + showTablesHas := func(table string) bool { + for _, r := range queryDoltCSV(t, dir, "SHOW TABLES") { + for _, v := range r { + if strings.EqualFold(v, table) { + return true + } + } + } + return false + } + agree := func(state, table string, want bool) { + t.Helper() + if got := showTablesHas(table); got != want { + t.Fatalf("%s: SHOW TABLES reported %s present=%v, want %v", state, table, got, want) + } + if got := infoSchemaHas(table); got != want { + t.Fatalf("%s: INFORMATION_SCHEMA reported %s present=%v, want %v", state, table, got, want) + } + } + + probed := []string{mainSource.cursorTable, ignoredSource.cursorTable, ignoredCursorUntrackTempTable} + probed = append(probed, ignoredSource.sentinelTables...) + for _, st := range ignoredSource.sentinelFlooredTables { + probed = append(probed, st.table) + } + for _, sc := range ignoredSource.sentinelColumns { + probed = append(probed, sc.table) + } + + for _, table := range probed { + agree("empty database", table, false) + } + + for _, pattern := range doltIgnorePatterns { + runDoltSQL(t, dir, fmt.Sprintf("REPLACE INTO dolt_ignore VALUES ('%s', true)", pattern)) + } + for _, gated := range versionGatedDoltIgnorePatterns { + runDoltSQL(t, dir, fmt.Sprintf("REPLACE INTO dolt_ignore VALUES ('%s', true)", gated.pattern)) + } + runDoltSQL(t, dir, "CREATE TABLE "+mainSource.cursorTable+" (version INT PRIMARY KEY)") + runDoltSQL(t, dir, "CALL DOLT_COMMIT('-Am', 'committed cursor')") + for _, table := range probed { + if table == mainSource.cursorTable || table == ignoredCursorUntrackTempTable { + continue + } + runDoltSQL(t, dir, "CREATE TABLE "+table+" (id VARCHAR(64) PRIMARY KEY)") + } + requireDoltNoRows(t, dir, "SELECT table_name FROM dolt_status", "dolt_status after creating only dolt_ignored tables") + for _, table := range probed { + agree("migrated shape, dolt_ignored tables in the working set", table, table != ignoredCursorUntrackTempTable) + } + + // 'schemaxmigrations' LIKE-matches 'schema_migrations'; only the name + // comparison keeps it from reading as the cursor table once that is gone. + runDoltSQL(t, dir, "CREATE TABLE schemaxmigrations (id INT PRIMARY KEY)") + runDoltSQL(t, dir, "DROP TABLE "+mainSource.cursorTable) + agree("LIKE wildcard lookalike", mainSource.cursorTable, false) + + // A table stored in another case is the same table to Dolt. + runDoltSQL(t, dir, "RENAME TABLE bd_events_seq TO bd_events_seq_case") + runDoltSQL(t, dir, "RENAME TABLE bd_events_seq_case TO BD_EVENTS_SEQ") + agree("table stored in another case", "bd_events_seq", true) +} diff --git a/internal/storage/uow/dolt_sql_provider_lock_test.go b/internal/storage/uow/dolt_sql_provider_lock_test.go index d276cbe57..e897f83f2 100644 --- a/internal/storage/uow/dolt_sql_provider_lock_test.go +++ b/internal/storage/uow/dolt_sql_provider_lock_test.go @@ -173,9 +173,10 @@ func TestInitSchemaConvergenceProbeRunsWithNoSessionDatabase(t *testing.T) { // migrationWorkNeeded's first question, reached only because the probe put // the session on the database itself. The cursor is far behind this // binary, so there is real work and the fast path declines. - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). - WithArgs("schema_migrations"). + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery("^SHOW TABLES$"). + WillReturnRows(sqlmock.NewRows([]string{"Tables_in_beads"}).AddRow("schema_migrations")) mock.ExpectQuery(regexp.QuoteMeta("SELECT COALESCE(MAX(version), 0) FROM schema_migrations")). WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) diff --git a/internal/storage/uow/team_server_schema_test.go b/internal/storage/uow/team_server_schema_test.go index 2b866eb23..9c45f9dad 100644 --- a/internal/storage/uow/team_server_schema_test.go +++ b/internal/storage/uow/team_server_schema_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "os" + "regexp" "strings" "testing" @@ -19,13 +20,13 @@ import ( // table. The probe was added by be-bv7x so a not-yet-created cursor table // never poisons the pooled Dolt session with a failing statement. func expectCursorProbe(mock sqlmock.Sqlmock, table string, exists bool) { - present := 0 + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = DATABASE()")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + rows := sqlmock.NewRows([]string{"Tables_in_beads"}).AddRow("issues") if exists { - present = 1 + rows.AddRow(table) } - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). - WithArgs(table). - WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(present)) + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) } func expectVersionQuery(mock sqlmock.Sqlmock, version int) {