fix(test): retry contended migration lock when re-running schema init on the shared test DB (#7235)

TestSchemaMigrationDoesNotCommitPreExistingStagedData flaked in the
server-Dolt storage tier (shard 4/16) with:

  initSchemaOnDB: schema migration: schema: acquire migration lock:
  schema migration lock unavailable: timeout

Every setupTestStore caller shares testSharedDB, and MigrateUpWithLock's
GET_LOCK name is database-scoped, so a test that re-runs schema init
contends with every parallel test's setup-time init for the same lock.
setupTestStore already uses initSchemaOnDBWithRetry for exactly this
reason, but schema_migration_dirty_test.go (4 sites) and
schema_version_test.go (2 sites) called the bare initSchemaOnDB, which
gives up after a single 5s wait. On a loaded runner a neighbour holds the
lock longer than that and the test fails. Production code retries this
case (isRetryableError covers schema.IsMigrationLockError), so this is
test-side only.

Route those calls through a rerunSchemaInitOnTestStore helper that uses
initSchemaOnDBWithRetry. It retries only a contended lock or a transient
connection error, never a migration result, so the asserted outcomes
(including the expected "pre-existing dirty tables changed" error) are
unchanged.

Seen on four unrelated PR branches in three days. Bazel repro (shard-4
test set, server pinned to 6 CPUs, --local_test_jobs=6): before 1/16
runs failed with the CI signature; after 32/32 passed.

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Julian Knutsen 2026-10-05 02:21:42 -07:00 • committed by GitHub
commit aaeaedc386
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 20 additions and 6 deletions

View file

@ -270,6 +270,20 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) {
return store, cleanup
}
// rerunSchemaInitOnTestStore re-runs schema initialization on a
// setupTestStore store, for tests that set up a schema state and then assert
// what one more migration pass does to it. Every setupTestStore caller shares
// testSharedDB, so this pass's GET_LOCK contends with every parallel test's
// own setup-time init for the same database-scoped lock name, exactly like the
// branch-local init in setupTestStore. The bare initSchemaOnDB gives up after
// one 5s wait (migrationLockAcquireTimeoutSeconds) and failed such tests with
// "schema migration lock unavailable: timeout" on loaded CI runners; the
// retrying helper retries only a contended lock or a transient connection
// error, never a migration result, so the outcome under test is unchanged.
func rerunSchemaInitOnTestStore(ctx context.Context, store *DoltStore) (int, error) {
return initSchemaOnDBWithRetry(ctx, store.db)
}
// setupConcurrentTestStore creates a test store with its own database for
// concurrent tests that need multiple connections. Branch-per-test isolation
// requires MaxOpenConns=1, which prevents concurrent transactions.

View file

@ -46,7 +46,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingDirtyData(t *testing.T) {
t.Fatalf("insert dirty label: %v", err)
}
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
t.Fatalf("initSchemaOnDB: %v", err)
}
@ -136,7 +136,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingStagedData(t *testing.T) {
t.Fatalf("stage label: %v", err)
}
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
t.Fatalf("initSchemaOnDB: %v", err)
}
@ -224,7 +224,7 @@ func TestSchemaMigrationDoesNotCommitIgnoredDirtyWispTables(t *testing.T) {
t.Fatalf("CreateIssue no-history wisp: %v", err)
}
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
t.Fatalf("initSchemaOnDB: %v", err)
}
@ -313,7 +313,7 @@ func TestSchemaMigrationRejectsChangedPreExistingDirtyTable(t *testing.T) {
t.Fatalf("dirty custom_statuses: %v", err)
}
_, err := initSchemaOnDB(ctx, store.db)
_, err := rerunSchemaInitOnTestStore(ctx, store)
if err == nil || !strings.Contains(err.Error(), "pre-existing dirty tables changed") {
t.Fatalf("initSchemaOnDB error = %v, want changed dirty table error", err)
}

View file

@ -46,7 +46,7 @@ func TestSchemaSkipsReinit(t *testing.T) {
}
// Run initSchemaOnDB again — should skip because migrations are current
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
t.Fatalf("initSchemaOnDB failed: %v", err)
}
@ -78,7 +78,7 @@ func TestSchemaRunsInitWhenStale(t *testing.T) {
}
// Run initSchemaOnDB — should detect stale and re-apply
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
t.Fatalf("initSchemaOnDB failed: %v", err)
}