From aaeaedc386dec58605a27706b62aa771db9d98cb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 02:21:42 -0700 Subject: [PATCH] fix(test): retry contended migration lock when re-running schema init on the shared test DB (#7235) TestSchemaMigrationDoesNotCommitPreExistingStagedData flaked in the server-Dolt storage tier (shard 4/16) with: initSchemaOnDB: schema migration: schema: acquire migration lock: schema migration lock unavailable: timeout Every setupTestStore caller shares testSharedDB, and MigrateUpWithLock's GET_LOCK name is database-scoped, so a test that re-runs schema init contends with every parallel test's setup-time init for the same lock. setupTestStore already uses initSchemaOnDBWithRetry for exactly this reason, but schema_migration_dirty_test.go (4 sites) and schema_version_test.go (2 sites) called the bare initSchemaOnDB, which gives up after a single 5s wait. On a loaded runner a neighbour holds the lock longer than that and the test fails. Production code retries this case (isRetryableError covers schema.IsMigrationLockError), so this is test-side only. Route those calls through a rerunSchemaInitOnTestStore helper that uses initSchemaOnDBWithRetry. It retries only a contended lock or a transient connection error, never a migration result, so the asserted outcomes (including the expected "pre-existing dirty tables changed" error) are unchanged. Seen on four unrelated PR branches in three days. Bazel repro (shard-4 test set, server pinned to 6 CPUs, --local_test_jobs=6): before 1/16 runs failed with the CI signature; after 32/32 passed. Co-authored-by: Claude Opus 5.5 (1M context) --- internal/storage/dolt/dolt_test.go | 14 ++++++++++++++ .../storage/dolt/schema_migration_dirty_test.go | 8 ++++---- internal/storage/dolt/schema_version_test.go | 4 ++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/internal/storage/dolt/dolt_test.go b/internal/storage/dolt/dolt_test.go index 38ef50fdb..a788b7ca7 100644 --- a/internal/storage/dolt/dolt_test.go +++ b/internal/storage/dolt/dolt_test.go @@ -270,6 +270,20 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) { return store, cleanup } +// rerunSchemaInitOnTestStore re-runs schema initialization on a +// setupTestStore store, for tests that set up a schema state and then assert +// what one more migration pass does to it. Every setupTestStore caller shares +// testSharedDB, so this pass's GET_LOCK contends with every parallel test's +// own setup-time init for the same database-scoped lock name, exactly like the +// branch-local init in setupTestStore. The bare initSchemaOnDB gives up after +// one 5s wait (migrationLockAcquireTimeoutSeconds) and failed such tests with +// "schema migration lock unavailable: timeout" on loaded CI runners; the +// retrying helper retries only a contended lock or a transient connection +// error, never a migration result, so the outcome under test is unchanged. +func rerunSchemaInitOnTestStore(ctx context.Context, store *DoltStore) (int, error) { + return initSchemaOnDBWithRetry(ctx, store.db) +} + // setupConcurrentTestStore creates a test store with its own database for // concurrent tests that need multiple connections. Branch-per-test isolation // requires MaxOpenConns=1, which prevents concurrent transactions. diff --git a/internal/storage/dolt/schema_migration_dirty_test.go b/internal/storage/dolt/schema_migration_dirty_test.go index 390642a33..6c5fcbc4e 100644 --- a/internal/storage/dolt/schema_migration_dirty_test.go +++ b/internal/storage/dolt/schema_migration_dirty_test.go @@ -46,7 +46,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingDirtyData(t *testing.T) { t.Fatalf("insert dirty label: %v", err) } - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB: %v", err) } @@ -136,7 +136,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingStagedData(t *testing.T) { t.Fatalf("stage label: %v", err) } - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB: %v", err) } @@ -224,7 +224,7 @@ func TestSchemaMigrationDoesNotCommitIgnoredDirtyWispTables(t *testing.T) { t.Fatalf("CreateIssue no-history wisp: %v", err) } - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB: %v", err) } @@ -313,7 +313,7 @@ func TestSchemaMigrationRejectsChangedPreExistingDirtyTable(t *testing.T) { t.Fatalf("dirty custom_statuses: %v", err) } - _, err := initSchemaOnDB(ctx, store.db) + _, err := rerunSchemaInitOnTestStore(ctx, store) if err == nil || !strings.Contains(err.Error(), "pre-existing dirty tables changed") { t.Fatalf("initSchemaOnDB error = %v, want changed dirty table error", err) } diff --git a/internal/storage/dolt/schema_version_test.go b/internal/storage/dolt/schema_version_test.go index e7b8deb25..974c4087d 100644 --- a/internal/storage/dolt/schema_version_test.go +++ b/internal/storage/dolt/schema_version_test.go @@ -46,7 +46,7 @@ func TestSchemaSkipsReinit(t *testing.T) { } // Run initSchemaOnDB again — should skip because migrations are current - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB failed: %v", err) } @@ -78,7 +78,7 @@ func TestSchemaRunsInitWhenStale(t *testing.T) { } // Run initSchemaOnDB — should detect stale and re-apply - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB failed: %v", err) }