fix(test): retry contended migration lock when re-running schema init on the shared test DB (#7235)
TestSchemaMigrationDoesNotCommitPreExistingStagedData flaked in the server-Dolt storage tier (shard 4/16) with: initSchemaOnDB: schema migration: schema: acquire migration lock: schema migration lock unavailable: timeout Every setupTestStore caller shares testSharedDB, and MigrateUpWithLock's GET_LOCK name is database-scoped, so a test that re-runs schema init contends with every parallel test's setup-time init for the same lock. setupTestStore already uses initSchemaOnDBWithRetry for exactly this reason, but schema_migration_dirty_test.go (4 sites) and schema_version_test.go (2 sites) called the bare initSchemaOnDB, which gives up after a single 5s wait. On a loaded runner a neighbour holds the lock longer than that and the test fails. Production code retries this case (isRetryableError covers schema.IsMigrationLockError), so this is test-side only. Route those calls through a rerunSchemaInitOnTestStore helper that uses initSchemaOnDBWithRetry. It retries only a contended lock or a transient connection error, never a migration result, so the asserted outcomes (including the expected "pre-existing dirty tables changed" error) are unchanged. Seen on four unrelated PR branches in three days. Bazel repro (shard-4 test set, server pinned to 6 CPUs, --local_test_jobs=6): before 1/16 runs failed with the CI signature; after 32/32 passed. Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
cd2507735e
commit
aaeaedc386
3 changed files with 20 additions and 6 deletions
|
|
@ -270,6 +270,20 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) {
|
|||
return store, cleanup
|
||||
}
|
||||
|
||||
// rerunSchemaInitOnTestStore re-runs schema initialization on a
|
||||
// setupTestStore store, for tests that set up a schema state and then assert
|
||||
// what one more migration pass does to it. Every setupTestStore caller shares
|
||||
// testSharedDB, so this pass's GET_LOCK contends with every parallel test's
|
||||
// own setup-time init for the same database-scoped lock name, exactly like the
|
||||
// branch-local init in setupTestStore. The bare initSchemaOnDB gives up after
|
||||
// one 5s wait (migrationLockAcquireTimeoutSeconds) and failed such tests with
|
||||
// "schema migration lock unavailable: timeout" on loaded CI runners; the
|
||||
// retrying helper retries only a contended lock or a transient connection
|
||||
// error, never a migration result, so the outcome under test is unchanged.
|
||||
func rerunSchemaInitOnTestStore(ctx context.Context, store *DoltStore) (int, error) {
|
||||
return initSchemaOnDBWithRetry(ctx, store.db)
|
||||
}
|
||||
|
||||
// setupConcurrentTestStore creates a test store with its own database for
|
||||
// concurrent tests that need multiple connections. Branch-per-test isolation
|
||||
// requires MaxOpenConns=1, which prevents concurrent transactions.
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingDirtyData(t *testing.T) {
|
|||
t.Fatalf("insert dirty label: %v", err)
|
||||
}
|
||||
|
||||
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
|
||||
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
|
||||
t.Fatalf("initSchemaOnDB: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -136,7 +136,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingStagedData(t *testing.T) {
|
|||
t.Fatalf("stage label: %v", err)
|
||||
}
|
||||
|
||||
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
|
||||
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
|
||||
t.Fatalf("initSchemaOnDB: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -224,7 +224,7 @@ func TestSchemaMigrationDoesNotCommitIgnoredDirtyWispTables(t *testing.T) {
|
|||
t.Fatalf("CreateIssue no-history wisp: %v", err)
|
||||
}
|
||||
|
||||
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
|
||||
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
|
||||
t.Fatalf("initSchemaOnDB: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -313,7 +313,7 @@ func TestSchemaMigrationRejectsChangedPreExistingDirtyTable(t *testing.T) {
|
|||
t.Fatalf("dirty custom_statuses: %v", err)
|
||||
}
|
||||
|
||||
_, err := initSchemaOnDB(ctx, store.db)
|
||||
_, err := rerunSchemaInitOnTestStore(ctx, store)
|
||||
if err == nil || !strings.Contains(err.Error(), "pre-existing dirty tables changed") {
|
||||
t.Fatalf("initSchemaOnDB error = %v, want changed dirty table error", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func TestSchemaSkipsReinit(t *testing.T) {
|
|||
}
|
||||
|
||||
// Run initSchemaOnDB again — should skip because migrations are current
|
||||
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
|
||||
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
|
||||
t.Fatalf("initSchemaOnDB failed: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -78,7 +78,7 @@ func TestSchemaRunsInitWhenStale(t *testing.T) {
|
|||
}
|
||||
|
||||
// Run initSchemaOnDB — should detect stale and re-apply
|
||||
if _, err := initSchemaOnDB(ctx, store.db); err != nil {
|
||||
if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil {
|
||||
t.Fatalf("initSchemaOnDB failed: %v", err)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue