fix(dolt): bind bootstrap recovery to database incarnation (#5252)
* fix(init): commit intentional config state * fix: bind bootstrap recovery to database incarnation * fix(uow): bind bootstrap recovery to incarnation --------- Co-authored-by: Steve Yegge <steve.yegge@gmail.com>
This commit is contained in:
parent
c680fe1bc4
commit
6cbd63359a
7 changed files with 846 additions and 84 deletions
|
|
@ -1762,11 +1762,13 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1):
|
|||
}
|
||||
|
||||
// Auto-commit Dolt state so bd doctor doesn't warn about uncommitted
|
||||
// changes and users don't need a separate "bd vc commit" step. Skipped in
|
||||
// gateway mode: the shared server owns its own history and the credential
|
||||
// may be read-only, so bd must not issue DOLT_ADD/DOLT_COMMIT against it.
|
||||
// changes and users don't need a separate "bd vc commit" step. Init
|
||||
// intentionally writes issue_prefix to config, while ordinary Commit
|
||||
// excludes config to avoid sweeping unrelated stale values. Skip this in
|
||||
// gateway mode: the shared server owns its history and the credential may
|
||||
// be read-only, so bd must not issue DOLT_ADD/DOLT_COMMIT against it.
|
||||
if shouldWriteInitStateToDB(doltCfg.Gateway) {
|
||||
if err := store.Commit(ctx, "bd init"); err != nil {
|
||||
if err := commitInitState(ctx, store); err != nil {
|
||||
// Non-fatal: some setups (e.g. no tables yet) may have nothing to commit
|
||||
if !strings.Contains(err.Error(), "nothing to commit") {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to commit initial state: %v\n", err)
|
||||
|
|
@ -2972,6 +2974,18 @@ func isEmptyRemoteCloneError(err error) bool {
|
|||
return strings.Contains(msg, "contains no dolt data")
|
||||
}
|
||||
|
||||
type initStateCommitter interface {
|
||||
CommitWithConfig(context.Context, string) error
|
||||
}
|
||||
|
||||
// commitInitState commits the configuration init deliberately created as well
|
||||
// as the initial schema state. The ordinary Commit contract excludes config to
|
||||
// avoid sweeping unrelated stale values, so using it here leaves every fresh
|
||||
// database dirty immediately after a successful init.
|
||||
func commitInitState(ctx context.Context, store initStateCommitter) error {
|
||||
return store.CommitWithConfig(ctx, "bd init")
|
||||
}
|
||||
|
||||
// verifyMetadata writes a metadata field and verifies the write succeeded.
|
||||
// Returns true if write+verify succeeded, false with warning if either failed.
|
||||
func verifyMetadata(ctx context.Context, store storage.DoltStorage, key, value string) bool {
|
||||
|
|
|
|||
35
cmd/bd/init_commit_test.go
Normal file
35
cmd/bd/init_commit_test.go
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type recordingInitCommitter struct {
|
||||
message string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *recordingInitCommitter) CommitWithConfig(_ context.Context, message string) error {
|
||||
s.message = message
|
||||
return s.err
|
||||
}
|
||||
|
||||
func TestCommitInitStateIncludesIntentionalConfig(t *testing.T) {
|
||||
store := &recordingInitCommitter{}
|
||||
if err := commitInitState(context.Background(), store); err != nil {
|
||||
t.Fatalf("commitInitState: %v", err)
|
||||
}
|
||||
if store.message != "bd init" {
|
||||
t.Fatalf("CommitWithConfig message = %q, want %q", store.message, "bd init")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommitInitStatePreservesCommitError(t *testing.T) {
|
||||
want := errors.New("commit failed")
|
||||
store := &recordingInitCommitter{err: want}
|
||||
if got := commitInitState(context.Background(), store); !errors.Is(got, want) {
|
||||
t.Fatalf("commitInitState error = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
324
internal/storage/dolt/fresh_bootstrap_heal_integration_test.go
Normal file
324
internal/storage/dolt/fresh_bootstrap_heal_integration_test.go
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
//go:build integration && !windows
|
||||
|
||||
package dolt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"github.com/steveyegge/beads/internal/doltserver"
|
||||
"github.com/steveyegge/beads/internal/storage/doltutil"
|
||||
"github.com/steveyegge/beads/internal/storage/schema"
|
||||
"github.com/steveyegge/beads/internal/testutil/integration"
|
||||
)
|
||||
|
||||
// TestFreshBootstrapHealIncarnation exercises the destructive bootstrap-heal
|
||||
// authorization against a real Dolt sql-server. In particular, the
|
||||
// drop/recreate subtest is the regression for stale creator authority crossing
|
||||
// a database-name ABA boundary and deleting another incarnation's work.
|
||||
func TestFreshBootstrapHealIncarnation(t *testing.T) {
|
||||
integration.RequireDolt(t)
|
||||
t.Setenv("BEADS_DOLT_SHARED_SERVER", "0")
|
||||
t.Setenv("BEADS_DOLT_SERVER_PORT", "")
|
||||
t.Setenv("BEADS_DOLT_PORT", "")
|
||||
|
||||
beadsDir := filepath.Join(t.TempDir(), ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0o700); err != nil {
|
||||
t.Fatalf("mkdir beads dir: %v", err)
|
||||
}
|
||||
state, err := doltserver.Start(beadsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("start local Dolt server: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
current, stateErr := doltserver.IsRunning(beadsDir)
|
||||
if stateErr != nil {
|
||||
t.Errorf("check local Dolt server before stop: %v", stateErr)
|
||||
return
|
||||
}
|
||||
if current == nil || !current.Running {
|
||||
return
|
||||
}
|
||||
if err := doltserver.Stop(beadsDir); err != nil {
|
||||
t.Errorf("stop local Dolt server: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
adminDSN := doltutil.ServerDSN{
|
||||
Host: "127.0.0.1",
|
||||
Port: state.Port,
|
||||
User: "root",
|
||||
}.String()
|
||||
admin, err := sql.Open("mysql", adminDSN)
|
||||
if err != nil {
|
||||
t.Fatalf("open admin connection: %v", err)
|
||||
}
|
||||
defer admin.Close()
|
||||
|
||||
t.Run("exactly one concurrent creator receives a capability", func(t *testing.T) {
|
||||
const workers = 16
|
||||
type result struct {
|
||||
hasCapability bool
|
||||
err error
|
||||
}
|
||||
|
||||
ready := make(chan struct{})
|
||||
results := make(chan result, workers)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-ready
|
||||
cfg := freshBootstrapIntegrationConfig(
|
||||
beadsDir, state.Port, "heal_creator_race", fmt.Sprintf("creator-%02d", i),
|
||||
)
|
||||
db, _, facts, err := openServerConnection(ctx, cfg)
|
||||
if db != nil {
|
||||
_ = db.Close()
|
||||
}
|
||||
results <- result{hasCapability: facts.bootstrapHeal != nil, err: err}
|
||||
}(i)
|
||||
}
|
||||
close(ready)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
capabilities := 0
|
||||
for got := range results {
|
||||
if got.err != nil {
|
||||
t.Errorf("concurrent open: %v", got.err)
|
||||
}
|
||||
if got.hasCapability {
|
||||
capabilities++
|
||||
}
|
||||
}
|
||||
if capabilities != 1 {
|
||||
t.Fatalf("creator capability grants = %d, want exactly 1", capabilities)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("exact creator heals its interrupted bootstrap", func(t *testing.T) {
|
||||
cfg := freshBootstrapIntegrationConfig(beadsDir, state.Port, "heal_exact_creator", "exact-creator")
|
||||
db, _, facts, err := openServerConnection(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("creator open: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
if facts.bootstrapHeal == nil {
|
||||
t.Fatal("exact bare CREATE did not return a capability")
|
||||
}
|
||||
|
||||
prepareFreshBootstrapV51Dirty(t, ctx, db, "creator_debris")
|
||||
if _, err := initSchemaOnDBWithBootstrapHeal(
|
||||
ctx, db, facts.bootstrapHeal, serverEndpointIdentity(cfg),
|
||||
); err != nil {
|
||||
t.Fatalf("creator-authorized migration: %v", err)
|
||||
}
|
||||
assertFreshBootstrapColumnCount(t, ctx, db, cfg.Database, "creator_debris", 0)
|
||||
assertFreshBootstrapConverged(t, ctx, db)
|
||||
})
|
||||
|
||||
t.Run("fresh store initializes and commits cleanly", func(t *testing.T) {
|
||||
cfg := freshBootstrapIntegrationConfig(beadsDir, state.Port, "heal_fresh_store", "fresh-store")
|
||||
cfg.CommitterName = "Beads Test"
|
||||
cfg.CommitterEmail = "beads@example.com"
|
||||
store, err := New(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("open fresh store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err := store.SetConfig(ctx, "issue_prefix", "heal"); err != nil {
|
||||
t.Fatalf("set intentional init config: %v", err)
|
||||
}
|
||||
if err := store.CommitWithConfig(ctx, "bd init"); err != nil {
|
||||
t.Fatalf("config-inclusive init commit: %v", err)
|
||||
}
|
||||
assertFreshBootstrapConverged(t, ctx, store.db)
|
||||
var prefix string
|
||||
if err := store.db.QueryRowContext(ctx,
|
||||
"SELECT value FROM config AS OF 'HEAD' WHERE `key` = 'issue_prefix'",
|
||||
).Scan(&prefix); err != nil {
|
||||
t.Fatalf("read committed issue_prefix: %v", err)
|
||||
}
|
||||
if prefix != "heal" {
|
||||
t.Fatalf("committed issue_prefix = %q, want heal", prefix)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("drop and recreate revokes creator capability", func(t *testing.T) {
|
||||
cfg := freshBootstrapIntegrationConfig(beadsDir, state.Port, "heal_creator_aba", "creator-aba")
|
||||
creatorDB, _, facts, err := openServerConnection(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("creator open: %v", err)
|
||||
}
|
||||
defer creatorDB.Close()
|
||||
if facts.bootstrapHeal == nil {
|
||||
t.Fatal("exact bare CREATE did not return a capability")
|
||||
}
|
||||
|
||||
if _, err := admin.ExecContext(ctx, "DROP DATABASE `heal_creator_aba`"); err != nil {
|
||||
t.Fatalf("drop creator database: %v", err)
|
||||
}
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `heal_creator_aba`"); err != nil {
|
||||
t.Fatalf("create replacement database: %v", err)
|
||||
}
|
||||
replacement := openFreshBootstrapIntegrationDB(t, state.Port, cfg.Database)
|
||||
defer replacement.Close()
|
||||
prepareFreshBootstrapV51Dirty(t, ctx, replacement, "foreign_debris")
|
||||
|
||||
_, migrateErr := initSchemaOnDBWithRetryAndGateBootstrapHeal(
|
||||
ctx, creatorDB, nil, facts.bootstrapHeal, serverEndpointIdentity(cfg),
|
||||
)
|
||||
var dirtyErr *schema.DirtyTablesError
|
||||
if !errors.As(migrateErr, &dirtyErr) {
|
||||
t.Fatalf("replacement migration error = %v, want DirtyTablesError", migrateErr)
|
||||
}
|
||||
assertFreshBootstrapColumnCount(t, ctx, replacement, cfg.Database, "foreign_debris", 1)
|
||||
if got := freshBootstrapSchemaVersion(t, ctx, replacement); got != 51 {
|
||||
t.Fatalf("replacement schema version = %d, want preserved v51", got)
|
||||
}
|
||||
if got := freshBootstrapDoltStatusCount(t, ctx, replacement); got == 0 {
|
||||
t.Fatal("replacement working set was cleared; wanted fail-closed preservation")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("preexisting dirty database has no reset authority", func(t *testing.T) {
|
||||
const database = "heal_preexisting_dirty"
|
||||
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+database+"`"); err != nil {
|
||||
t.Fatalf("create preexisting database: %v", err)
|
||||
}
|
||||
db := openFreshBootstrapIntegrationDB(t, state.Port, database)
|
||||
defer db.Close()
|
||||
prepareFreshBootstrapV51Dirty(t, ctx, db, "preexisting_debris")
|
||||
|
||||
_, migrateErr := initSchemaOnDB(ctx, db)
|
||||
var dirtyErr *schema.DirtyTablesError
|
||||
if !errors.As(migrateErr, &dirtyErr) {
|
||||
t.Fatalf("preexisting migration error = %v, want DirtyTablesError", migrateErr)
|
||||
}
|
||||
assertFreshBootstrapColumnCount(t, ctx, db, database, "preexisting_debris", 1)
|
||||
if got := freshBootstrapDoltStatusCount(t, ctx, db); got == 0 {
|
||||
t.Fatal("preexisting working set was cleared; wanted fail-closed preservation")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func freshBootstrapIntegrationConfig(beadsDir string, port int, database, pathSuffix string) *Config {
|
||||
return &Config{
|
||||
Path: filepath.Join(beadsDir, pathSuffix),
|
||||
BeadsDir: beadsDir,
|
||||
ServerHost: "127.0.0.1",
|
||||
ServerPort: port,
|
||||
ServerUser: "root",
|
||||
Database: database,
|
||||
CreateIfMissing: true,
|
||||
MaxOpenConns: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func prepareFreshBootstrapV51Dirty(t *testing.T, ctx context.Context, db *sql.DB, debrisColumn string) {
|
||||
t.Helper()
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("pin database: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := schema.MigrateUpTo(ctx, conn, 51); err != nil {
|
||||
t.Fatalf("migrate database to v51: %v", err)
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx,
|
||||
"CALL DOLT_COMMIT('-Am', 'test: v51 baseline', '--author', 'Beads Test <beads@example.com>')",
|
||||
); err != nil {
|
||||
t.Fatalf("commit v51 baseline: %v", err)
|
||||
}
|
||||
if got := freshBootstrapSchemaVersion(t, ctx, conn); got != 51 {
|
||||
t.Fatalf("baseline schema version = %d, want 51", got)
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, "ALTER TABLE issues ADD COLUMN "+debrisColumn+" INT"); err != nil {
|
||||
t.Fatalf("dirty bootstrap database: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func openFreshBootstrapIntegrationDB(t *testing.T, port int, database string) *sql.DB {
|
||||
t.Helper()
|
||||
dsn := doltutil.ServerDSN{
|
||||
Host: "127.0.0.1",
|
||||
Port: port,
|
||||
User: "root",
|
||||
Database: database,
|
||||
}.String()
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", database, err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
type freshBootstrapQueryer interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
func freshBootstrapSchemaVersion(t *testing.T, ctx context.Context, db freshBootstrapQueryer) int {
|
||||
t.Helper()
|
||||
var version int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
"SELECT COALESCE(MAX(version), 0) FROM schema_migrations",
|
||||
).Scan(&version); err != nil {
|
||||
t.Fatalf("read schema version: %v", err)
|
||||
}
|
||||
return version
|
||||
}
|
||||
|
||||
func freshBootstrapDoltStatusCount(t *testing.T, ctx context.Context, db freshBootstrapQueryer) int {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_status").Scan(&count); err != nil {
|
||||
t.Fatalf("count dolt_status: %v", err)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func assertFreshBootstrapColumnCount(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
db freshBootstrapQueryer,
|
||||
database string,
|
||||
column string,
|
||||
want int,
|
||||
) {
|
||||
t.Helper()
|
||||
var count int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = ? AND table_name = 'issues' AND column_name = ?",
|
||||
database, column,
|
||||
).Scan(&count); err != nil {
|
||||
t.Fatalf("count %s.%s: %v", database, column, err)
|
||||
}
|
||||
if count != want {
|
||||
t.Fatalf("column %s.%s count = %d, want %d", database, column, count, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFreshBootstrapConverged(t *testing.T, ctx context.Context, db freshBootstrapQueryer) {
|
||||
t.Helper()
|
||||
want := schema.LatestVersion()
|
||||
if got := freshBootstrapSchemaVersion(t, ctx, db); got != want {
|
||||
t.Fatalf("schema version = %d, want %d", got, want)
|
||||
}
|
||||
if got := freshBootstrapDoltStatusCount(t, ctx, db); got != 0 {
|
||||
t.Fatalf("dolt_status rows after heal = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -306,15 +306,16 @@ var _ storage.ExternalRefHistoryQuerier = (*DoltStore)(nil)
|
|||
|
||||
// DoltStore implements the Storage interface using Dolt
|
||||
type DoltStore struct {
|
||||
db *sql.DB
|
||||
dbPath string // Path to Dolt data directory (server root, e.g. .beads/dolt/)
|
||||
beadsDir string // Path to .beads directory (parent of dbPath)
|
||||
database string // Database name (subdirectory under dbPath)
|
||||
closed atomic.Bool // Tracks whether Close() has been called
|
||||
connStr string // Connection string for reconnection
|
||||
mu sync.RWMutex // Protects concurrent access
|
||||
readOnly bool // True if opened in read-only mode
|
||||
credentialKey []byte // Random encryption key for federation credentials
|
||||
db *sql.DB
|
||||
dbPath string // Path to Dolt data directory (server root, e.g. .beads/dolt/)
|
||||
beadsDir string // Path to .beads directory (parent of dbPath)
|
||||
database string // Database name (subdirectory under dbPath)
|
||||
closed atomic.Bool // Tracks whether Close() has been called
|
||||
connStr string // Connection string for reconnection
|
||||
serverEndpoint string // Exact endpoint bound to bootstrap reset authority
|
||||
mu sync.RWMutex // Protects concurrent access
|
||||
readOnly bool // True if opened in read-only mode
|
||||
credentialKey []byte // Random encryption key for federation credentials
|
||||
|
||||
// localActiveDatabaseDir is the exact active database directory when this
|
||||
// store instance has authoritative local filesystem access. It is resolved
|
||||
|
|
@ -1744,6 +1745,7 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) {
|
|||
database: cfg.Database,
|
||||
localActiveDatabaseDir: resolveLocalActiveDatabaseDir(cfg),
|
||||
connStr: connStr,
|
||||
serverEndpoint: serverEndpointIdentity(cfg),
|
||||
breaker: breaker,
|
||||
committerName: cfg.CommitterName,
|
||||
committerEmail: cfg.CommitterEmail,
|
||||
|
|
@ -1813,7 +1815,7 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) {
|
|||
// ReadOnly for schema — the forward-drift guard above still protects a stale client
|
||||
// binary.
|
||||
if !cfg.ReadOnly && !cfg.Gateway {
|
||||
if err := store.initSchema(ctx, dbFacts.created); err != nil {
|
||||
if err := store.initSchema(ctx, dbFacts.bootstrapHeal); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize schema: %w", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -2092,17 +2094,20 @@ func applyPoolLimits(db *sql.DB, cfg *Config) {
|
|||
}
|
||||
|
||||
// serverConnFacts reports what openServerConnection established about the
|
||||
// target database while connecting. The two fields are deliberately NOT
|
||||
// inverses of each other: for a gateway database, existence is never probed,
|
||||
// so both are false — neither "we created it" nor "it was already there" is
|
||||
// proven, and each caller's gate must fail closed on its own terms.
|
||||
// target database while connecting. Creation and prior existence are
|
||||
// deliberately NOT inverses: for a gateway database, existence is never
|
||||
// probed, so neither is proven and each caller's gate fails closed.
|
||||
type serverConnFacts struct {
|
||||
// created reports whether THIS call's bare CREATE DATABASE won the
|
||||
// ownership arbitration and actually created the database — server-mode's
|
||||
// analog of the uow path's `created` signal (#5042). Threaded through to
|
||||
// initSchema so only the proven creator arms schema.WithFreshBootstrapHeal.
|
||||
// ownership arbitration and actually created the database.
|
||||
created bool
|
||||
|
||||
// bootstrapHeal carries one-shot reset authority bound to the exact
|
||||
// endpoint, server UUID, database, and initial HEAD captured after this
|
||||
// call created and successfully connected to a pristine database. A nil
|
||||
// capability always fails closed.
|
||||
bootstrapHeal *schema.FreshBootstrapHealCapability
|
||||
|
||||
// alreadyExisted reports whether the database was proven to exist on the
|
||||
// server before this call: either the SHOW DATABASES probe found it, or
|
||||
// our CREATE DATABASE was refused with "database exists" (1007). Callers
|
||||
|
|
@ -2272,8 +2277,37 @@ func openServerConnection(ctx context.Context, cfg *Config) (*sql.DB, string, se
|
|||
return nil, "", serverConnFacts{}, fmt.Errorf("database %q not available after CREATE DATABASE: %w", cfg.Database, err)
|
||||
}
|
||||
|
||||
var bootstrapHeal *schema.FreshBootstrapHealCapability
|
||||
if created {
|
||||
conn, connErr := db.Conn(ctx)
|
||||
if connErr != nil {
|
||||
return nil, "", serverConnFacts{}, fmt.Errorf("capture fresh database identity: pin connection: %w", connErr)
|
||||
}
|
||||
bootstrapHeal, err = schema.CaptureFreshBootstrapHealCapability(
|
||||
ctx, conn, serverEndpointIdentity(cfg), cfg.Database,
|
||||
)
|
||||
_ = conn.Close()
|
||||
if err != nil {
|
||||
return nil, "", serverConnFacts{}, fmt.Errorf("capture fresh database identity: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
connReady = true
|
||||
return db, connStr, serverConnFacts{created: created, alreadyExisted: dbExists}, nil
|
||||
return db, connStr, serverConnFacts{
|
||||
created: created,
|
||||
bootstrapHeal: bootstrapHeal,
|
||||
alreadyExisted: dbExists,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// serverEndpointIdentity returns the exact configured transport endpoint. It
|
||||
// is captured with fresh-bootstrap authority and must match the migration
|
||||
// connection's endpoint before that authority can be consumed.
|
||||
func serverEndpointIdentity(cfg *Config) string {
|
||||
if cfg.ServerSocket != "" {
|
||||
return "unix:" + cfg.ServerSocket
|
||||
}
|
||||
return "tcp:" + net.JoinHostPort(cfg.ServerHost, strconv.Itoa(cfg.ServerPort))
|
||||
}
|
||||
|
||||
// databaseExistsOnServer checks if a database with the exact given name exists
|
||||
|
|
@ -2302,20 +2336,17 @@ func databaseExistsOnServer(ctx context.Context, db *sql.DB, name string) (bool,
|
|||
// applied versions in schema_migrations and backfills legacy config-driven
|
||||
// tables. Returns the number of migrations applied.
|
||||
func initSchemaOnDB(ctx context.Context, db *sql.DB) (int, error) {
|
||||
return initSchemaOnDBOwnership(ctx, db, false)
|
||||
return initSchemaOnDBWithBootstrapHeal(ctx, db, nil, "")
|
||||
}
|
||||
|
||||
// initSchemaOnDBOwnership is initSchemaOnDB with the fresh-bootstrap
|
||||
// ownership signal for the #4566 dirty-table guard self-heal
|
||||
// (gastownhall/beads#5012), ported from the uow path's initSchema (#5042):
|
||||
// createdDatabase must be true only when the caller proved — via the
|
||||
// bare-CREATE-DATABASE ownership arbitration in openServerConnection — that
|
||||
// THIS process's open created the target database. On a database this init
|
||||
// created, dirty tables a retry finds can only be this same process's own
|
||||
// half-applied migration step (a session that died between a step's SQL and
|
||||
// its per-step Dolt commit), never pre-existing user data, so
|
||||
// schema.WithFreshBootstrapHeal is safe to arm.
|
||||
func initSchemaOnDBOwnership(ctx context.Context, db *sql.DB, createdDatabase bool) (int, error) {
|
||||
// initSchemaOnDBWithBootstrapHeal threads one-shot, incarnation-bound reset
|
||||
// authority into the migration lock. A nil capability always fails closed.
|
||||
func initSchemaOnDBWithBootstrapHeal(
|
||||
ctx context.Context,
|
||||
db *sql.DB,
|
||||
bootstrapHeal *schema.FreshBootstrapHealCapability,
|
||||
endpoint string,
|
||||
) (int, error) {
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("schema: pin connection: %w", err)
|
||||
|
|
@ -2328,8 +2359,8 @@ func initSchemaOnDBOwnership(ctx context.Context, db *sql.DB, createdDatabase bo
|
|||
}
|
||||
|
||||
var opts []schema.MigrateLockOption
|
||||
if createdDatabase {
|
||||
opts = append(opts, schema.WithFreshBootstrapHeal())
|
||||
if bootstrapHeal != nil {
|
||||
opts = append(opts, schema.WithFreshBootstrapHeal(bootstrapHeal, endpoint))
|
||||
}
|
||||
applied, err := schema.MigrateUpWithLock(ctx, conn, dbName, opts...)
|
||||
if err != nil {
|
||||
|
|
@ -2349,17 +2380,18 @@ func initSchemaOnDBWithRetry(ctx context.Context, db *sql.DB) (int, error) {
|
|||
// retried with them instead of failing the open fast (bd-6dnrw.30); a
|
||||
// *schema.RemoteMigrateGateError refusal stays permanent.
|
||||
func initSchemaOnDBWithRetryAndGate(ctx context.Context, db *sql.DB, gate func(context.Context, *sql.DB) error) (int, error) {
|
||||
return initSchemaOnDBWithRetryAndGateOwnership(ctx, db, gate, false)
|
||||
return initSchemaOnDBWithRetryAndGateBootstrapHeal(ctx, db, gate, nil, "")
|
||||
}
|
||||
|
||||
// initSchemaOnDBWithRetryAndGateOwnership is initSchemaOnDBWithRetryAndGate
|
||||
// with the fresh-bootstrap ownership signal (see initSchemaOnDBOwnership)
|
||||
// threaded to every retry attempt. createdDatabase is fixed for the whole
|
||||
// retry loop: unlike the uow path, where database creation is retried inside
|
||||
// the same loop that runs the migration, server mode creates the database
|
||||
// exactly once in openServerConnection, before this loop starts — so there is
|
||||
// no "re-assert on retry" case to port here.
|
||||
func initSchemaOnDBWithRetryAndGateOwnership(ctx context.Context, db *sql.DB, gate func(context.Context, *sql.DB) error, createdDatabase bool) (int, error) {
|
||||
// initSchemaOnDBWithRetryAndGateBootstrapHeal shares one capability across the
|
||||
// outer retry loop. Once consumed, no later retry can issue another reset.
|
||||
func initSchemaOnDBWithRetryAndGateBootstrapHeal(
|
||||
ctx context.Context,
|
||||
db *sql.DB,
|
||||
gate func(context.Context, *sql.DB) error,
|
||||
bootstrapHeal *schema.FreshBootstrapHealCapability,
|
||||
endpoint string,
|
||||
) (int, error) {
|
||||
// Schema initialization for server mode is idempotent. Retry transient
|
||||
// Dolt startup/catalog races and contended migration-lock attempts so
|
||||
// concurrent bd processes converge instead of failing one unlucky waiter.
|
||||
|
|
@ -2379,7 +2411,7 @@ func initSchemaOnDBWithRetryAndGateOwnership(ctx context.Context, db *sql.DB, ga
|
|||
}
|
||||
}
|
||||
var schemaErr error
|
||||
applied, schemaErr = initSchemaOnDBOwnership(ctx, db, createdDatabase)
|
||||
applied, schemaErr = initSchemaOnDBWithBootstrapHeal(ctx, db, bootstrapHeal, endpoint)
|
||||
if schemaErr != nil && isRetryableError(schemaErr) {
|
||||
return schemaErr
|
||||
}
|
||||
|
|
@ -2391,12 +2423,7 @@ func initSchemaOnDBWithRetryAndGateOwnership(ctx context.Context, db *sql.DB, ga
|
|||
return applied, err
|
||||
}
|
||||
|
||||
// initSchema runs pending schema migrations for a freshly opened store.
|
||||
// createdDatabase must be true only when this open's openServerConnection
|
||||
// call proved (via bare-CREATE ownership arbitration) that it created the
|
||||
// target database — see initSchemaOnDBOwnership for why that arms
|
||||
// schema.WithFreshBootstrapHeal (gastownhall/beads#5012, #5042).
|
||||
func (s *DoltStore) initSchema(ctx context.Context, createdDatabase bool) error {
|
||||
func (s *DoltStore) initSchema(ctx context.Context, bootstrapHeal *schema.FreshBootstrapHealCapability) error {
|
||||
// Schema migrations can run arbitrarily long (e.g. full-table recomputes
|
||||
// such as the is_blocked backfill in migration 0047). The main connection
|
||||
// pool sets a 10s ReadTimeout (see buildServerDSN); a slow migration over
|
||||
|
|
@ -2447,7 +2474,7 @@ func (s *DoltStore) initSchema(ctx context.Context, createdDatabase bool) error
|
|||
gate := func(ctx context.Context, db *sql.DB) error {
|
||||
return schema.CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(ctx, db, s.remote, s.hasPersistedCLIRemote, adopt)
|
||||
}
|
||||
_, err = initSchemaOnDBWithRetryAndGateOwnership(ctx, migDB, gate, createdDatabase)
|
||||
_, err = initSchemaOnDBWithRetryAndGateBootstrapHeal(ctx, migDB, gate, bootstrapHeal, s.serverEndpoint)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"fmt"
|
||||
"hash/fnv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -52,21 +53,100 @@ func MigrationLockName(databaseName string) string {
|
|||
type MigrateLockOption func(*migrateLockOptions)
|
||||
|
||||
type migrateLockOptions struct {
|
||||
freshBootstrapHeal bool
|
||||
freshBootstrapHeal *freshBootstrapHealRequest
|
||||
}
|
||||
|
||||
type freshBootstrapHealRequest struct {
|
||||
capability *FreshBootstrapHealCapability
|
||||
endpoint string
|
||||
}
|
||||
|
||||
// FreshBootstrapHealCapability is a one-shot authorization to discard an
|
||||
// interrupted bootstrap working set. It is bound to the exact endpoint,
|
||||
// server, database, and initial Dolt commit observed immediately after a
|
||||
// successful bare CREATE DATABASE. Its fields are intentionally private so a
|
||||
// caller cannot manufacture authority from a boolean.
|
||||
//
|
||||
// A capability may be shared by the server-mode migration retry loop. An
|
||||
// atomic consumed bit ensures that the whole logical open can attempt at most
|
||||
// one destructive reset, even when a later transient error causes an outer
|
||||
// retry.
|
||||
type FreshBootstrapHealCapability struct {
|
||||
endpoint string
|
||||
serverUUID string
|
||||
databaseName string
|
||||
initialHead string
|
||||
consumed atomic.Bool
|
||||
}
|
||||
|
||||
// CaptureFreshBootstrapHealCapability records the identity of a database that
|
||||
// the caller has just created with an exact, successful bare CREATE DATABASE.
|
||||
// The database must still be pristine: one initial commit and an empty working
|
||||
// set. Any uncertainty fails closed instead of issuing reset authority.
|
||||
func CaptureFreshBootstrapHealCapability(
|
||||
ctx context.Context,
|
||||
conn *sql.Conn,
|
||||
endpoint string,
|
||||
expectedDatabase string,
|
||||
) (*FreshBootstrapHealCapability, error) {
|
||||
if endpoint == "" {
|
||||
return nil, errors.New("schema: capture fresh-bootstrap identity: empty endpoint")
|
||||
}
|
||||
if expectedDatabase == "" {
|
||||
return nil, errors.New("schema: capture fresh-bootstrap identity: empty database name")
|
||||
}
|
||||
|
||||
var databaseName, serverUUID, initialHead string
|
||||
if err := conn.QueryRowContext(ctx,
|
||||
"SELECT DATABASE(), @@server_uuid, DOLT_HASHOF('HEAD')",
|
||||
).Scan(&databaseName, &serverUUID, &initialHead); err != nil {
|
||||
return nil, fmt.Errorf("schema: capture fresh-bootstrap identity: %w", err)
|
||||
}
|
||||
if databaseName != expectedDatabase {
|
||||
return nil, fmt.Errorf("schema: capture fresh-bootstrap identity: database is %q, want %q", databaseName, expectedDatabase)
|
||||
}
|
||||
if serverUUID == "" {
|
||||
return nil, errors.New("schema: capture fresh-bootstrap identity: empty server UUID")
|
||||
}
|
||||
if initialHead == "" {
|
||||
return nil, errors.New("schema: capture fresh-bootstrap identity: empty initial HEAD")
|
||||
}
|
||||
|
||||
var commitCount, dirtyCount int
|
||||
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_log").Scan(&commitCount); err != nil {
|
||||
return nil, fmt.Errorf("schema: verify fresh-bootstrap history: %w", err)
|
||||
}
|
||||
if commitCount != 1 {
|
||||
return nil, fmt.Errorf("schema: verify fresh-bootstrap history: got %d commits, want 1", commitCount)
|
||||
}
|
||||
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_status").Scan(&dirtyCount); err != nil {
|
||||
return nil, fmt.Errorf("schema: verify fresh-bootstrap working set: %w", err)
|
||||
}
|
||||
if dirtyCount != 0 {
|
||||
return nil, fmt.Errorf("schema: verify fresh-bootstrap working set: got %d dirty entries, want 0", dirtyCount)
|
||||
}
|
||||
|
||||
return &FreshBootstrapHealCapability{
|
||||
endpoint: endpoint,
|
||||
serverUUID: serverUUID,
|
||||
databaseName: databaseName,
|
||||
initialHead: initialHead,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// WithFreshBootstrapHeal enables the fresh-bootstrap self-heal for the #4566
|
||||
// dirty-table guard (gastownhall/beads#5012). Callers must pass it only when
|
||||
// they created the database within the current logical operation (it did not
|
||||
// exist before this init began), so any working-set dirt the guard reads can
|
||||
// only be a previous migration attempt's own half-applied step — a session
|
||||
// that died between a migration's SQL and its per-step Dolt commit — never
|
||||
// pre-existing user data. When the guard fires under this option, the working
|
||||
// set is discarded with DOLT_RESET('--hard') (per-step commits already in
|
||||
// history survive) and the migration pass is re-run once, all on the same
|
||||
// locked session so no concurrent migrator can be mid-pass.
|
||||
func WithFreshBootstrapHeal() MigrateLockOption {
|
||||
return func(o *migrateLockOptions) { o.freshBootstrapHeal = true }
|
||||
// dirty-table guard (gastownhall/beads#5012). The capability must have been
|
||||
// captured immediately after this logical open's exact CREATE DATABASE, and
|
||||
// endpoint must identify the connection pool used for migration. Before any
|
||||
// DOLT_RESET, MigrateUpWithLock revalidates the capability against the pinned,
|
||||
// locked session and consumes it atomically.
|
||||
func WithFreshBootstrapHeal(capability *FreshBootstrapHealCapability, endpoint string) MigrateLockOption {
|
||||
return func(o *migrateLockOptions) {
|
||||
o.freshBootstrapHeal = &freshBootstrapHealRequest{
|
||||
capability: capability,
|
||||
endpoint: endpoint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MigrateUpWithLock serializes schema migrations for a single Dolt sql-server
|
||||
|
|
@ -93,12 +173,20 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string,
|
|||
|
||||
applied, err = MigrateUp(ctx, conn)
|
||||
var dirtyErr *DirtyTablesError
|
||||
if err != nil && o.freshBootstrapHeal && errors.As(err, &dirtyErr) {
|
||||
// The database was created by this init and we hold its migration
|
||||
// lock: the dirt the guard refused is a previous attempt's own
|
||||
// bootstrap debris (mid-step session death), not user data. Discard
|
||||
// it and re-run the pass instead of failing an init that a plain
|
||||
// retry can never converge (gastownhall/beads#5012).
|
||||
if err != nil && o.freshBootstrapHeal != nil && errors.As(err, &dirtyErr) {
|
||||
// Authorization is checked after the dirty guard fires and while the
|
||||
// database-scoped migration lock is still held. A mismatch, missing
|
||||
// ancestor, probe error, or previously consumed capability returns the
|
||||
// original DirtyTablesError without attempting a destructive reset.
|
||||
if !o.freshBootstrapHeal.capability.consumeIfCurrentIncarnation(
|
||||
ctx, conn, databaseName, o.freshBootstrapHeal.endpoint,
|
||||
) {
|
||||
return applied, err
|
||||
}
|
||||
|
||||
// Consume occurs before the reset call. Thus a reset error or a
|
||||
// transient failure in the following migration pass cannot re-arm a
|
||||
// second reset in the caller's outer retry loop.
|
||||
fmt.Fprintf(stderr, "Discarding interrupted-bootstrap working set (%s) and re-running migrations…\n",
|
||||
strings.Join(dirtyErr.Tables, ", "))
|
||||
// Drained, not Exec'd: the very next thing this path does is re-run the
|
||||
|
|
@ -112,6 +200,45 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string,
|
|||
return applied, err
|
||||
}
|
||||
|
||||
// consumeIfCurrentIncarnation validates and atomically consumes c. All probes
|
||||
// run on the same pinned session while MigrateUpWithLock holds the migration
|
||||
// lock. Query errors deliberately collapse to false so callers preserve and
|
||||
// return the original DirtyTablesError.
|
||||
func (c *FreshBootstrapHealCapability) consumeIfCurrentIncarnation(
|
||||
ctx context.Context,
|
||||
conn *sql.Conn,
|
||||
databaseName string,
|
||||
endpoint string,
|
||||
) bool {
|
||||
if c == nil || c.consumed.Load() {
|
||||
return false
|
||||
}
|
||||
if endpoint == "" || endpoint != c.endpoint || databaseName != c.databaseName {
|
||||
return false
|
||||
}
|
||||
|
||||
var currentDatabase, currentServerUUID string
|
||||
if err := conn.QueryRowContext(ctx, "SELECT DATABASE(), @@server_uuid").
|
||||
Scan(¤tDatabase, ¤tServerUUID); err != nil {
|
||||
return false
|
||||
}
|
||||
if currentDatabase != databaseName || currentDatabase != c.databaseName || currentServerUUID != c.serverUUID {
|
||||
return false
|
||||
}
|
||||
|
||||
var ancestorCount int
|
||||
if err := conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?", c.initialHead,
|
||||
).Scan(&ancestorCount); err != nil {
|
||||
return false
|
||||
}
|
||||
if ancestorCount != 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
return c.consumed.CompareAndSwap(false, true)
|
||||
}
|
||||
|
||||
// AcquireMigrationLock acquires the named schema migration lock on the pinned
|
||||
// connection's current Dolt/MySQL session.
|
||||
func AcquireMigrationLock(ctx context.Context, conn *sql.Conn, lockName string) error {
|
||||
|
|
|
|||
|
|
@ -363,6 +363,30 @@ func expectDoltStatusDirtyEvents(mock sqlmock.Sqlmock) {
|
|||
WillReturnRows(sqlmock.NewRows([]string{"table_name", "staged"}).AddRow("events", false))
|
||||
}
|
||||
|
||||
const (
|
||||
testBootstrapEndpoint = "tcp:127.0.0.1:3307"
|
||||
testBootstrapServerUUID = "11111111-2222-3333-4444-555555555555"
|
||||
testBootstrapInitialHead = "abcdefghijklmnopqrstuvwx12345678"
|
||||
)
|
||||
|
||||
func testFreshBootstrapHealCapability() *FreshBootstrapHealCapability {
|
||||
return &FreshBootstrapHealCapability{
|
||||
endpoint: testBootstrapEndpoint,
|
||||
serverUUID: testBootstrapServerUUID,
|
||||
databaseName: "testdb",
|
||||
initialHead: testBootstrapInitialHead,
|
||||
}
|
||||
}
|
||||
|
||||
func expectFreshBootstrapIdentityMatch(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
|
||||
AddRow("testdb", testBootstrapServerUUID))
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?")).
|
||||
WithArgs(testBootstrapInitialHead).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
|
||||
}
|
||||
|
||||
// TestMigrateUpWithLockDirtyGuardStaysFatalWithoutHeal pins the default
|
||||
// behavior: without WithFreshBootstrapHeal, the #4566 guard refusal surfaces
|
||||
// as *DirtyTablesError and no DOLT_RESET runs (sqlmock's ordered expectations
|
||||
|
|
@ -429,7 +453,9 @@ func TestMigrateUpWithLockFreshBootstrapHealResetsAndRetries(t *testing.T) {
|
|||
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
|
||||
// First MigrateUp: refused by the dirty guard.
|
||||
expectDirtyGuardRefusal(t, mock)
|
||||
// Heal: discard the bootstrap debris on the same locked session.
|
||||
// Heal: revalidate the exact database incarnation, atomically consume the
|
||||
// capability, and discard the bootstrap debris on the same locked session.
|
||||
expectFreshBootstrapIdentityMatch(mock)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"status"}))
|
||||
// Second MigrateUp: clean working set, one pending migration applies.
|
||||
|
|
@ -438,7 +464,8 @@ func TestMigrateUpWithLockFreshBootstrapHealResetsAndRetries(t *testing.T) {
|
|||
WithArgs(lockName).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
|
||||
|
||||
applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithFreshBootstrapHeal())
|
||||
applied, err := MigrateUpWithLock(ctx, conn, "testdb",
|
||||
WithFreshBootstrapHeal(testFreshBootstrapHealCapability(), testBootstrapEndpoint))
|
||||
if err != nil {
|
||||
t.Fatalf("MigrateUpWithLock() error = %v", err)
|
||||
}
|
||||
|
|
@ -464,3 +491,186 @@ func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) {
|
|||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(count))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateUpWithLockFreshBootstrapHealProbeFailuresStayFatal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
expectProbe func(sqlmock.Sqlmock)
|
||||
}{
|
||||
{
|
||||
name: "endpoint mismatch",
|
||||
endpoint: "tcp:127.0.0.1:9999",
|
||||
},
|
||||
{
|
||||
name: "identity query failure",
|
||||
endpoint: testBootstrapEndpoint,
|
||||
expectProbe: func(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
|
||||
WillReturnError(errors.New("identity probe failed"))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "database mismatch",
|
||||
endpoint: testBootstrapEndpoint,
|
||||
expectProbe: func(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
|
||||
AddRow("replacement", testBootstrapServerUUID))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "server mismatch",
|
||||
endpoint: testBootstrapEndpoint,
|
||||
expectProbe: func(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
|
||||
AddRow("testdb", "replacement-server"))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ancestry query failure",
|
||||
endpoint: testBootstrapEndpoint,
|
||||
expectProbe: func(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
|
||||
AddRow("testdb", testBootstrapServerUUID))
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?")).
|
||||
WithArgs(testBootstrapInitialHead).
|
||||
WillReturnError(errors.New("ancestry probe failed"))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "initial head missing from ancestry",
|
||||
endpoint: testBootstrapEndpoint,
|
||||
expectProbe: func(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
|
||||
AddRow("testdb", testBootstrapServerUUID))
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?")).
|
||||
WithArgs(testBootstrapInitialHead).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("create sql mock: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("pin mock connection: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
lockName := MigrationLockName("testdb")
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")).
|
||||
WithArgs(lockName, migrationLockAcquireTimeoutSeconds).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
|
||||
expectDirtyGuardRefusal(t, mock)
|
||||
if tt.expectProbe != nil {
|
||||
tt.expectProbe(mock)
|
||||
}
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")).
|
||||
WithArgs(lockName).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
|
||||
|
||||
capability := testFreshBootstrapHealCapability()
|
||||
applied, err := MigrateUpWithLock(ctx, conn, "testdb",
|
||||
WithFreshBootstrapHeal(capability, tt.endpoint))
|
||||
if applied != 0 {
|
||||
t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied)
|
||||
}
|
||||
var dirtyErr *DirtyTablesError
|
||||
if !errors.As(err, &dirtyErr) {
|
||||
t.Fatalf("MigrateUpWithLock() error = %v, want original *DirtyTablesError", err)
|
||||
}
|
||||
if err != dirtyErr {
|
||||
t.Fatalf("MigrateUpWithLock() wrapped or replaced DirtyTablesError: %T %v", err, err)
|
||||
}
|
||||
if capability.consumed.Load() {
|
||||
t.Fatal("failed identity probe consumed capability")
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet SQL expectations: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateUpWithLockFreshBootstrapHealCapabilityIsOneShot(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("create sql mock: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("pin mock connection: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
lockName := MigrationLockName("testdb")
|
||||
capability := testFreshBootstrapHealCapability()
|
||||
option := WithFreshBootstrapHeal(capability, testBootstrapEndpoint)
|
||||
|
||||
// First locked pass consumes the capability before reset, then its single
|
||||
// rerun fails transiently. This is the shape that causes an outer retry.
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")).
|
||||
WithArgs(lockName, migrationLockAcquireTimeoutSeconds).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
|
||||
expectDirtyGuardRefusal(t, mock)
|
||||
expectFreshBootstrapIdentityMatch(mock)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"status"}))
|
||||
// The reset returns to the v60 HEAD used by expectDirtyGuardRefusal. The
|
||||
// rerun repeats main's unconditional ignore-pattern seed before reaching
|
||||
// migrationWorkNeeded, where the injected transient failure occurs.
|
||||
expectIgnorePatternSeedNoop(mock, LatestVersion()-2)
|
||||
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()-2)
|
||||
mock.ExpectQuery("(?s)SELECT s\\.table_name, s\\.staged\\s+FROM dolt_status s").
|
||||
WillReturnError(errors.New("connection reset"))
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")).
|
||||
WithArgs(lockName).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
|
||||
|
||||
if _, err := MigrateUpWithLock(ctx, conn, "testdb", option); err == nil || !strings.Contains(err.Error(), "connection reset") {
|
||||
t.Fatalf("first MigrateUpWithLock() error = %v, want transient rerun failure", err)
|
||||
}
|
||||
if !capability.consumed.Load() {
|
||||
t.Fatal("successful reset attempt did not consume capability")
|
||||
}
|
||||
|
||||
// A later outer retry may encounter another dirty guard, but the consumed
|
||||
// capability returns that original refusal without a second reset.
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")).
|
||||
WithArgs(lockName, migrationLockAcquireTimeoutSeconds).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
|
||||
expectDirtyGuardRefusal(t, mock)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")).
|
||||
WithArgs(lockName).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
|
||||
|
||||
applied, err := MigrateUpWithLock(ctx, conn, "testdb", option)
|
||||
if applied != 0 {
|
||||
t.Fatalf("second MigrateUpWithLock() applied = %d, want 0", applied)
|
||||
}
|
||||
var dirtyErr *DirtyTablesError
|
||||
if !errors.As(err, &dirtyErr) {
|
||||
t.Fatalf("second MigrateUpWithLock() error = %v, want original *DirtyTablesError", err)
|
||||
}
|
||||
if err != dirtyErr {
|
||||
t.Fatalf("second MigrateUpWithLock() wrapped or replaced DirtyTablesError: %T %v", err, err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("unmet SQL expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ const (
|
|||
type doltSQLProvider struct {
|
||||
defaultBranch string
|
||||
db *sql.DB
|
||||
// serverEndpoint is the exact transport endpoint used by the migration
|
||||
// connection. Fresh-bootstrap reset authority is bound to it together with
|
||||
// the Dolt server UUID, database name, and initial HEAD.
|
||||
serverEndpoint string
|
||||
// teamServer: schema is owned by beads-team-server (bts) — bd never
|
||||
// creates the database or migrates, only verifies the schema version.
|
||||
teamServer bool
|
||||
|
|
@ -117,17 +121,22 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
|
|||
// DATABASE (no IF NOT EXISTS), so the server arbitrates creation
|
||||
// atomically — success proves THIS init created the database, and an
|
||||
// already-exists refusal (1007) proves it did not. Only the proven
|
||||
// creator passes WithFreshBootstrapHeal: on a database this init
|
||||
// creator captures and passes a one-shot FreshBootstrapHealCapability: on
|
||||
// a database this init
|
||||
// created, a retry attempt that finds dirty tables can only be seeing a
|
||||
// previous attempt's own half-applied migration step (a session that
|
||||
// died between a step's SQL and its per-step Dolt commit — the "busy
|
||||
// buffer" shape on a loaded shared server), never pre-existing user
|
||||
// data, so the migrate call may discard that debris and converge instead
|
||||
// of failing the init permanently. A concurrent initializer that loses
|
||||
// the create race keeps the guard's refusal unchanged. `created` is
|
||||
// sticky across retry attempts: it is set exactly when this init's
|
||||
// CREATE succeeded, which no later attempt can re-learn from probing.
|
||||
// of failing the init permanently. The capability also binds that proof to
|
||||
// the endpoint, server UUID, database, and initial HEAD, preventing a stale
|
||||
// creator from resetting a drop/recreated name. A concurrent initializer
|
||||
// that loses the create race keeps the guard's refusal unchanged. `created`
|
||||
// is sticky across retry attempts for availability re-creation; reset
|
||||
// authority is captured only in the exact CREATE-winning attempt and is
|
||||
// never inferred again from probing or CREATE IF NOT EXISTS.
|
||||
created := false
|
||||
var bootstrapHeal *schema.FreshBootstrapHealCapability
|
||||
return backoff.Retry(func() error {
|
||||
conn, err := p.db.Conn(ctx)
|
||||
if err != nil {
|
||||
|
|
@ -179,6 +188,7 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
|
|||
}
|
||||
return nil
|
||||
}
|
||||
justCreated := false
|
||||
if created {
|
||||
// Re-assert on retries so a database dropped between attempts
|
||||
// (e.g. a concurrent clean-databases) is recreated rather than
|
||||
|
|
@ -190,6 +200,7 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
|
|||
switch err := ddl.CreateDatabase(ctx, database); {
|
||||
case err == nil:
|
||||
created = true
|
||||
justCreated = true
|
||||
case isDatabaseExistsError(err):
|
||||
// Pre-existing (or a concurrent initializer won the create
|
||||
// race): not ours, heal stays off.
|
||||
|
|
@ -202,10 +213,22 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
|
|||
if err := ddl.UseDatabase(ctx, database); err != nil {
|
||||
return backoff.Permanent(fmt.Errorf("uow: switching to database: %w", err))
|
||||
}
|
||||
if justCreated {
|
||||
// Capture authority only in the same attempt that won the exact bare
|
||||
// CREATE. A later retry may re-create a missing database for
|
||||
// availability, but it must never infer ownership of a replacement
|
||||
// incarnation from CREATE IF NOT EXISTS.
|
||||
bootstrapHeal, err = schema.CaptureFreshBootstrapHealCapability(
|
||||
ctx, conn, p.serverEndpoint, database,
|
||||
)
|
||||
if err != nil {
|
||||
return backoff.Permanent(fmt.Errorf("uow: capture fresh database identity: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
var migrateOpts []schema.MigrateLockOption
|
||||
if created {
|
||||
migrateOpts = append(migrateOpts, schema.WithFreshBootstrapHeal())
|
||||
if bootstrapHeal != nil {
|
||||
migrateOpts = append(migrateOpts, schema.WithFreshBootstrapHeal(bootstrapHeal, p.serverEndpoint))
|
||||
}
|
||||
if _, err := schema.MigrateUpWithLock(ctx, conn, database, migrateOpts...); err != nil {
|
||||
if isSerializationError(err) || schema.IsMigrationLockError(err) {
|
||||
|
|
@ -249,6 +272,7 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse
|
|||
initProvider := &doltSQLProvider{
|
||||
defaultBranch: defaultBranch,
|
||||
db: initDB,
|
||||
serverEndpoint: "tcp:" + ep.Address(),
|
||||
teamServer: teamServer,
|
||||
expectedProjectID: expectedProjectID,
|
||||
preview: opts.preview,
|
||||
|
|
@ -271,6 +295,7 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse
|
|||
return &doltSQLProvider{
|
||||
defaultBranch: defaultBranch,
|
||||
db: dbConn,
|
||||
serverEndpoint: "tcp:" + ep.Address(),
|
||||
teamServer: teamServer,
|
||||
expectedProjectID: expectedProjectID,
|
||||
preview: opts.preview,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue