fix(dolt): bind bootstrap recovery to database incarnation (#5252)

* fix(init): commit intentional config state

* fix: bind bootstrap recovery to database incarnation

* fix(uow): bind bootstrap recovery to incarnation

---------

Co-authored-by: Steve Yegge <steve.yegge@gmail.com>
This commit is contained in:
Zachary Taylor 2026-08-02 09:25:13 -04:00 • committed by GitHub
commit 6cbd63359a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 846 additions and 84 deletions

View file

@ -1762,11 +1762,13 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1):
}
// Auto-commit Dolt state so bd doctor doesn't warn about uncommitted
// changes and users don't need a separate "bd vc commit" step. Skipped in
// gateway mode: the shared server owns its own history and the credential
// may be read-only, so bd must not issue DOLT_ADD/DOLT_COMMIT against it.
// changes and users don't need a separate "bd vc commit" step. Init
// intentionally writes issue_prefix to config, while ordinary Commit
// excludes config to avoid sweeping unrelated stale values. Skip this in
// gateway mode: the shared server owns its history and the credential may
// be read-only, so bd must not issue DOLT_ADD/DOLT_COMMIT against it.
if shouldWriteInitStateToDB(doltCfg.Gateway) {
if err := store.Commit(ctx, "bd init"); err != nil {
if err := commitInitState(ctx, store); err != nil {
// Non-fatal: some setups (e.g. no tables yet) may have nothing to commit
if !strings.Contains(err.Error(), "nothing to commit") {
fmt.Fprintf(os.Stderr, "Warning: failed to commit initial state: %v\n", err)
@ -2972,6 +2974,18 @@ func isEmptyRemoteCloneError(err error) bool {
return strings.Contains(msg, "contains no dolt data")
}
type initStateCommitter interface {
CommitWithConfig(context.Context, string) error
}
// commitInitState commits the configuration init deliberately created as well
// as the initial schema state. The ordinary Commit contract excludes config to
// avoid sweeping unrelated stale values, so using it here leaves every fresh
// database dirty immediately after a successful init.
func commitInitState(ctx context.Context, store initStateCommitter) error {
return store.CommitWithConfig(ctx, "bd init")
}
// verifyMetadata writes a metadata field and verifies the write succeeded.
// Returns true if write+verify succeeded, false with warning if either failed.
func verifyMetadata(ctx context.Context, store storage.DoltStorage, key, value string) bool {

View file

@ -0,0 +1,35 @@
package main
import (
"context"
"errors"
"testing"
)
type recordingInitCommitter struct {
message string
err error
}
func (s *recordingInitCommitter) CommitWithConfig(_ context.Context, message string) error {
s.message = message
return s.err
}
func TestCommitInitStateIncludesIntentionalConfig(t *testing.T) {
store := &recordingInitCommitter{}
if err := commitInitState(context.Background(), store); err != nil {
t.Fatalf("commitInitState: %v", err)
}
if store.message != "bd init" {
t.Fatalf("CommitWithConfig message = %q, want %q", store.message, "bd init")
}
}
func TestCommitInitStatePreservesCommitError(t *testing.T) {
want := errors.New("commit failed")
store := &recordingInitCommitter{err: want}
if got := commitInitState(context.Background(), store); !errors.Is(got, want) {
t.Fatalf("commitInitState error = %v, want %v", got, want)
}
}

View file

@ -0,0 +1,324 @@
//go:build integration && !windows
package dolt
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/steveyegge/beads/internal/doltserver"
"github.com/steveyegge/beads/internal/storage/doltutil"
"github.com/steveyegge/beads/internal/storage/schema"
"github.com/steveyegge/beads/internal/testutil/integration"
)
// TestFreshBootstrapHealIncarnation exercises the destructive bootstrap-heal
// authorization against a real Dolt sql-server. In particular, the
// drop/recreate subtest is the regression for stale creator authority crossing
// a database-name ABA boundary and deleting another incarnation's work.
func TestFreshBootstrapHealIncarnation(t *testing.T) {
integration.RequireDolt(t)
t.Setenv("BEADS_DOLT_SHARED_SERVER", "0")
t.Setenv("BEADS_DOLT_SERVER_PORT", "")
t.Setenv("BEADS_DOLT_PORT", "")
beadsDir := filepath.Join(t.TempDir(), ".beads")
if err := os.MkdirAll(beadsDir, 0o700); err != nil {
t.Fatalf("mkdir beads dir: %v", err)
}
state, err := doltserver.Start(beadsDir)
if err != nil {
t.Fatalf("start local Dolt server: %v", err)
}
t.Cleanup(func() {
current, stateErr := doltserver.IsRunning(beadsDir)
if stateErr != nil {
t.Errorf("check local Dolt server before stop: %v", stateErr)
return
}
if current == nil || !current.Running {
return
}
if err := doltserver.Stop(beadsDir); err != nil {
t.Errorf("stop local Dolt server: %v", err)
}
})
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
adminDSN := doltutil.ServerDSN{
Host: "127.0.0.1",
Port: state.Port,
User: "root",
}.String()
admin, err := sql.Open("mysql", adminDSN)
if err != nil {
t.Fatalf("open admin connection: %v", err)
}
defer admin.Close()
t.Run("exactly one concurrent creator receives a capability", func(t *testing.T) {
const workers = 16
type result struct {
hasCapability bool
err error
}
ready := make(chan struct{})
results := make(chan result, workers)
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
<-ready
cfg := freshBootstrapIntegrationConfig(
beadsDir, state.Port, "heal_creator_race", fmt.Sprintf("creator-%02d", i),
)
db, _, facts, err := openServerConnection(ctx, cfg)
if db != nil {
_ = db.Close()
}
results <- result{hasCapability: facts.bootstrapHeal != nil, err: err}
}(i)
}
close(ready)
wg.Wait()
close(results)
capabilities := 0
for got := range results {
if got.err != nil {
t.Errorf("concurrent open: %v", got.err)
}
if got.hasCapability {
capabilities++
}
}
if capabilities != 1 {
t.Fatalf("creator capability grants = %d, want exactly 1", capabilities)
}
})
t.Run("exact creator heals its interrupted bootstrap", func(t *testing.T) {
cfg := freshBootstrapIntegrationConfig(beadsDir, state.Port, "heal_exact_creator", "exact-creator")
db, _, facts, err := openServerConnection(ctx, cfg)
if err != nil {
t.Fatalf("creator open: %v", err)
}
defer db.Close()
if facts.bootstrapHeal == nil {
t.Fatal("exact bare CREATE did not return a capability")
}
prepareFreshBootstrapV51Dirty(t, ctx, db, "creator_debris")
if _, err := initSchemaOnDBWithBootstrapHeal(
ctx, db, facts.bootstrapHeal, serverEndpointIdentity(cfg),
); err != nil {
t.Fatalf("creator-authorized migration: %v", err)
}
assertFreshBootstrapColumnCount(t, ctx, db, cfg.Database, "creator_debris", 0)
assertFreshBootstrapConverged(t, ctx, db)
})
t.Run("fresh store initializes and commits cleanly", func(t *testing.T) {
cfg := freshBootstrapIntegrationConfig(beadsDir, state.Port, "heal_fresh_store", "fresh-store")
cfg.CommitterName = "Beads Test"
cfg.CommitterEmail = "beads@example.com"
store, err := New(ctx, cfg)
if err != nil {
t.Fatalf("open fresh store: %v", err)
}
defer store.Close()
if err := store.SetConfig(ctx, "issue_prefix", "heal"); err != nil {
t.Fatalf("set intentional init config: %v", err)
}
if err := store.CommitWithConfig(ctx, "bd init"); err != nil {
t.Fatalf("config-inclusive init commit: %v", err)
}
assertFreshBootstrapConverged(t, ctx, store.db)
var prefix string
if err := store.db.QueryRowContext(ctx,
"SELECT value FROM config AS OF 'HEAD' WHERE `key` = 'issue_prefix'",
).Scan(&prefix); err != nil {
t.Fatalf("read committed issue_prefix: %v", err)
}
if prefix != "heal" {
t.Fatalf("committed issue_prefix = %q, want heal", prefix)
}
})
t.Run("drop and recreate revokes creator capability", func(t *testing.T) {
cfg := freshBootstrapIntegrationConfig(beadsDir, state.Port, "heal_creator_aba", "creator-aba")
creatorDB, _, facts, err := openServerConnection(ctx, cfg)
if err != nil {
t.Fatalf("creator open: %v", err)
}
defer creatorDB.Close()
if facts.bootstrapHeal == nil {
t.Fatal("exact bare CREATE did not return a capability")
}
if _, err := admin.ExecContext(ctx, "DROP DATABASE `heal_creator_aba`"); err != nil {
t.Fatalf("drop creator database: %v", err)
}
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `heal_creator_aba`"); err != nil {
t.Fatalf("create replacement database: %v", err)
}
replacement := openFreshBootstrapIntegrationDB(t, state.Port, cfg.Database)
defer replacement.Close()
prepareFreshBootstrapV51Dirty(t, ctx, replacement, "foreign_debris")
_, migrateErr := initSchemaOnDBWithRetryAndGateBootstrapHeal(
ctx, creatorDB, nil, facts.bootstrapHeal, serverEndpointIdentity(cfg),
)
var dirtyErr *schema.DirtyTablesError
if !errors.As(migrateErr, &dirtyErr) {
t.Fatalf("replacement migration error = %v, want DirtyTablesError", migrateErr)
}
assertFreshBootstrapColumnCount(t, ctx, replacement, cfg.Database, "foreign_debris", 1)
if got := freshBootstrapSchemaVersion(t, ctx, replacement); got != 51 {
t.Fatalf("replacement schema version = %d, want preserved v51", got)
}
if got := freshBootstrapDoltStatusCount(t, ctx, replacement); got == 0 {
t.Fatal("replacement working set was cleared; wanted fail-closed preservation")
}
})
t.Run("preexisting dirty database has no reset authority", func(t *testing.T) {
const database = "heal_preexisting_dirty"
if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+database+"`"); err != nil {
t.Fatalf("create preexisting database: %v", err)
}
db := openFreshBootstrapIntegrationDB(t, state.Port, database)
defer db.Close()
prepareFreshBootstrapV51Dirty(t, ctx, db, "preexisting_debris")
_, migrateErr := initSchemaOnDB(ctx, db)
var dirtyErr *schema.DirtyTablesError
if !errors.As(migrateErr, &dirtyErr) {
t.Fatalf("preexisting migration error = %v, want DirtyTablesError", migrateErr)
}
assertFreshBootstrapColumnCount(t, ctx, db, database, "preexisting_debris", 1)
if got := freshBootstrapDoltStatusCount(t, ctx, db); got == 0 {
t.Fatal("preexisting working set was cleared; wanted fail-closed preservation")
}
})
}
func freshBootstrapIntegrationConfig(beadsDir string, port int, database, pathSuffix string) *Config {
return &Config{
Path: filepath.Join(beadsDir, pathSuffix),
BeadsDir: beadsDir,
ServerHost: "127.0.0.1",
ServerPort: port,
ServerUser: "root",
Database: database,
CreateIfMissing: true,
MaxOpenConns: 1,
}
}
func prepareFreshBootstrapV51Dirty(t *testing.T, ctx context.Context, db *sql.DB, debrisColumn string) {
t.Helper()
conn, err := db.Conn(ctx)
if err != nil {
t.Fatalf("pin database: %v", err)
}
defer conn.Close()
if _, err := schema.MigrateUpTo(ctx, conn, 51); err != nil {
t.Fatalf("migrate database to v51: %v", err)
}
if _, err := conn.ExecContext(ctx,
"CALL DOLT_COMMIT('-Am', 'test: v51 baseline', '--author', 'Beads Test <beads@example.com>')",
); err != nil {
t.Fatalf("commit v51 baseline: %v", err)
}
if got := freshBootstrapSchemaVersion(t, ctx, conn); got != 51 {
t.Fatalf("baseline schema version = %d, want 51", got)
}
if _, err := conn.ExecContext(ctx, "ALTER TABLE issues ADD COLUMN "+debrisColumn+" INT"); err != nil {
t.Fatalf("dirty bootstrap database: %v", err)
}
}
func openFreshBootstrapIntegrationDB(t *testing.T, port int, database string) *sql.DB {
t.Helper()
dsn := doltutil.ServerDSN{
Host: "127.0.0.1",
Port: port,
User: "root",
Database: database,
}.String()
db, err := sql.Open("mysql", dsn)
if err != nil {
t.Fatalf("open %s: %v", database, err)
}
return db
}
type freshBootstrapQueryer interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func freshBootstrapSchemaVersion(t *testing.T, ctx context.Context, db freshBootstrapQueryer) int {
t.Helper()
var version int
if err := db.QueryRowContext(ctx,
"SELECT COALESCE(MAX(version), 0) FROM schema_migrations",
).Scan(&version); err != nil {
t.Fatalf("read schema version: %v", err)
}
return version
}
func freshBootstrapDoltStatusCount(t *testing.T, ctx context.Context, db freshBootstrapQueryer) int {
t.Helper()
var count int
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_status").Scan(&count); err != nil {
t.Fatalf("count dolt_status: %v", err)
}
return count
}
func assertFreshBootstrapColumnCount(
t *testing.T,
ctx context.Context,
db freshBootstrapQueryer,
database string,
column string,
want int,
) {
t.Helper()
var count int
if err := db.QueryRowContext(ctx,
"SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = ? AND table_name = 'issues' AND column_name = ?",
database, column,
).Scan(&count); err != nil {
t.Fatalf("count %s.%s: %v", database, column, err)
}
if count != want {
t.Fatalf("column %s.%s count = %d, want %d", database, column, count, want)
}
}
func assertFreshBootstrapConverged(t *testing.T, ctx context.Context, db freshBootstrapQueryer) {
t.Helper()
want := schema.LatestVersion()
if got := freshBootstrapSchemaVersion(t, ctx, db); got != want {
t.Fatalf("schema version = %d, want %d", got, want)
}
if got := freshBootstrapDoltStatusCount(t, ctx, db); got != 0 {
t.Fatalf("dolt_status rows after heal = %d, want 0", got)
}
}

View file

@ -306,15 +306,16 @@ var _ storage.ExternalRefHistoryQuerier = (*DoltStore)(nil)
// DoltStore implements the Storage interface using Dolt
type DoltStore struct {
db *sql.DB
dbPath string // Path to Dolt data directory (server root, e.g. .beads/dolt/)
beadsDir string // Path to .beads directory (parent of dbPath)
database string // Database name (subdirectory under dbPath)
closed atomic.Bool // Tracks whether Close() has been called
connStr string // Connection string for reconnection
mu sync.RWMutex // Protects concurrent access
readOnly bool // True if opened in read-only mode
credentialKey []byte // Random encryption key for federation credentials
db *sql.DB
dbPath string // Path to Dolt data directory (server root, e.g. .beads/dolt/)
beadsDir string // Path to .beads directory (parent of dbPath)
database string // Database name (subdirectory under dbPath)
closed atomic.Bool // Tracks whether Close() has been called
connStr string // Connection string for reconnection
serverEndpoint string // Exact endpoint bound to bootstrap reset authority
mu sync.RWMutex // Protects concurrent access
readOnly bool // True if opened in read-only mode
credentialKey []byte // Random encryption key for federation credentials
// localActiveDatabaseDir is the exact active database directory when this
// store instance has authoritative local filesystem access. It is resolved
@ -1744,6 +1745,7 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) {
database: cfg.Database,
localActiveDatabaseDir: resolveLocalActiveDatabaseDir(cfg),
connStr: connStr,
serverEndpoint: serverEndpointIdentity(cfg),
breaker: breaker,
committerName: cfg.CommitterName,
committerEmail: cfg.CommitterEmail,
@ -1813,7 +1815,7 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) {
// ReadOnly for schema — the forward-drift guard above still protects a stale client
// binary.
if !cfg.ReadOnly && !cfg.Gateway {
if err := store.initSchema(ctx, dbFacts.created); err != nil {
if err := store.initSchema(ctx, dbFacts.bootstrapHeal); err != nil {
return nil, fmt.Errorf("failed to initialize schema: %w", err)
}
}
@ -2092,17 +2094,20 @@ func applyPoolLimits(db *sql.DB, cfg *Config) {
}
// serverConnFacts reports what openServerConnection established about the
// target database while connecting. The two fields are deliberately NOT
// inverses of each other: for a gateway database, existence is never probed,
// so both are false — neither "we created it" nor "it was already there" is
// proven, and each caller's gate must fail closed on its own terms.
// target database while connecting. Creation and prior existence are
// deliberately NOT inverses: for a gateway database, existence is never
// probed, so neither is proven and each caller's gate fails closed.
type serverConnFacts struct {
// created reports whether THIS call's bare CREATE DATABASE won the
// ownership arbitration and actually created the database — server-mode's
// analog of the uow path's `created` signal (#5042). Threaded through to
// initSchema so only the proven creator arms schema.WithFreshBootstrapHeal.
// ownership arbitration and actually created the database.
created bool
// bootstrapHeal carries one-shot reset authority bound to the exact
// endpoint, server UUID, database, and initial HEAD captured after this
// call created and successfully connected to a pristine database. A nil
// capability always fails closed.
bootstrapHeal *schema.FreshBootstrapHealCapability
// alreadyExisted reports whether the database was proven to exist on the
// server before this call: either the SHOW DATABASES probe found it, or
// our CREATE DATABASE was refused with "database exists" (1007). Callers
@ -2272,8 +2277,37 @@ func openServerConnection(ctx context.Context, cfg *Config) (*sql.DB, string, se
return nil, "", serverConnFacts{}, fmt.Errorf("database %q not available after CREATE DATABASE: %w", cfg.Database, err)
}
var bootstrapHeal *schema.FreshBootstrapHealCapability
if created {
conn, connErr := db.Conn(ctx)
if connErr != nil {
return nil, "", serverConnFacts{}, fmt.Errorf("capture fresh database identity: pin connection: %w", connErr)
}
bootstrapHeal, err = schema.CaptureFreshBootstrapHealCapability(
ctx, conn, serverEndpointIdentity(cfg), cfg.Database,
)
_ = conn.Close()
if err != nil {
return nil, "", serverConnFacts{}, fmt.Errorf("capture fresh database identity: %w", err)
}
}
connReady = true
return db, connStr, serverConnFacts{created: created, alreadyExisted: dbExists}, nil
return db, connStr, serverConnFacts{
created: created,
bootstrapHeal: bootstrapHeal,
alreadyExisted: dbExists,
}, nil
}
// serverEndpointIdentity returns the exact configured transport endpoint. It
// is captured with fresh-bootstrap authority and must match the migration
// connection's endpoint before that authority can be consumed.
func serverEndpointIdentity(cfg *Config) string {
if cfg.ServerSocket != "" {
return "unix:" + cfg.ServerSocket
}
return "tcp:" + net.JoinHostPort(cfg.ServerHost, strconv.Itoa(cfg.ServerPort))
}
// databaseExistsOnServer checks if a database with the exact given name exists
@ -2302,20 +2336,17 @@ func databaseExistsOnServer(ctx context.Context, db *sql.DB, name string) (bool,
// applied versions in schema_migrations and backfills legacy config-driven
// tables. Returns the number of migrations applied.
func initSchemaOnDB(ctx context.Context, db *sql.DB) (int, error) {
return initSchemaOnDBOwnership(ctx, db, false)
return initSchemaOnDBWithBootstrapHeal(ctx, db, nil, "")
}
// initSchemaOnDBOwnership is initSchemaOnDB with the fresh-bootstrap
// ownership signal for the #4566 dirty-table guard self-heal
// (gastownhall/beads#5012), ported from the uow path's initSchema (#5042):
// createdDatabase must be true only when the caller proved — via the
// bare-CREATE-DATABASE ownership arbitration in openServerConnection — that
// THIS process's open created the target database. On a database this init
// created, dirty tables a retry finds can only be this same process's own
// half-applied migration step (a session that died between a step's SQL and
// its per-step Dolt commit), never pre-existing user data, so
// schema.WithFreshBootstrapHeal is safe to arm.
func initSchemaOnDBOwnership(ctx context.Context, db *sql.DB, createdDatabase bool) (int, error) {
// initSchemaOnDBWithBootstrapHeal threads one-shot, incarnation-bound reset
// authority into the migration lock. A nil capability always fails closed.
func initSchemaOnDBWithBootstrapHeal(
ctx context.Context,
db *sql.DB,
bootstrapHeal *schema.FreshBootstrapHealCapability,
endpoint string,
) (int, error) {
conn, err := db.Conn(ctx)
if err != nil {
return 0, fmt.Errorf("schema: pin connection: %w", err)
@ -2328,8 +2359,8 @@ func initSchemaOnDBOwnership(ctx context.Context, db *sql.DB, createdDatabase bo
}
var opts []schema.MigrateLockOption
if createdDatabase {
opts = append(opts, schema.WithFreshBootstrapHeal())
if bootstrapHeal != nil {
opts = append(opts, schema.WithFreshBootstrapHeal(bootstrapHeal, endpoint))
}
applied, err := schema.MigrateUpWithLock(ctx, conn, dbName, opts...)
if err != nil {
@ -2349,17 +2380,18 @@ func initSchemaOnDBWithRetry(ctx context.Context, db *sql.DB) (int, error) {
// retried with them instead of failing the open fast (bd-6dnrw.30); a
// *schema.RemoteMigrateGateError refusal stays permanent.
func initSchemaOnDBWithRetryAndGate(ctx context.Context, db *sql.DB, gate func(context.Context, *sql.DB) error) (int, error) {
return initSchemaOnDBWithRetryAndGateOwnership(ctx, db, gate, false)
return initSchemaOnDBWithRetryAndGateBootstrapHeal(ctx, db, gate, nil, "")
}
// initSchemaOnDBWithRetryAndGateOwnership is initSchemaOnDBWithRetryAndGate
// with the fresh-bootstrap ownership signal (see initSchemaOnDBOwnership)
// threaded to every retry attempt. createdDatabase is fixed for the whole
// retry loop: unlike the uow path, where database creation is retried inside
// the same loop that runs the migration, server mode creates the database
// exactly once in openServerConnection, before this loop starts — so there is
// no "re-assert on retry" case to port here.
func initSchemaOnDBWithRetryAndGateOwnership(ctx context.Context, db *sql.DB, gate func(context.Context, *sql.DB) error, createdDatabase bool) (int, error) {
// initSchemaOnDBWithRetryAndGateBootstrapHeal shares one capability across the
// outer retry loop. Once consumed, no later retry can issue another reset.
func initSchemaOnDBWithRetryAndGateBootstrapHeal(
ctx context.Context,
db *sql.DB,
gate func(context.Context, *sql.DB) error,
bootstrapHeal *schema.FreshBootstrapHealCapability,
endpoint string,
) (int, error) {
// Schema initialization for server mode is idempotent. Retry transient
// Dolt startup/catalog races and contended migration-lock attempts so
// concurrent bd processes converge instead of failing one unlucky waiter.
@ -2379,7 +2411,7 @@ func initSchemaOnDBWithRetryAndGateOwnership(ctx context.Context, db *sql.DB, ga
}
}
var schemaErr error
applied, schemaErr = initSchemaOnDBOwnership(ctx, db, createdDatabase)
applied, schemaErr = initSchemaOnDBWithBootstrapHeal(ctx, db, bootstrapHeal, endpoint)
if schemaErr != nil && isRetryableError(schemaErr) {
return schemaErr
}
@ -2391,12 +2423,7 @@ func initSchemaOnDBWithRetryAndGateOwnership(ctx context.Context, db *sql.DB, ga
return applied, err
}
// initSchema runs pending schema migrations for a freshly opened store.
// createdDatabase must be true only when this open's openServerConnection
// call proved (via bare-CREATE ownership arbitration) that it created the
// target database — see initSchemaOnDBOwnership for why that arms
// schema.WithFreshBootstrapHeal (gastownhall/beads#5012, #5042).
func (s *DoltStore) initSchema(ctx context.Context, createdDatabase bool) error {
func (s *DoltStore) initSchema(ctx context.Context, bootstrapHeal *schema.FreshBootstrapHealCapability) error {
// Schema migrations can run arbitrarily long (e.g. full-table recomputes
// such as the is_blocked backfill in migration 0047). The main connection
// pool sets a 10s ReadTimeout (see buildServerDSN); a slow migration over
@ -2447,7 +2474,7 @@ func (s *DoltStore) initSchema(ctx context.Context, createdDatabase bool) error
gate := func(ctx context.Context, db *sql.DB) error {
return schema.CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(ctx, db, s.remote, s.hasPersistedCLIRemote, adopt)
}
_, err = initSchemaOnDBWithRetryAndGateOwnership(ctx, migDB, gate, createdDatabase)
_, err = initSchemaOnDBWithRetryAndGateBootstrapHeal(ctx, migDB, gate, bootstrapHeal, s.serverEndpoint)
return err
}

View file

@ -8,6 +8,7 @@ import (
"fmt"
"hash/fnv"
"strings"
"sync/atomic"
"time"
)
@ -52,21 +53,100 @@ func MigrationLockName(databaseName string) string {
type MigrateLockOption func(*migrateLockOptions)
type migrateLockOptions struct {
freshBootstrapHeal bool
freshBootstrapHeal *freshBootstrapHealRequest
}
type freshBootstrapHealRequest struct {
capability *FreshBootstrapHealCapability
endpoint string
}
// FreshBootstrapHealCapability is a one-shot authorization to discard an
// interrupted bootstrap working set. It is bound to the exact endpoint,
// server, database, and initial Dolt commit observed immediately after a
// successful bare CREATE DATABASE. Its fields are intentionally private so a
// caller cannot manufacture authority from a boolean.
//
// A capability may be shared by the server-mode migration retry loop. An
// atomic consumed bit ensures that the whole logical open can attempt at most
// one destructive reset, even when a later transient error causes an outer
// retry.
type FreshBootstrapHealCapability struct {
endpoint string
serverUUID string
databaseName string
initialHead string
consumed atomic.Bool
}
// CaptureFreshBootstrapHealCapability records the identity of a database that
// the caller has just created with an exact, successful bare CREATE DATABASE.
// The database must still be pristine: one initial commit and an empty working
// set. Any uncertainty fails closed instead of issuing reset authority.
func CaptureFreshBootstrapHealCapability(
ctx context.Context,
conn *sql.Conn,
endpoint string,
expectedDatabase string,
) (*FreshBootstrapHealCapability, error) {
if endpoint == "" {
return nil, errors.New("schema: capture fresh-bootstrap identity: empty endpoint")
}
if expectedDatabase == "" {
return nil, errors.New("schema: capture fresh-bootstrap identity: empty database name")
}
var databaseName, serverUUID, initialHead string
if err := conn.QueryRowContext(ctx,
"SELECT DATABASE(), @@server_uuid, DOLT_HASHOF('HEAD')",
).Scan(&databaseName, &serverUUID, &initialHead); err != nil {
return nil, fmt.Errorf("schema: capture fresh-bootstrap identity: %w", err)
}
if databaseName != expectedDatabase {
return nil, fmt.Errorf("schema: capture fresh-bootstrap identity: database is %q, want %q", databaseName, expectedDatabase)
}
if serverUUID == "" {
return nil, errors.New("schema: capture fresh-bootstrap identity: empty server UUID")
}
if initialHead == "" {
return nil, errors.New("schema: capture fresh-bootstrap identity: empty initial HEAD")
}
var commitCount, dirtyCount int
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_log").Scan(&commitCount); err != nil {
return nil, fmt.Errorf("schema: verify fresh-bootstrap history: %w", err)
}
if commitCount != 1 {
return nil, fmt.Errorf("schema: verify fresh-bootstrap history: got %d commits, want 1", commitCount)
}
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_status").Scan(&dirtyCount); err != nil {
return nil, fmt.Errorf("schema: verify fresh-bootstrap working set: %w", err)
}
if dirtyCount != 0 {
return nil, fmt.Errorf("schema: verify fresh-bootstrap working set: got %d dirty entries, want 0", dirtyCount)
}
return &FreshBootstrapHealCapability{
endpoint: endpoint,
serverUUID: serverUUID,
databaseName: databaseName,
initialHead: initialHead,
}, nil
}
// WithFreshBootstrapHeal enables the fresh-bootstrap self-heal for the #4566
// dirty-table guard (gastownhall/beads#5012). Callers must pass it only when
// they created the database within the current logical operation (it did not
// exist before this init began), so any working-set dirt the guard reads can
// only be a previous migration attempt's own half-applied step — a session
// that died between a migration's SQL and its per-step Dolt commit — never
// pre-existing user data. When the guard fires under this option, the working
// set is discarded with DOLT_RESET('--hard') (per-step commits already in
// history survive) and the migration pass is re-run once, all on the same
// locked session so no concurrent migrator can be mid-pass.
func WithFreshBootstrapHeal() MigrateLockOption {
return func(o *migrateLockOptions) { o.freshBootstrapHeal = true }
// dirty-table guard (gastownhall/beads#5012). The capability must have been
// captured immediately after this logical open's exact CREATE DATABASE, and
// endpoint must identify the connection pool used for migration. Before any
// DOLT_RESET, MigrateUpWithLock revalidates the capability against the pinned,
// locked session and consumes it atomically.
func WithFreshBootstrapHeal(capability *FreshBootstrapHealCapability, endpoint string) MigrateLockOption {
return func(o *migrateLockOptions) {
o.freshBootstrapHeal = &freshBootstrapHealRequest{
capability: capability,
endpoint: endpoint,
}
}
}
// MigrateUpWithLock serializes schema migrations for a single Dolt sql-server
@ -93,12 +173,20 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string,
applied, err = MigrateUp(ctx, conn)
var dirtyErr *DirtyTablesError
if err != nil && o.freshBootstrapHeal && errors.As(err, &dirtyErr) {
// The database was created by this init and we hold its migration
// lock: the dirt the guard refused is a previous attempt's own
// bootstrap debris (mid-step session death), not user data. Discard
// it and re-run the pass instead of failing an init that a plain
// retry can never converge (gastownhall/beads#5012).
if err != nil && o.freshBootstrapHeal != nil && errors.As(err, &dirtyErr) {
// Authorization is checked after the dirty guard fires and while the
// database-scoped migration lock is still held. A mismatch, missing
// ancestor, probe error, or previously consumed capability returns the
// original DirtyTablesError without attempting a destructive reset.
if !o.freshBootstrapHeal.capability.consumeIfCurrentIncarnation(
ctx, conn, databaseName, o.freshBootstrapHeal.endpoint,
) {
return applied, err
}
// Consume occurs before the reset call. Thus a reset error or a
// transient failure in the following migration pass cannot re-arm a
// second reset in the caller's outer retry loop.
fmt.Fprintf(stderr, "Discarding interrupted-bootstrap working set (%s) and re-running migrations…\n",
strings.Join(dirtyErr.Tables, ", "))
// Drained, not Exec'd: the very next thing this path does is re-run the
@ -112,6 +200,45 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string,
return applied, err
}
// consumeIfCurrentIncarnation validates and atomically consumes c. All probes
// run on the same pinned session while MigrateUpWithLock holds the migration
// lock. Query errors deliberately collapse to false so callers preserve and
// return the original DirtyTablesError.
func (c *FreshBootstrapHealCapability) consumeIfCurrentIncarnation(
ctx context.Context,
conn *sql.Conn,
databaseName string,
endpoint string,
) bool {
if c == nil || c.consumed.Load() {
return false
}
if endpoint == "" || endpoint != c.endpoint || databaseName != c.databaseName {
return false
}
var currentDatabase, currentServerUUID string
if err := conn.QueryRowContext(ctx, "SELECT DATABASE(), @@server_uuid").
Scan(&currentDatabase, &currentServerUUID); err != nil {
return false
}
if currentDatabase != databaseName || currentDatabase != c.databaseName || currentServerUUID != c.serverUUID {
return false
}
var ancestorCount int
if err := conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?", c.initialHead,
).Scan(&ancestorCount); err != nil {
return false
}
if ancestorCount != 1 {
return false
}
return c.consumed.CompareAndSwap(false, true)
}
// AcquireMigrationLock acquires the named schema migration lock on the pinned
// connection's current Dolt/MySQL session.
func AcquireMigrationLock(ctx context.Context, conn *sql.Conn, lockName string) error {

View file

@ -363,6 +363,30 @@ func expectDoltStatusDirtyEvents(mock sqlmock.Sqlmock) {
WillReturnRows(sqlmock.NewRows([]string{"table_name", "staged"}).AddRow("events", false))
}
const (
testBootstrapEndpoint = "tcp:127.0.0.1:3307"
testBootstrapServerUUID = "11111111-2222-3333-4444-555555555555"
testBootstrapInitialHead = "abcdefghijklmnopqrstuvwx12345678"
)
func testFreshBootstrapHealCapability() *FreshBootstrapHealCapability {
return &FreshBootstrapHealCapability{
endpoint: testBootstrapEndpoint,
serverUUID: testBootstrapServerUUID,
databaseName: "testdb",
initialHead: testBootstrapInitialHead,
}
}
func expectFreshBootstrapIdentityMatch(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
AddRow("testdb", testBootstrapServerUUID))
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?")).
WithArgs(testBootstrapInitialHead).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1))
}
// TestMigrateUpWithLockDirtyGuardStaysFatalWithoutHeal pins the default
// behavior: without WithFreshBootstrapHeal, the #4566 guard refusal surfaces
// as *DirtyTablesError and no DOLT_RESET runs (sqlmock's ordered expectations
@ -429,7 +453,9 @@ func TestMigrateUpWithLockFreshBootstrapHealResetsAndRetries(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
// First MigrateUp: refused by the dirty guard.
expectDirtyGuardRefusal(t, mock)
// Heal: discard the bootstrap debris on the same locked session.
// Heal: revalidate the exact database incarnation, atomically consume the
// capability, and discard the bootstrap debris on the same locked session.
expectFreshBootstrapIdentityMatch(mock)
mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")).
WillReturnRows(sqlmock.NewRows([]string{"status"}))
// Second MigrateUp: clean working set, one pending migration applies.
@ -438,7 +464,8 @@ func TestMigrateUpWithLockFreshBootstrapHealResetsAndRetries(t *testing.T) {
WithArgs(lockName).
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithFreshBootstrapHeal())
applied, err := MigrateUpWithLock(ctx, conn, "testdb",
WithFreshBootstrapHeal(testFreshBootstrapHealCapability(), testBootstrapEndpoint))
if err != nil {
t.Fatalf("MigrateUpWithLock() error = %v", err)
}
@ -464,3 +491,186 @@ func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) {
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(count))
}
}
func TestMigrateUpWithLockFreshBootstrapHealProbeFailuresStayFatal(t *testing.T) {
tests := []struct {
name string
endpoint string
expectProbe func(sqlmock.Sqlmock)
}{
{
name: "endpoint mismatch",
endpoint: "tcp:127.0.0.1:9999",
},
{
name: "identity query failure",
endpoint: testBootstrapEndpoint,
expectProbe: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
WillReturnError(errors.New("identity probe failed"))
},
},
{
name: "database mismatch",
endpoint: testBootstrapEndpoint,
expectProbe: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
AddRow("replacement", testBootstrapServerUUID))
},
},
{
name: "server mismatch",
endpoint: testBootstrapEndpoint,
expectProbe: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
AddRow("testdb", "replacement-server"))
},
},
{
name: "ancestry query failure",
endpoint: testBootstrapEndpoint,
expectProbe: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
AddRow("testdb", testBootstrapServerUUID))
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?")).
WithArgs(testBootstrapInitialHead).
WillReturnError(errors.New("ancestry probe failed"))
},
},
{
name: "initial head missing from ancestry",
endpoint: testBootstrapEndpoint,
expectProbe: func(mock sqlmock.Sqlmock) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT DATABASE(), @@server_uuid")).
WillReturnRows(sqlmock.NewRows([]string{"database", "server_uuid"}).
AddRow("testdb", testBootstrapServerUUID))
mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(*) FROM dolt_log WHERE commit_hash = ?")).
WithArgs(testBootstrapInitialHead).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sql mock: %v", err)
}
defer db.Close()
ctx := context.Background()
conn, err := db.Conn(ctx)
if err != nil {
t.Fatalf("pin mock connection: %v", err)
}
defer conn.Close()
lockName := MigrationLockName("testdb")
mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")).
WithArgs(lockName, migrationLockAcquireTimeoutSeconds).
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
expectDirtyGuardRefusal(t, mock)
if tt.expectProbe != nil {
tt.expectProbe(mock)
}
mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")).
WithArgs(lockName).
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
capability := testFreshBootstrapHealCapability()
applied, err := MigrateUpWithLock(ctx, conn, "testdb",
WithFreshBootstrapHeal(capability, tt.endpoint))
if applied != 0 {
t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied)
}
var dirtyErr *DirtyTablesError
if !errors.As(err, &dirtyErr) {
t.Fatalf("MigrateUpWithLock() error = %v, want original *DirtyTablesError", err)
}
if err != dirtyErr {
t.Fatalf("MigrateUpWithLock() wrapped or replaced DirtyTablesError: %T %v", err, err)
}
if capability.consumed.Load() {
t.Fatal("failed identity probe consumed capability")
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
})
}
}
func TestMigrateUpWithLockFreshBootstrapHealCapabilityIsOneShot(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("create sql mock: %v", err)
}
defer db.Close()
ctx := context.Background()
conn, err := db.Conn(ctx)
if err != nil {
t.Fatalf("pin mock connection: %v", err)
}
defer conn.Close()
lockName := MigrationLockName("testdb")
capability := testFreshBootstrapHealCapability()
option := WithFreshBootstrapHeal(capability, testBootstrapEndpoint)
// First locked pass consumes the capability before reset, then its single
// rerun fails transiently. This is the shape that causes an outer retry.
mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")).
WithArgs(lockName, migrationLockAcquireTimeoutSeconds).
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
expectDirtyGuardRefusal(t, mock)
expectFreshBootstrapIdentityMatch(mock)
mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")).
WillReturnRows(sqlmock.NewRows([]string{"status"}))
// The reset returns to the v60 HEAD used by expectDirtyGuardRefusal. The
// rerun repeats main's unconditional ignore-pattern seed before reaching
// migrationWorkNeeded, where the injected transient failure occurs.
expectIgnorePatternSeedNoop(mock, LatestVersion()-2)
expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", LatestVersion()-2)
mock.ExpectQuery("(?s)SELECT s\\.table_name, s\\.staged\\s+FROM dolt_status s").
WillReturnError(errors.New("connection reset"))
mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")).
WithArgs(lockName).
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
if _, err := MigrateUpWithLock(ctx, conn, "testdb", option); err == nil || !strings.Contains(err.Error(), "connection reset") {
t.Fatalf("first MigrateUpWithLock() error = %v, want transient rerun failure", err)
}
if !capability.consumed.Load() {
t.Fatal("successful reset attempt did not consume capability")
}
// A later outer retry may encounter another dirty guard, but the consumed
// capability returns that original refusal without a second reset.
mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")).
WithArgs(lockName, migrationLockAcquireTimeoutSeconds).
WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1))
expectDirtyGuardRefusal(t, mock)
mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")).
WithArgs(lockName).
WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
applied, err := MigrateUpWithLock(ctx, conn, "testdb", option)
if applied != 0 {
t.Fatalf("second MigrateUpWithLock() applied = %d, want 0", applied)
}
var dirtyErr *DirtyTablesError
if !errors.As(err, &dirtyErr) {
t.Fatalf("second MigrateUpWithLock() error = %v, want original *DirtyTablesError", err)
}
if err != dirtyErr {
t.Fatalf("second MigrateUpWithLock() wrapped or replaced DirtyTablesError: %T %v", err, err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("unmet SQL expectations: %v", err)
}
}

View file

@ -24,6 +24,10 @@ const (
type doltSQLProvider struct {
defaultBranch string
db *sql.DB
// serverEndpoint is the exact transport endpoint used by the migration
// connection. Fresh-bootstrap reset authority is bound to it together with
// the Dolt server UUID, database name, and initial HEAD.
serverEndpoint string
// teamServer: schema is owned by beads-team-server (bts) — bd never
// creates the database or migrates, only verifies the schema version.
teamServer bool
@ -117,17 +121,22 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
// DATABASE (no IF NOT EXISTS), so the server arbitrates creation
// atomically — success proves THIS init created the database, and an
// already-exists refusal (1007) proves it did not. Only the proven
// creator passes WithFreshBootstrapHeal: on a database this init
// creator captures and passes a one-shot FreshBootstrapHealCapability: on
// a database this init
// created, a retry attempt that finds dirty tables can only be seeing a
// previous attempt's own half-applied migration step (a session that
// died between a step's SQL and its per-step Dolt commit — the "busy
// buffer" shape on a loaded shared server), never pre-existing user
// data, so the migrate call may discard that debris and converge instead
// of failing the init permanently. A concurrent initializer that loses
// the create race keeps the guard's refusal unchanged. `created` is
// sticky across retry attempts: it is set exactly when this init's
// CREATE succeeded, which no later attempt can re-learn from probing.
// of failing the init permanently. The capability also binds that proof to
// the endpoint, server UUID, database, and initial HEAD, preventing a stale
// creator from resetting a drop/recreated name. A concurrent initializer
// that loses the create race keeps the guard's refusal unchanged. `created`
// is sticky across retry attempts for availability re-creation; reset
// authority is captured only in the exact CREATE-winning attempt and is
// never inferred again from probing or CREATE IF NOT EXISTS.
created := false
var bootstrapHeal *schema.FreshBootstrapHealCapability
return backoff.Retry(func() error {
conn, err := p.db.Conn(ctx)
if err != nil {
@ -179,6 +188,7 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
}
return nil
}
justCreated := false
if created {
// Re-assert on retries so a database dropped between attempts
// (e.g. a concurrent clean-databases) is recreated rather than
@ -190,6 +200,7 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
switch err := ddl.CreateDatabase(ctx, database); {
case err == nil:
created = true
justCreated = true
case isDatabaseExistsError(err):
// Pre-existing (or a concurrent initializer won the create
// race): not ours, heal stays off.
@ -202,10 +213,22 @@ func (p *doltSQLProvider) initSchema(ctx context.Context, database string) error
if err := ddl.UseDatabase(ctx, database); err != nil {
return backoff.Permanent(fmt.Errorf("uow: switching to database: %w", err))
}
if justCreated {
// Capture authority only in the same attempt that won the exact bare
// CREATE. A later retry may re-create a missing database for
// availability, but it must never infer ownership of a replacement
// incarnation from CREATE IF NOT EXISTS.
bootstrapHeal, err = schema.CaptureFreshBootstrapHealCapability(
ctx, conn, p.serverEndpoint, database,
)
if err != nil {
return backoff.Permanent(fmt.Errorf("uow: capture fresh database identity: %w", err))
}
}
var migrateOpts []schema.MigrateLockOption
if created {
migrateOpts = append(migrateOpts, schema.WithFreshBootstrapHeal())
if bootstrapHeal != nil {
migrateOpts = append(migrateOpts, schema.WithFreshBootstrapHeal(bootstrapHeal, p.serverEndpoint))
}
if _, err := schema.MigrateUpWithLock(ctx, conn, database, migrateOpts...); err != nil {
if isSerializationError(err) || schema.IsMigrationLockError(err) {
@ -249,6 +272,7 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse
initProvider := &doltSQLProvider{
defaultBranch: defaultBranch,
db: initDB,
serverEndpoint: "tcp:" + ep.Address(),
teamServer: teamServer,
expectedProjectID: expectedProjectID,
preview: opts.preview,
@ -271,6 +295,7 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse
return &doltSQLProvider{
defaultBranch: defaultBranch,
db: dbConn,
serverEndpoint: "tcp:" + ep.Address(),
teamServer: teamServer,
expectedProjectID: expectedProjectID,
preview: opts.preview,