* cmd/bd: extract sanitizeDBName to its own file for CGO_ENABLED=0 lint sanitizeDBName lived in store_factory.go which is //go:build cgo, but init.go (no tag) calls it. The pre-commit golangci-lint runs with CGO_ENABLED=0 and flagged it as undefined. Move the pure-Go function to its own untagged file and drop the now-unused strings import from store_factory.go. No behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * schema: tolerate "does not have column" errors in migration runner Broadens isConcurrentInitError to also treat "does not have column" as a benign migration error, symmetric with the existing "duplicate column" tolerance for ADD races. Makes DROP COLUMN migrations idempotent across databases bootstrapped with different base schemas. Observed failure: migration 0032 (DROP COLUMN applied_at) loops forever on any database that lands at v31 without the column. Dolt has no DROP COLUMN IF EXISTS, so the SQL itself can't be guarded; handling it in the runner keeps the migration file trivial. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * Add migration 0032 idempotence tests (bd-7mh.3) * Document bd execution metadata workflow (bd-b1a) * Revert "Document bd execution metadata workflow (bd-b1a)" This reverts commit0d6a96babd. * Reapply "Document bd execution metadata workflow (bd-b1a)" This reverts commitdf90cf5404. --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: matt wilkie <maphew@gmail.com>
240 lines
8.2 KiB
Go
240 lines
8.2 KiB
Go
// Package schema provides the unified schema definitions and migration runner
|
|
// for both DoltStore and EmbeddedDoltStore. The embedded .up.sql migration
|
|
// files are the single source of truth for the database schema.
|
|
package schema
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"embed"
|
|
"fmt"
|
|
"io/fs"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
// DBConn is the minimal interface satisfied by *sql.DB, *sql.Tx, and *sql.Conn.
|
|
// It provides query and exec methods needed by the migration runner.
|
|
type DBConn interface {
|
|
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
|
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
|
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
|
}
|
|
|
|
//go:embed migrations/*.up.sql
|
|
var upMigrations embed.FS
|
|
|
|
var (
|
|
latestOnce sync.Once
|
|
latestVer int
|
|
)
|
|
|
|
const schemaMigrationsBootstrapSQL = `CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
version INT PRIMARY KEY,
|
|
applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
)`
|
|
|
|
// LatestVersion returns the highest version number among the embedded .up.sql files.
|
|
// Computed once and cached.
|
|
func LatestVersion() int {
|
|
latestOnce.Do(func() {
|
|
entries, err := fs.ReadDir(upMigrations, "migrations")
|
|
if err != nil {
|
|
panic(fmt.Sprintf("schema: failed to read embedded migrations: %v", err))
|
|
}
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") {
|
|
continue
|
|
}
|
|
v, err := parseVersion(e.Name())
|
|
if err != nil {
|
|
panic(fmt.Sprintf("schema: invalid migration filename %q: %v", e.Name(), err))
|
|
}
|
|
if v > latestVer {
|
|
latestVer = v
|
|
}
|
|
}
|
|
})
|
|
return latestVer
|
|
}
|
|
|
|
// AllMigrationsSQL returns the schema_migrations bootstrap plus all .up.sql
|
|
// migration contents concatenated in order. Used by integration tests that need
|
|
// to initialize a schema via dolt sql CLI.
|
|
func AllMigrationsSQL() string {
|
|
entries, err := fs.ReadDir(upMigrations, "migrations")
|
|
if err != nil {
|
|
panic(fmt.Sprintf("schema: failed to read embedded migrations: %v", err))
|
|
}
|
|
|
|
type mf struct {
|
|
version int
|
|
name string
|
|
}
|
|
var files []mf
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") {
|
|
continue
|
|
}
|
|
v, err := parseVersion(e.Name())
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, mf{version: v, name: e.Name()})
|
|
}
|
|
sort.Slice(files, func(i, j int) bool { return files[i].version < files[j].version })
|
|
|
|
var b strings.Builder
|
|
b.WriteString(schemaMigrationsBootstrapSQL)
|
|
b.WriteString(";\n")
|
|
for _, f := range files {
|
|
data, err := upMigrations.ReadFile("migrations/" + f.name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
b.Write(data)
|
|
b.WriteByte('\n')
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// parseVersion extracts the leading integer from a migration filename like "0001_create_issues.up.sql".
|
|
func parseVersion(name string) (int, error) {
|
|
parts := strings.SplitN(name, "_", 2)
|
|
if len(parts) == 0 {
|
|
return 0, fmt.Errorf("no version prefix")
|
|
}
|
|
return strconv.Atoi(parts[0])
|
|
}
|
|
|
|
// MigrateUp applies all embedded .up.sql migrations that haven't been applied yet.
|
|
// Returns the number of migrations applied. Safe for use with both *sql.Tx and
|
|
// *sql.DB — the caller controls transaction boundaries.
|
|
func MigrateUp(ctx context.Context, db DBConn) (int, error) {
|
|
// Bootstrap the tracking table.
|
|
// Bootstrap with applied_at so migration 0032 can unconditionally drop it.
|
|
// After 0032 runs, the table has only the version column.
|
|
if _, err := db.ExecContext(ctx, schemaMigrationsBootstrapSQL); err != nil {
|
|
return 0, fmt.Errorf("creating schema_migrations table: %w", err)
|
|
}
|
|
|
|
// Find the current version.
|
|
var current int
|
|
err := db.QueryRowContext(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(¤t)
|
|
if err == sql.ErrNoRows {
|
|
current = 0
|
|
} else if err != nil {
|
|
return 0, fmt.Errorf("reading current migration version: %w", err)
|
|
}
|
|
|
|
// Fast path: if current version matches the highest embedded migration, nothing to do.
|
|
if current >= LatestVersion() {
|
|
return 0, nil
|
|
}
|
|
|
|
// If schema_migrations is empty but core tables already exist (e.g. restored
|
|
// from a DoltStore backup that doesn't track embedded migrations), backfill
|
|
// all versions so we don't re-run migrations that would fail on "already exists".
|
|
if current == 0 {
|
|
var tableCount int
|
|
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'issues' AND table_schema = DATABASE()").Scan(&tableCount); err == nil && tableCount > 0 {
|
|
return backfillMigrations(ctx, db)
|
|
}
|
|
}
|
|
|
|
return runMigrations(ctx, db, current, false)
|
|
}
|
|
|
|
// backfillMigrations runs all migrations in order, ignoring "already exists"
|
|
// errors, and records each version. Used when a database is restored from a
|
|
// backup that predates the schema_migrations tracking table — most of the
|
|
// schema is already correct, but dolt_ignore'd tables (wisps) may be missing.
|
|
func backfillMigrations(ctx context.Context, db DBConn) (int, error) {
|
|
return runMigrations(ctx, db, 0, true)
|
|
}
|
|
|
|
type migrationFile struct {
|
|
version int
|
|
name string
|
|
}
|
|
|
|
// runMigrations collects all embedded .up.sql files with version > minVersion,
|
|
// sorts them, and executes each one. DDL "already exists" errors and duplicate
|
|
// version inserts are always tolerated to support concurrent initialization
|
|
// (multiple processes racing to apply the same migration). When tolerateExisting
|
|
// is true, ALL "already exists" errors are silently ignored (backfill path).
|
|
func runMigrations(ctx context.Context, db DBConn, minVersion int, tolerateExisting bool) (int, error) {
|
|
entries, err := fs.ReadDir(upMigrations, "migrations")
|
|
if err != nil {
|
|
return 0, fmt.Errorf("reading embedded migrations: %w", err)
|
|
}
|
|
|
|
var pending []migrationFile
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") {
|
|
continue
|
|
}
|
|
v, err := parseVersion(e.Name())
|
|
if err != nil {
|
|
return 0, fmt.Errorf("parsing migration filename %q: %w", e.Name(), err)
|
|
}
|
|
if v > minVersion {
|
|
pending = append(pending, migrationFile{version: v, name: e.Name()})
|
|
}
|
|
}
|
|
|
|
sort.Slice(pending, func(i, j int) bool { return pending[i].version < pending[j].version })
|
|
|
|
if len(pending) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
for _, mf := range pending {
|
|
data, err := upMigrations.ReadFile("migrations/" + mf.name)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("reading migration %s: %w", mf.name, err)
|
|
}
|
|
|
|
// Execute statements individually. Multi-statement Exec can abort the
|
|
// batch on the first error, which under tolerateExisting silently skips
|
|
// subsequent DDL while still recording the version as applied (GH#3363).
|
|
for _, stmt := range splitStatements(string(data)) {
|
|
if _, err := db.ExecContext(ctx, stmt); err != nil {
|
|
if !tolerateExisting && !isConcurrentInitError(err) {
|
|
return 0, fmt.Errorf("migration %s: statement failed: %w", mf.name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Always use INSERT IGNORE — concurrent processes may race to record
|
|
// the same migration version. Duplicate PK is expected and harmless.
|
|
if _, err := db.ExecContext(ctx, "INSERT IGNORE INTO schema_migrations (version) VALUES (?)", mf.version); err != nil {
|
|
if !isConcurrentInitError(err) {
|
|
return 0, fmt.Errorf("recording migration %s: %w", mf.name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
return len(pending), nil
|
|
}
|
|
|
|
// isConcurrentInitError returns true for errors that are expected and harmless
|
|
// during concurrent schema initialization, or when the desired state is already
|
|
// present because a db was bootstrapped out-of-band:
|
|
// - "already exists" — table/index/key created by another process (1050, 1061)
|
|
// - "duplicate column" — ALTER TABLE ADD COLUMN raced (1060)
|
|
// - "duplicate key name" — CREATE INDEX raced (1061)
|
|
// - "serialization failure" — Dolt write conflict from concurrent transaction
|
|
// - "does not have column" — ALTER TABLE DROP COLUMN on already-missing column
|
|
// (Dolt 1105); symmetric with "duplicate column" above. Makes DROP COLUMN
|
|
// migrations idempotent across dbs bootstrapped with different base schemas.
|
|
func isConcurrentInitError(err error) bool {
|
|
msg := strings.ToLower(err.Error())
|
|
return strings.Contains(msg, "already exists") ||
|
|
strings.Contains(msg, "duplicate column") ||
|
|
strings.Contains(msg, "duplicate key name") ||
|
|
strings.Contains(msg, "serialization failure") ||
|
|
strings.Contains(msg, "does not have column")
|
|
}
|