fix: prevent concurrent embedded Dolt panic with exclusive flock (#2571)
When two bd commands ran simultaneously against the same embedded Dolt database, the second process could panic with a nil-pointer dereference in DoltDB.SetCrashOnFatalError during engine initialization. The race occurs because dolthub/driver.NewConnector is not safe for concurrent initialization on the same data directory. The fix acquires an exclusive flock in embeddeddolt.New() for the entire store lifetime. The second process now receives a clear error message instead of panicking. The lock is released in Close(). A WithLock option allows bd init (which acquires the lock earlier) to pass its pre-held lock and avoid double-locking on macOS where flock is per-fd. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Executed-By: beads/crew/emma
This commit is contained in:
parent
2a71d7fc41
commit
543e1943b6
6 changed files with 149 additions and 9 deletions
|
|
@ -21,6 +21,7 @@ import (
|
|||
"github.com/steveyegge/beads/internal/git"
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/storage/dolt"
|
||||
"github.com/steveyegge/beads/internal/storage/embeddeddolt"
|
||||
"github.com/steveyegge/beads/internal/templates/agents"
|
||||
"github.com/steveyegge/beads/internal/ui"
|
||||
"github.com/steveyegge/beads/internal/utils"
|
||||
|
|
@ -523,7 +524,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1):
|
|||
}
|
||||
}
|
||||
|
||||
store, err := newDoltStore(ctx, doltCfg)
|
||||
store, err := newDoltStore(ctx, doltCfg, embeddeddolt.WithLock(initLock))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed to open Dolt store: %v\n", err)
|
||||
os.Exit(1)
|
||||
|
|
|
|||
|
|
@ -38,11 +38,11 @@ func isEmbeddedMode() bool {
|
|||
// When cfg.ServerMode is true, connects to an external dolt sql-server;
|
||||
// otherwise uses the embedded Dolt engine (default).
|
||||
// Used by bd init and PersistentPreRun.
|
||||
func newDoltStore(ctx context.Context, cfg *dolt.Config) (storage.DoltStorage, error) {
|
||||
func newDoltStore(ctx context.Context, cfg *dolt.Config, opts ...embeddeddolt.Option) (storage.DoltStorage, error) {
|
||||
if cfg.ServerMode {
|
||||
return dolt.New(ctx, cfg)
|
||||
}
|
||||
return embeddeddolt.New(ctx, cfg.BeadsDir, cfg.Database, "main")
|
||||
return embeddeddolt.New(ctx, cfg.BeadsDir, cfg.Database, "main", opts...)
|
||||
}
|
||||
|
||||
// acquireEmbeddedLock acquires an exclusive flock on the embeddeddolt data
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ func isEmbeddedMode() bool {
|
|||
|
||||
// newDoltStore creates a server-mode storage backend. Embedded Dolt is not
|
||||
// available without CGO.
|
||||
func newDoltStore(ctx context.Context, cfg *dolt.Config) (storage.DoltStorage, error) {
|
||||
func newDoltStore(ctx context.Context, cfg *dolt.Config, _ ...embeddeddolt.Option) (storage.DoltStorage, error) {
|
||||
if !cfg.ServerMode {
|
||||
return nil, fmt.Errorf("embedded Dolt requires CGO; use server mode (bd init --mode server)")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -211,6 +211,78 @@ func mustQueryRow(t *testing.T, db *sql.DB, ctx context.Context, query string, a
|
|||
return row
|
||||
}
|
||||
|
||||
// TestConcurrentNewReturnsError verifies that opening a second EmbeddedDoltStore
|
||||
// on the same data directory returns a clear error instead of panicking with a
|
||||
// nil-pointer dereference in the embedded Dolt engine (GH#2571).
|
||||
func TestConcurrentNewReturnsError(t *testing.T) {
|
||||
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
|
||||
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt concurrency tests")
|
||||
}
|
||||
|
||||
ctx := t.Context()
|
||||
beadsDir := t.TempDir()
|
||||
|
||||
// First store opens successfully and holds the exclusive flock.
|
||||
store1, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
|
||||
if err != nil {
|
||||
t.Fatalf("first New: %v", err)
|
||||
}
|
||||
defer store1.Close()
|
||||
|
||||
// Second store on the same directory must fail with a lock error, not panic.
|
||||
store2, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
|
||||
if err == nil {
|
||||
store2.Close()
|
||||
t.Fatal("second New succeeded; expected lock contention error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "another process holds the exclusive lock") {
|
||||
t.Fatalf("unexpected error from second New: %v", err)
|
||||
}
|
||||
t.Logf("second New correctly returned: %v", err)
|
||||
|
||||
// After closing the first store, a third open must succeed.
|
||||
store1.Close()
|
||||
store3, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
|
||||
if err != nil {
|
||||
t.Fatalf("third New (after close): %v", err)
|
||||
}
|
||||
store3.Close()
|
||||
}
|
||||
|
||||
// TestWithLockBypassesDoubleLock verifies that WithLock allows the caller to
|
||||
// supply a pre-acquired lock so New does not attempt a second flock (which
|
||||
// would fail on macOS where flock is per-fd, not per-process).
|
||||
func TestWithLockBypassesDoubleLock(t *testing.T) {
|
||||
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
|
||||
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt concurrency tests")
|
||||
}
|
||||
|
||||
ctx := t.Context()
|
||||
beadsDir := t.TempDir()
|
||||
dataDir := filepath.Join(beadsDir, "embeddeddolt")
|
||||
|
||||
// Acquire the lock externally (as bd init does).
|
||||
lock, err := embeddeddolt.TryLock(dataDir)
|
||||
if err != nil {
|
||||
t.Fatalf("TryLock: %v", err)
|
||||
}
|
||||
defer lock.Unlock()
|
||||
|
||||
// New with WithLock must succeed without double-locking.
|
||||
store, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main", embeddeddolt.WithLock(lock))
|
||||
if err != nil {
|
||||
t.Fatalf("New with WithLock: %v", err)
|
||||
}
|
||||
// Close must NOT release the caller-supplied lock.
|
||||
store.Close()
|
||||
|
||||
// The lock should still be held — verify by trying to lock again (should fail).
|
||||
_, err = embeddeddolt.TryLock(dataDir)
|
||||
if err == nil {
|
||||
t.Fatal("TryLock succeeded after Close; expected lock to still be held by caller")
|
||||
}
|
||||
}
|
||||
|
||||
func envInt(name string, def int) int {
|
||||
v := strings.TrimSpace(os.Getenv(name))
|
||||
if v == "" {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ var _ storage.Compactor = (*EmbeddedDoltStore)(nil)
|
|||
// SQL transaction, and closes the connection immediately. This minimizes the
|
||||
// time the embedded engine's write lock is held, reducing contention when
|
||||
// multiple processes access the same database concurrently.
|
||||
//
|
||||
// The store holds an exclusive flock on the data directory for its entire
|
||||
// lifetime. This prevents concurrent processes from initializing the embedded
|
||||
// Dolt engine on the same directory, which causes a nil-pointer panic in
|
||||
// DoltDB.SetCrashOnFatalError (GH#2571).
|
||||
type EmbeddedDoltStore struct {
|
||||
dataDir string
|
||||
beadsDir string
|
||||
|
|
@ -38,15 +43,43 @@ type EmbeddedDoltStore struct {
|
|||
branch string
|
||||
credentialKey []byte
|
||||
closed atomic.Bool
|
||||
lock Unlocker // exclusive flock held for the store's lifetime
|
||||
ownsLock bool // true when New acquired the lock (false when caller supplied it via WithLock)
|
||||
}
|
||||
|
||||
// errClosed is returned when a method is called after Close.
|
||||
var errClosed = errors.New("embeddeddolt: store is closed")
|
||||
|
||||
// Option configures optional behavior for New.
|
||||
type Option func(*options)
|
||||
|
||||
type options struct {
|
||||
lock Unlocker // pre-acquired lock; nil means New acquires its own
|
||||
}
|
||||
|
||||
// WithLock passes a pre-acquired exclusive lock to New so it does not attempt
|
||||
// to acquire a second one. The caller retains ownership — Close will NOT
|
||||
// release a caller-supplied lock. This is used by bd init, which acquires the
|
||||
// lock earlier to protect pre-initialization steps.
|
||||
func WithLock(lock Unlocker) Option {
|
||||
return func(o *options) { o.lock = lock }
|
||||
}
|
||||
|
||||
// New creates an EmbeddedDoltStore using the embedded Dolt engine.
|
||||
// beadsDir is the .beads/ root; the data directory is derived as <beadsDir>/embeddeddolt/.
|
||||
// The database is created automatically if it doesn't exist (initSchema handles this).
|
||||
func New(ctx context.Context, beadsDir, database, branch string) (*EmbeddedDoltStore, error) {
|
||||
//
|
||||
// An exclusive flock is held on the data directory for the store's entire
|
||||
// lifetime. If another process already holds the lock, New returns an error
|
||||
// instead of panicking during concurrent engine initialization (GH#2571).
|
||||
// The lock is released when Close is called, unless a pre-acquired lock was
|
||||
// supplied via WithLock (in which case the caller is responsible for it).
|
||||
func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*EmbeddedDoltStore, error) {
|
||||
var o options
|
||||
for _, fn := range opts {
|
||||
fn(&o)
|
||||
}
|
||||
|
||||
// Resolve to absolute path — the embedded dolt driver resolves file://
|
||||
// DSN paths relative to its data directory, so relative paths cause
|
||||
// doubled-path errors on subsequent opens.
|
||||
|
|
@ -59,14 +92,32 @@ func New(ctx context.Context, beadsDir, database, branch string) (*EmbeddedDoltS
|
|||
return nil, fmt.Errorf("embeddeddolt: creating data directory: %w", err)
|
||||
}
|
||||
|
||||
// Acquire an exclusive flock before initializing the embedded engine.
|
||||
// Without this, concurrent processes race through NewConnector →
|
||||
// DoltDB.SetCrashOnFatalError → newDatabase → CollectDBs and one of
|
||||
// them panics with a nil-pointer dereference (GH#2571).
|
||||
lock := o.lock
|
||||
ownsLock := lock == nil
|
||||
if ownsLock {
|
||||
lock, err = TryLock(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
s := &EmbeddedDoltStore{
|
||||
dataDir: dataDir,
|
||||
beadsDir: absBeadsDir,
|
||||
database: database,
|
||||
branch: branch,
|
||||
lock: lock,
|
||||
ownsLock: ownsLock,
|
||||
}
|
||||
|
||||
if err := s.initSchema(ctx); err != nil {
|
||||
if ownsLock {
|
||||
lock.Unlock()
|
||||
}
|
||||
return nil, fmt.Errorf("embeddeddolt: init schema: %w", err)
|
||||
}
|
||||
|
||||
|
|
@ -346,10 +397,18 @@ func (s *EmbeddedDoltStore) GetAllEventsSince(ctx context.Context, since time.Ti
|
|||
|
||||
// RunInTransaction is implemented in transaction.go.
|
||||
|
||||
// Close marks the store as closed. Subsequent method calls will return errClosed.
|
||||
// It is safe to call multiple times.
|
||||
// Close marks the store as closed and releases the exclusive flock on the data
|
||||
// directory (if the store owns it). Subsequent method calls will return errClosed.
|
||||
// It is safe to call multiple times. When the lock was supplied by the caller
|
||||
// via WithLock, Close does NOT release it — the caller retains ownership.
|
||||
func (s *EmbeddedDoltStore) Close() error {
|
||||
s.closed.Store(true)
|
||||
// Use CompareAndSwap so we only unlock once even if Close is called
|
||||
// multiple times (the Lock.Unlock method panics on double-unlock).
|
||||
if s.closed.CompareAndSwap(false, true) {
|
||||
if s.lock != nil && s.ownsLock {
|
||||
s.lock.Unlock()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,15 @@ type EmbeddedDoltStore struct {
|
|||
branch string
|
||||
}
|
||||
|
||||
// Option configures optional behavior for New (stub: no-op).
|
||||
type Option func(*struct{})
|
||||
|
||||
// WithLock is a no-op in non-CGO builds.
|
||||
func WithLock(_ Unlocker) Option {
|
||||
return func(*struct{}) {}
|
||||
}
|
||||
|
||||
// New returns an error when CGO is not enabled.
|
||||
func New(_ context.Context, _, _, _ string) (*EmbeddedDoltStore, error) {
|
||||
func New(_ context.Context, _, _, _ string, _ ...Option) (*EmbeddedDoltStore, error) {
|
||||
return nil, errors.New("embeddeddolt: requires CGO (build with CGO_ENABLED=1)")
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue