/{beads_cgo,cmd,internal}: cache open embedded stores deterministically to prevent deadlocking when multiple store open attempts are made against an opened store

This commit is contained in:
coffeegoddd☕️✨ 2026-04-29 15:49:51 -07:00
commit 58dc2e1b8b
18 changed files with 287 additions and 71 deletions

View file

@ -51,7 +51,7 @@ func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, embeddedd
if cfg != nil {
database = cfg.GetDoltDatabase()
}
store, err := embeddeddolt.New(ctx, beadsDir, database, "main", embeddeddolt.WithLock(lock))
store, err := embeddeddolt.Open(ctx, beadsDir, database, "main", embeddeddolt.WithLock(lock))
if err != nil {
lock.Unlock()
return nil, nil, err

View file

@ -96,7 +96,7 @@ func bdShow(t *testing.T, bd, dir, id string) *types.Issue {
// openStore opens an EmbeddedDoltStore for direct verification queries.
func openStore(t *testing.T, beadsDir, database string) *embeddeddolt.EmbeddedDoltStore {
t.Helper()
store, err := embeddeddolt.New(t.Context(), beadsDir, database, "main")
store, err := embeddeddolt.Open(t.Context(), beadsDir, database, "main")
if err != nil {
t.Fatalf("openStore: %v", err)
}

View file

@ -67,7 +67,7 @@ func bdDiffJSON(t *testing.T, bd, dir string, args ...string) []map[string]inter
// bd subprocess commands to acquire it.
func getCommitHash(t *testing.T, beadsDir, database string) string {
t.Helper()
s, err := embeddeddolt.New(t.Context(), beadsDir, database, "main")
s, err := embeddeddolt.Open(t.Context(), beadsDir, database, "main")
if err != nil {
t.Fatalf("openStore for getCommitHash: %v", err)
}

View file

@ -184,7 +184,7 @@ func readBackOnce(t *testing.T, beadsDir, database, key string, metadata bool) (
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
store, err := embeddeddolt.New(ctx, beadsDir, database, "main")
store, err := embeddeddolt.Open(ctx, beadsDir, database, "main")
if err != nil {
return "", fmt.Errorf("New failed: %w", err)
}
@ -758,7 +758,7 @@ func TestEmbeddedInit(t *testing.T) {
func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
store, err := embeddeddolt.New(ctx, beadsDir, "meta", "main")
store, err := embeddeddolt.Open(ctx, beadsDir, "meta", "main")
if err != nil {
t.Fatalf("failed to open store for bd_version check: %v", err)
}

View file

@ -21,7 +21,7 @@ func TestSpawnMolecule_PreservesStepLabels(t *testing.T) {
}
ctx := t.Context()
s, err := embeddeddolt.New(ctx, t.TempDir(), "beads", "main")
s, err := embeddeddolt.Open(ctx, t.TempDir(), "beads", "main")
if err != nil {
t.Fatalf("embeddeddolt.New failed: %v", err)
}

View file

@ -62,7 +62,7 @@ func simulateCompaction(t *testing.T, bd, dir, beadsDir, database string) string
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
for i := 0; i < 5; i++ {
store, err = embeddeddolt.New(ctx, beadsDir, database, "main")
store, err = embeddeddolt.Open(ctx, beadsDir, database, "main")
if err == nil {
break
}
@ -181,7 +181,7 @@ func TestEmbeddedRestoreConcurrent(t *testing.T) {
var store *embeddeddolt.EmbeddedDoltStore
var err error
for i := 0; i < 5; i++ {
store, err = embeddeddolt.New(ctx, beadsDir, "rstcon", "main")
store, err = embeddeddolt.Open(ctx, beadsDir, "rstcon", "main")
if err == nil {
break
}

View file

@ -44,7 +44,7 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config, opts ...embeddeddolt.Op
if cfg.ServerMode {
return dolt.New(ctx, cfg)
}
return embeddeddolt.New(ctx, cfg.BeadsDir, cfg.Database, "main", opts...)
return embeddeddolt.Open(ctx, cfg.BeadsDir, cfg.Database, "main", opts...)
}
// acquireEmbeddedLock acquires an exclusive flock on the embeddeddolt data
@ -80,7 +80,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS
}
database = sanitized
}
return embeddeddolt.New(ctx, beadsDir, database, "main")
return embeddeddolt.Open(ctx, beadsDir, database, "main")
}
// migrateHyphenatedDB renames a legacy hyphenated database directory and
@ -141,5 +141,5 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D
if sanitized := sanitizeDBName(database); sanitized != database {
database = sanitized
}
return embeddeddolt.New(ctx, beadsDir, database, "main")
return embeddeddolt.Open(ctx, beadsDir, database, "main")
}

View file

@ -46,7 +46,7 @@ func TestNewDoltStoreFromConfig_NoMetadata(t *testing.T) {
// deferring to a confusing "no database selected" SQL error.
// Belt-and-suspenders defense for be-sy8 / GH#2988.
func TestEmbeddedNew_EmptyDatabaseRejected(t *testing.T) {
_, err := embeddeddolt.New(t.Context(), t.TempDir(), "", "main")
_, err := embeddeddolt.Open(t.Context(), t.TempDir(), "", "main")
if err == nil {
t.Fatal("expected error for empty database name")
}

View file

@ -47,7 +47,7 @@ func TestWhereCommand_ReadsPrefixFromEmbeddedStore(t *testing.T) {
t.Fatalf("save metadata: %v", err)
}
store, err := embeddeddolt.New(context.Background(), beadsDir, "embedcfg", "main")
store, err := embeddeddolt.Open(context.Background(), beadsDir, "embedcfg", "main")
if err != nil {
t.Fatalf("embeddeddolt.New: %v", err)
}

View file

@ -0,0 +1,108 @@
//go:build cgo
package embeddeddolt
import (
"context"
"fmt"
"path/filepath"
"sync"
)
var (
cacheMu sync.Mutex
cache = make(map[string]*cacheEntry) // keyed by absolute dataDir
)
type cacheEntry struct {
store *EmbeddedDoltStore
refCount int // guarded by cacheMu
}
// Open returns a cached EmbeddedDoltStore for the given data directory, creating
// one via New if no cached instance exists. Subsequent calls with the same
// resolved dataDir return the existing store and increment a reference count.
//
// Each Open must be paired with a Close. The underlying store is only truly
// closed (flock released, resources freed) when the last reference calls Close.
//
// This prevents the same-process deadlock that occurs when two code paths open
// connectors against the same data directory: the embedded Dolt driver's
// internal engine lock combined with infinite-backoff retry means the second
// connector spins forever waiting for the first to release.
func Open(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*EmbeddedDoltStore, error) {
key, err := cacheKey(beadsDir)
if err != nil {
return nil, err
}
cacheMu.Lock()
if entry, ok := cache[key]; ok {
entry.refCount++
cacheMu.Unlock()
return entry.store, nil
}
cacheMu.Unlock()
// Slow path: create a new store outside the lock. New() acquires the
// flock and initializes the schema, which can take significant time.
s, err := newStore(ctx, beadsDir, database, branch, opts...)
if err != nil {
return nil, err
}
cacheMu.Lock()
// Double-check: another goroutine may have inserted while we created.
if entry, ok := cache[key]; ok {
cacheMu.Unlock()
// Discard the store we just created; use the cached one.
_ = s.closeUnderlying()
entry.refCount++
return entry.store, nil
}
cache[key] = &cacheEntry{store: s, refCount: 1}
cacheMu.Unlock()
return s, nil
}
// closeCached decrements the reference count for a cached store and removes it
// from the cache when the count reaches zero, running the real close logic.
// Returns true if this store was managed by the cache (caller should not close
// it directly), false if the store was not in the cache (caller owns cleanup).
func closeCached(s *EmbeddedDoltStore) bool {
cacheMu.Lock()
defer cacheMu.Unlock()
for key, entry := range cache {
if entry.store == s {
entry.refCount--
if entry.refCount <= 0 {
delete(cache, key)
// Actual close happens after releasing cacheMu (via caller).
return false
}
// Other references remain — suppress the real close.
return true
}
}
// Not in cache — let the caller close normally.
return false
}
// cacheKey resolves beadsDir to an absolute dataDir path for use as a cache key.
func cacheKey(beadsDir string) (string, error) {
absBeadsDir, err := filepath.Abs(beadsDir)
if err != nil {
return "", fmt.Errorf("embeddeddolt: resolving beads dir for cache key: %w", err)
}
return filepath.Join(absBeadsDir, "embeddeddolt"), nil
}
// ResetCache removes all entries from the store cache without closing any
// stores. This is intended for use in tests only, to prevent cross-test
// cache pollution.
func ResetCache() {
cacheMu.Lock()
clear(cache)
cacheMu.Unlock()
}

View file

@ -0,0 +1,109 @@
//go:build cgo
package embeddeddolt_test
import (
"os"
"path/filepath"
"testing"
"github.com/steveyegge/beads/internal/storage/embeddeddolt"
)
func TestOpenReturnsCachedStore(t *testing.T) {
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests")
}
t.Cleanup(embeddeddolt.ResetCache)
ctx := t.Context()
beadsDir := filepath.Join(t.TempDir(), ".beads")
// First Open creates a new store.
store1, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("first Open: %v", err)
}
// Second Open with same beadsDir returns the same pointer.
store2, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("second Open: %v", err)
}
if store1 != store2 {
t.Fatal("expected second Open to return the same store instance")
}
// First Close is a no-op (refcount > 0).
if err := store1.Close(); err != nil {
t.Fatalf("first Close: %v", err)
}
// Store should still be usable via store2 — GetIssue exercises withConn.
_, getErr := store2.GetIssue(t.Context(), "nonexistent")
if getErr != nil && getErr.Error() != "not found: issue nonexistent" {
t.Fatalf("GetIssue after first Close should work (got: %v)", getErr)
}
// Second Close actually releases the store.
if err := store2.Close(); err != nil {
t.Fatalf("second Close: %v", err)
}
}
func TestOpenDifferentDirsReturnsDifferentStores(t *testing.T) {
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests")
}
t.Cleanup(embeddeddolt.ResetCache)
ctx := t.Context()
beadsDir1 := filepath.Join(t.TempDir(), ".beads")
beadsDir2 := filepath.Join(t.TempDir(), ".beads")
store1, err := embeddeddolt.Open(ctx, beadsDir1, "testdb", "main")
if err != nil {
t.Fatalf("Open dir1: %v", err)
}
defer store1.Close()
store2, err := embeddeddolt.Open(ctx, beadsDir2, "testdb", "main")
if err != nil {
t.Fatalf("Open dir2: %v", err)
}
defer store2.Close()
if store1 == store2 {
t.Fatal("expected different stores for different directories")
}
}
func TestOpenAfterCloseCreatesNewStore(t *testing.T) {
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests")
}
t.Cleanup(embeddeddolt.ResetCache)
ctx := t.Context()
beadsDir := filepath.Join(t.TempDir(), ".beads")
store1, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("first Open: %v", err)
}
// Close releases the last reference — evicts from cache.
store1.Close()
// Re-opening should create a fresh store, not return the closed one.
store2, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("Open after Close: %v", err)
}
defer store2.Close()
if store1 == store2 {
t.Fatal("expected a new store after full close, got the same pointer")
}
}

View file

@ -31,7 +31,7 @@ func main() {
}
ctx := context.Background()
store, err := embeddeddolt.New(ctx, absDir, *database, *branch)
store, err := embeddeddolt.Open(ctx, absDir, *database, *branch)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)

View file

@ -37,7 +37,7 @@ func TestEmbeddedOpenRunsCompatMigrations(t *testing.T) {
dataDir := filepath.Join(beadsDir, "embeddeddolt")
// Step 1: fresh init — schema.MigrateUp adds all columns including started_at.
store, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("initial New: %v", err)
}
@ -62,7 +62,7 @@ func TestEmbeddedOpenRunsCompatMigrations(t *testing.T) {
// Step 3: reopen. If compat migrations are wired in, this repairs the
// column via MigrateAddStartedAtColumn.
store2, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store2, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("reopen New: %v", err)
}

View file

@ -216,59 +216,36 @@ func mustQueryRow(t *testing.T, db *sql.DB, ctx context.Context, query string, a
return row
}
// TestConcurrentNewQueues verifies that opening a second EmbeddedDoltStore
// on the same data directory blocks until the first is closed, rather than
// failing immediately or panicking (GH#2571).
func TestConcurrentNewQueues(t *testing.T) {
// TestConcurrentOpenReturnsCached verifies that opening a second store on
// the same data directory returns the cached instance immediately rather than
// blocking on the flock (the cache prevents same-process deadlock from the
// driver's infinite backoff — GH#2571).
func TestConcurrentOpenReturnsCached(t *testing.T) {
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt concurrency tests")
}
t.Cleanup(embeddeddolt.ResetCache)
ctx := t.Context()
beadsDir := t.TempDir()
// First store opens successfully and holds the exclusive flock.
store1, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store1, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("first New: %v", err)
t.Fatalf("first Open: %v", err)
}
// Launch second New in a goroutine — it should block on the lock.
type result struct {
store *embeddeddolt.EmbeddedDoltStore
err error
// Second Open should return immediately with the same cached store.
store2, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("second Open: %v", err)
}
ch := make(chan result, 1)
go func() {
s, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
ch <- result{s, err}
}()
// Give the goroutine time to start waiting, then release the first store.
time.Sleep(200 * time.Millisecond)
select {
case r := <-ch:
if r.err != nil {
t.Fatalf("second New returned early with error: %v", r.err)
}
t.Fatal("second New returned before first store was closed")
default:
// Good — still blocking.
if store1 != store2 {
t.Fatal("expected second Open to return the cached store")
}
store1.Close()
// The second New should now succeed.
select {
case r := <-ch:
if r.err != nil {
t.Fatalf("second New failed after first close: %v", r.err)
}
r.store.Close()
t.Log("second New succeeded after first store closed")
case <-time.After(10 * time.Second):
t.Fatal("second New did not complete within 10s after first store closed")
}
store2.Close()
}
// TestWaitLockBlocksAndSucceeds verifies that WaitLock blocks until the lock
@ -373,7 +350,7 @@ func TestWithLockBypassesDoubleLock(t *testing.T) {
defer lock.Unlock()
// New with WithLock must succeed without double-locking.
store, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main", embeddeddolt.WithLock(lock))
store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main", embeddeddolt.WithLock(lock))
if err != nil {
t.Fatalf("New with WithLock: %v", err)
}

View file

@ -28,7 +28,7 @@ func newTestEnv(t *testing.T, prefix string) *testEnv {
t.Helper()
ctx := t.Context()
beadsDir := filepath.Join(t.TempDir(), ".beads")
store, err := embeddeddolt.New(ctx, beadsDir, prefix, "main")
store, err := embeddeddolt.Open(ctx, beadsDir, prefix, "main")
if err != nil {
t.Fatalf("New: %v", err)
}
@ -214,7 +214,7 @@ func TestCreateIssue(t *testing.T) {
t.Run("missing_prefix_errors", func(t *testing.T) {
ctx := t.Context()
beadsDir := filepath.Join(t.TempDir(), ".beads")
store, err := embeddeddolt.New(ctx, beadsDir, "noprefix", "main")
store, err := embeddeddolt.Open(ctx, beadsDir, "noprefix", "main")
if err != nil {
t.Fatalf("New: %v", err)
}

View file

@ -23,7 +23,7 @@ func TestSchemaAfterInit(t *testing.T) {
dataDir := filepath.Join(beadsDir, "embeddeddolt")
// Initialize store — creates database and runs all migrations.
store, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("New: %v", err)
}
@ -166,7 +166,7 @@ func TestSchemaAfterInit(t *testing.T) {
// --- Verify idempotency: New on same dir succeeds ---
store2, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store2, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("second New (idempotency): %v", err)
}
@ -211,7 +211,7 @@ func TestBackfillCreatesWispTables(t *testing.T) {
dataDir := filepath.Join(beadsDir, "embeddeddolt")
// Step 1: Normal init — creates everything including wisp tables.
store, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("New: %v", err)
}
@ -242,7 +242,7 @@ func TestBackfillCreatesWispTables(t *testing.T) {
// Step 3: Re-open. This triggers the backfill path (schema_migrations
// empty, issues table exists). The old code would mark all migrations
// as applied without executing them, leaving wisp tables missing.
store2, err := embeddeddolt.New(ctx, beadsDir, "testdb", "main")
store2, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main")
if err != nil {
t.Fatalf("New after backfill: %v", err)
}

View file

@ -68,17 +68,20 @@ func WithLock(lock Unlocker) Option {
return func(o *options) { o.lock = lock }
}
// New creates an EmbeddedDoltStore using the embedded Dolt engine.
// newStore creates an EmbeddedDoltStore using the embedded Dolt engine.
// beadsDir is the .beads/ root; the data directory is derived as <beadsDir>/embeddeddolt/.
// The database is created automatically if it doesn't exist (initSchema handles this).
//
// An exclusive flock is held on the data directory for the store's entire
// lifetime. If another process already holds the lock, New queues with
// lifetime. If another process already holds the lock, newStore queues with
// exponential backoff until the lock becomes available or the context is
// canceled, instead of panicking during concurrent engine initialization
// (GH#2571). The lock is released when Close is called, unless a pre-acquired
// lock was supplied via WithLock (in which case the caller is responsible for it).
func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*EmbeddedDoltStore, error) {
//
// Production code should use Open, which routes through a process-scoped cache
// to prevent same-process deadlocks from the driver's infinite backoff.
func newStore(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*EmbeddedDoltStore, error) {
if database == "" {
return nil, fmt.Errorf("embeddeddolt: database name must not be empty (caller should default to %q)", "beads")
}
@ -463,12 +466,26 @@ func (s *EmbeddedDoltStore) GetAllEventsSince(ctx context.Context, since time.Ti
// RunInTransaction is implemented in transaction.go.
// Close marks the store as closed, cleans up orphaned git-remote-cache
// garbage, and releases the exclusive flock on the data directory (if the
// store owns it). Subsequent method calls will return errClosed.
// Close decrements the reference count if this store was opened via Open (the
// process-scoped cache). When other references remain, Close is a no-op — the
// store stays alive for the remaining callers. When the last reference calls
// Close (or if the store was created directly via New), the underlying
// resources are released: orphaned git-remote-cache garbage is cleaned up and
// the exclusive flock on the data directory is released (if the store owns it).
//
// It is safe to call multiple times. When the lock was supplied by the caller
// via WithLock, Close does NOT release it — the caller retains ownership.
func (s *EmbeddedDoltStore) Close() error {
if closeCached(s) {
// Other references remain — suppress the real close.
return nil
}
return s.closeUnderlying()
}
// closeUnderlying performs the actual close: marks the store as closed,
// cleans up orphaned garbage, and releases the flock.
func (s *EmbeddedDoltStore) closeUnderlying() error {
// Use CompareAndSwap so we only unlock once even if Close is called
// multiple times (the Lock.Unlock method panics on double-unlock).
if s.closed.CompareAndSwap(false, true) {

View file

@ -14,7 +14,7 @@ type EmbeddedDoltStore struct {
branch string
}
// Option configures optional behavior for New (stub: no-op).
// Option configures optional behavior for Open (stub: no-op).
type Option func(*struct{})
// WithLock is a no-op in non-CGO builds.
@ -22,7 +22,12 @@ func WithLock(_ Unlocker) Option {
return func(*struct{}) {}
}
// New returns an error when CGO is not enabled.
func New(_ context.Context, _, _, _ string, _ ...Option) (*EmbeddedDoltStore, error) {
return nil, errors.New("embeddeddolt: requires CGO (build with CGO_ENABLED=1)")
var errNoCGO = errors.New("embeddeddolt: requires CGO (build with CGO_ENABLED=1)")
// Open returns an error when CGO is not enabled.
func Open(_ context.Context, _, _, _ string, _ ...Option) (*EmbeddedDoltStore, error) {
return nil, errNoCGO
}
// ResetCache is a no-op in non-CGO builds.
func ResetCache() {}