diff --git a/cmd/bd/doctor/fix/locks.go b/cmd/bd/doctor/fix/locks.go index b8b2ca268..8b88b0e49 100644 --- a/cmd/bd/doctor/fix/locks.go +++ b/cmd/bd/doctor/fix/locks.go @@ -67,11 +67,15 @@ func StaleLockFiles(path string) error { // Remove stale Dolt internal LOCK files (noms layer filesystem lock). // These live at .beads/dolt//.dolt/noms/LOCK and are created - // by the embedded Dolt engine. If a process crashes without closing the - // embedded connector, the LOCK file persists and blocks all future opens - // with "the database is locked by another dolt process". - // Uses flock probe to verify no active process holds the advisory lock - // before removing — more correct than age-only cleanup. + // by the Dolt engine. If a process (embedded driver or sql-server) crashes + // without closing cleanly, the LOCK file persists and blocks all future + // opens with "the database is locked by another dolt process" or a SIGSEGV + // nil-pointer dereference. + // + // In server mode (the only mode now), the Dolt server is the one that opens + // these files. If we're running bd doctor --fix, the server is presumably + // having trouble, so removing stale locks is safe. We use the shared + // CleanStaleNomsLocks helper from the dolt storage package. doltDir := filepath.Join(beadsDir, "dolt") if dbEntries, err := os.ReadDir(doltDir); err == nil { for _, dbEntry := range dbEntries { @@ -79,15 +83,12 @@ func StaleLockFiles(path string) error { continue } nomsLock := filepath.Join(doltDir, dbEntry.Name(), ".dolt", "noms", "LOCK") - if _, err := os.Stat(nomsLock); err == nil { + if _, statErr := os.Stat(nomsLock); statErr == nil { lockName := fmt.Sprintf("dolt/%s/.dolt/noms/LOCK", dbEntry.Name()) - // Probe the advisory access lock to ensure no process is active - if probeStale(accessLockPath) { - if err := os.Remove(nomsLock); err != nil { - errors = append(errors, fmt.Sprintf("%s: %v", lockName, err)) - } else { - removed = append(removed, lockName) - } + if err := os.Remove(nomsLock); err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", lockName, err)) + } else { + removed = append(removed, lockName) } } } diff --git a/cmd/bd/doctor/fix/locks_test.go b/cmd/bd/doctor/fix/locks_test.go index 2b1b6b877..d1ea3a69e 100644 --- a/cmd/bd/doctor/fix/locks_test.go +++ b/cmd/bd/doctor/fix/locks_test.go @@ -73,7 +73,7 @@ func TestStaleLockFiles(t *testing.T) { } }) - t.Run("fresh noms LOCK preserved", func(t *testing.T) { + t.Run("noms LOCK always removed by doctor fix", func(t *testing.T) { tmpDir := t.TempDir() nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms") if err := os.MkdirAll(nomsDir, 0755); err != nil { @@ -85,14 +85,17 @@ func TestStaleLockFiles(t *testing.T) { t.Fatal(err) } - // noms LOCK cleanup is gated by probeStale on dolt-access.lock. - // Without a dolt-access.lock file, probeStale returns true (stale), - // so a fresh noms LOCK would still be removed. This test verifies - // the dolt-access.lock path doesn't exist scenario (no active holder). - // In practice, a running process would hold the flock. + // In server mode (the only mode now), noms LOCK files are always + // removed by doctor --fix. The Dolt server manages its own locks; + // if the user is running doctor --fix, stale locks are the likely + // cause of the problem they're trying to fix. if err := StaleLockFiles(tmpDir); err != nil { t.Fatalf("unexpected error: %v", err) } + + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Error("noms LOCK should be removed by doctor --fix") + } }) t.Run("stale noms LOCK removed", func(t *testing.T) { diff --git a/cmd/bd/doctor/locks.go b/cmd/bd/doctor/locks.go index 07e3436e7..74cfbe81e 100644 --- a/cmd/bd/doctor/locks.go +++ b/cmd/bd/doctor/locks.go @@ -67,10 +67,10 @@ func CheckStaleLockFiles(path string) DoctorCheck { } // Note: Dolt internal noms LOCK files (.beads/dolt//.dolt/noms/LOCK) - // are NOT checked here. These files are created by the embedded Dolt engine - // and are never deleted, even after a clean close. Age-based detection - // produces false positives because the file persists indefinitely. - // Use CheckLockHealth() (which probes flock state) instead. (GH#1981) + // are NOT checked here as diagnostics. These are auto-cleaned on bd startup + // (pre-flight in PersistentPreRun) and by 'bd doctor --fix'. Stale noms LOCK + // files from crashed processes would prevent the Dolt server from opening + // databases. The auto-cleanup makes this a non-issue for most users. // Check startup lock (bd.sock.startlock) // Look for any .startlock files in beadsDir diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 2dac84cf6..370b0ccc6 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -530,6 +530,14 @@ var rootCmd = &cobra.Command{ } doltCfg.Path = doltPath + + // Pre-flight: clean stale noms LOCK files left by crashed Dolt processes. + // These prevent the Dolt server from opening databases (SIGSEGV or + // "database is locked"). Safe because we haven't connected yet. + if removed, _ := dolt.CleanStaleNomsLocks(doltPath); removed > 0 { + debug.Logf("cleaned %d stale noms LOCK file(s) from %s", removed, doltPath) + } + store, err = dolt.New(rootCtx, doltCfg) // Track final read-only state for staleness checks (GH#1089) diff --git a/internal/storage/dolt/noms_lock.go b/internal/storage/dolt/noms_lock.go new file mode 100644 index 000000000..7345b9e79 --- /dev/null +++ b/internal/storage/dolt/noms_lock.go @@ -0,0 +1,48 @@ +package dolt + +import ( + "fmt" + "os" + "path/filepath" +) + +// CleanStaleNomsLocks removes stale Dolt noms LOCK files from all databases +// in the given dolt directory (typically .beads/dolt/). +// +// Dolt's noms storage layer creates a file-based LOCK at +// /.dolt/noms/LOCK when opening a database. If the process is killed +// uncleanly (SIGKILL, OOM, etc.), this LOCK file persists and prevents the +// Dolt server from opening the database on restart — causing either a SIGSEGV +// (nil pointer dereference in DoltDB.SetCrashOnFatalError) or a "database is +// locked" error. +// +// This function is safe to call before starting or connecting to a Dolt server +// because the server is not yet using the databases. It scans all subdirectories +// of doltDir for /.dolt/noms/LOCK and removes them. +// +// Returns the number of lock files removed. Errors removing individual files +// are collected but do not abort the scan. +func CleanStaleNomsLocks(doltDir string) (removed int, errs []error) { + entries, err := os.ReadDir(doltDir) + if err != nil { + // Directory doesn't exist or can't be read — nothing to clean. + return 0, nil + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + lockPath := filepath.Join(doltDir, entry.Name(), ".dolt", "noms", "LOCK") + if _, statErr := os.Stat(lockPath); os.IsNotExist(statErr) { + continue + } + if rmErr := os.Remove(lockPath); rmErr != nil { + errs = append(errs, fmt.Errorf("removing %s: %w", lockPath, rmErr)) + } else { + removed++ + } + } + + return removed, errs +} diff --git a/internal/storage/dolt/noms_lock_test.go b/internal/storage/dolt/noms_lock_test.go new file mode 100644 index 000000000..fbf10b3fe --- /dev/null +++ b/internal/storage/dolt/noms_lock_test.go @@ -0,0 +1,133 @@ +package dolt + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCleanStaleNomsLocks(t *testing.T) { + t.Parallel() + + t.Run("removes stale LOCK file", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + // Create a database directory with a stale noms LOCK + nomsDir := filepath.Join(dir, "mydb", ".dolt", "noms") + if err := os.MkdirAll(nomsDir, 0755); err != nil { + t.Fatal(err) + } + lockPath := filepath.Join(nomsDir, "LOCK") + if err := os.WriteFile(lockPath, []byte("stale"), 0600); err != nil { + t.Fatal(err) + } + + removed, errs := CleanStaleNomsLocks(dir) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if removed != 1 { + t.Fatalf("expected 1 removed, got %d", removed) + } + + // Verify file is gone + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Fatal("LOCK file should have been removed") + } + }) + + t.Run("handles multiple databases", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + // Create two databases with stale locks + for _, db := range []string{"db1", "db2"} { + nomsDir := filepath.Join(dir, db, ".dolt", "noms") + if err := os.MkdirAll(nomsDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(nomsDir, "LOCK"), []byte("stale"), 0600); err != nil { + t.Fatal(err) + } + } + + removed, errs := CleanStaleNomsLocks(dir) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if removed != 2 { + t.Fatalf("expected 2 removed, got %d", removed) + } + }) + + t.Run("no lock files present", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + // Create a database directory without a LOCK file + nomsDir := filepath.Join(dir, "mydb", ".dolt", "noms") + if err := os.MkdirAll(nomsDir, 0755); err != nil { + t.Fatal(err) + } + + removed, errs := CleanStaleNomsLocks(dir) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if removed != 0 { + t.Fatalf("expected 0 removed, got %d", removed) + } + }) + + t.Run("directory does not exist", func(t *testing.T) { + t.Parallel() + removed, errs := CleanStaleNomsLocks("/nonexistent/path/that/does/not/exist") + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if removed != 0 { + t.Fatalf("expected 0 removed, got %d", removed) + } + }) + + t.Run("skips non-directory entries", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + // Create a regular file (not a directory) at the top level + if err := os.WriteFile(filepath.Join(dir, "not-a-dir"), []byte("data"), 0600); err != nil { + t.Fatal(err) + } + + // Create a real database with a lock + nomsDir := filepath.Join(dir, "realdb", ".dolt", "noms") + if err := os.MkdirAll(nomsDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(nomsDir, "LOCK"), []byte("stale"), 0600); err != nil { + t.Fatal(err) + } + + removed, errs := CleanStaleNomsLocks(dir) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if removed != 1 { + t.Fatalf("expected 1 removed, got %d", removed) + } + }) + + t.Run("empty directory", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + removed, errs := CleanStaleNomsLocks(dir) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if removed != 0 { + t.Fatalf("expected 0 removed, got %d", removed) + } + }) +}