fix: auto-clean stale Dolt noms LOCK files on bd startup (#2059)

When a Dolt server process is killed uncleanly (SIGKILL, OOM, tmux
kill-session), it leaves behind .dolt/noms/LOCK files that prevent the
server from reopening databases on restart, causing SIGSEGV or
"database is locked" errors.

Add CleanStaleNomsLocks() to internal/storage/dolt/ that scans a dolt
data directory and removes any noms LOCK files. Call it pre-flight in
PersistentPreRun before connecting to the server.

Also update doctor/fix/locks.go to remove the flock probe dependency
for noms LOCK cleanup — the probe was designed for the now-removed
embedded mode and is unreliable in server mode.

Co-authored-by: mayor <mayor@gastown.local>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Danno 2026-02-24 23:09:51 -07:00 • committed by GitHub
commit 76e01b270d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 216 additions and 23 deletions

View file

@ -67,11 +67,15 @@ func StaleLockFiles(path string) error {
// Remove stale Dolt internal LOCK files (noms layer filesystem lock).
// These live at .beads/dolt/<database>/.dolt/noms/LOCK and are created
// by the embedded Dolt engine. If a process crashes without closing the
// embedded connector, the LOCK file persists and blocks all future opens
// with "the database is locked by another dolt process".
// Uses flock probe to verify no active process holds the advisory lock
// before removing — more correct than age-only cleanup.
// by the Dolt engine. If a process (embedded driver or sql-server) crashes
// without closing cleanly, the LOCK file persists and blocks all future
// opens with "the database is locked by another dolt process" or a SIGSEGV
// nil-pointer dereference.
//
// In server mode (the only mode now), the Dolt server is the one that opens
// these files. If we're running bd doctor --fix, the server is presumably
// having trouble, so removing stale locks is safe. We use the shared
// CleanStaleNomsLocks helper from the dolt storage package.
doltDir := filepath.Join(beadsDir, "dolt")
if dbEntries, err := os.ReadDir(doltDir); err == nil {
for _, dbEntry := range dbEntries {
@ -79,15 +83,12 @@ func StaleLockFiles(path string) error {
continue
}
nomsLock := filepath.Join(doltDir, dbEntry.Name(), ".dolt", "noms", "LOCK")
if _, err := os.Stat(nomsLock); err == nil {
if _, statErr := os.Stat(nomsLock); statErr == nil {
lockName := fmt.Sprintf("dolt/%s/.dolt/noms/LOCK", dbEntry.Name())
// Probe the advisory access lock to ensure no process is active
if probeStale(accessLockPath) {
if err := os.Remove(nomsLock); err != nil {
errors = append(errors, fmt.Sprintf("%s: %v", lockName, err))
} else {
removed = append(removed, lockName)
}
if err := os.Remove(nomsLock); err != nil {
errors = append(errors, fmt.Sprintf("%s: %v", lockName, err))
} else {
removed = append(removed, lockName)
}
}
}

View file

@ -73,7 +73,7 @@ func TestStaleLockFiles(t *testing.T) {
}
})
t.Run("fresh noms LOCK preserved", func(t *testing.T) {
t.Run("noms LOCK always removed by doctor fix", func(t *testing.T) {
tmpDir := t.TempDir()
nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms")
if err := os.MkdirAll(nomsDir, 0755); err != nil {
@ -85,14 +85,17 @@ func TestStaleLockFiles(t *testing.T) {
t.Fatal(err)
}
// noms LOCK cleanup is gated by probeStale on dolt-access.lock.
// Without a dolt-access.lock file, probeStale returns true (stale),
// so a fresh noms LOCK would still be removed. This test verifies
// the dolt-access.lock path doesn't exist scenario (no active holder).
// In practice, a running process would hold the flock.
// In server mode (the only mode now), noms LOCK files are always
// removed by doctor --fix. The Dolt server manages its own locks;
// if the user is running doctor --fix, stale locks are the likely
// cause of the problem they're trying to fix.
if err := StaleLockFiles(tmpDir); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
t.Error("noms LOCK should be removed by doctor --fix")
}
})
t.Run("stale noms LOCK removed", func(t *testing.T) {

View file

@ -67,10 +67,10 @@ func CheckStaleLockFiles(path string) DoctorCheck {
}
// Note: Dolt internal noms LOCK files (.beads/dolt/<db>/.dolt/noms/LOCK)
// are NOT checked here. These files are created by the embedded Dolt engine
// and are never deleted, even after a clean close. Age-based detection
// produces false positives because the file persists indefinitely.
// Use CheckLockHealth() (which probes flock state) instead. (GH#1981)
// are NOT checked here as diagnostics. These are auto-cleaned on bd startup
// (pre-flight in PersistentPreRun) and by 'bd doctor --fix'. Stale noms LOCK
// files from crashed processes would prevent the Dolt server from opening
// databases. The auto-cleanup makes this a non-issue for most users.
// Check startup lock (bd.sock.startlock)
// Look for any .startlock files in beadsDir

View file

@ -530,6 +530,14 @@ var rootCmd = &cobra.Command{
}
doltCfg.Path = doltPath
// Pre-flight: clean stale noms LOCK files left by crashed Dolt processes.
// These prevent the Dolt server from opening databases (SIGSEGV or
// "database is locked"). Safe because we haven't connected yet.
if removed, _ := dolt.CleanStaleNomsLocks(doltPath); removed > 0 {
debug.Logf("cleaned %d stale noms LOCK file(s) from %s", removed, doltPath)
}
store, err = dolt.New(rootCtx, doltCfg)
// Track final read-only state for staleness checks (GH#1089)

View file

@ -0,0 +1,48 @@
package dolt
import (
"fmt"
"os"
"path/filepath"
)
// CleanStaleNomsLocks removes stale Dolt noms LOCK files from all databases
// in the given dolt directory (typically .beads/dolt/).
//
// Dolt's noms storage layer creates a file-based LOCK at
// <db>/.dolt/noms/LOCK when opening a database. If the process is killed
// uncleanly (SIGKILL, OOM, etc.), this LOCK file persists and prevents the
// Dolt server from opening the database on restart — causing either a SIGSEGV
// (nil pointer dereference in DoltDB.SetCrashOnFatalError) or a "database is
// locked" error.
//
// This function is safe to call before starting or connecting to a Dolt server
// because the server is not yet using the databases. It scans all subdirectories
// of doltDir for <db>/.dolt/noms/LOCK and removes them.
//
// Returns the number of lock files removed. Errors removing individual files
// are collected but do not abort the scan.
func CleanStaleNomsLocks(doltDir string) (removed int, errs []error) {
entries, err := os.ReadDir(doltDir)
if err != nil {
// Directory doesn't exist or can't be read — nothing to clean.
return 0, nil
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
lockPath := filepath.Join(doltDir, entry.Name(), ".dolt", "noms", "LOCK")
if _, statErr := os.Stat(lockPath); os.IsNotExist(statErr) {
continue
}
if rmErr := os.Remove(lockPath); rmErr != nil {
errs = append(errs, fmt.Errorf("removing %s: %w", lockPath, rmErr))
} else {
removed++
}
}
return removed, errs
}

View file

@ -0,0 +1,133 @@
package dolt
import (
"os"
"path/filepath"
"testing"
)
func TestCleanStaleNomsLocks(t *testing.T) {
t.Parallel()
t.Run("removes stale LOCK file", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
// Create a database directory with a stale noms LOCK
nomsDir := filepath.Join(dir, "mydb", ".dolt", "noms")
if err := os.MkdirAll(nomsDir, 0755); err != nil {
t.Fatal(err)
}
lockPath := filepath.Join(nomsDir, "LOCK")
if err := os.WriteFile(lockPath, []byte("stale"), 0600); err != nil {
t.Fatal(err)
}
removed, errs := CleanStaleNomsLocks(dir)
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if removed != 1 {
t.Fatalf("expected 1 removed, got %d", removed)
}
// Verify file is gone
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
t.Fatal("LOCK file should have been removed")
}
})
t.Run("handles multiple databases", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
// Create two databases with stale locks
for _, db := range []string{"db1", "db2"} {
nomsDir := filepath.Join(dir, db, ".dolt", "noms")
if err := os.MkdirAll(nomsDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(nomsDir, "LOCK"), []byte("stale"), 0600); err != nil {
t.Fatal(err)
}
}
removed, errs := CleanStaleNomsLocks(dir)
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if removed != 2 {
t.Fatalf("expected 2 removed, got %d", removed)
}
})
t.Run("no lock files present", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
// Create a database directory without a LOCK file
nomsDir := filepath.Join(dir, "mydb", ".dolt", "noms")
if err := os.MkdirAll(nomsDir, 0755); err != nil {
t.Fatal(err)
}
removed, errs := CleanStaleNomsLocks(dir)
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if removed != 0 {
t.Fatalf("expected 0 removed, got %d", removed)
}
})
t.Run("directory does not exist", func(t *testing.T) {
t.Parallel()
removed, errs := CleanStaleNomsLocks("/nonexistent/path/that/does/not/exist")
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if removed != 0 {
t.Fatalf("expected 0 removed, got %d", removed)
}
})
t.Run("skips non-directory entries", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
// Create a regular file (not a directory) at the top level
if err := os.WriteFile(filepath.Join(dir, "not-a-dir"), []byte("data"), 0600); err != nil {
t.Fatal(err)
}
// Create a real database with a lock
nomsDir := filepath.Join(dir, "realdb", ".dolt", "noms")
if err := os.MkdirAll(nomsDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(nomsDir, "LOCK"), []byte("stale"), 0600); err != nil {
t.Fatal(err)
}
removed, errs := CleanStaleNomsLocks(dir)
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if removed != 1 {
t.Fatalf("expected 1 removed, got %d", removed)
}
})
t.Run("empty directory", func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
removed, errs := CleanStaleNomsLocks(dir)
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if removed != 0 {
t.Fatalf("expected 0 removed, got %d", removed)
}
})
}