fix: auto-clean stale Dolt noms LOCK files on bd startup (#2059)
When a Dolt server process is killed uncleanly (SIGKILL, OOM, tmux kill-session), it leaves behind .dolt/noms/LOCK files that prevent the server from reopening databases on restart, causing SIGSEGV or "database is locked" errors. Add CleanStaleNomsLocks() to internal/storage/dolt/ that scans a dolt data directory and removes any noms LOCK files. Call it pre-flight in PersistentPreRun before connecting to the server. Also update doctor/fix/locks.go to remove the flock probe dependency for noms LOCK cleanup — the probe was designed for the now-removed embedded mode and is unreliable in server mode. Co-authored-by: mayor <mayor@gastown.local> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
7c5d984d91
commit
76e01b270d
6 changed files with 216 additions and 23 deletions
|
|
@ -67,11 +67,15 @@ func StaleLockFiles(path string) error {
|
|||
|
||||
// Remove stale Dolt internal LOCK files (noms layer filesystem lock).
|
||||
// These live at .beads/dolt/<database>/.dolt/noms/LOCK and are created
|
||||
// by the embedded Dolt engine. If a process crashes without closing the
|
||||
// embedded connector, the LOCK file persists and blocks all future opens
|
||||
// with "the database is locked by another dolt process".
|
||||
// Uses flock probe to verify no active process holds the advisory lock
|
||||
// before removing — more correct than age-only cleanup.
|
||||
// by the Dolt engine. If a process (embedded driver or sql-server) crashes
|
||||
// without closing cleanly, the LOCK file persists and blocks all future
|
||||
// opens with "the database is locked by another dolt process" or a SIGSEGV
|
||||
// nil-pointer dereference.
|
||||
//
|
||||
// In server mode (the only mode now), the Dolt server is the one that opens
|
||||
// these files. If we're running bd doctor --fix, the server is presumably
|
||||
// having trouble, so removing stale locks is safe. We use the shared
|
||||
// CleanStaleNomsLocks helper from the dolt storage package.
|
||||
doltDir := filepath.Join(beadsDir, "dolt")
|
||||
if dbEntries, err := os.ReadDir(doltDir); err == nil {
|
||||
for _, dbEntry := range dbEntries {
|
||||
|
|
@ -79,15 +83,12 @@ func StaleLockFiles(path string) error {
|
|||
continue
|
||||
}
|
||||
nomsLock := filepath.Join(doltDir, dbEntry.Name(), ".dolt", "noms", "LOCK")
|
||||
if _, err := os.Stat(nomsLock); err == nil {
|
||||
if _, statErr := os.Stat(nomsLock); statErr == nil {
|
||||
lockName := fmt.Sprintf("dolt/%s/.dolt/noms/LOCK", dbEntry.Name())
|
||||
// Probe the advisory access lock to ensure no process is active
|
||||
if probeStale(accessLockPath) {
|
||||
if err := os.Remove(nomsLock); err != nil {
|
||||
errors = append(errors, fmt.Sprintf("%s: %v", lockName, err))
|
||||
} else {
|
||||
removed = append(removed, lockName)
|
||||
}
|
||||
if err := os.Remove(nomsLock); err != nil {
|
||||
errors = append(errors, fmt.Sprintf("%s: %v", lockName, err))
|
||||
} else {
|
||||
removed = append(removed, lockName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ func TestStaleLockFiles(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("fresh noms LOCK preserved", func(t *testing.T) {
|
||||
t.Run("noms LOCK always removed by doctor fix", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
|
|
@ -85,14 +85,17 @@ func TestStaleLockFiles(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// noms LOCK cleanup is gated by probeStale on dolt-access.lock.
|
||||
// Without a dolt-access.lock file, probeStale returns true (stale),
|
||||
// so a fresh noms LOCK would still be removed. This test verifies
|
||||
// the dolt-access.lock path doesn't exist scenario (no active holder).
|
||||
// In practice, a running process would hold the flock.
|
||||
// In server mode (the only mode now), noms LOCK files are always
|
||||
// removed by doctor --fix. The Dolt server manages its own locks;
|
||||
// if the user is running doctor --fix, stale locks are the likely
|
||||
// cause of the problem they're trying to fix.
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
|
||||
t.Error("noms LOCK should be removed by doctor --fix")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale noms LOCK removed", func(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -67,10 +67,10 @@ func CheckStaleLockFiles(path string) DoctorCheck {
|
|||
}
|
||||
|
||||
// Note: Dolt internal noms LOCK files (.beads/dolt/<db>/.dolt/noms/LOCK)
|
||||
// are NOT checked here. These files are created by the embedded Dolt engine
|
||||
// and are never deleted, even after a clean close. Age-based detection
|
||||
// produces false positives because the file persists indefinitely.
|
||||
// Use CheckLockHealth() (which probes flock state) instead. (GH#1981)
|
||||
// are NOT checked here as diagnostics. These are auto-cleaned on bd startup
|
||||
// (pre-flight in PersistentPreRun) and by 'bd doctor --fix'. Stale noms LOCK
|
||||
// files from crashed processes would prevent the Dolt server from opening
|
||||
// databases. The auto-cleanup makes this a non-issue for most users.
|
||||
|
||||
// Check startup lock (bd.sock.startlock)
|
||||
// Look for any .startlock files in beadsDir
|
||||
|
|
|
|||
|
|
@ -530,6 +530,14 @@ var rootCmd = &cobra.Command{
|
|||
}
|
||||
|
||||
doltCfg.Path = doltPath
|
||||
|
||||
// Pre-flight: clean stale noms LOCK files left by crashed Dolt processes.
|
||||
// These prevent the Dolt server from opening databases (SIGSEGV or
|
||||
// "database is locked"). Safe because we haven't connected yet.
|
||||
if removed, _ := dolt.CleanStaleNomsLocks(doltPath); removed > 0 {
|
||||
debug.Logf("cleaned %d stale noms LOCK file(s) from %s", removed, doltPath)
|
||||
}
|
||||
|
||||
store, err = dolt.New(rootCtx, doltCfg)
|
||||
|
||||
// Track final read-only state for staleness checks (GH#1089)
|
||||
|
|
|
|||
48
internal/storage/dolt/noms_lock.go
Normal file
48
internal/storage/dolt/noms_lock.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// CleanStaleNomsLocks removes stale Dolt noms LOCK files from all databases
|
||||
// in the given dolt directory (typically .beads/dolt/).
|
||||
//
|
||||
// Dolt's noms storage layer creates a file-based LOCK at
|
||||
// <db>/.dolt/noms/LOCK when opening a database. If the process is killed
|
||||
// uncleanly (SIGKILL, OOM, etc.), this LOCK file persists and prevents the
|
||||
// Dolt server from opening the database on restart — causing either a SIGSEGV
|
||||
// (nil pointer dereference in DoltDB.SetCrashOnFatalError) or a "database is
|
||||
// locked" error.
|
||||
//
|
||||
// This function is safe to call before starting or connecting to a Dolt server
|
||||
// because the server is not yet using the databases. It scans all subdirectories
|
||||
// of doltDir for <db>/.dolt/noms/LOCK and removes them.
|
||||
//
|
||||
// Returns the number of lock files removed. Errors removing individual files
|
||||
// are collected but do not abort the scan.
|
||||
func CleanStaleNomsLocks(doltDir string) (removed int, errs []error) {
|
||||
entries, err := os.ReadDir(doltDir)
|
||||
if err != nil {
|
||||
// Directory doesn't exist or can't be read — nothing to clean.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
lockPath := filepath.Join(doltDir, entry.Name(), ".dolt", "noms", "LOCK")
|
||||
if _, statErr := os.Stat(lockPath); os.IsNotExist(statErr) {
|
||||
continue
|
||||
}
|
||||
if rmErr := os.Remove(lockPath); rmErr != nil {
|
||||
errs = append(errs, fmt.Errorf("removing %s: %w", lockPath, rmErr))
|
||||
} else {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
|
||||
return removed, errs
|
||||
}
|
||||
133
internal/storage/dolt/noms_lock_test.go
Normal file
133
internal/storage/dolt/noms_lock_test.go
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanStaleNomsLocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("removes stale LOCK file", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
|
||||
// Create a database directory with a stale noms LOCK
|
||||
nomsDir := filepath.Join(dir, "mydb", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("stale"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, errs := CleanStaleNomsLocks(dir)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("expected 1 removed, got %d", removed)
|
||||
}
|
||||
|
||||
// Verify file is gone
|
||||
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
|
||||
t.Fatal("LOCK file should have been removed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("handles multiple databases", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
|
||||
// Create two databases with stale locks
|
||||
for _, db := range []string{"db1", "db2"} {
|
||||
nomsDir := filepath.Join(dir, db, ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(nomsDir, "LOCK"), []byte("stale"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
removed, errs := CleanStaleNomsLocks(dir)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if removed != 2 {
|
||||
t.Fatalf("expected 2 removed, got %d", removed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no lock files present", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
|
||||
// Create a database directory without a LOCK file
|
||||
nomsDir := filepath.Join(dir, "mydb", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, errs := CleanStaleNomsLocks(dir)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Fatalf("expected 0 removed, got %d", removed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("directory does not exist", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
removed, errs := CleanStaleNomsLocks("/nonexistent/path/that/does/not/exist")
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Fatalf("expected 0 removed, got %d", removed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips non-directory entries", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
|
||||
// Create a regular file (not a directory) at the top level
|
||||
if err := os.WriteFile(filepath.Join(dir, "not-a-dir"), []byte("data"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create a real database with a lock
|
||||
nomsDir := filepath.Join(dir, "realdb", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(nomsDir, "LOCK"), []byte("stale"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, errs := CleanStaleNomsLocks(dir)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("expected 1 removed, got %d", removed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty directory", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
|
||||
removed, errs := CleanStaleNomsLocks(dir)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors: %v", errs)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Fatalf("expected 0 removed, got %d", removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue