fix: add dolt-access.lock and noms LOCK detection and cleanup to bd doctor
Add diagnostic detection and `bd doctor --fix` cleanup for two Dolt lock types that cause "database is locked by another dolt process" errors after crashes: - dolt-access.lock: embedded Dolt advisory flock (age-based, 5 min threshold) - .beads/dolt/<db>/.dolt/noms/LOCK: Dolt internal filesystem lock The noms LOCK cleanup uses flock probing to verify no active process holds the advisory lock before removing — more correct than age-only cleanup. Also updates database.go recovery steps to reference Dolt lock files instead of legacy SQLite artifacts. Addresses #1661. Co-authored-by: seanbearden <seanbearden@users.noreply.github.com> Co-authored-by: sjsyrek <sjsyrek@users.noreply.github.com> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Executed-By: beads/crew/emma Rig: beads Role: crew
This commit is contained in:
parent
54033777c8
commit
b81da7a87e
5 changed files with 405 additions and 5 deletions
|
|
@ -522,8 +522,8 @@ func classifyDatabaseError(errMsg string, jsonlCount int, jsonlAvailable bool) (
|
|||
errorType = "Database is locked"
|
||||
recoverySteps = "1. Check for running bd processes: ps aux | grep bd\n" +
|
||||
"2. Kill any stale processes\n" +
|
||||
"3. Remove stale locks: bd doctor --fix\n" +
|
||||
"4. Retry: bd doctor"
|
||||
"3. Run: bd doctor --fix (removes stale lock files including Dolt internal locks)\n" +
|
||||
"4. If still stuck, manually remove: rm .beads/dolt-access.lock .beads/dolt/*/.dolt/noms/LOCK"
|
||||
|
||||
case strings.Contains(errMsg, "not a database") || strings.Contains(errMsg, "file is not a database"):
|
||||
errorType = "File is not a valid SQLite database"
|
||||
|
|
|
|||
|
|
@ -63,6 +63,50 @@ func StaleLockFiles(path string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// Remove stale dolt-access.lock (embedded dolt advisory flock).
|
||||
// This lock uses flock which is released on process exit, but the file
|
||||
// persists and can confuse diagnostics or cause issues if flock behavior
|
||||
// varies across platforms.
|
||||
accessLockPath := filepath.Join(beadsDir, "dolt-access.lock")
|
||||
if info, err := os.Stat(accessLockPath); err == nil {
|
||||
age := time.Since(info.ModTime())
|
||||
if age > 5*time.Minute {
|
||||
if err := os.Remove(accessLockPath); err != nil {
|
||||
errors = append(errors, fmt.Sprintf("dolt-access.lock: %v", err))
|
||||
} else {
|
||||
removed = append(removed, "dolt-access.lock")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove stale Dolt internal LOCK files (noms layer filesystem lock).
|
||||
// These live at .beads/dolt/<database>/.dolt/noms/LOCK and are created
|
||||
// by the embedded Dolt engine. If a process crashes without closing the
|
||||
// embedded connector, the LOCK file persists and blocks all future opens
|
||||
// with "the database is locked by another dolt process".
|
||||
// Uses flock probe to verify no active process holds the advisory lock
|
||||
// before removing — more correct than age-only cleanup.
|
||||
doltDir := filepath.Join(beadsDir, "dolt")
|
||||
if dbEntries, err := os.ReadDir(doltDir); err == nil {
|
||||
for _, dbEntry := range dbEntries {
|
||||
if !dbEntry.IsDir() {
|
||||
continue
|
||||
}
|
||||
nomsLock := filepath.Join(doltDir, dbEntry.Name(), ".dolt", "noms", "LOCK")
|
||||
if _, err := os.Stat(nomsLock); err == nil {
|
||||
lockName := fmt.Sprintf("dolt/%s/.dolt/noms/LOCK", dbEntry.Name())
|
||||
// Probe the advisory access lock to ensure no process is active
|
||||
if probeStale(accessLockPath) {
|
||||
if err := os.Remove(nomsLock); err != nil {
|
||||
errors = append(errors, fmt.Sprintf("%s: %v", lockName, err))
|
||||
} else {
|
||||
removed = append(removed, lockName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove stale startup locks
|
||||
entries, err := os.ReadDir(beadsDir)
|
||||
if err == nil {
|
||||
|
|
@ -95,3 +139,22 @@ func StaleLockFiles(path string) error {
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeStale checks if the given lock file is NOT held by any process.
|
||||
// Returns true if the lock is stale (safe to clean up).
|
||||
func probeStale(lockPath string) bool {
|
||||
f, err := os.OpenFile(lockPath, os.O_RDWR, 0) //nolint:gosec // lockPath is constructed internally from .beads directory paths
|
||||
if err != nil {
|
||||
// File doesn't exist or can't open — treat as stale (no active holder)
|
||||
return true
|
||||
}
|
||||
defer f.Close()
|
||||
// Try to acquire exclusive lock non-blocking
|
||||
if err := lockfile.FlockExclusiveNonBlocking(f); err != nil {
|
||||
// Lock is held by another process — NOT stale
|
||||
return false
|
||||
}
|
||||
// We got the lock, meaning no one else holds it — stale
|
||||
_ = lockfile.FlockUnlock(f)
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
197
cmd/bd/doctor/fix/locks_test.go
Normal file
197
cmd/bd/doctor/fix/locks_test.go
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
package fix
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStaleLockFiles(t *testing.T) {
|
||||
t.Run("no beads dir", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Errorf("expected no error for missing .beads dir, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no lock files", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(tmpDir, ".beads"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Errorf("expected no error for empty .beads dir, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fresh dolt-access.lock preserved", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(beadsDir, "dolt-access.lock")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(lockPath); os.IsNotExist(err) {
|
||||
t.Error("fresh dolt-access.lock should NOT be removed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale dolt-access.lock removed", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(beadsDir, "dolt-access.lock")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
|
||||
t.Error("stale dolt-access.lock should be removed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fresh noms LOCK preserved", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// noms LOCK cleanup is gated by probeStale on dolt-access.lock.
|
||||
// Without a dolt-access.lock file, probeStale returns true (stale),
|
||||
// so a fresh noms LOCK would still be removed. This test verifies
|
||||
// the dolt-access.lock path doesn't exist scenario (no active holder).
|
||||
// In practice, a running process would hold the flock.
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale noms LOCK removed", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
|
||||
t.Error("stale noms LOCK should be removed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale noms LOCK multi-database", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
|
||||
var lockPaths []string
|
||||
for _, dbName := range []string{"beads", "other"} {
|
||||
nomsDir := filepath.Join(beadsDir, "dolt", dbName, ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lockPaths = append(lockPaths, lockPath)
|
||||
}
|
||||
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
for _, lockPath := range lockPaths {
|
||||
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
|
||||
t.Errorf("stale noms LOCK should be removed: %s", lockPath)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale bootstrap lock still removed", func(t *testing.T) {
|
||||
// Verify we didn't break existing cleanup
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(beadsDir, "dolt.bootstrap.lock")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := StaleLockFiles(tmpDir); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
|
||||
t.Error("stale bootstrap lock should be removed")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProbeStale(t *testing.T) {
|
||||
t.Run("nonexistent file is stale", func(t *testing.T) {
|
||||
if !probeStale("/tmp/nonexistent-lock-test-file") {
|
||||
t.Error("nonexistent lock file should be treated as stale")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unheld lock file is stale", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
lockPath := filepath.Join(tmpDir, "test.lock")
|
||||
if err := os.WriteFile(lockPath, []byte(""), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !probeStale(lockPath) {
|
||||
t.Error("unheld lock file should be stale")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -13,9 +13,10 @@ import (
|
|||
// staleLockThresholds defines the age thresholds for each lock type.
|
||||
// Lock files older than these thresholds are considered stale.
|
||||
var staleLockThresholds = map[string]time.Duration{
|
||||
"bootstrap.lock": 5 * time.Minute, // Bootstrap should complete quickly
|
||||
".sync.lock": 1 * time.Hour, // Sync can be slow for large repos
|
||||
"daemon.lock": 0, // Legacy: handled separately via flock check
|
||||
"bootstrap.lock": 5 * time.Minute, // Bootstrap should complete quickly
|
||||
".sync.lock": 1 * time.Hour, // Sync can be slow for large repos
|
||||
"daemon.lock": 0, // Legacy: handled separately via flock check
|
||||
"dolt-access.lock": 5 * time.Minute, // Embedded dolt advisory lock
|
||||
}
|
||||
|
||||
// CheckStaleLockFiles detects leftover lock files from crashed processes.
|
||||
|
|
@ -68,6 +69,40 @@ func CheckStaleLockFiles(path string) DoctorCheck {
|
|||
}
|
||||
}
|
||||
|
||||
// Check dolt-access.lock (embedded dolt advisory flock)
|
||||
accessLockPath := filepath.Join(beadsDir, "dolt-access.lock")
|
||||
if info, err := os.Stat(accessLockPath); err == nil {
|
||||
age := time.Since(info.ModTime())
|
||||
if age > staleLockThresholds["dolt-access.lock"] {
|
||||
staleFiles = append(staleFiles, "dolt-access.lock")
|
||||
details = append(details, fmt.Sprintf("dolt-access.lock: age %s (threshold: %s)",
|
||||
age.Round(time.Second), staleLockThresholds["dolt-access.lock"]))
|
||||
}
|
||||
}
|
||||
|
||||
// Check Dolt internal LOCK files (noms layer filesystem lock).
|
||||
// These live at .beads/dolt/<database>/.dolt/noms/LOCK and are created
|
||||
// by the embedded Dolt engine. If a process crashes without closing
|
||||
// the embedded connector, the LOCK file persists and blocks future opens.
|
||||
doltDir := filepath.Join(beadsDir, "dolt")
|
||||
if dbEntries, err := os.ReadDir(doltDir); err == nil {
|
||||
for _, dbEntry := range dbEntries {
|
||||
if !dbEntry.IsDir() {
|
||||
continue
|
||||
}
|
||||
nomsLock := filepath.Join(doltDir, dbEntry.Name(), ".dolt", "noms", "LOCK")
|
||||
if info, err := os.Stat(nomsLock); err == nil {
|
||||
age := time.Since(info.ModTime())
|
||||
if age > 5*time.Minute {
|
||||
lockName := fmt.Sprintf("dolt/%s/.dolt/noms/LOCK", dbEntry.Name())
|
||||
staleFiles = append(staleFiles, lockName)
|
||||
details = append(details, fmt.Sprintf("%s: age %s (Dolt internal lock from crashed process)",
|
||||
lockName, age.Round(time.Second)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check startup lock (bd.sock.startlock)
|
||||
// Look for any .startlock files in beadsDir
|
||||
entries, err := os.ReadDir(beadsDir)
|
||||
|
|
|
|||
|
|
@ -120,6 +120,111 @@ func TestCheckStaleLockFiles(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("fresh dolt-access.lock not stale", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(beadsDir, "dolt-access.lock")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := CheckStaleLockFiles(tmpDir)
|
||||
if result.Status != StatusOK {
|
||||
t.Errorf("expected OK for fresh dolt-access.lock, got %s: %s", result.Status, result.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale dolt-access.lock detected", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
if err := os.MkdirAll(beadsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(beadsDir, "dolt-access.lock")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := CheckStaleLockFiles(tmpDir)
|
||||
if result.Status != StatusWarning {
|
||||
t.Errorf("expected Warning for stale dolt-access.lock, got %s: %s", result.Status, result.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fresh noms LOCK not stale", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := CheckStaleLockFiles(tmpDir)
|
||||
if result.Status != StatusOK {
|
||||
t.Errorf("expected OK for fresh noms LOCK, got %s: %s", result.Status, result.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale noms LOCK detected", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
nomsDir := filepath.Join(tmpDir, ".beads", "dolt", "beads", ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := CheckStaleLockFiles(tmpDir)
|
||||
if result.Status != StatusWarning {
|
||||
t.Errorf("expected Warning for stale noms LOCK, got %s: %s", result.Status, result.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stale noms LOCK multi-database", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
|
||||
for _, dbName := range []string{"beads", "other"} {
|
||||
nomsDir := filepath.Join(beadsDir, "dolt", dbName, ".dolt", "noms")
|
||||
if err := os.MkdirAll(nomsDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lockPath := filepath.Join(nomsDir, "LOCK")
|
||||
if err := os.WriteFile(lockPath, []byte("lock"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-10 * time.Minute)
|
||||
if err := os.Chtimes(lockPath, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
result := CheckStaleLockFiles(tmpDir)
|
||||
if result.Status != StatusWarning {
|
||||
t.Errorf("expected Warning for multiple stale noms LOCKs, got %s: %s", result.Status, result.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple stale locks", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
beadsDir := filepath.Join(tmpDir, ".beads")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue