beads/cmd/bd/daemon_sync.go
Steve Yegge a769e70ccc
Merge pull request #1487 from Robin831/fix/daemon-sync-improvements
fix(daemon): move hasJSONLChanged check after git pull and add auto-import before mutation exports
2026-02-05 09:19:45 -08:00

1124 lines
39 KiB
Go

package main
import (
"bufio"
"cmp"
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/storage/sqlite"
"github.com/steveyegge/beads/internal/syncbranch"
"github.com/steveyegge/beads/internal/types"
)
// warnIfSyncBranchMisconfigured logs a warning at daemon startup if sync-branch
// equals the current branch. This is a one-time warning to alert users about
// the misconfiguration. The daemon continues to start (warn only, don't block).
// Returns true if misconfigured (warning was logged), false otherwise.
// GH#1258: Prevents silent failure when sync-branch == current-branch.
func warnIfSyncBranchMisconfigured(ctx context.Context, store storage.Storage, log *slog.Logger) bool {
syncBranch, err := syncbranch.Get(ctx, store)
if err != nil || syncBranch == "" {
return false // No sync branch configured, not misconfigured
}
if syncbranch.IsSyncBranchSameAsCurrent(ctx, syncBranch) {
log.Warn("sync-branch misconfiguration detected",
"sync_branch", syncBranch,
"message", "sync-branch is your current branch; daemon sync operations will be skipped; configure a dedicated sync branch (e.g., 'beads-sync') to enable sync")
return true
}
return false
}
// shouldSkipDueToSameBranch checks if operation should be skipped because
// sync-branch == current-branch. Returns true if should skip, logs reason.
// Uses fail-open pattern: if branch detection fails, allows operation to proceed.
func shouldSkipDueToSameBranch(ctx context.Context, store storage.Storage, operation string, log *slog.Logger) bool {
syncBranch, err := syncbranch.Get(ctx, store)
if err != nil || syncBranch == "" {
return false // No sync branch configured, allow
}
if syncbranch.IsSyncBranchSameAsCurrent(ctx, syncBranch) {
log.Info("Skipping operation: sync-branch is current branch", "operation", operation, "sync_branch", syncBranch)
return true
}
return false
}
// exportToJSONLWithStore exports issues to JSONL using the provided store.
// If multi-repo mode is configured, routes issues to their respective JSONL files.
// Otherwise, exports to a single JSONL file.
// Always writes local JSONL as a safety net (even in dolt-native mode).
func exportToJSONLWithStore(ctx context.Context, store storage.Storage, jsonlPath string) error {
// Try multi-repo export first
sqliteStore, ok := store.(*sqlite.SQLiteStorage)
if ok {
results, err := sqliteStore.ExportToMultiRepo(ctx)
if err != nil {
return fmt.Errorf("multi-repo export failed: %w", err)
}
if results != nil {
// Multi-repo mode active - export succeeded
return nil
}
}
// Single-repo mode - use existing logic
// Get all issues including tombstones for sync propagation
// Tombstones must be exported so they propagate to other clones and prevent resurrection
issues, err := store.SearchIssues(ctx, "", types.IssueFilter{IncludeTombstones: true})
if err != nil {
return fmt.Errorf("failed to get issues: %w", err)
}
// Safety check: prevent exporting empty database over non-empty JSONL
// Note: The main protection is in sync.go's reverse ZFC check which runs BEFORE export.
// Here we only block the most catastrophic case (empty DB) to allow legitimate deletions.
if len(issues) == 0 {
existingCount, err := countIssuesInJSONL(jsonlPath)
if err != nil {
// If we can't read the file, it might not exist yet, which is fine
if !os.IsNotExist(err) {
return fmt.Errorf("warning: failed to read existing JSONL: %w", err)
}
} else if existingCount > 0 {
return fmt.Errorf("refusing to export empty database over non-empty JSONL file (database: 0 issues, JSONL: %d issues). This would result in data loss", existingCount)
}
}
// Sort by ID for consistent output
slices.SortFunc(issues, func(a, b *types.Issue) int {
return cmp.Compare(a.ID, b.ID)
})
// Populate dependencies for all issues
allDeps, err := store.GetAllDependencyRecords(ctx)
if err != nil {
return fmt.Errorf("failed to get dependencies: %w", err)
}
for _, issue := range issues {
issue.Dependencies = allDeps[issue.ID]
}
// Populate labels for all issues
for _, issue := range issues {
labels, err := store.GetLabels(ctx, issue.ID)
if err != nil {
return fmt.Errorf("failed to get labels for %s: %w", issue.ID, err)
}
issue.Labels = labels
}
// Populate comments for all issues
for _, issue := range issues {
comments, err := store.GetIssueComments(ctx, issue.ID)
if err != nil {
return fmt.Errorf("failed to get comments for %s: %w", issue.ID, err)
}
issue.Comments = comments
}
// Create temp file for atomic write
dir := filepath.Dir(jsonlPath)
base := filepath.Base(jsonlPath)
tempFile, err := os.CreateTemp(dir, base+".tmp.*")
if err != nil {
return fmt.Errorf("failed to create temp file: %w", err)
}
tempPath := tempFile.Name()
// Use defer pattern for proper cleanup
var writeErr error
defer func() {
_ = tempFile.Close()
if writeErr != nil {
_ = os.Remove(tempPath) // Remove temp file on error
}
}()
// Write JSONL
for _, issue := range issues {
data, marshalErr := json.Marshal(issue)
if marshalErr != nil {
writeErr = fmt.Errorf("failed to marshal issue %s: %w", issue.ID, marshalErr)
return writeErr
}
if _, writeErr = tempFile.Write(data); writeErr != nil {
writeErr = fmt.Errorf("failed to write issue %s: %w", issue.ID, writeErr)
return writeErr
}
if _, writeErr = tempFile.WriteString("\n"); writeErr != nil {
writeErr = fmt.Errorf("failed to write newline: %w", writeErr)
return writeErr
}
}
// Close before rename
if writeErr = tempFile.Close(); writeErr != nil {
writeErr = fmt.Errorf("failed to close temp file: %w", writeErr)
return writeErr
}
// Atomic rename
if writeErr = os.Rename(tempPath, jsonlPath); writeErr != nil {
writeErr = fmt.Errorf("failed to rename temp file: %w", writeErr)
return writeErr
}
// Update export_hashes for all exported issues (GH#1278)
// This ensures child issues created with --parent are properly registered
for _, issue := range issues {
if issue.ContentHash != "" {
if err := store.SetExportHash(ctx, issue.ID, issue.ContentHash); err != nil {
// Non-fatal warning - continue with other issues
fmt.Fprintf(os.Stderr, "Warning: failed to set export hash for %s: %v\n", issue.ID, err)
}
}
}
return nil
}
// importToJSONLWithStore imports issues from JSONL using the provided store
func importToJSONLWithStore(ctx context.Context, store storage.Storage, jsonlPath string) error {
// Try multi-repo import first
sqliteStore, ok := store.(*sqlite.SQLiteStorage)
if ok {
results, err := sqliteStore.HydrateFromMultiRepo(ctx)
if err != nil {
return fmt.Errorf("multi-repo import failed: %w", err)
}
if results != nil {
// Multi-repo mode active - import succeeded
return nil
}
}
// Single-repo mode - use existing logic
// Read JSONL file
file, err := os.Open(jsonlPath) // #nosec G304 - controlled path from config
if err != nil {
return fmt.Errorf("failed to open JSONL: %w", err)
}
defer file.Close()
// Parse all issues
var issues []*types.Issue
scanner := bufio.NewScanner(file)
lineNum := 0
for scanner.Scan() {
lineNum++
line := scanner.Text()
// Skip empty lines
if line == "" {
continue
}
// Parse JSON
var issue types.Issue
if err := json.Unmarshal([]byte(line), &issue); err != nil {
// Log error but continue - don't fail entire import
fmt.Fprintf(os.Stderr, "Warning: failed to parse JSONL line %d: %v\n", lineNum, err)
continue
}
issue.SetDefaults() // Apply defaults for omitted fields
// Migrate old JSONL format: auto-correct deleted status to tombstone
// This handles JSONL files from versions that used "deleted" instead of "tombstone"
// (GH#1223: Stuck in sync diversion loop)
if issue.Status == types.Status("deleted") && issue.DeletedAt != nil {
issue.Status = types.StatusTombstone
}
// Fix: Any non-tombstone issue with deleted_at set is malformed and should be tombstone
// This catches issues that may have been corrupted or migrated incorrectly
if issue.Status != types.StatusTombstone && issue.DeletedAt != nil {
issue.Status = types.StatusTombstone
}
if issue.Status == types.StatusClosed && issue.ClosedAt == nil {
now := time.Now()
issue.ClosedAt = &now
}
// Ensure tombstones have deleted_at set (fix for malformed data)
if issue.Status == types.StatusTombstone && issue.DeletedAt == nil {
now := time.Now()
issue.DeletedAt = &now
}
issues = append(issues, &issue)
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("failed to read JSONL: %w", err)
}
// Use existing import logic with auto-conflict resolution
opts := ImportOptions{
DryRun: false,
SkipUpdate: false,
Strict: false,
SkipPrefixValidation: true, // Skip prefix validation for auto-import
}
_, err = importIssuesCore(ctx, "", store, issues, opts)
return err
}
// getRepoKeyForPath extracts the stable repo identifier from a JSONL path.
// For single-repo mode, returns empty string (no suffix needed).
// For multi-repo mode, extracts the repo path (e.g., ".", "../frontend").
// This creates portable metadata keys that work across different machine paths.
func getRepoKeyForPath(jsonlPath string) string {
multiRepo := config.GetMultiRepoConfig()
if multiRepo == nil {
return "" // Single-repo mode
}
// Normalize the jsonlPath for comparison
// Remove trailing "/.beads/issues.jsonl" to get repo path
const suffix = "/.beads/issues.jsonl"
if strings.HasSuffix(jsonlPath, suffix) {
repoPath := strings.TrimSuffix(jsonlPath, suffix)
// Try to match against primary repo
primaryPath := multiRepo.Primary
if primaryPath == "" {
primaryPath = "."
}
if repoPath == primaryPath {
return primaryPath
}
// Try to match against additional repos
for _, additional := range multiRepo.Additional {
if repoPath == additional {
return additional
}
}
}
// Fallback: return empty string for single-repo mode behavior
return ""
}
// sanitizeMetadataKey removes or replaces characters that conflict with metadata key format.
// On Windows, absolute paths contain colons (e.g., C:\...) which conflict with the ':' separator
// used in multi-repo metadata keys. This function replaces colons with underscores to make
// paths safe for use as metadata key suffixes.
func sanitizeMetadataKey(key string) string {
return strings.ReplaceAll(key, ":", "_")
}
// updateExportMetadata updates jsonl_content_hash and related metadata after a successful export.
// This prevents "JSONL content has changed since last import" errors on subsequent exports.
// In multi-repo mode, keySuffix should be the stable repo identifier (e.g., ".", "../frontend").
//
// Metadata key format:
// - Single-repo mode: "jsonl_content_hash", "last_import_time"
// - Multi-repo mode: "jsonl_content_hash:<repo_key>", "last_import_time:<repo_key>", etc.
// where <repo_key> is a stable repo identifier like "." or "../frontend"
// - Windows paths: Colons in absolute paths (e.g., C:\...) are replaced with underscores
// - Note: "last_import_mtime" was removed (git doesn't preserve mtime)
// - Note: "last_import_hash" renamed to "jsonl_content_hash" - more accurate name
//
// Transaction boundaries:
// This function does NOT provide atomicity between JSONL write, metadata updates, and DB mtime.
// If a crash occurs between these operations, metadata may be inconsistent. However, this is
// acceptable because:
// 1. The worst case is "JSONL content has changed" error on next export
// 2. User can fix by running 'bd import' (safe, no data loss)
// 3. Current approach is simple and doesn't require complex WAL or format changes
//
// Future: Consider defensive checks on startup if this becomes a common issue.
func updateExportMetadata(ctx context.Context, store storage.Storage, jsonlPath string, log *slog.Logger, keySuffix string) {
// Sanitize keySuffix to handle Windows paths with colons
if keySuffix != "" {
keySuffix = sanitizeMetadataKey(keySuffix)
}
currentHash, err := computeJSONLHash(jsonlPath)
if err != nil {
log.Info("Warning: failed to compute JSONL hash for metadata update", "error", err)
return
}
// Build metadata keys with optional suffix for per-repo tracking
// Renamed from last_import_hash to jsonl_content_hash
hashKey := "jsonl_content_hash"
timeKey := "last_import_time"
if keySuffix != "" {
hashKey += ":" + keySuffix
timeKey += ":" + keySuffix
}
// Note: Metadata update failures are treated as warnings, not errors.
// This is acceptable because the worst case is the next export will require
// an import first, which is safe and prevents data loss.
// Alternative: Make this critical and fail the export if metadata updates fail,
// but this makes exports more fragile and doesn't prevent data corruption.
if err := store.SetMetadata(ctx, hashKey, currentHash); err != nil {
log.Info("Warning: failed to update metadata", "key", hashKey, "error", err)
log.Info("Next export may require running 'bd import' first")
}
// Use RFC3339Nano for nanosecond precision to avoid race with file mtime (fixes #399)
exportTime := time.Now().Format(time.RFC3339Nano)
if err := store.SetMetadata(ctx, timeKey, exportTime); err != nil {
log.Info("Warning: failed to update metadata", "key", timeKey, "error", err)
}
// Note: mtime tracking removed (git doesn't preserve mtime)
}
// validateDatabaseFingerprint checks that the database belongs to this repository
func validateDatabaseFingerprint(ctx context.Context, store storage.Storage, log *slog.Logger) error {
// Get stored repo ID
storedRepoID, err := store.GetMetadata(ctx, "repo_id")
if err != nil && err.Error() != "metadata key not found: repo_id" {
return fmt.Errorf("failed to read repo_id: %w", err)
}
// If no repo_id, this is a legacy database - require explicit migration
if storedRepoID == "" {
return fmt.Errorf(`
LEGACY DATABASE DETECTED!
This database was created before version 0.17.5 and lacks a repository fingerprint.
To continue using this database, you must explicitly set its repository ID:
bd migrate --update-repo-id
This ensures the database is bound to this repository and prevents accidental
database sharing between different repositories.
If this is a fresh clone, run:
rm -rf .beads && bd init
Note: Auto-claiming legacy databases is intentionally disabled to prevent
silent corruption when databases are copied between repositories.
`)
}
// Validate repo ID matches current repository
currentRepoID, err := beads.ComputeRepoID()
if err != nil {
log.Info("Warning: could not compute current repository ID", "error", err)
return nil
}
if storedRepoID != currentRepoID {
return fmt.Errorf(`
DATABASE MISMATCH DETECTED!
This database belongs to a different repository:
Database repo ID: %s
Current repo ID: %s
This usually means:
1. You copied a .beads directory from another repo (don't do this!)
2. Git remote URL changed (run 'bd migrate --update-repo-id')
3. Database corruption
4. bd was upgraded and URL canonicalization changed
⚠️ CRITICAL: This mismatch can cause beads to incorrectly delete issues during sync!
The git-history-backfill mechanism may treat your local issues as deleted
because they don't exist in the remote repository's history.
Solutions:
- If remote URL changed: bd migrate --update-repo-id
- If bd was upgraded: bd migrate --update-repo-id
- If wrong database: rm -rf .beads && bd init
- If correct database: BEADS_IGNORE_REPO_MISMATCH=1 bd daemon
(Warning: This can cause data corruption and unwanted deletions across clones!)
`, storedRepoID[:8], currentRepoID[:8])
}
log.Info("Repository fingerprint validated", "repo_id", currentRepoID[:8])
return nil
}
// createExportFunc creates a function that only exports database to JSONL
// and optionally commits/pushes (no git pull or import). Used for mutation events.
func createExportFunc(ctx context.Context, store storage.Storage, autoCommit, autoPush bool, log *slog.Logger) func() {
return performExport(ctx, store, autoCommit, autoPush, false, log)
}
// createLocalExportFunc creates a function that only exports database to JSONL
// without any git operations. Used for local-only mode with mutation events.
func createLocalExportFunc(ctx context.Context, store storage.Storage, log *slog.Logger) func() {
return performExport(ctx, store, false, false, true, log)
}
// performExport is the shared implementation for export-only functions.
// skipGit: if true, skips all git operations (commits, pushes).
func performExport(ctx context.Context, store storage.Storage, autoCommit, autoPush, skipGit bool, log *slog.Logger) func() {
return func() {
exportCtx, exportCancel := context.WithTimeout(ctx, 30*time.Second)
defer exportCancel()
mode := "export"
if skipGit {
mode = "local export"
}
// Guard: Skip if sync-branch == current-branch (GH#1258)
// Local-only mode (skipGit) doesn't use sync-branch, so skip the guard
if !skipGit && shouldSkipDueToSameBranch(exportCtx, store, mode, log) {
return
}
log.Info("Starting", "mode", mode)
jsonlPath := findJSONLPath()
if jsonlPath == "" {
log.Info("Error: beads storage file not found")
return
}
// Check for exclusive lock
beadsDir := filepath.Dir(jsonlPath)
skip, holder, err := types.ShouldSkipDatabase(beadsDir)
if skip {
if err != nil {
log.Info("Skipping (lock check failed)", "mode", mode, "error", err)
} else {
log.Info("Skipping (locked)", "mode", mode, "holder", holder)
}
return
}
if holder != "" {
log.Info("Removed stale lock, proceeding", "holder", holder)
}
// Auto-import if JSONL has changed since last import (GH#XXXX)
// This prevents "refusing to export: JSONL content has changed" errors
// when external changes (git pull, other clones) modify JSONL between imports.
// Check JSONL content hash to detect external changes
repoKey := getRepoKeyForPath(jsonlPath)
if hasJSONLChanged(exportCtx, store, jsonlPath, repoKey) {
log.Info("JSONL changed externally, importing before export")
// Count issues before import for validation
beforeCount, err := countDBIssues(exportCtx, store)
if err != nil {
log.Info("Failed to count issues before auto-import", "error", err)
return
}
// Import external changes
if err := importToJSONLWithStore(exportCtx, store, jsonlPath); err != nil {
log.Info("Auto-import failed", "error", err)
return
}
// Validate import didn't cause data loss
afterCount, err := countDBIssues(exportCtx, store)
if err != nil {
log.Info("Failed to count issues after auto-import", "error", err)
return
}
if err := validatePostImport(beforeCount, afterCount, jsonlPath); err != nil {
log.Info("Post-import validation failed", "error", err)
return
}
// Update metadata hash to match imported JSONL
// This prevents pre-export validation from failing
if currentHash, err := computeJSONLHash(jsonlPath); err == nil {
hashKey := "jsonl_content_hash"
if repoKey != "" {
hashKey += ":" + repoKey
}
if err := store.SetMetadata(exportCtx, hashKey, currentHash); err != nil {
log.Info("Failed to update JSONL hash after auto-import", "error", err)
}
}
log.Info("Auto-import complete, proceeding with export")
}
// Pre-export validation
if err := validatePreExport(exportCtx, store, jsonlPath); err != nil {
log.Info("Pre-export validation failed", "error", err)
return
}
// Export to JSONL
if err := exportToJSONLWithStore(exportCtx, store, jsonlPath); err != nil {
log.Info("Export failed", "error", err)
return
}
log.Info("Exported to JSONL")
// Export events to JSONL (non-fatal, opt-in via config)
if config.GetBool("events-export") {
eventsPath := filepath.Join(filepath.Dir(jsonlPath), "events.jsonl")
if err := exportEventsToJSONL(exportCtx, store, eventsPath); err != nil {
log.Info("Warning: events export failed", "error", err)
}
}
// GH#885: Defer metadata updates until AFTER git commit succeeds.
// This is a helper to finalize the export after git operations.
finalizeExportMetadata := func() {
// Update export metadata for multi-repo support with stable keys
multiRepoPaths := getMultiRepoJSONLPaths()
if multiRepoPaths != nil {
// Multi-repo mode: update metadata for each JSONL with stable repo key
for _, path := range multiRepoPaths {
repoKey := getRepoKeyForPath(path)
updateExportMetadata(exportCtx, store, path, log, repoKey)
}
} else {
// Single-repo mode: update metadata for main JSONL
updateExportMetadata(exportCtx, store, jsonlPath, log, "")
}
// Update database mtime to be >= JSONL mtime (fixes #278, #301, #321)
// This prevents validatePreExport from incorrectly blocking on next export
// with "JSONL is newer than database" after daemon auto-export
// Dolt backend does not have a SQLite DB file; mtime touch is SQLite-only.
// Use store.Path() to get the actual database location, not the JSONL directory,
// since sync-branch exports write JSONL to a worktree but the DB stays in the main repo.
if sqliteStore, ok := store.(*sqlite.SQLiteStorage); ok {
dbPath := sqliteStore.Path()
if err := TouchDatabaseFile(dbPath, jsonlPath); err != nil {
log.Info("Warning: failed to update database mtime", "error", err)
}
}
}
// Auto-commit if enabled (skip in git-free mode)
if autoCommit && !skipGit {
// Try sync branch commit first
// Use forceOverwrite=true because mutation-triggered exports (create, update, delete)
// mean the local state is authoritative and should not be merged with worktree.
// This is critical for delete mutations to be properly reflected in the sync branch.
committed, err := syncBranchCommitAndPushWithOptions(exportCtx, store, autoPush, true, log)
if err != nil {
log.Info("Sync branch commit failed", "error", err)
return
}
if committed {
// GH#885: Finalize after sync branch commit succeeded
finalizeExportMetadata()
} else {
// If sync branch not configured, use regular commit
hasChanges, err := gitHasChanges(exportCtx, jsonlPath)
if err != nil {
log.Info("Error checking git status", "error", err)
return
}
if hasChanges {
message := fmt.Sprintf("bd daemon export: %s", time.Now().Format("2006-01-02 15:04:05"))
if err := gitCommit(exportCtx, jsonlPath, message); err != nil {
log.Info("Commit failed", "error", err)
return
}
log.Info("Committed changes")
// GH#885: Finalize after git commit succeeded, before push
// Push failure shouldn't prevent metadata update since commit succeeded
finalizeExportMetadata()
// Auto-push if enabled (GH#872: use sync.remote config)
if autoPush {
configuredRemote, _ := store.GetConfig(exportCtx, "sync.remote")
if err := gitPush(exportCtx, configuredRemote); err != nil {
log.Info("Push failed", "error", err)
return
}
log.Info("Pushed to remote")
}
} else {
// No git changes but export happened - finalize metadata
finalizeExportMetadata()
}
}
} else if skipGit {
// Git-free mode: finalize immediately since there's no git to wait for
finalizeExportMetadata()
}
if skipGit {
log.Info("Local export complete")
} else {
log.Info("Export complete")
}
}
}
// createAutoImportFunc creates a function that pulls from git and imports JSONL
// to database (no export). Used for file system change events.
func createAutoImportFunc(ctx context.Context, store storage.Storage, log *slog.Logger) func() {
return performAutoImport(ctx, store, false, log)
}
// createLocalAutoImportFunc creates a function that imports from JSONL to database
// without any git operations. Used for local-only mode with file system change events.
func createLocalAutoImportFunc(ctx context.Context, store storage.Storage, log *slog.Logger) func() {
return performAutoImport(ctx, store, true, log)
}
// performAutoImport is the shared implementation for import-only functions.
// skipGit: if true, skips git pull operations.
func performAutoImport(ctx context.Context, store storage.Storage, skipGit bool, log *slog.Logger) func() {
return func() {
importCtx, importCancel := context.WithTimeout(ctx, 1*time.Minute)
defer importCancel()
mode := "auto-import"
if skipGit {
mode = "local auto-import"
}
// Skip JSONL import in dolt-native mode (JSONL is export-only backup)
if !ShouldImportJSONL(importCtx, store) {
log.Info("Skipping (dolt-native mode)", "mode", mode)
return
}
// Guard: Skip if sync-branch == current-branch (GH#1258)
// Local-only mode (skipGit) doesn't use sync-branch, so skip the guard
if !skipGit && shouldSkipDueToSameBranch(importCtx, store, mode, log) {
return
}
// Check backoff before attempting sync (skip for local mode)
if !skipGit {
jsonlPath := findJSONLPath()
if jsonlPath != "" {
beadsDir := filepath.Dir(jsonlPath)
if ShouldSkipSync(beadsDir) {
log.Info("Skipping: in backoff period", "mode", mode)
return
}
}
}
log.Info("Starting auto-import", "mode", mode)
jsonlPath := findJSONLPath()
if jsonlPath == "" {
log.Info("Error: beads storage file not found")
return
}
// Check for exclusive lock
beadsDir := filepath.Dir(jsonlPath)
skip, holder, err := types.ShouldSkipDatabase(beadsDir)
if skip {
if err != nil {
log.Info("Skipping (lock check failed)", "mode", mode, "error", err)
} else {
log.Info("Skipping (locked)", "mode", mode, "holder", holder)
}
return
}
if holder != "" {
log.Info("Removed stale lock, proceeding", "holder", holder)
}
// Pull from git if not in git-free mode (do this BEFORE checking if JSONL changed)
if !skipGit {
// SAFETY CHECK: Warn if there are uncommitted local changes
// This helps detect race conditions where local work hasn't been pushed yet
jsonlPath := findJSONLPath()
if jsonlPath != "" {
if hasLocalChanges, err := gitHasChanges(importCtx, jsonlPath); err == nil && hasLocalChanges {
log.Info("WARNING: Uncommitted local changes detected", "path", jsonlPath)
log.Info(" Pulling from remote may overwrite local unpushed changes.")
log.Info(" Consider running 'bd sync' to commit and push your changes first.")
// Continue anyway, but user has been warned
}
}
// Try sync branch first
pulled, err := syncBranchPull(importCtx, store, log)
if err != nil {
backoff := RecordSyncFailure(beadsDir, err.Error())
log.Info("Sync branch pull failed", "error", err, "backoff", backoff)
return
}
// If sync branch not configured, use regular pull (GH#872: use sync.remote config)
if !pulled {
configuredRemote, _ := store.GetConfig(importCtx, "sync.remote")
if err := gitPull(importCtx, configuredRemote); err != nil {
backoff := RecordSyncFailure(beadsDir, err.Error())
log.Info("Pull failed", "error", err, "backoff", backoff)
return
}
log.Info("Pulled from remote")
}
}
// Check JSONL content hash AFTER pulling to detect changes from remote
// Use content-based check (not mtime) to avoid git resurrection bug
// Use getRepoKeyForPath for multi-repo support
log.Info("🔍 Checking JSONL content hash AFTER pull (fix: auto-pull-not-pulling)")
repoKey := getRepoKeyForPath(jsonlPath)
if !hasJSONLChanged(importCtx, store, jsonlPath, repoKey) {
log.Info("Skipping: JSONL content unchanged", "mode", mode)
return
}
log.Info("JSONL content changed, proceeding with operation...", "mode", mode)
// Count issues before import
beforeCount, err := countDBIssues(importCtx, store)
if err != nil {
log.Info("Failed to count issues before import", "error", err)
return
}
// Import from JSONL
if err := importToJSONLWithStore(importCtx, store, jsonlPath); err != nil {
log.Info("Import failed", "error", err)
return
}
log.Info("Imported from JSONL")
// Validate import
afterCount, err := countDBIssues(importCtx, store)
if err != nil {
log.Info("Failed to count issues after import", "error", err)
return
}
if err := validatePostImport(beforeCount, afterCount, jsonlPath); err != nil {
log.Info("Post-import validation failed", "error", err)
return
}
if skipGit {
log.Info("Local auto-import complete")
} else {
// Record success to clear backoff state
RecordSyncSuccess(beadsDir)
log.Info("Auto-import complete")
}
}
}
// createSyncFunc creates a function that performs full sync cycle (export, commit, pull, import, push)
func createSyncFunc(ctx context.Context, store storage.Storage, autoCommit, autoPush bool, log *slog.Logger) func() {
return performSync(ctx, store, autoCommit, autoPush, false, log)
}
// createLocalSyncFunc creates a function that performs local-only sync (export only, no git).
// Used when daemon is started with --local flag.
func createLocalSyncFunc(ctx context.Context, store storage.Storage, log *slog.Logger) func() {
return performSync(ctx, store, false, false, true, log)
}
// performSync is the shared implementation for sync functions.
// skipGit: if true, skips all git operations (commits, pulls, pushes, snapshot capture, 3-way merge, import).
// Local-only mode only performs validation and export since there's no remote to sync with.
func performSync(ctx context.Context, store storage.Storage, autoCommit, autoPush, skipGit bool, log *slog.Logger) func() {
return func() {
syncCtx, syncCancel := context.WithTimeout(ctx, 2*time.Minute)
defer syncCancel()
mode := "sync cycle"
if skipGit {
mode = "local sync cycle"
}
// Guard: Skip if sync-branch == current-branch (GH#1258)
// Local-only mode (skipGit) doesn't use sync-branch, so skip the guard
if !skipGit && shouldSkipDueToSameBranch(syncCtx, store, mode, log) {
return
}
log.Info("Starting sync cycle", "mode", mode)
jsonlPath := findJSONLPath()
if jsonlPath == "" {
log.Info("Error: beads storage file not found")
return
}
// Cache multi-repo paths to avoid redundant calls
multiRepoPaths := getMultiRepoJSONLPaths()
// Check for exclusive lock before processing database
beadsDir := filepath.Dir(jsonlPath)
skip, holder, err := types.ShouldSkipDatabase(beadsDir)
if skip {
if err != nil {
log.Info("Skipping database (lock check failed)", "error", err)
} else {
log.Info("Skipping database (locked)", "holder", holder)
}
return
}
if holder != "" {
log.Info("Removed stale lock, proceeding", "holder", holder, "mode", mode)
}
// Integrity check: validate before export
if err := validatePreExport(syncCtx, store, jsonlPath); err != nil {
log.Info("Pre-export validation failed", "error", err)
return
}
// Check for duplicate IDs (database corruption)
if err := checkDuplicateIDs(syncCtx, store); err != nil {
log.Info("Duplicate ID check failed", "error", err)
return
}
// Check for orphaned dependencies (warns but doesn't fail)
if orphaned, err := checkOrphanedDeps(syncCtx, store); err != nil {
log.Info("Orphaned dependency check failed", "error", err)
} else if len(orphaned) > 0 {
log.Info("Found orphaned dependencies", "count", len(orphaned), "orphaned", orphaned)
}
if err := exportToJSONLWithStore(syncCtx, store, jsonlPath); err != nil {
log.Info("Export failed", "error", err)
return
}
log.Info("Exported to JSONL")
// Export events to JSONL (non-fatal, opt-in via config)
if config.GetBool("events-export") {
syncEventsPath := filepath.Join(beadsDir, "events.jsonl")
if err := exportEventsToJSONL(syncCtx, store, syncEventsPath); err != nil {
log.Info("Warning: events export failed", "error", err)
}
}
// GH#885: Defer metadata updates until AFTER git commit succeeds.
// Define helper to finalize after git operations.
finalizeExportMetadata := func() {
// Update export metadata for multi-repo support with stable keys
if multiRepoPaths != nil {
// Multi-repo mode: update metadata for each JSONL with stable repo key
for _, path := range multiRepoPaths {
repoKey := getRepoKeyForPath(path)
updateExportMetadata(syncCtx, store, path, log, repoKey)
}
} else {
// Single-repo mode: update metadata for main JSONL
updateExportMetadata(syncCtx, store, jsonlPath, log, "")
}
// Update database mtime to be >= JSONL mtime
// This prevents validatePreExport from incorrectly blocking on next export
// Dolt backend does not have a SQLite DB file; mtime touch is SQLite-only.
// Use store.Path() to get the actual database location, not the JSONL directory,
// since sync-branch exports write JSONL to a worktree but the DB stays in the main repo.
if sqliteStore, ok := store.(*sqlite.SQLiteStorage); ok {
dbPath := sqliteStore.Path()
if err := TouchDatabaseFile(dbPath, jsonlPath); err != nil {
log.Info("Warning: failed to update database mtime", "error", err)
}
}
}
// Skip git operations, snapshot capture, deletion tracking, and import in local-only mode
// Local-only sync is export-only since there's no remote to sync with
if skipGit {
// Git-free mode: finalize immediately since there's no git to wait for
finalizeExportMetadata()
log.Info("Local sync complete", "mode", mode)
return
}
// In dolt-native mode, JSONL is export-only backup — skip git sync and import
if !ShouldImportJSONL(syncCtx, store) {
finalizeExportMetadata()
log.Info("Sync complete (dolt-native mode, export-only)", "mode", mode)
return
}
// ---- Git operations start here ----
// Capture left snapshot (pre-pull state) for 3-way merge
// This is mandatory for deletion tracking integrity
// In multi-repo mode, capture snapshots for all JSONL files
if multiRepoPaths != nil {
// Multi-repo mode: snapshot each JSONL file
for _, path := range multiRepoPaths {
if err := captureLeftSnapshot(path); err != nil {
log.Info("Error: failed to capture snapshot", "path", path, "error", err)
return
}
}
log.Info("Captured snapshots (multi-repo mode)", "count", len(multiRepoPaths))
} else {
// Single-repo mode: snapshot the main JSONL
if err := captureLeftSnapshot(jsonlPath); err != nil {
log.Info("Error: failed to capture snapshot", "error", err)
return
}
}
if autoCommit {
// Try sync branch commit first
committed, err := syncBranchCommitAndPush(syncCtx, store, autoPush, log)
if err != nil {
log.Info("Sync branch commit failed", "error", err)
return
}
// If sync branch not configured, use regular commit
if !committed {
hasChanges, err := gitHasChanges(syncCtx, jsonlPath)
if err != nil {
log.Info("Error checking git status", "error", err)
return
}
if hasChanges {
message := fmt.Sprintf("bd daemon sync: %s", time.Now().Format("2006-01-02 15:04:05"))
if err := gitCommit(syncCtx, jsonlPath, message); err != nil {
log.Info("Commit failed", "error", err)
return
}
log.Info("Committed changes")
}
}
// GH#885: NOW finalize metadata after git commit succeeded
finalizeExportMetadata()
}
// Pull (try sync branch first)
pulled, err := syncBranchPull(syncCtx, store, log)
if err != nil {
log.Info("Sync branch pull failed", "error", err)
return
}
// If sync branch not configured, use regular pull (GH#872: use sync.remote config)
if !pulled {
configuredRemote, _ := store.GetConfig(syncCtx, "sync.remote")
if err := gitPull(syncCtx, configuredRemote); err != nil {
log.Info("Pull failed", "error", err)
return
}
log.Info("Pulled from remote")
}
// Count issues before import for validation
beforeCount, err := countDBIssues(syncCtx, store)
if err != nil {
log.Info("Failed to count issues before import", "error", err)
return
}
// Perform 3-way merge and prune deletions
// In multi-repo mode, apply deletions for each JSONL file
if multiRepoPaths != nil {
// Multi-repo mode: merge/prune for each JSONL
for _, path := range multiRepoPaths {
if err := applyDeletionsFromMerge(syncCtx, store, path); err != nil {
log.Info("Error during 3-way merge", "path", path, "error", err)
return
}
}
log.Info("Applied deletions", "repo_count", len(multiRepoPaths))
} else {
// Single-repo mode
if err := applyDeletionsFromMerge(syncCtx, store, jsonlPath); err != nil {
log.Info("Error during 3-way merge", "error", err)
return
}
}
if err := importToJSONLWithStore(syncCtx, store, jsonlPath); err != nil {
log.Info("Import failed", "error", err)
return
}
log.Info("Imported from JSONL")
// Update database mtime after import (fixes #278, #301, #321)
// Sync branch import can update JSONL timestamp, so ensure DB >= JSONL
// Dolt backend does not have a SQLite DB file; mtime touch is SQLite-only.
// Use store.Path() to get the actual database location, not the JSONL directory,
// since sync-branch imports read JSONL from a worktree but the DB stays in the main repo.
if sqliteStore, ok := store.(*sqlite.SQLiteStorage); ok {
dbPath := sqliteStore.Path()
if err := TouchDatabaseFile(dbPath, jsonlPath); err != nil {
log.Info("Warning: failed to update database mtime", "error", err)
}
}
// Validate import didn't cause data loss
afterCount, err := countDBIssues(syncCtx, store)
if err != nil {
log.Info("Failed to count issues after import", "error", err)
return
}
if err := validatePostImport(beforeCount, afterCount, jsonlPath); err != nil {
log.Info("Post-import validation failed", "error", err)
return
}
// Update base snapshot after successful import
// In multi-repo mode, update snapshots for all JSONL files
if multiRepoPaths != nil {
for _, path := range multiRepoPaths {
if err := updateBaseSnapshot(path); err != nil {
log.Info("Warning: failed to update base snapshot", "path", path, "error", err)
}
}
} else {
if err := updateBaseSnapshot(jsonlPath); err != nil {
log.Info("Warning: failed to update base snapshot", "error", err)
}
}
// Clean up temporary snapshot files after successful merge
// In multi-repo mode, clean up snapshots for all JSONL files
if multiRepoPaths != nil {
for _, path := range multiRepoPaths {
sm := NewSnapshotManager(path)
if err := sm.Cleanup(); err != nil {
log.Info("Warning: failed to clean up snapshots", "path", path, "error", err)
}
}
} else {
sm := NewSnapshotManager(jsonlPath)
if err := sm.Cleanup(); err != nil {
log.Info("Warning: failed to clean up snapshots", "error", err)
}
}
// GH#872: use sync.remote config
if autoPush && autoCommit {
configuredRemote, _ := store.GetConfig(syncCtx, "sync.remote")
if err := gitPush(syncCtx, configuredRemote); err != nil {
log.Info("Push failed", "error", err)
return
}
log.Info("Pushed to remote")
}
log.Info("Sync cycle complete")
}
}