diff --git a/CHANGELOG.md b/CHANGELOG.md index 01e816187..4b2ec795a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 its server through the proxy, so an ambient port does not describe its lifecycle. +- **`bd` caps how large the auto-backup destination can grow, instead of + growing it forever** ([#6071](https://github.com/gastownhall/beads/pull/6071)). + `CALL DOLT_BACKUP('sync', ...)` only ever adds new chunks to the + destination — it never prunes ones that became unreachable on the source + (history rewrites, superseded data) — and Dolt exposes no supported way to + GC a backup destination in place. Left uncapped, the destination could + only grow until disk filled; this is the root cause of the 2026-06-19 + outage, where a 1.7GB store produced a 43GB backup directory. Auto-backup + now pauses (nothing is deleted) once the destination reaches + `backup.size-cap-mb` (default 2048MB); set it to `0` to disable the cap + entirely. The pause is no longer stderr-only: `bd backup status` and its + `--json` output now report a `size_cap` object (`enabled`, `cap_mb`, + `current_bytes`, `exceeded`), so an agent/CI caller relying on `--json` or + `--quiet` can see that auto-backup has stopped instead of reading a + reassuring "Last backup" line while nothing further syncs. The remediation + advice no longer suggests deleting the backup directory — nothing + guarantees a deleted destination is cleanly recreated by the next sync, and + the server-side backup remote stays registered against that path; it now + points at `backup.size-cap-mb` / `bd backup init ` instead. The + size-cap check itself runs after both the interval throttle and change + detection, so it costs nothing on any path that is not about to sync — an + idle workspace never reaches it at all, and a paused destination re-arms + the interval throttle on the skip, so it is measured at most once per + `backup.interval` rather than on every command. `backup.size-warn-interval` + (default 24h) controls how often the pause is re-announced. Manual `bd + backup` / `bd backup sync` are not capped. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 76f738ea5..29a89b369 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "backend_support.go", "backup.go", "backup_auto.go", + "backup_cap.go", "backup_dolt.go", "backup_export.go", "backup_lock.go", @@ -512,6 +513,7 @@ go_test( "backup_auto_proxied_local_integration_test.go", "backup_auto_proxied_test.go", "backup_auto_test.go", + "backup_cap_test.go", "backup_dolt_test.go", "backup_embedded_test.go", "backup_export_failure_test.go", diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 0e8e02fcc..55b5b812f 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -88,8 +88,9 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { if jsonOutput { result := map[string]interface{}{ - "backup": state, - "dolt": showDoltBackupStatusJSON(), + "backup": state, + "dolt": showDoltBackupStatusJSON(), + "size_cap": showSizeCapStatusJSON(dir), } if sizeAvailable { result["database_size"] = showDBSizeJSON(databaseSize) @@ -137,6 +138,7 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { } fmt.Printf("\nConfig: enabled=%v%s interval=%s\n", enabled, enabledNote, interval) + showSizeCapStatus(dir) // Show Dolt backup info showDoltBackupStatus() diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 988c0dcfb..eff435400 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -189,7 +189,12 @@ func maybeAutoBackup(ctx context.Context) { return } - // Throttle: skip if we backed up recently + // Throttle: skip if we backed up recently. Checked before the size-cap + // walk below so the common case — the majority of bd invocations, + // still inside the interval — pays zero directory-walk cost (ga-y6gjv + // PR #6071 review: getDirSize's filepath.Walk cost 65-100ms per + // invocation at 20k files when the cap check ran unconditionally, + // before this throttle, on every single command). interval := config.GetDuration("backup.interval") if interval == 0 { interval = 15 * time.Minute @@ -211,6 +216,32 @@ func maybeAutoBackup(ctx context.Context) { return } + // Size cap: skip entirely once the destination has grown past + // backup.size-cap-mb (ga-y6gjv) — see backupSizeCapExceeded for why a + // cap, not an in-place prune, is the safe fix here. + // + // Placed AFTER change detection, which is the order + // docs/reference/configuration.md:"How it works" documents. Ahead of + // it, the walk ran on EVERY bd invocation indefinitely in an idle + // workspace: nothing on the idle path advances state.Timestamp, so the + // interval throttle above can never re-arm while nothing is changing + // (ga-y6gjv PR #6071 review). Keep it above any lock acquisition — + // a skip should not first take a lock it is about to release. + if exceeded, size, err := backupSizeCapExceeded(dir); err != nil { + warnBackupSizeCapUnavailable(err) + // Re-arm the interval throttle before proceeding uncapped: not + // every runBackupExport exit persists state.Timestamp (see + // warnBackupSizeCapUnavailable), and without this the walk and + // its warning would repeat on every bd command. + state.Timestamp = time.Now().UTC() + if saveErr := saveBackupState(dir, state); saveErr != nil { + debug.Logf("backup: failed to persist throttle state after size cap error: %v\n", saveErr) + } + } else if exceeded { + pauseAutoBackupForSizeCap(dir, state, size) + return + } + // One backup at a time per workspace (backup_lock.go). Auto-backup never // waits: a held lock means a backup is already running. release, err := acquireBackupLock(0) diff --git a/cmd/bd/backup_cap.go b/cmd/bd/backup_cap.go new file mode 100644 index 000000000..64854731f --- /dev/null +++ b/cmd/bd/backup_cap.go @@ -0,0 +1,232 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/debug" +) + +// effectiveSizeCapMB returns the configured backup.size-cap-mb, or 0 if the +// cap is disabled. internal/config/config.go registers a viper default of +// 2048 for this key, so GetInt only returns <= 0 when the operator has +// explicitly set 0 (or an unparseable value, which viper's GetInt also +// resolves to 0) — per the ga-y6gjv PR #6071 review, that means "no cap", +// not "use the 2048 default instead" (an operator with a legitimately +// larger destination needs an off switch). +func effectiveSizeCapMB() int { + capMB := config.GetInt("backup.size-cap-mb") + if capMB <= 0 { + return 0 + } + return capMB +} + +// backupSizeCapExceeded reports whether dir's on-disk size has crossed +// backup.size-cap-mb (default 2048MB / 2GB; 0 disables the cap). +// +// A hard cap exists because BackupSync/CALL DOLT_BACKUP('sync', ...) only +// ever transfers new chunks into dir — it never prunes ones that became +// unreachable on the source DB (history rewrites, superseded data) — and +// Dolt exposes no supported way to GC a backup destination in place: it is +// a bare chunk-store directory with no .dolt repo-root marker (confirmed +// against dolt_backup.go's SQL procedures — add/sync/restore only, no gc; +// and empirically, `dolt gc` run with its working directory set to a real +// backup destination fails "not a valid dolt repository" because there is +// no .dolt for the CLI to find). Absent a cap, the destination can only +// grow forever until disk fills (ga-y6gjv, the 2026-06-19 outage: 43GB +// backup dir from a 1.7GB store). formatBytes is shared with +// runCompactDolt (compact.go); getDirSize is defined below. +func backupSizeCapExceeded(dir string) (exceeded bool, size int64, err error) { + capMB := effectiveSizeCapMB() + if capMB == 0 { + return false, 0, nil + } + size, err = getDirSize(dir) + if err != nil { + return false, 0, err + } + return size >= int64(capMB)*1024*1024, size, nil +} + +// walkBackupDestination is getDirSize, indirected so tests can inject the +// mid-walk ENOENT that a real filesystem produces only under a race. +var walkBackupDestination = getDirSize + +// backupDestinationSize measures dir for the status readers. Only a +// destination that does not exist yet counts as empty; every walk error is +// returned, ENOENT included. getDirSize aborts on the first per-entry +// error, so a chunk file removed between listing its directory and +// stat-ing it would otherwise read as 0 bytes: a false all-clear on a +// destination that may be well over its cap. +func backupDestinationSize(dir string) (int64, error) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return 0, nil + } + return walkBackupDestination(dir) +} + +// showSizeCapStatus prints size-cap info as part of `bd backup status` +// (ga-y6gjv PR #6071 review: status previously said nothing about the cap, +// so an agent/CI caller watching a paused destination saw only a +// reassuring "Last backup" line). +func showSizeCapStatus(dir string) { + capMB := effectiveSizeCapMB() + if capMB == 0 { + // Echo the configured value rather than a literal 0: an + // unparseable one ("2048MB") also disables the cap (see + // effectiveSizeCapMB), and this line is where an operator + // debugging that looks. + fmt.Printf(" Size cap: disabled (backup.size-cap-mb=%s)\n", config.GetString("backup.size-cap-mb")) + return + } + size, err := backupDestinationSize(dir) + if err != nil { + // Say so rather than dropping the line: a silent omission + // looks identical to "no cap configured" to an operator + // debugging a permission problem on the destination, and + // --json already reports this case (showSizeCapStatusJSON). + debug.Logf("backup status: size cap check failed (non-fatal): %v\n", err) + fmt.Printf(" Size cap: unavailable (%v)\n", err) + return + } + capBytes := int64(capMB) * 1024 * 1024 + if size >= capBytes { + fmt.Printf(" Size cap: PAUSED (cap exceeded) — %s / %s. Raise backup.size-cap-mb "+ + "or run `bd backup init ` to switch destinations.\n", + formatBytes(size), formatBytes(capBytes)) + return + } + fmt.Printf(" Size cap: %s / %s\n", formatBytes(size), formatBytes(capBytes)) +} + +// showSizeCapStatusJSON returns size-cap info for `bd backup status --json` +// — see showSizeCapStatus. +func showSizeCapStatusJSON(dir string) map[string]interface{} { + capMB := effectiveSizeCapMB() + if capMB == 0 { + return map[string]interface{}{"enabled": false} + } + size, err := backupDestinationSize(dir) + if err != nil { + // exceeded is explicitly null, not omitted: a consumer that + // reads a missing key as false would see "not exceeded" + // during exactly the failure that makes it unmeasurable. + return map[string]interface{}{ + "enabled": true, + "cap_mb": capMB, + "error": err.Error(), + "exceeded": nil, + } + } + capBytes := int64(capMB) * 1024 * 1024 + return map[string]interface{}{ + "enabled": true, + "cap_mb": capMB, + "current_bytes": size, + "exceeded": size >= capBytes, + } +} + +// warnBackupSizeCapUnavailable reports, on stderr, that the cap could not +// be measured and the backup is proceeding uncapped. +// +// getDirSize returns the first per-file error straight out of its +// filepath.Walk callback, so one unreadable entry — or a chunk file removed +// mid-walk — aborts the whole measurement. maybeAutoBackup deliberately +// fails OPEN there (blocking backups outright on an unreadable destination +// would be worse), but a debug-only line made that silent: the cap can +// disable itself and restore the unbounded growth this feature exists to +// prevent with nothing on the operator's terminal, while `bd backup status +// --json` reports the very same error (ga-y6gjv PR #6071 review). +// +// It is throttled by the backup interval, not by the PAUSED warning's +// LastCapWarnAt slot — spending that slot here would let a transient walk +// error suppress the more important "auto-backup PAUSED" notice for a full +// warn interval. maybeAutoBackup reaches the walk only after the interval +// throttle and change detection have both passed, and re-arms +// state.Timestamp itself after this warning rather than relying on the +// fall-through into runBackupExport: not every runBackupExport exit +// persists it (a failed post-sync GetCurrentCommit returns without saving +// state), and the next command would then walk and warn again. That bounds +// this warning to once per backup.interval whenever backup_state.json can +// be written. +func warnBackupSizeCapUnavailable(err error) { + if !isQuiet() && !jsonOutput { + fmt.Fprintf(os.Stderr, + "Warning: backup size cap could not be measured (%v); "+ + "proceeding without the cap. Auto-backup is unbounded until this clears.\n", err) + } + debug.Logf("backup: size cap check failed (non-fatal): %v\n", err) +} + +// pauseAutoBackupForSizeCap records the skip that happens when the +// destination is over backup.size-cap-mb, and announces it to stderr at +// most once per backup.size-warn-interval (default 24h). +// +// It re-arms the interval throttle (state.Timestamp) even though no backup +// ran, leaving LastDoltCommit untouched exactly as runBackupExport's own +// failure path does (backup_export.go, wy-zrmqr). Without that, the paused +// state is self-perpetuating and pathological: Timestamp is only ever +// advanced by a backup attempt, so a destination that is over cap — by +// definition the largest one — would pay the full getDirSize walk on every +// single bd command, forever, until an operator intervenes (ga-y6gjv +// PR #6071 review). LastDoltCommit is deliberately left alone so change +// detection still sees the pending work once the cap is raised. +// +// The warn timestamp, by contrast, is recorded only when the message was +// actually printed. --json/--quiet callers — i.e. every agent-driven +// command — would otherwise consume the 24h slot silently and the +// human-visible warning could effectively never fire; those callers have +// `bd backup status --json`'s size_cap field instead. +// +// Never returns an error: a failure to persist must not block the +// (already-decided) skip of the backup attempt itself. +func pauseAutoBackupForSizeCap(dir string, state *backupState, size int64) { + warnInterval := config.GetDuration("backup.size-warn-interval") + if warnInterval == 0 { + warnInterval = 24 * time.Hour + } + + state.Timestamp = time.Now().UTC() + + lastWarn := state.LastCapWarnAt + throttled := !lastWarn.IsZero() && time.Since(lastWarn) < warnInterval + announce := !throttled && !isQuiet() && !jsonOutput + if announce { + state.LastCapWarnAt = time.Now().UTC() + } + + if err := saveBackupState(dir, state); err != nil { + debug.Logf("backup: failed to persist size-cap skip state: %v\n", err) + } + + if announce { + fmt.Fprintf(os.Stderr, + "Warning: auto-backup PAUSED — destination %s has reached %s. "+ + "No further syncs will run until you raise backup.size-cap-mb "+ + "or switch to a fresh destination with `bd backup init `.\n", + dir, formatBytes(size)) + debug.Logf("backup: size cap exceeded (%s), auto-backup paused\n", formatBytes(size)) + return + } + debug.Logf("backup: size cap exceeded (%s), auto-backup paused (warning throttled)\n", formatBytes(size)) +} + +// getDirSize calculates the total size of a directory recursively. +func getDirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} diff --git a/cmd/bd/backup_cap_test.go b/cmd/bd/backup_cap_test.go new file mode 100644 index 000000000..d44ac0438 --- /dev/null +++ b/cmd/bd/backup_cap_test.go @@ -0,0 +1,824 @@ +package main + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/config" +) + +// TestBackupSizeCapExceeded pins the threshold check itself against a real +// directory (no stubbing needed — getDirSize is a pure filesystem read and +// formatBytes a pure formatter). +func TestBackupSizeCapExceeded(t *testing.T) { + tests := []struct { + name string + fileBytes int + capMB string // config value; "" = use default (2048) + wantExceeded bool + }{ + { + name: "tiny dir, default cap → not exceeded", + fileBytes: 1024, + wantExceeded: false, + }, + { + name: "dir over a small explicit cap → exceeded", + fileBytes: 2 * 1024 * 1024, // 2MB + capMB: "1", + wantExceeded: true, + }, + { + name: "dir under a small explicit cap → not exceeded", + fileBytes: 1024, + capMB: "1", + wantExceeded: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if tt.capMB != "" { + t.Setenv("BD_BACKUP_SIZE_CAP_MB", tt.capMB) + } else { + os.Unsetenv("BD_BACKUP_SIZE_CAP_MB") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_SIZE_CAP_MB") }) + } + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "data"), make([]byte, tt.fileBytes), 0o600); err != nil { + t.Fatal(err) + } + + exceeded, size, err := backupSizeCapExceeded(dir) + if err != nil { + t.Fatalf("backupSizeCapExceeded: %v", err) + } + if exceeded != tt.wantExceeded { + t.Errorf("exceeded = %v (size=%d), want %v", exceeded, size, tt.wantExceeded) + } + }) + } +} + +// TestPauseAutoBackupForSizeCap_WarnThrottle pins the warning throttle: the +// stderr warning must not repeat on every single call once the cap is +// already known to be exceeded — only once per backup.size-warn-interval. +// Mirrors the throttle-persistence pattern already used for the backup +// interval itself (backup_export.go, wy-zrmqr). +// +// It also pins the half of the PR #6071 review fix that is independent of +// the warning: every call, throttled or not, re-arms the backup interval +// throttle (state.Timestamp) and persists it, so a paused destination pays +// the getDirSize walk at most once per backup.interval instead of once per +// bd command. +func TestPauseAutoBackupForSizeCap_WarnThrottle(t *testing.T) { + tests := []struct { + name string + lastWarnAt time.Time + warnInterval string // "" = use default (24h) + wantUpdated bool // whether LastCapWarnAt should advance + }{ + { + name: "never warned → warns now", + lastWarnAt: time.Time{}, + wantUpdated: true, + }, + { + name: "warned 1h ago, default 24h interval → throttled", + lastWarnAt: time.Now().UTC().Add(-1 * time.Hour), + wantUpdated: false, + }, + { + name: "warned 25h ago, default 24h interval → warns again", + lastWarnAt: time.Now().UTC().Add(-25 * time.Hour), + wantUpdated: true, + }, + { + name: "warned 1h ago, custom 30m interval → warns again", + lastWarnAt: time.Now().UTC().Add(-1 * time.Hour), + warnInterval: "30m", + wantUpdated: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if tt.warnInterval != "" { + t.Setenv("BD_BACKUP_SIZE_WARN_INTERVAL", tt.warnInterval) + } else { + os.Unsetenv("BD_BACKUP_SIZE_WARN_INTERVAL") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_SIZE_WARN_INTERVAL") }) + } + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + before := tt.lastWarnAt + state := &backupState{LastCapWarnAt: tt.lastWarnAt, LastDoltCommit: "deadbeef"} + + pauseAutoBackupForSizeCap(dir, state, 3*1024*1024*1024) + + updated := !state.LastCapWarnAt.Equal(before) + if updated != tt.wantUpdated { + t.Errorf("LastCapWarnAt updated = %v (before=%v after=%v), want %v", + updated, before, state.LastCapWarnAt, tt.wantUpdated) + } + + // The interval throttle is re-armed and persisted on EVERY + // skip, throttled warning or not — that is what stops a + // paused destination from walking on every bd command. + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if st.Timestamp.IsZero() { + t.Error("timestamp not persisted to backup_state.json: the interval throttle never re-arms, so the cap walk reruns on every command") + } + if st.LastDoltCommit != "deadbeef" { + t.Errorf("last_dolt_commit = %q, want it left untouched so change detection still sees pending work", st.LastDoltCommit) + } + if tt.wantUpdated { + // Persisted state must reflect the new warning time too. + if st.LastCapWarnAt.IsZero() { + t.Error("last_cap_warn_at not persisted to backup_state.json") + } + } + }) + } +} + +// TestMaybeAutoBackup_SkipsWhenCapExceeded is the wiring test: a backup +// destination already over the size cap must never attempt a sync at all +// (runDoltGCCommand-equivalent risk avoided entirely — there is nothing to +// stub here because the whole point is that BackupDatabase must NOT be +// called once capped). +func TestMaybeAutoBackup_SkipsWhenCapExceeded(t *testing.T) { + // Isolate CWD/BEADS_DIR: unlike runBackupExport (used by the other + // tests in this file), maybeAutoBackup also calls + // clientServerShareFilesystem → beads.FindBeadsDir before ever + // reaching backupDir(), so an unisolated CWD could walk up into this + // repo's own real .beads/ directory (be-yjp4z; see backup_auto_test.go). + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // Push the destination over the 1MB cap before any backup attempt. + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + + oldStore := store + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called once the size cap is exceeded, got %d calls", fake.backupCalls) + } +} + +// TestBackupSizeCapExceeded_DisabledWithZero pins the PR #6071 review fix: +// backup.size-cap-mb: 0 must mean "no cap", not "use the legacy 2048MB +// default" — an operator with a legitimately larger destination has no off +// switch otherwise. Uses a sparse file (Truncate, not a real write) to +// cross the legacy 2048MB threshold without allocating 2GB of real disk or +// memory. +func TestBackupSizeCapExceeded_DisabledWithZero(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "0") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + f, err := os.Create(filepath.Join(dir, "sparse-filler")) + if err != nil { + t.Fatal(err) + } + const overLegacyDefault = int64(2049) * 1024 * 1024 // just over the old 2048MB fallback + if err := f.Truncate(overLegacyDefault); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + exceeded, _, err := backupSizeCapExceeded(dir) + if err != nil { + t.Fatalf("backupSizeCapExceeded: %v", err) + } + if exceeded { + t.Error("exceeded = true with backup.size-cap-mb=0, want false (cap disabled)") + } +} + +// TestPauseAutoBackupForSizeCap_RemediationAdvice pins the PR #6071 review +// fix: the warning must not tell operators to delete the backup directory +// — nothing confirms the destination is cleanly recreated by the next +// sync, and Dolt's server-side backup remote stays registered against that +// path. It should point at the safe levers instead: raising the cap, or a +// fresh destination via `bd backup init`. +func TestPauseAutoBackupForSizeCap_RemediationAdvice(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + state := &backupState{} + + stderr := captureStderr(t, func() { + pauseAutoBackupForSizeCap(dir, state, 3*1024*1024*1024) + }) + + if strings.Contains(stderr, "delete") { + t.Errorf("warning suggests deleting the backup directory (unsafe — see PR #6071 review): %q", stderr) + } + if !strings.Contains(stderr, "backup.size-cap-mb") { + t.Errorf("warning missing backup.size-cap-mb pointer: %q", stderr) + } + if !strings.Contains(stderr, "bd backup init") { + t.Errorf("warning missing `bd backup init ` pointer: %q", stderr) + } +} + +// TestPauseAutoBackupForSizeCap_PersistFailureIsNonFatal pins the contract +// the whole pause path rests on: it persists the throttle by writing +// backup_state.json INTO the directory it just declared over-cap, and in +// the disk-full case this cap exists for that write fails. The skip and +// its operator warning must still happen — a failure to record the +// throttle must never block the already-decided skip, nor escape. +func TestPauseAutoBackupForSizeCap_PersistFailureIsNonFatal(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; chmod does not deny writes") + } + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + // Make the destination read-only so saveBackupState can never persist + // the throttle timestamp — the same failure mode as the disk-full case + // this cap exists for. + if err := os.Chmod(dir, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) // let t.TempDir() clean up + + state := &backupState{} + stderr := captureStderr(t, func() { + pauseAutoBackupForSizeCap(dir, state, 3*1024*1024*1024) + }) + if !strings.Contains(stderr, "PAUSED") { + t.Fatalf("expected a PAUSED warning even when the throttle cannot be persisted, got %q", stderr) + } + // The in-memory state still carries the re-armed throttle, so the rest + // of this process behaves as though it had been recorded. + if state.Timestamp.IsZero() { + t.Error("interval throttle not re-armed in memory when persistence failed") + } + if state.LastCapWarnAt.IsZero() { + t.Error("warn timestamp not recorded in memory when persistence failed") + } +} + +// TestMaybeAutoBackup_CapCheckSkippedWhenThrottled pins the ga-y6gjv PR +// review's performance fix: the size-cap directory walk must not run when +// the interval throttle would already skip the backup — the reviewer +// measured 65-100ms per bd invocation at 20k files in the backup dir if +// the cap check runs unconditionally before the throttle. Detected +// indirectly: if the cap check ran, it would find the destination over +// cap and persist LastCapWarnAt via pauseAutoBackupForSizeCap; if the +// interval throttle short-circuits first (as it must), LastCapWarnAt +// stays exactly as pre-seeded (zero). The cap check now sits after change +// detection too, so the store reports a commit that differs from the +// seeded watermark: otherwise change detection alone would stop the walk +// and this test would pass without the throttle. +func TestMaybeAutoBackup_CapCheckSkippedWhenThrottled(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // Push the destination over the 1MB cap... + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + // ...but seed a fresh backup timestamp so the interval throttle (15m + // default) fires first, before the cap check ever gets a chance to run. + seeded := &backupState{Timestamp: time.Now().UTC(), LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + // Different commit from the watermark ⇒ change detection would pass, + // so only the interval throttle can keep the walk from running. + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called while throttled, got %d calls", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.LastCapWarnAt.IsZero() { + t.Error("LastCapWarnAt was set — size-cap check ran despite the interval throttle, want it skipped entirely") + } +} + +// TestMaybeAutoBackup_CapCheckSkippedWhenUnchanged pins the first half of +// the PR #6071 review's gating finding: the size-cap walk must sit AFTER +// change detection, so an IDLE workspace — nothing new committed since the +// last backup — never pays it. +// +// The interval throttle alone cannot cover this case, because only a +// backup attempt advances state.Timestamp: an idle workspace never reaches +// runBackupExport, so the throttle can never re-arm and, with the walk +// ahead of change detection, every single bd command paid the full +// filepath.Walk indefinitely. +// +// Detected the same way as the throttle test above: if the cap check ran, +// it would find the destination over cap and record LastCapWarnAt. +func TestMaybeAutoBackup_CapCheckSkippedWhenUnchanged(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // Push the destination over the 1MB cap, so any walk that runs finds + // it exceeded and leaves a mark. + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + // Last backup an hour ago: the 15m interval throttle has passed, so + // change detection — not the throttle — is what must stop this. + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "deadbeef"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + want, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + + oldStore := store + // Same commit as the recorded watermark ⇒ nothing changed ⇒ idle. + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called when nothing changed, got %d calls", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.LastCapWarnAt.IsZero() { + t.Error("LastCapWarnAt was set — the size-cap walk ran on an idle workspace, want it skipped entirely (it would then run on every bd command, forever)") + } + if !st.Timestamp.Equal(want.Timestamp) { + t.Errorf("timestamp = %v, want it untouched at %v: the idle path must not write state at all, or an idle period would cost up to one backup.interval of extra latency", + st.Timestamp, want.Timestamp) + } +} + +// TestMaybeAutoBackup_PausedSkipArmsIntervalThrottle pins the second half +// of the PR #6071 review's gating finding, and is the positive control for +// the idle test above: with data genuinely changed, the cap walk DOES run +// and finds the destination over cap — and that skip must re-arm the +// interval throttle it would otherwise never reach. +// +// PAUSED is the pathological state: an over-cap destination is by +// definition the largest one, so without this the feature's terminal state +// is its most expensive, paying the full walk on every bd command until an +// operator intervenes. LastDoltCommit stays untouched so change detection +// still sees the pending work once the cap is raised. +func TestMaybeAutoBackup_PausedSkipArmsIntervalThrottle(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + seededAt := time.Now().UTC().Add(-time.Hour) + seeded := &backupState{Timestamp: seededAt, LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + // Different commit from the watermark ⇒ data changed ⇒ the cap check + // is reached. + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called once the size cap is exceeded, got %d calls", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if st.LastCapWarnAt.IsZero() { + t.Fatal("LastCapWarnAt not set — the cap path never ran, so this test proves nothing about the skip") + } + if !st.Timestamp.After(seededAt) { + t.Errorf("timestamp = %v, want it advanced past %v: the cap skip must re-arm the interval throttle or the walk reruns on every bd command", + st.Timestamp, seededAt) + } + if st.LastDoltCommit != "oldcommit" { + t.Errorf("last_dolt_commit = %q, want %q left untouched so change detection still sees pending work once the cap is raised", + st.LastDoltCommit, "oldcommit") + } +} + +// TestWarnBackupSizeCapUnavailable_IsOperatorVisible pins the PR #6071 +// review's error-handling minor: getDirSize aborts its whole walk on the +// first unreadable entry, and maybeAutoBackup then proceeds UNCAPPED. That +// is the right direction (fail open), but it used to happen behind a +// debug-only line, so the cap could silently disable itself and restore +// the unbounded growth the feature exists to prevent. The operator must be +// told, on stderr, both that the measurement failed and what it costs. +func TestWarnBackupSizeCapUnavailable_IsOperatorVisible(t *testing.T) { + stderr := captureStderr(t, func() { + warnBackupSizeCapUnavailable(errors.New("permission denied")) + }) + + if !strings.Contains(stderr, "permission denied") { + t.Errorf("warning does not name the underlying error: %q", stderr) + } + if !strings.Contains(stderr, "cap") { + t.Errorf("warning does not say the size cap is what failed: %q", stderr) + } + if !strings.Contains(stderr, "unbounded") { + t.Errorf("warning does not state the consequence — the backup proceeds uncapped: %q", stderr) + } +} + +// TestSizeCapStatus_UnavailableOnWalkError pins the PR #6071 review's +// render-asymmetry nit: on a non-ENOENT getDirSize error the human status +// path used to drop the "Size cap:" line entirely — indistinguishable from +// "no cap configured" for an operator debugging a permission problem — +// while --json reported the error all along. The JSON error object must +// also carry an explicit null `exceeded`, because a consumer reading a +// missing key as false sees "not exceeded" during exactly the failure that +// makes it unmeasurable. +func TestSizeCapStatus_UnavailableOnWalkError(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; chmod does not deny reads") + } + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + // Unreadable, but present: getDirSize fails with a permission error, + // which is NOT os.IsNotExist, so it takes the error branch rather than + // the "destination not created yet" branch. + if err := os.Chmod(dir, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) // let t.TempDir() clean up + + stdout := captureStdout(t, func() error { + showSizeCapStatus(dir) + return nil + }) + if !strings.Contains(stdout, "Size cap: unavailable") { + t.Errorf("human status dropped the size-cap line on a walk error, want an explicit `unavailable`: %q", stdout) + } + + got := showSizeCapStatusJSON(dir) + if got["error"] == nil || got["error"] == "" { + t.Errorf("JSON status omits the walk error: %#v", got) + } + exceeded, ok := got["exceeded"] + if !ok { + t.Errorf("JSON error object omits `exceeded`; a consumer reading the missing key as false sees `not exceeded`: %#v", got) + } + if exceeded != nil { + t.Errorf("exceeded = %#v, want an explicit null while the size is unmeasurable", exceeded) + } +} + +// postSyncCommitFailingStore syncs successfully but then fails the +// GetCurrentCommit that runBackupExport needs to record its watermark — one +// of the runBackupExport exits that returns without persisting state. Every +// other GetCurrentCommit (change detection) succeeds. +type postSyncCommitFailingStore struct { + failingBackupStore + justSynced bool +} + +func (f *postSyncCommitFailingStore) BackupDatabase(ctx context.Context, dir string) error { + f.justSynced = true + return f.failingBackupStore.BackupDatabase(ctx, dir) +} + +func (f *postSyncCommitFailingStore) GetCurrentCommit(context.Context) (string, error) { + if f.justSynced { + f.justSynced = false + return "", errors.New("dolt server unavailable") + } + return f.commit, nil +} + +// TestMaybeAutoBackup_WalkErrorArmsIntervalThrottle pins the PR #6071 +// iteration-2 review's behavioral minor: when the size-cap walk fails, +// maybeAutoBackup warns and proceeds uncapped, and it must re-arm the +// interval throttle itself before doing so. runBackupExport does not persist +// state.Timestamp on every exit — here the post-sync GetCurrentCommit fails +// after a successful sync — so without the re-arm the walk, its warning and +// the sync itself all repeat on every bd command instead of once per +// backup.interval. +func TestMaybeAutoBackup_WalkErrorArmsIntervalThrottle(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; chmod does not deny reads") + } + t.Chdir(t.TempDir()) + // The sync has to actually run for this test to reach runBackupExport's + // non-persisting exit, so give the command a real workspace: a + // workspace-scoped step on the way (a backup lock, for one) must not + // skip the sync before it starts. + prepareBackupStatusTest(t) + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // One unreadable entry inside an otherwise writable destination: the + // walk fails, but backup_state.json can still be read and written. + locked := filepath.Join(dir, "locked") + if err := os.Mkdir(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) // let t.TempDir() clean up + seededAt := time.Now().UTC().Add(-time.Hour) + seeded := &backupState{Timestamp: seededAt, LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + // Different commit from the watermark ⇒ data changed ⇒ the walk runs. + fake := &postSyncCommitFailingStore{failingBackupStore: failingBackupStore{commit: "deadbeef"}} + store = fake + t.Cleanup(func() { store = oldStore }) + + first := captureStderr(t, func() { maybeAutoBackup(context.Background()) }) + if !strings.Contains(first, "could not be measured") { + t.Fatalf("the walk error never reached its warning, so this test proves nothing: %q", first) + } + if fake.backupCalls != 1 { + t.Fatalf("BackupDatabase calls = %d, want 1: an unmeasurable destination must fail open and still sync", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.Timestamp.After(seededAt) { + t.Errorf("timestamp = %v, want it advanced past %v: runBackupExport returned without persisting it, so the walk-error path must re-arm the interval throttle", + st.Timestamp, seededAt) + } + if st.LastDoltCommit != "oldcommit" { + t.Errorf("last_dolt_commit = %q, want %q left untouched: no watermark was recorded, so change detection must still see the pending work", + st.LastDoltCommit, "oldcommit") + } + + second := captureStderr(t, func() { maybeAutoBackup(context.Background()) }) + if strings.Contains(second, "could not be measured") { + t.Errorf("the walk-error warning repeated on the very next command, want it bounded by the interval throttle: %q", second) + } + if fake.backupCalls != 1 { + t.Errorf("BackupDatabase calls = %d after a second command, want still 1: the interval throttle should have stopped it", fake.backupCalls) + } +} + +// TestShowSizeCapStatus_DisabledEchoesConfiguredValue pins the PR #6071 +// iteration-2 review's debuggability minor: viper's GetInt resolves an +// unparseable backup.size-cap-mb ("2048MB") to 0, which disables the cap, +// so the status line must echo what is actually configured rather than +// asserting "backup.size-cap-mb=0", a value the operator never wrote. +func TestShowSizeCapStatus_DisabledEchoesConfiguredValue(t *testing.T) { + for _, raw := range []string{"0", "2048MB"} { + t.Run(raw, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", raw) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + if got := effectiveSizeCapMB(); got != 0 { + t.Fatalf("effectiveSizeCapMB() = %d for %q, want 0 (cap disabled), or this test proves nothing", got, raw) + } + + stdout := captureStdout(t, func() error { + showSizeCapStatus(t.TempDir()) + return nil + }) + want := "Size cap: disabled (backup.size-cap-mb=" + raw + ")" + if !strings.Contains(stdout, want) { + t.Errorf("status = %q, want it to contain %q", stdout, want) + } + }) + } +} + +// TestSizeCapStatus_MidWalkNotExistIsUnavailable pins the PR #6071 +// iteration-2 review's error-handling minor: only a destination that does +// not exist yet may read as empty. An ENOENT from inside the walk — a chunk +// file removed between listing and stat-ing it — used to be swallowed as +// size 0, reporting `"exceeded": false` for a destination that may be well +// over its cap, which is exactly what the explicit-null `exceeded` on the +// error branch exists to prevent. +func TestSizeCapStatus_MidWalkNotExistIsUnavailable(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + t.Run("missing destination reads as empty", func(t *testing.T) { + missing := filepath.Join(t.TempDir(), "not-created-yet") + + stdout := captureStdout(t, func() error { + showSizeCapStatus(missing) + return nil + }) + if !strings.Contains(stdout, "Size cap: 0 B / ") { + t.Errorf("status for a not-yet-created destination = %q, want an empty `Size cap: 0 B / ...`", stdout) + } + got := showSizeCapStatusJSON(missing) + if got["exceeded"] != false || got["current_bytes"] != int64(0) { + t.Errorf("JSON status for a not-yet-created destination = %#v, want current_bytes 0 and exceeded false", got) + } + }) + + t.Run("ENOENT inside an existing destination is unavailable", func(t *testing.T) { + dir := t.TempDir() + vanished := &fs.PathError{Op: "lstat", Path: filepath.Join(dir, "chunk"), Err: fs.ErrNotExist} + if !os.IsNotExist(vanished) { + t.Fatalf("injected error is not an os.IsNotExist error, so this test proves nothing: %v", vanished) + } + oldWalk := walkBackupDestination + walkBackupDestination = func(string) (int64, error) { return 0, vanished } + t.Cleanup(func() { walkBackupDestination = oldWalk }) + + stdout := captureStdout(t, func() error { + showSizeCapStatus(dir) + return nil + }) + if !strings.Contains(stdout, "Size cap: unavailable") { + t.Errorf("status after a mid-walk ENOENT = %q, want `Size cap: unavailable`, not a size", stdout) + } + got := showSizeCapStatusJSON(dir) + exceeded, ok := got["exceeded"] + if !ok || exceeded != nil { + t.Errorf("JSON status after a mid-walk ENOENT = %#v, want an explicit null `exceeded`", got) + } + if got["error"] == nil || got["error"] == "" { + t.Errorf("JSON status omits the walk error: %#v", got) + } + }) +} diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index 654ba7c47..1c6924914 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -19,6 +19,7 @@ import ( type backupState struct { LastDoltCommit string `json:"last_dolt_commit"` Timestamp time.Time `json:"timestamp"` + LastCapWarnAt time.Time `json:"last_cap_warn_at"` } // backupDir returns the backup directory path, creating it if needed. diff --git a/cmd/bd/backup_status_test.go b/cmd/bd/backup_status_test.go index d06d0db78..513603cc8 100644 --- a/cmd/bd/backup_status_test.go +++ b/cmd/bd/backup_status_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/configfile" @@ -160,6 +161,135 @@ func TestBackupStatusIncludesAvailableDatabaseSize(t *testing.T) { }) } +// TestBackupStatusShowsSizeCapPaused pins the PR #6071 review's second +// major point: `bd --json` / `--quiet` callers never saw the size-cap +// pause — the warning is stderr-only and throttled, and status previously +// said nothing about the cap, so an agent/CI caller saw a reassuring +// "Last backup" line while auto-backup was silently, permanently paused. +func TestBackupStatusShowsSizeCapPaused(t *testing.T) { + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + prepareBackupStatusTest(t) + sizeDatabase := func(context.Context) (int64, bool, error) { return 0, false, nil } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "deadbeef"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + t.Run("json", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status", "--json"}) + stdout, stderr, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + if stderr != "" { + t.Errorf("stderr = %q, want empty", stderr) + } + var response struct { + SizeCap struct { + Enabled bool `json:"enabled"` + CapMB int `json:"cap_mb"` + CurrentBytes int64 `json:"current_bytes"` + Exceeded bool `json:"exceeded"` + } `json:"size_cap"` + } + if err := json.Unmarshal([]byte(stdout), &response); err != nil { + t.Fatalf("stdout is not valid JSON: %v\nstdout: %s", err, stdout) + } + if !response.SizeCap.Enabled { + t.Errorf("size_cap.enabled = false, want true: %s", stdout) + } + if !response.SizeCap.Exceeded { + t.Errorf("size_cap.exceeded = false, want true: %s", stdout) + } + if response.SizeCap.CapMB != 1 { + t.Errorf("size_cap.cap_mb = %d, want 1: %s", response.SizeCap.CapMB, stdout) + } + if response.SizeCap.CurrentBytes < 2*1024*1024 { + t.Errorf("size_cap.current_bytes = %d, want >= 2MB: %s", response.SizeCap.CurrentBytes, stdout) + } + }) + + t.Run("human", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status"}) + stdout, stderr, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + if stderr != "" { + t.Errorf("stderr = %q, want empty", stderr) + } + if !strings.Contains(stdout, "PAUSED (cap exceeded)") { + t.Errorf("status text missing PAUSED indicator: %s", stdout) + } + }) +} + +// TestBackupStatusShowsSizeCapDisabled pins the PR #6071 review's first +// major point from the status side: backup.size-cap-mb: 0 must render as +// disabled in status, not silently report the legacy 2048MB default. +func TestBackupStatusShowsSizeCapDisabled(t *testing.T) { + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "0") + prepareBackupStatusTest(t) + sizeDatabase := func(context.Context) (int64, bool, error) { return 0, false, nil } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "deadbeef"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + t.Run("json", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status", "--json"}) + stdout, _, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(stdout), &raw); err != nil { + t.Fatalf("stdout is not valid JSON: %v\nstdout: %s", err, stdout) + } + sizeCapRaw, ok := raw["size_cap"] + if !ok { + t.Fatalf("size_cap key missing from status JSON: %s", stdout) + } + var sizeCap struct { + Enabled bool `json:"enabled"` + } + if err := json.Unmarshal(sizeCapRaw, &sizeCap); err != nil { + t.Fatalf("size_cap is not valid JSON: %v\nraw: %s", err, sizeCapRaw) + } + if sizeCap.Enabled { + t.Errorf("size_cap.enabled = true with backup.size-cap-mb=0, want false: %s", stdout) + } + }) + + t.Run("human", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status"}) + stdout, _, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + if !strings.Contains(stdout, "Size cap: disabled") { + t.Errorf("status text missing disabled size-cap line: %s", stdout) + } + }) +} + // TestBackupStatusProxiedGuardFollowsLocality replaces a test that asserted a // blanket proxied refusal. Since slice S3 the guard is scoped to locality: bd // owns the dolt process on a managed-local workspace and can honor the command diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 20d9e001e..33803808c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -5,7 +5,7 @@ description: Complete reference for bd configuration across config.yaml and data Complete configuration reference for beads. -Last reviewed: 2026-08-28 +Last reviewed: 2026-09-10 Freshness source: `cmd/bd/main.go`, `cmd/bd/config.go`, and `internal/configfile/`. @@ -126,6 +126,8 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `backup.interval` | — | `BD_BACKUP_INTERVAL` | `15m` | Minimum time between auto-backups | | `backup.git-push` | — | — | `false` | Auto-push backup repo | | `backup.git-repo` | — | `BD_BACKUP_GIT_REPO` | (none) | Backup git repo URL; when set, backups go to a `backup/` directory inside that repo | +| `backup.size-cap-mb` | — | `BD_BACKUP_SIZE_CAP_MB` | `2048` | Pause auto-backup once the destination directory reaches this size; `0` disables the cap (see [below](#auto-backup)) | +| `backup.size-warn-interval` | — | `BD_BACKUP_SIZE_WARN_INTERVAL` | `24h` | Minimum time between repeated "auto-backup paused" warnings | | `audit.enabled` | — | `BD_AUDIT_ENABLED` | `false` | Enable the optional JSONL interaction sidecar at `.beads/interactions.jsonl`, written by `bd audit record` / `bd audit label`. While disabled, `bd init` does not create the file and `bd audit record` / `bd audit label` refuse to write. Issue history is always recorded in the database either way — see `bd history --events` | | `export.auto` | — | — | `false` | Refresh `.beads/issues.jsonl` export after every write; not cross-machine sync | | `export.path` | — | — | `issues.jsonl` | Output filename relative to `.beads/` | @@ -224,15 +226,19 @@ Periodic Dolt-native backup to `.beads/backup/` provides a recovery path indepen ```yaml backup: - enabled: true # Enable auto-backup after write commands - interval: 15m # Minimum time between auto-backups + enabled: true # Enable auto-backup after write commands + interval: 15m # Minimum time between auto-backups + size-cap-mb: 2048 # Pause auto-backup once the destination reaches this size; 0 disables the cap + size-warn-interval: 24h # Minimum time between repeated "paused" warnings ``` How it works: - After each write command, `bd` compares the Dolt HEAD commit hash against the last backup state. -- If data changed and the throttle interval has passed, a Dolt-native backup is synced to `.beads/backup/` (or to a `backup/` directory inside `backup.git-repo` when configured). -- State is tracked in `backup_state.json` inside the backup directory. +- If data changed and the throttle interval has passed, `bd` checks the destination directory's on-disk size against `backup.size-cap-mb`. `CALL DOLT_BACKUP('sync', ...)` only ever adds new chunks to the destination — it never prunes ones that became unreachable on the source (history rewrites, superseded data), and Dolt exposes no supported way to GC a backup destination in place. Left uncapped, the destination can only grow until disk fills; this was the root cause of a 2026-06-19 outage where a 1.7GB store produced a 43GB backup directory. Once the cap is reached, auto-backup pauses — nothing is deleted, but no further syncs run until you raise `backup.size-cap-mb` or point to a fresh destination with `bd backup init `. Set `size-cap-mb: 0` to disable the cap entirely. +- If the destination is under its cap, a Dolt-native backup is synced to `.beads/backup/` (or to a `backup/` directory inside `backup.git-repo` when configured). +- State is tracked in `backup_state.json` inside the backup directory. `bd backup status` prints `PAUSED (cap exceeded)` when the destination is over its cap, and its `--json` output reports the same state as `size_cap.exceeded: true` — so an agent/CI caller relying on `--json` or `--quiet` can see that auto-backup has stopped instead of reading a reassuring "Last backup" line while nothing further syncs. +- The cap applies only to auto-backup; manual `bd backup` / `bd backup sync` are not capped. Manual commands (see [bd backup](/cli-reference/backup)): diff --git a/internal/config/config.go b/internal/config/config.go index 7a209329b..6095bd983 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -341,6 +341,17 @@ func Initialize() error { // Backup configuration defaults (JSONL export to .beads/backup/) v.SetDefault("backup.enabled", false) v.SetDefault("backup.interval", "15m") + // size-cap-mb pauses auto-backup once the destination exceeds this size + // (ga-y6gjv). BackupSync only ever transfers new chunks — it never + // prunes ones that became unreachable on the source DB — and Dolt + // exposes no way to GC a backup destination in place (it is a bare + // chunk-store directory with no .dolt repo-root marker, so neither the + // standalone dolt CLI nor CALL DOLT_GC can operate on it directly), so + // without a hard cap the directory can only grow forever. Default + // 2048MB; size-warn-interval throttles how often the pause is + // re-announced once the cap is hit, so it doesn't spam every command. + v.SetDefault("backup.size-cap-mb", 2048) + v.SetDefault("backup.size-warn-interval", "24h") v.SetDefault("backup.git-push", false) v.SetDefault("backup.git-repo", "") diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 345457a67..50c2f436a 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -80,10 +80,12 @@ var YamlOnlyKeys = map[string]bool{ "hierarchy.max-depth": true, // Backup settings (must be in yaml so GetValueSource can detect overrides) - "backup.enabled": true, - "backup.interval": true, - "backup.git-push": true, - "backup.git-repo": true, + "backup.enabled": true, + "backup.interval": true, + "backup.git-push": true, + "backup.git-repo": true, + "backup.size-cap-mb": true, + "backup.size-warn-interval": true, // Import settings "import.auto": true,