diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 461a4f757..64b5f5d10 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -29,7 +29,17 @@ Commands: DoltHub is recommended for cloud backup: bd backup init https://doltremoteapi.dolthub.com// - Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication.`, + Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication. + +Auto-backup default: + When backup.enabled is unset, auto-backup turns ON in embedded mode if a + git remote exists, and stays OFF in sql-server / shared-server mode. In + server mode many bd clients share one Dolt server, and each would register + a server-side backup remote under the same name pointing at its own local + dir and full-sync the whole database — a self-amplifying storm. To back up + a shared server, run 'bd backup' explicitly (or set backup.enabled=true and + coordinate destinations). 'bd config get backup.enabled' shows the effective + value and its source.`, GroupID: "sync", } diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index d94dd8898..0cfd05a1e 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -14,11 +14,26 @@ import ( // isBackupAutoEnabled returns whether backup should run. // If user explicitly configured backup.enabled, use that. -// Otherwise, auto-enable when a git remote exists. +// Otherwise auto-enable when a git remote exists — BUT only in +// embedded mode. +// +// In sql-server / shared-server mode (usesSQLServer()) the default is +// OFF: N bd clients share a single Dolt server, and the Dolt-native +// backup path (store.BackupDatabase) registers a server-side backup +// remote under one fixed name pointing at THIS client's local +// .beads/backup dir, then full-syncs the whole DB. With many clients +// that means racing remove/add of the same name plus every client +// full-syncing the entire history into its own dir — the amplifier +// behind the 2026-07 shared-dolt CPU-pin incident. Operators who want +// backups in server mode must opt in explicitly (backup.enabled=true +// / BD_BACKUP_ENABLED=1) and coordinate destinations themselves. func isBackupAutoEnabled() bool { if config.GetValueSource("backup.enabled") != config.SourceDefault { return config.GetBool("backup.enabled") } + if usesSQLServer() { + return false + } return primeHasGitRemote() } diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 9ed8415da..f1c161d63 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -14,10 +14,11 @@ func TestIsBackupAutoEnabled(t *testing.T) { // Cannot be parallel: modifies global primeHasGitRemote and env vars. tests := []struct { - name string - envVal string // "\x00" = not set, "" = set to empty, "true"/"false"/"0" = explicit - hasRemote bool - wantResult bool + name string + envVal string // "\x00" = not set, "" = set to empty, "true"/"false"/"0" = explicit + hasRemote bool + sharedServer bool // BEADS_DOLT_SHARED_SERVER=1 → usesSQLServer() true + wantResult bool }{ { name: "default + git remote → enabled", @@ -55,6 +56,25 @@ func TestIsBackupAutoEnabled(t *testing.T) { hasRemote: true, wantResult: false, }, + { + // wy-zrmqr: unset default must NOT auto-enable in sql-server + // mode even with a git remote — N clients racing one backup + // name was the storm amplifier in the 2026-07 CPU-pin incident. + name: "default + git remote + sql-server mode → disabled", + envVal: "\x00", + hasRemote: true, + sharedServer: true, + wantResult: false, + }, + { + // Explicit opt-in still honored in sql-server mode: operators + // who coordinate destinations themselves may turn it on. + name: "explicit true + sql-server mode → enabled", + envVal: "true", + hasRemote: false, + sharedServer: true, + wantResult: true, + }, } for _, tt := range tests { @@ -64,6 +84,13 @@ func TestIsBackupAutoEnabled(t *testing.T) { primeHasGitRemote = func() bool { return tt.hasRemote } t.Cleanup(func() { primeHasGitRemote = orig }) + if tt.sharedServer { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + } else { + os.Unsetenv("BEADS_DOLT_SHARED_SERVER") + t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") }) + } + // Set env var: "\x00" = unset, anything else = set to that value if tt.envVal == "\x00" { os.Unsetenv("BD_BACKUP_ENABLED") diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index b550f52d2..baeaa442d 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -140,6 +140,19 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } if err := bs.BackupDatabase(ctx, dir); err != nil { + // Persist the attempt time even on failure so the throttle + // interval (checked by maybeAutoBackup via state.Timestamp) + // applies to the next command. Without this, a sync that keeps + // failing — e.g. a slow/overloaded shared Dolt server — retries + // on EVERY bd command instead of once per interval, turning a + // transient slowdown into a self-amplifying storm (the 2026-07 + // shared-dolt CPU-pin incident). LastDoltCommit is deliberately + // left unchanged so change-detection still sees pending work and + // a real backup runs once the failure clears. + state.Timestamp = time.Now().UTC() + if saveErr := saveBackupState(dir, state); saveErr != nil { + debug.Logf("backup: failed to persist throttle state after error: %v\n", saveErr) + } return nil, err } diff --git a/cmd/bd/backup_export_failure_test.go b/cmd/bd/backup_export_failure_test.go new file mode 100644 index 000000000..174756bf8 --- /dev/null +++ b/cmd/bd/backup_export_failure_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/storage" +) + +// failingBackupStore is a minimal DoltStorage stand-in whose Dolt-native +// backup always fails. It embeds a nil DoltStorage for identity/type +// assertion and overrides only the methods runBackupExport touches. +type failingBackupStore struct { + storage.DoltStorage + commit string + backupErr error + backupCalls int +} + +func (f *failingBackupStore) GetCurrentCommit(context.Context) (string, error) { return f.commit, nil } + +func (f *failingBackupStore) BackupDatabase(context.Context, string) error { + f.backupCalls++ + return f.backupErr +} +func (f *failingBackupStore) BackupAdd(context.Context, string, string) error { return nil } +func (f *failingBackupStore) BackupSync(context.Context, string) error { return nil } +func (f *failingBackupStore) BackupRemove(context.Context, string) error { return nil } +func (f *failingBackupStore) RestoreDatabase(context.Context, string, bool) error { return nil } + +// TestRunBackupExport_PersistsThrottleOnFailure is the wy-zrmqr repro: a +// failing Dolt backup sync must still advance the throttle timestamp so the +// next bd command is throttled instead of retrying immediately. Before the +// fix, saveBackupState ran only on success, so a slow/failing shared Dolt +// server was hammered on EVERY subsequent command — the amplifier behind the +// 2026-07 CPU-pin incident. +func TestRunBackupExport_PersistsThrottleOnFailure(t *testing.T) { + // Point backupDir() at a temp git repo via backup.git-repo so we don't + // need a live beads workspace. + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + + // Sanity: no prior state, so the throttle would NOT block a first attempt. + if st, _ := loadBackupState(dir); !st.Timestamp.IsZero() { + t.Fatalf("precondition: expected zero timestamp, got %v", st.Timestamp) + } + + oldStore := store + fake := &failingBackupStore{commit: "deadbeef", backupErr: errors.New("sync to backup: server too busy")} + store = fake + t.Cleanup(func() { store = oldStore }) + + before := time.Now().UTC() + if _, err := runBackupExport(context.Background(), true); err == nil { + t.Fatal("expected runBackupExport to return the sync error, got nil") + } + if fake.backupCalls != 1 { + t.Fatalf("expected exactly 1 BackupDatabase call, got %d", fake.backupCalls) + } + + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if st.Timestamp.IsZero() { + t.Fatal("throttle timestamp not persisted on failure — next command would retry unthrottled (storm)") + } + if st.Timestamp.Before(before) { + t.Errorf("throttle timestamp %v predates the failed attempt %v", st.Timestamp, before) + } + // The recorded timestamp must be recent enough that the default 15m + // throttle window blocks the next attempt. + if time.Since(st.Timestamp) >= 15*time.Minute { + t.Errorf("persisted timestamp %v is already outside the throttle window", st.Timestamp) + } + // LastDoltCommit must stay empty: the backup did NOT succeed, so change + // detection must still see pending work once the failure clears. + if st.LastDoltCommit != "" { + t.Errorf("LastDoltCommit should remain empty after a failed backup, got %q", st.LastDoltCommit) + } +} diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 440e969b1..924db0631 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -246,6 +246,17 @@ var configGetCmd = &cobra.Command{ key := args[0] + if key == "backup.enabled" { + // backup.enabled has an auto-detected effective value that + // differs from the stored value: when unset it auto-enables + // in embedded mode with a git remote, and is forced OFF in + // sql-server mode (see isBackupAutoEnabled). Reporting the raw + // stored "false"/"not set" here misled operators during the + // 2026-07 shared-dolt incident, so show the EFFECTIVE value + // and its source. + return runConfigGetBackupEnabled() + } + if config.IsYamlOnlyKey(key) { // User-global keys (e.g. metrics.*) must be read from the user-global // config.yaml only — the same source the runtime uses for metrics @@ -343,6 +354,45 @@ var configGetCmd = &cobra.Command{ }, } +// runConfigGetBackupEnabled reports the EFFECTIVE value of +// backup.enabled together with its source, rather than the raw stored +// value. The stored value is misleading because isBackupAutoEnabled() +// derives the runtime value: unset → auto-enabled in embedded mode +// when a git remote exists, and forced OFF in sql-server mode. +func runConfigGetBackupEnabled() error { + const key = "backup.enabled" + source := config.GetValueSource(key) + effective := isBackupAutoEnabled() + + var sourceDesc string + switch source { + case config.SourceEnvVar: + sourceDesc = "env var" + case config.SourceConfigFile: + sourceDesc = "config.yaml" + default: // SourceDefault — value came from auto-detection + switch { + case usesSQLServer(): + sourceDesc = "default (auto: off in sql-server mode)" + case effective: + sourceDesc = "default (auto: on — git remote detected)" + default: + sourceDesc = "default (auto: off — no git remote)" + } + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "key": key, + "value": effective, + "effective": effective, + "source": string(source), + }) + } + fmt.Printf("%t (%s)\n", effective, sourceDesc) + return nil +} + var configListCmd = &cobra.Command{ Use: "list", Short: "List all configuration", diff --git a/cmd/bd/config_get_backup_enabled_test.go b/cmd/bd/config_get_backup_enabled_test.go new file mode 100644 index 000000000..724254dd6 --- /dev/null +++ b/cmd/bd/config_get_backup_enabled_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/config" +) + +// TestConfigGetBackupEnabled_EffectiveValue pins wy-zrmqr fix #3: `bd config +// get backup.enabled` must report the EFFECTIVE value (what isBackupAutoEnabled +// actually returns) plus its source, not the raw stored value. Before the fix +// it printed "false"/"not set" even while auto-backup was running via +// primeHasGitRemote — the mismatch that hid the storm from operators. +func TestConfigGetBackupEnabled_EffectiveValue(t *testing.T) { + tests := []struct { + name string + envVal string // "\x00" = unset + hasRemote bool + sharedServer bool + wantPrefix string // leading "true "/"false " + wantContains string // source annotation substring + }{ + { + name: "unset + no remote → off (no git remote)", + envVal: "\x00", + hasRemote: false, + wantPrefix: "false ", + wantContains: "no git remote", + }, + { + name: "unset + remote → on (git remote)", + envVal: "\x00", + hasRemote: true, + wantPrefix: "true ", + wantContains: "git remote detected", + }, + { + name: "unset + remote + sql-server → off (server mode)", + envVal: "\x00", + hasRemote: true, + sharedServer: true, + wantPrefix: "false ", + wantContains: "sql-server mode", + }, + { + name: "explicit true + sql-server → on (env var)", + envVal: "true", + hasRemote: false, + sharedServer: true, + wantPrefix: "true ", + wantContains: "env var", + }, + { + name: "explicit false → off (env var)", + envVal: "false", + hasRemote: true, + wantPrefix: "false ", + wantContains: "env var", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + orig := primeHasGitRemote + primeHasGitRemote = func() bool { return tt.hasRemote } + t.Cleanup(func() { primeHasGitRemote = orig }) + + if tt.sharedServer { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + } else { + os.Unsetenv("BEADS_DOLT_SHARED_SERVER") + t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") }) + } + + if tt.envVal == "\x00" { + os.Unsetenv("BD_BACKUP_ENABLED") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_ENABLED") }) + } else { + t.Setenv("BD_BACKUP_ENABLED", tt.envVal) + } + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + out := captureStdout(t, runConfigGetBackupEnabled) + + if !strings.HasPrefix(out, tt.wantPrefix) { + t.Errorf("output %q: want prefix %q", out, tt.wantPrefix) + } + if !strings.Contains(out, tt.wantContains) { + t.Errorf("output %q: want to contain %q", out, tt.wantContains) + } + }) + } +}