fix(backup): make auto-backup safe under a shared Dolt sql-server (wy-zrmqr)

Three defects turned a transient shared-Dolt slowdown into the 2026-07
CPU-pin incident, where ~31 bd clients on one sql-server hammered it with
full-DB backup syncs:

1. maybeAutoBackup throttle bypass on failure. runBackupExport only wrote
   backup_state.json (the throttle watermark) on SUCCESS, so a failing sync
   left Timestamp stale and retried on EVERY subsequent bd command — a
   self-amplifying storm. Now the attempt time is persisted on failure too
   (LastDoltCommit left unchanged so change-detection still runs a real
   backup once the failure clears).

2. Auto-backup on by default in sql-server mode. isBackupAutoEnabled
   auto-enabled whenever a git remote existed, even in shared-server mode
   where N clients race one server-side backup name and each full-syncs the
   whole DB into its own dir. Default is now OFF in sql-server /
   shared-server mode (usesSQLServer()); explicit backup.enabled=true still
   honored for operators who coordinate destinations.

3. `bd config get backup.enabled` printed the raw stored value ("false" /
   "not set") while the runtime auto-enabled — the mismatch that hid the
   storm. It now reports the EFFECTIVE value plus its source (env var /
   config.yaml / auto: git-remote / auto: off in sql-server mode).

Tests: server-mode + failure cases added to TestIsBackupAutoEnabled; new
TestRunBackupExport_PersistsThrottleOnFailure repro (failing sync advances
the throttle, does not clear LastDoltCommit); TestConfigGetBackupEnabled_
EffectiveValue pins the new get output. backup help documents the server
default. golangci-lint clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Steve Yegge 2026-07-14 17:16:03 -07:00
commit 5f685959cd
7 changed files with 324 additions and 6 deletions

View file

@ -29,7 +29,17 @@ Commands:
DoltHub is recommended for cloud backup:
bd backup init https://doltremoteapi.dolthub.com/<user>/<repo>
Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication.`,
Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication.
Auto-backup default:
When backup.enabled is unset, auto-backup turns ON in embedded mode if a
git remote exists, and stays OFF in sql-server / shared-server mode. In
server mode many bd clients share one Dolt server, and each would register
a server-side backup remote under the same name pointing at its own local
dir and full-sync the whole database — a self-amplifying storm. To back up
a shared server, run 'bd backup' explicitly (or set backup.enabled=true and
coordinate destinations). 'bd config get backup.enabled' shows the effective
value and its source.`,
GroupID: "sync",
}

View file

@ -14,11 +14,26 @@ import (
// isBackupAutoEnabled returns whether backup should run.
// If user explicitly configured backup.enabled, use that.
// Otherwise, auto-enable when a git remote exists.
// Otherwise auto-enable when a git remote exists — BUT only in
// embedded mode.
//
// In sql-server / shared-server mode (usesSQLServer()) the default is
// OFF: N bd clients share a single Dolt server, and the Dolt-native
// backup path (store.BackupDatabase) registers a server-side backup
// remote under one fixed name pointing at THIS client's local
// .beads/backup dir, then full-syncs the whole DB. With many clients
// that means racing remove/add of the same name plus every client
// full-syncing the entire history into its own dir — the amplifier
// behind the 2026-07 shared-dolt CPU-pin incident. Operators who want
// backups in server mode must opt in explicitly (backup.enabled=true
// / BD_BACKUP_ENABLED=1) and coordinate destinations themselves.
func isBackupAutoEnabled() bool {
if config.GetValueSource("backup.enabled") != config.SourceDefault {
return config.GetBool("backup.enabled")
}
if usesSQLServer() {
return false
}
return primeHasGitRemote()
}

View file

@ -14,10 +14,11 @@ func TestIsBackupAutoEnabled(t *testing.T) {
// Cannot be parallel: modifies global primeHasGitRemote and env vars.
tests := []struct {
name string
envVal string // "\x00" = not set, "" = set to empty, "true"/"false"/"0" = explicit
hasRemote bool
wantResult bool
name string
envVal string // "\x00" = not set, "" = set to empty, "true"/"false"/"0" = explicit
hasRemote bool
sharedServer bool // BEADS_DOLT_SHARED_SERVER=1 → usesSQLServer() true
wantResult bool
}{
{
name: "default + git remote → enabled",
@ -55,6 +56,25 @@ func TestIsBackupAutoEnabled(t *testing.T) {
hasRemote: true,
wantResult: false,
},
{
// wy-zrmqr: unset default must NOT auto-enable in sql-server
// mode even with a git remote — N clients racing one backup
// name was the storm amplifier in the 2026-07 CPU-pin incident.
name: "default + git remote + sql-server mode → disabled",
envVal: "\x00",
hasRemote: true,
sharedServer: true,
wantResult: false,
},
{
// Explicit opt-in still honored in sql-server mode: operators
// who coordinate destinations themselves may turn it on.
name: "explicit true + sql-server mode → enabled",
envVal: "true",
hasRemote: false,
sharedServer: true,
wantResult: true,
},
}
for _, tt := range tests {
@ -64,6 +84,13 @@ func TestIsBackupAutoEnabled(t *testing.T) {
primeHasGitRemote = func() bool { return tt.hasRemote }
t.Cleanup(func() { primeHasGitRemote = orig })
if tt.sharedServer {
t.Setenv("BEADS_DOLT_SHARED_SERVER", "1")
} else {
os.Unsetenv("BEADS_DOLT_SHARED_SERVER")
t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") })
}
// Set env var: "\x00" = unset, anything else = set to that value
if tt.envVal == "\x00" {
os.Unsetenv("BD_BACKUP_ENABLED")

View file

@ -140,6 +140,19 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) {
}
if err := bs.BackupDatabase(ctx, dir); err != nil {
// Persist the attempt time even on failure so the throttle
// interval (checked by maybeAutoBackup via state.Timestamp)
// applies to the next command. Without this, a sync that keeps
// failing — e.g. a slow/overloaded shared Dolt server — retries
// on EVERY bd command instead of once per interval, turning a
// transient slowdown into a self-amplifying storm (the 2026-07
// shared-dolt CPU-pin incident). LastDoltCommit is deliberately
// left unchanged so change-detection still sees pending work and
// a real backup runs once the failure clears.
state.Timestamp = time.Now().UTC()
if saveErr := saveBackupState(dir, state); saveErr != nil {
debug.Logf("backup: failed to persist throttle state after error: %v\n", saveErr)
}
return nil, err
}

View file

@ -0,0 +1,99 @@
package main
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/storage"
)
// failingBackupStore is a minimal DoltStorage stand-in whose Dolt-native
// backup always fails. It embeds a nil DoltStorage for identity/type
// assertion and overrides only the methods runBackupExport touches.
type failingBackupStore struct {
storage.DoltStorage
commit string
backupErr error
backupCalls int
}
func (f *failingBackupStore) GetCurrentCommit(context.Context) (string, error) { return f.commit, nil }
func (f *failingBackupStore) BackupDatabase(context.Context, string) error {
f.backupCalls++
return f.backupErr
}
func (f *failingBackupStore) BackupAdd(context.Context, string, string) error { return nil }
func (f *failingBackupStore) BackupSync(context.Context, string) error { return nil }
func (f *failingBackupStore) BackupRemove(context.Context, string) error { return nil }
func (f *failingBackupStore) RestoreDatabase(context.Context, string, bool) error { return nil }
// TestRunBackupExport_PersistsThrottleOnFailure is the wy-zrmqr repro: a
// failing Dolt backup sync must still advance the throttle timestamp so the
// next bd command is throttled instead of retrying immediately. Before the
// fix, saveBackupState ran only on success, so a slow/failing shared Dolt
// server was hammered on EVERY subsequent command — the amplifier behind the
// 2026-07 CPU-pin incident.
func TestRunBackupExport_PersistsThrottleOnFailure(t *testing.T) {
// Point backupDir() at a temp git repo via backup.git-repo so we don't
// need a live beads workspace.
repo := t.TempDir()
if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("BD_BACKUP_GIT_REPO", repo)
config.ResetForTesting()
t.Cleanup(config.ResetForTesting)
if err := config.Initialize(); err != nil {
t.Fatalf("config.Initialize: %v", err)
}
dir, err := backupDir()
if err != nil {
t.Fatalf("backupDir: %v", err)
}
// Sanity: no prior state, so the throttle would NOT block a first attempt.
if st, _ := loadBackupState(dir); !st.Timestamp.IsZero() {
t.Fatalf("precondition: expected zero timestamp, got %v", st.Timestamp)
}
oldStore := store
fake := &failingBackupStore{commit: "deadbeef", backupErr: errors.New("sync to backup: server too busy")}
store = fake
t.Cleanup(func() { store = oldStore })
before := time.Now().UTC()
if _, err := runBackupExport(context.Background(), true); err == nil {
t.Fatal("expected runBackupExport to return the sync error, got nil")
}
if fake.backupCalls != 1 {
t.Fatalf("expected exactly 1 BackupDatabase call, got %d", fake.backupCalls)
}
st, err := loadBackupState(dir)
if err != nil {
t.Fatalf("loadBackupState: %v", err)
}
if st.Timestamp.IsZero() {
t.Fatal("throttle timestamp not persisted on failure — next command would retry unthrottled (storm)")
}
if st.Timestamp.Before(before) {
t.Errorf("throttle timestamp %v predates the failed attempt %v", st.Timestamp, before)
}
// The recorded timestamp must be recent enough that the default 15m
// throttle window blocks the next attempt.
if time.Since(st.Timestamp) >= 15*time.Minute {
t.Errorf("persisted timestamp %v is already outside the throttle window", st.Timestamp)
}
// LastDoltCommit must stay empty: the backup did NOT succeed, so change
// detection must still see pending work once the failure clears.
if st.LastDoltCommit != "" {
t.Errorf("LastDoltCommit should remain empty after a failed backup, got %q", st.LastDoltCommit)
}
}

View file

@ -246,6 +246,17 @@ var configGetCmd = &cobra.Command{
key := args[0]
if key == "backup.enabled" {
// backup.enabled has an auto-detected effective value that
// differs from the stored value: when unset it auto-enables
// in embedded mode with a git remote, and is forced OFF in
// sql-server mode (see isBackupAutoEnabled). Reporting the raw
// stored "false"/"not set" here misled operators during the
// 2026-07 shared-dolt incident, so show the EFFECTIVE value
// and its source.
return runConfigGetBackupEnabled()
}
if config.IsYamlOnlyKey(key) {
// User-global keys (e.g. metrics.*) must be read from the user-global
// config.yaml only — the same source the runtime uses for metrics
@ -343,6 +354,45 @@ var configGetCmd = &cobra.Command{
},
}
// runConfigGetBackupEnabled reports the EFFECTIVE value of
// backup.enabled together with its source, rather than the raw stored
// value. The stored value is misleading because isBackupAutoEnabled()
// derives the runtime value: unset → auto-enabled in embedded mode
// when a git remote exists, and forced OFF in sql-server mode.
func runConfigGetBackupEnabled() error {
const key = "backup.enabled"
source := config.GetValueSource(key)
effective := isBackupAutoEnabled()
var sourceDesc string
switch source {
case config.SourceEnvVar:
sourceDesc = "env var"
case config.SourceConfigFile:
sourceDesc = "config.yaml"
default: // SourceDefault — value came from auto-detection
switch {
case usesSQLServer():
sourceDesc = "default (auto: off in sql-server mode)"
case effective:
sourceDesc = "default (auto: on — git remote detected)"
default:
sourceDesc = "default (auto: off — no git remote)"
}
}
if jsonOutput {
return outputJSON(map[string]interface{}{
"key": key,
"value": effective,
"effective": effective,
"source": string(source),
})
}
fmt.Printf("%t (%s)\n", effective, sourceDesc)
return nil
}
var configListCmd = &cobra.Command{
Use: "list",
Short: "List all configuration",

View file

@ -0,0 +1,104 @@
package main
import (
"os"
"strings"
"testing"
"github.com/steveyegge/beads/internal/config"
)
// TestConfigGetBackupEnabled_EffectiveValue pins wy-zrmqr fix #3: `bd config
// get backup.enabled` must report the EFFECTIVE value (what isBackupAutoEnabled
// actually returns) plus its source, not the raw stored value. Before the fix
// it printed "false"/"not set" even while auto-backup was running via
// primeHasGitRemote — the mismatch that hid the storm from operators.
func TestConfigGetBackupEnabled_EffectiveValue(t *testing.T) {
tests := []struct {
name string
envVal string // "\x00" = unset
hasRemote bool
sharedServer bool
wantPrefix string // leading "true "/"false "
wantContains string // source annotation substring
}{
{
name: "unset + no remote → off (no git remote)",
envVal: "\x00",
hasRemote: false,
wantPrefix: "false ",
wantContains: "no git remote",
},
{
name: "unset + remote → on (git remote)",
envVal: "\x00",
hasRemote: true,
wantPrefix: "true ",
wantContains: "git remote detected",
},
{
name: "unset + remote + sql-server → off (server mode)",
envVal: "\x00",
hasRemote: true,
sharedServer: true,
wantPrefix: "false ",
wantContains: "sql-server mode",
},
{
name: "explicit true + sql-server → on (env var)",
envVal: "true",
hasRemote: false,
sharedServer: true,
wantPrefix: "true ",
wantContains: "env var",
},
{
name: "explicit false → off (env var)",
envVal: "false",
hasRemote: true,
wantPrefix: "false ",
wantContains: "env var",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
orig := primeHasGitRemote
primeHasGitRemote = func() bool { return tt.hasRemote }
t.Cleanup(func() { primeHasGitRemote = orig })
if tt.sharedServer {
t.Setenv("BEADS_DOLT_SHARED_SERVER", "1")
} else {
os.Unsetenv("BEADS_DOLT_SHARED_SERVER")
t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") })
}
if tt.envVal == "\x00" {
os.Unsetenv("BD_BACKUP_ENABLED")
t.Cleanup(func() { os.Unsetenv("BD_BACKUP_ENABLED") })
} else {
t.Setenv("BD_BACKUP_ENABLED", tt.envVal)
}
config.ResetForTesting()
t.Cleanup(config.ResetForTesting)
if err := config.Initialize(); err != nil {
t.Fatalf("config.Initialize: %v", err)
}
oldJSON := jsonOutput
jsonOutput = false
t.Cleanup(func() { jsonOutput = oldJSON })
out := captureStdout(t, runConfigGetBackupEnabled)
if !strings.HasPrefix(out, tt.wantPrefix) {
t.Errorf("output %q: want prefix %q", out, tt.wantPrefix)
}
if !strings.Contains(out, tt.wantContains) {
t.Errorf("output %q: want to contain %q", out, tt.wantContains)
}
})
}
}