fix(backup): make auto-backup safe under a shared Dolt sql-server (wy-zrmqr)
Three defects turned a transient shared-Dolt slowdown into the 2026-07
CPU-pin incident, where ~31 bd clients on one sql-server hammered it with
full-DB backup syncs:
1. maybeAutoBackup throttle bypass on failure. runBackupExport only wrote
backup_state.json (the throttle watermark) on SUCCESS, so a failing sync
left Timestamp stale and retried on EVERY subsequent bd command — a
self-amplifying storm. Now the attempt time is persisted on failure too
(LastDoltCommit left unchanged so change-detection still runs a real
backup once the failure clears).
2. Auto-backup on by default in sql-server mode. isBackupAutoEnabled
auto-enabled whenever a git remote existed, even in shared-server mode
where N clients race one server-side backup name and each full-syncs the
whole DB into its own dir. Default is now OFF in sql-server /
shared-server mode (usesSQLServer()); explicit backup.enabled=true still
honored for operators who coordinate destinations.
3. `bd config get backup.enabled` printed the raw stored value ("false" /
"not set") while the runtime auto-enabled — the mismatch that hid the
storm. It now reports the EFFECTIVE value plus its source (env var /
config.yaml / auto: git-remote / auto: off in sql-server mode).
Tests: server-mode + failure cases added to TestIsBackupAutoEnabled; new
TestRunBackupExport_PersistsThrottleOnFailure repro (failing sync advances
the throttle, does not clear LastDoltCommit); TestConfigGetBackupEnabled_
EffectiveValue pins the new get output. backup help documents the server
default. golangci-lint clean.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
27a382e1bc
commit
5f685959cd
7 changed files with 324 additions and 6 deletions
|
|
@ -29,7 +29,17 @@ Commands:
|
|||
|
||||
DoltHub is recommended for cloud backup:
|
||||
bd backup init https://doltremoteapi.dolthub.com/<user>/<repo>
|
||||
Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication.`,
|
||||
Set DOLT_REMOTE_USER and DOLT_REMOTE_PASSWORD for authentication.
|
||||
|
||||
Auto-backup default:
|
||||
When backup.enabled is unset, auto-backup turns ON in embedded mode if a
|
||||
git remote exists, and stays OFF in sql-server / shared-server mode. In
|
||||
server mode many bd clients share one Dolt server, and each would register
|
||||
a server-side backup remote under the same name pointing at its own local
|
||||
dir and full-sync the whole database — a self-amplifying storm. To back up
|
||||
a shared server, run 'bd backup' explicitly (or set backup.enabled=true and
|
||||
coordinate destinations). 'bd config get backup.enabled' shows the effective
|
||||
value and its source.`,
|
||||
GroupID: "sync",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,11 +14,26 @@ import (
|
|||
|
||||
// isBackupAutoEnabled returns whether backup should run.
|
||||
// If user explicitly configured backup.enabled, use that.
|
||||
// Otherwise, auto-enable when a git remote exists.
|
||||
// Otherwise auto-enable when a git remote exists — BUT only in
|
||||
// embedded mode.
|
||||
//
|
||||
// In sql-server / shared-server mode (usesSQLServer()) the default is
|
||||
// OFF: N bd clients share a single Dolt server, and the Dolt-native
|
||||
// backup path (store.BackupDatabase) registers a server-side backup
|
||||
// remote under one fixed name pointing at THIS client's local
|
||||
// .beads/backup dir, then full-syncs the whole DB. With many clients
|
||||
// that means racing remove/add of the same name plus every client
|
||||
// full-syncing the entire history into its own dir — the amplifier
|
||||
// behind the 2026-07 shared-dolt CPU-pin incident. Operators who want
|
||||
// backups in server mode must opt in explicitly (backup.enabled=true
|
||||
// / BD_BACKUP_ENABLED=1) and coordinate destinations themselves.
|
||||
func isBackupAutoEnabled() bool {
|
||||
if config.GetValueSource("backup.enabled") != config.SourceDefault {
|
||||
return config.GetBool("backup.enabled")
|
||||
}
|
||||
if usesSQLServer() {
|
||||
return false
|
||||
}
|
||||
return primeHasGitRemote()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,10 +14,11 @@ func TestIsBackupAutoEnabled(t *testing.T) {
|
|||
// Cannot be parallel: modifies global primeHasGitRemote and env vars.
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
envVal string // "\x00" = not set, "" = set to empty, "true"/"false"/"0" = explicit
|
||||
hasRemote bool
|
||||
wantResult bool
|
||||
name string
|
||||
envVal string // "\x00" = not set, "" = set to empty, "true"/"false"/"0" = explicit
|
||||
hasRemote bool
|
||||
sharedServer bool // BEADS_DOLT_SHARED_SERVER=1 → usesSQLServer() true
|
||||
wantResult bool
|
||||
}{
|
||||
{
|
||||
name: "default + git remote → enabled",
|
||||
|
|
@ -55,6 +56,25 @@ func TestIsBackupAutoEnabled(t *testing.T) {
|
|||
hasRemote: true,
|
||||
wantResult: false,
|
||||
},
|
||||
{
|
||||
// wy-zrmqr: unset default must NOT auto-enable in sql-server
|
||||
// mode even with a git remote — N clients racing one backup
|
||||
// name was the storm amplifier in the 2026-07 CPU-pin incident.
|
||||
name: "default + git remote + sql-server mode → disabled",
|
||||
envVal: "\x00",
|
||||
hasRemote: true,
|
||||
sharedServer: true,
|
||||
wantResult: false,
|
||||
},
|
||||
{
|
||||
// Explicit opt-in still honored in sql-server mode: operators
|
||||
// who coordinate destinations themselves may turn it on.
|
||||
name: "explicit true + sql-server mode → enabled",
|
||||
envVal: "true",
|
||||
hasRemote: false,
|
||||
sharedServer: true,
|
||||
wantResult: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
@ -64,6 +84,13 @@ func TestIsBackupAutoEnabled(t *testing.T) {
|
|||
primeHasGitRemote = func() bool { return tt.hasRemote }
|
||||
t.Cleanup(func() { primeHasGitRemote = orig })
|
||||
|
||||
if tt.sharedServer {
|
||||
t.Setenv("BEADS_DOLT_SHARED_SERVER", "1")
|
||||
} else {
|
||||
os.Unsetenv("BEADS_DOLT_SHARED_SERVER")
|
||||
t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") })
|
||||
}
|
||||
|
||||
// Set env var: "\x00" = unset, anything else = set to that value
|
||||
if tt.envVal == "\x00" {
|
||||
os.Unsetenv("BD_BACKUP_ENABLED")
|
||||
|
|
|
|||
|
|
@ -140,6 +140,19 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) {
|
|||
}
|
||||
|
||||
if err := bs.BackupDatabase(ctx, dir); err != nil {
|
||||
// Persist the attempt time even on failure so the throttle
|
||||
// interval (checked by maybeAutoBackup via state.Timestamp)
|
||||
// applies to the next command. Without this, a sync that keeps
|
||||
// failing — e.g. a slow/overloaded shared Dolt server — retries
|
||||
// on EVERY bd command instead of once per interval, turning a
|
||||
// transient slowdown into a self-amplifying storm (the 2026-07
|
||||
// shared-dolt CPU-pin incident). LastDoltCommit is deliberately
|
||||
// left unchanged so change-detection still sees pending work and
|
||||
// a real backup runs once the failure clears.
|
||||
state.Timestamp = time.Now().UTC()
|
||||
if saveErr := saveBackupState(dir, state); saveErr != nil {
|
||||
debug.Logf("backup: failed to persist throttle state after error: %v\n", saveErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
|
|||
99
cmd/bd/backup_export_failure_test.go
Normal file
99
cmd/bd/backup_export_failure_test.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/config"
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
)
|
||||
|
||||
// failingBackupStore is a minimal DoltStorage stand-in whose Dolt-native
|
||||
// backup always fails. It embeds a nil DoltStorage for identity/type
|
||||
// assertion and overrides only the methods runBackupExport touches.
|
||||
type failingBackupStore struct {
|
||||
storage.DoltStorage
|
||||
commit string
|
||||
backupErr error
|
||||
backupCalls int
|
||||
}
|
||||
|
||||
func (f *failingBackupStore) GetCurrentCommit(context.Context) (string, error) { return f.commit, nil }
|
||||
|
||||
func (f *failingBackupStore) BackupDatabase(context.Context, string) error {
|
||||
f.backupCalls++
|
||||
return f.backupErr
|
||||
}
|
||||
func (f *failingBackupStore) BackupAdd(context.Context, string, string) error { return nil }
|
||||
func (f *failingBackupStore) BackupSync(context.Context, string) error { return nil }
|
||||
func (f *failingBackupStore) BackupRemove(context.Context, string) error { return nil }
|
||||
func (f *failingBackupStore) RestoreDatabase(context.Context, string, bool) error { return nil }
|
||||
|
||||
// TestRunBackupExport_PersistsThrottleOnFailure is the wy-zrmqr repro: a
|
||||
// failing Dolt backup sync must still advance the throttle timestamp so the
|
||||
// next bd command is throttled instead of retrying immediately. Before the
|
||||
// fix, saveBackupState ran only on success, so a slow/failing shared Dolt
|
||||
// server was hammered on EVERY subsequent command — the amplifier behind the
|
||||
// 2026-07 CPU-pin incident.
|
||||
func TestRunBackupExport_PersistsThrottleOnFailure(t *testing.T) {
|
||||
// Point backupDir() at a temp git repo via backup.git-repo so we don't
|
||||
// need a live beads workspace.
|
||||
repo := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("BD_BACKUP_GIT_REPO", repo)
|
||||
config.ResetForTesting()
|
||||
t.Cleanup(config.ResetForTesting)
|
||||
if err := config.Initialize(); err != nil {
|
||||
t.Fatalf("config.Initialize: %v", err)
|
||||
}
|
||||
|
||||
dir, err := backupDir()
|
||||
if err != nil {
|
||||
t.Fatalf("backupDir: %v", err)
|
||||
}
|
||||
|
||||
// Sanity: no prior state, so the throttle would NOT block a first attempt.
|
||||
if st, _ := loadBackupState(dir); !st.Timestamp.IsZero() {
|
||||
t.Fatalf("precondition: expected zero timestamp, got %v", st.Timestamp)
|
||||
}
|
||||
|
||||
oldStore := store
|
||||
fake := &failingBackupStore{commit: "deadbeef", backupErr: errors.New("sync to backup: server too busy")}
|
||||
store = fake
|
||||
t.Cleanup(func() { store = oldStore })
|
||||
|
||||
before := time.Now().UTC()
|
||||
if _, err := runBackupExport(context.Background(), true); err == nil {
|
||||
t.Fatal("expected runBackupExport to return the sync error, got nil")
|
||||
}
|
||||
if fake.backupCalls != 1 {
|
||||
t.Fatalf("expected exactly 1 BackupDatabase call, got %d", fake.backupCalls)
|
||||
}
|
||||
|
||||
st, err := loadBackupState(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("loadBackupState: %v", err)
|
||||
}
|
||||
if st.Timestamp.IsZero() {
|
||||
t.Fatal("throttle timestamp not persisted on failure — next command would retry unthrottled (storm)")
|
||||
}
|
||||
if st.Timestamp.Before(before) {
|
||||
t.Errorf("throttle timestamp %v predates the failed attempt %v", st.Timestamp, before)
|
||||
}
|
||||
// The recorded timestamp must be recent enough that the default 15m
|
||||
// throttle window blocks the next attempt.
|
||||
if time.Since(st.Timestamp) >= 15*time.Minute {
|
||||
t.Errorf("persisted timestamp %v is already outside the throttle window", st.Timestamp)
|
||||
}
|
||||
// LastDoltCommit must stay empty: the backup did NOT succeed, so change
|
||||
// detection must still see pending work once the failure clears.
|
||||
if st.LastDoltCommit != "" {
|
||||
t.Errorf("LastDoltCommit should remain empty after a failed backup, got %q", st.LastDoltCommit)
|
||||
}
|
||||
}
|
||||
|
|
@ -246,6 +246,17 @@ var configGetCmd = &cobra.Command{
|
|||
|
||||
key := args[0]
|
||||
|
||||
if key == "backup.enabled" {
|
||||
// backup.enabled has an auto-detected effective value that
|
||||
// differs from the stored value: when unset it auto-enables
|
||||
// in embedded mode with a git remote, and is forced OFF in
|
||||
// sql-server mode (see isBackupAutoEnabled). Reporting the raw
|
||||
// stored "false"/"not set" here misled operators during the
|
||||
// 2026-07 shared-dolt incident, so show the EFFECTIVE value
|
||||
// and its source.
|
||||
return runConfigGetBackupEnabled()
|
||||
}
|
||||
|
||||
if config.IsYamlOnlyKey(key) {
|
||||
// User-global keys (e.g. metrics.*) must be read from the user-global
|
||||
// config.yaml only — the same source the runtime uses for metrics
|
||||
|
|
@ -343,6 +354,45 @@ var configGetCmd = &cobra.Command{
|
|||
},
|
||||
}
|
||||
|
||||
// runConfigGetBackupEnabled reports the EFFECTIVE value of
|
||||
// backup.enabled together with its source, rather than the raw stored
|
||||
// value. The stored value is misleading because isBackupAutoEnabled()
|
||||
// derives the runtime value: unset → auto-enabled in embedded mode
|
||||
// when a git remote exists, and forced OFF in sql-server mode.
|
||||
func runConfigGetBackupEnabled() error {
|
||||
const key = "backup.enabled"
|
||||
source := config.GetValueSource(key)
|
||||
effective := isBackupAutoEnabled()
|
||||
|
||||
var sourceDesc string
|
||||
switch source {
|
||||
case config.SourceEnvVar:
|
||||
sourceDesc = "env var"
|
||||
case config.SourceConfigFile:
|
||||
sourceDesc = "config.yaml"
|
||||
default: // SourceDefault — value came from auto-detection
|
||||
switch {
|
||||
case usesSQLServer():
|
||||
sourceDesc = "default (auto: off in sql-server mode)"
|
||||
case effective:
|
||||
sourceDesc = "default (auto: on — git remote detected)"
|
||||
default:
|
||||
sourceDesc = "default (auto: off — no git remote)"
|
||||
}
|
||||
}
|
||||
|
||||
if jsonOutput {
|
||||
return outputJSON(map[string]interface{}{
|
||||
"key": key,
|
||||
"value": effective,
|
||||
"effective": effective,
|
||||
"source": string(source),
|
||||
})
|
||||
}
|
||||
fmt.Printf("%t (%s)\n", effective, sourceDesc)
|
||||
return nil
|
||||
}
|
||||
|
||||
var configListCmd = &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List all configuration",
|
||||
|
|
|
|||
104
cmd/bd/config_get_backup_enabled_test.go
Normal file
104
cmd/bd/config_get_backup_enabled_test.go
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/config"
|
||||
)
|
||||
|
||||
// TestConfigGetBackupEnabled_EffectiveValue pins wy-zrmqr fix #3: `bd config
|
||||
// get backup.enabled` must report the EFFECTIVE value (what isBackupAutoEnabled
|
||||
// actually returns) plus its source, not the raw stored value. Before the fix
|
||||
// it printed "false"/"not set" even while auto-backup was running via
|
||||
// primeHasGitRemote — the mismatch that hid the storm from operators.
|
||||
func TestConfigGetBackupEnabled_EffectiveValue(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
envVal string // "\x00" = unset
|
||||
hasRemote bool
|
||||
sharedServer bool
|
||||
wantPrefix string // leading "true "/"false "
|
||||
wantContains string // source annotation substring
|
||||
}{
|
||||
{
|
||||
name: "unset + no remote → off (no git remote)",
|
||||
envVal: "\x00",
|
||||
hasRemote: false,
|
||||
wantPrefix: "false ",
|
||||
wantContains: "no git remote",
|
||||
},
|
||||
{
|
||||
name: "unset + remote → on (git remote)",
|
||||
envVal: "\x00",
|
||||
hasRemote: true,
|
||||
wantPrefix: "true ",
|
||||
wantContains: "git remote detected",
|
||||
},
|
||||
{
|
||||
name: "unset + remote + sql-server → off (server mode)",
|
||||
envVal: "\x00",
|
||||
hasRemote: true,
|
||||
sharedServer: true,
|
||||
wantPrefix: "false ",
|
||||
wantContains: "sql-server mode",
|
||||
},
|
||||
{
|
||||
name: "explicit true + sql-server → on (env var)",
|
||||
envVal: "true",
|
||||
hasRemote: false,
|
||||
sharedServer: true,
|
||||
wantPrefix: "true ",
|
||||
wantContains: "env var",
|
||||
},
|
||||
{
|
||||
name: "explicit false → off (env var)",
|
||||
envVal: "false",
|
||||
hasRemote: true,
|
||||
wantPrefix: "false ",
|
||||
wantContains: "env var",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
orig := primeHasGitRemote
|
||||
primeHasGitRemote = func() bool { return tt.hasRemote }
|
||||
t.Cleanup(func() { primeHasGitRemote = orig })
|
||||
|
||||
if tt.sharedServer {
|
||||
t.Setenv("BEADS_DOLT_SHARED_SERVER", "1")
|
||||
} else {
|
||||
os.Unsetenv("BEADS_DOLT_SHARED_SERVER")
|
||||
t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") })
|
||||
}
|
||||
|
||||
if tt.envVal == "\x00" {
|
||||
os.Unsetenv("BD_BACKUP_ENABLED")
|
||||
t.Cleanup(func() { os.Unsetenv("BD_BACKUP_ENABLED") })
|
||||
} else {
|
||||
t.Setenv("BD_BACKUP_ENABLED", tt.envVal)
|
||||
}
|
||||
|
||||
config.ResetForTesting()
|
||||
t.Cleanup(config.ResetForTesting)
|
||||
if err := config.Initialize(); err != nil {
|
||||
t.Fatalf("config.Initialize: %v", err)
|
||||
}
|
||||
|
||||
oldJSON := jsonOutput
|
||||
jsonOutput = false
|
||||
t.Cleanup(func() { jsonOutput = oldJSON })
|
||||
|
||||
out := captureStdout(t, runConfigGetBackupEnabled)
|
||||
|
||||
if !strings.HasPrefix(out, tt.wantPrefix) {
|
||||
t.Errorf("output %q: want prefix %q", out, tt.wantPrefix)
|
||||
}
|
||||
if !strings.Contains(out, tt.wantContains) {
|
||||
t.Errorf("output %q: want to contain %q", out, tt.wantContains)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue