diff --git a/CHANGELOG.md b/CHANGELOG.md index 00e318de1..66b2bc4ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ownership check itself — foreign unclaim requires `--force` — landed in [#4675](https://github.com/gastownhall/beads/pull/4675).) +### Fixed + +- **`bd dolt clean-databases` gains an opt-in `--purge-dropped` flag to + reclaim disk from what it drops** (be-pq5, + [#3663](https://github.com/gastownhall/beads/pull/3663)). `DROP DATABASE` + only moves a database's directory under `.dolt_dropped_databases/`; Dolt + keeps the on-disk data there — recoverable via `CALL DOLT_UNDROP(name)` — + until an explicit `CALL DOLT_PURGE_DROPPED_DATABASES()`. The operator-facing + cleanup command dropped stale `testdb_*`/`benchdb_*`/etc. databases from + `SHOW DATABASES` but never issued that purge, so disk usage on the shared + Dolt server stayed high across repeated runs. `--purge-dropped` runs the + purge after cleanup (a failed purge is reported as a warning rather than + aborting the command, since the databases are already dropped from the + server's active set at that point) and fires even on a run that finds + nothing stale to drop, since a prior run's dropped-but-unpurged residue + isn't visible to `SHOW DATABASES` either. **`CALL + DOLT_PURGE_DROPPED_DATABASES()` is server-global and irreversible** — Dolt + has no way to scope it to only the databases a given run dropped, so + `--purge-dropped` also permanently deletes every other dropped-but-not-yet- + purged database on the server, removing `DOLT_UNDROP` recovery for all of + them. It defaults to off for that reason; without it, dropped databases are + left in place, recoverable via `DOLT_UNDROP`, same as before this change. + ### Added - **Pool-aware claiming via the `claim.pools` config key** (bd-bguz6). diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index b843b7e53..4aba07671 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -1004,7 +1004,23 @@ on the shared Dolt server from interrupted test runs and terminated agents. Stale database prefixes: testdb_*, beads_test*, beads_pt*, beads_vr*, doctest_*, doctortest_*, benchdb_* These waste server memory and can degrade performance under concurrent load. -Use --dry-run to see what would be dropped without actually dropping.`, +Use --dry-run to see what would be dropped without actually dropping. + +DROP DATABASE only marks a database as dropped; Dolt keeps its directory +under .dolt_dropped_databases/ so it can be restored with +CALL DOLT_UNDROP(name) until an explicit purge — disk is not reclaimed +until then. Pass --purge-dropped to run CALL DOLT_PURGE_DROPPED_DATABASES() +after cleanup. + +--purge-dropped is SERVER-GLOBAL and IRREVERSIBLE. Dolt has no way to scope +the purge to only the databases this run dropped: it permanently deletes +every dropped-but-not-yet-purged database on the server, including ones +dropped by something else entirely (e.g. an operator's accidental +DROP DATABASE on an unrelated database that was still recoverable via +DOLT_UNDROP). It also purges pre-existing residue from earlier +clean-databases runs even if this run finds no stale databases to drop. +Only pass it when nothing else on the server may be relying on DOLT_UNDROP +recovery.`, RunE: func(cmd *cobra.Command, args []string) error { beadsDir := selectedDoltBeadsDir() if beadsDir == "" { @@ -1017,6 +1033,7 @@ Use --dry-run to see what would be dropped without actually dropping.`, return HandleError("'bd dolt clean-databases' is not supported in embedded mode (no Dolt server)") } dryRun, _ := cmd.Flags().GetBool("dry-run") + purgeDropped, _ := cmd.Flags().GetBool("purge-dropped") if usesProxiedServer() { return runDoltCleanDatabasesProxied(rootCtx, beadsDir, dryRun) @@ -1055,78 +1072,133 @@ Use --dry-run to see what would be dropped without actually dropping.`, if len(stale) == 0 { fmt.Println("No stale databases found.") - return nil - } - - fmt.Printf("Found %d stale databases:\n", len(stale)) - for _, name := range stale { - fmt.Printf(" %s\n", name) + } else { + fmt.Printf("Found %d stale databases:\n", len(stale)) + for _, name := range stale { + fmt.Printf(" %s\n", name) + } } if dryRun { - fmt.Println("\n(dry run — no databases dropped)") + if len(stale) > 0 { + fmt.Println("\n(dry run — no databases dropped)") + } + if purgeDropped { + fmt.Println("(dry run — --purge-dropped ignored; no purge performed)") + } return nil } - fmt.Println() dropped := 0 - failures := 0 - consecutiveTimeouts := 0 - const ( - batchSize = 5 // Drop this many before pausing - batchPause = 2 * time.Second - backoffPause = 10 * time.Second - timeoutThreshold = 3 // Consecutive timeouts before backoff - perDropTimeout = 30 * time.Second - maxConsecFailures = 10 // Stop after this many consecutive failures - ) + if len(stale) > 0 { + fmt.Println() + failures := 0 + consecutiveTimeouts := 0 + const ( + batchSize = 5 // Drop this many before pausing + batchPause = 2 * time.Second + backoffPause = 10 * time.Second + timeoutThreshold = 3 // Consecutive timeouts before backoff + perDropTimeout = 30 * time.Second + maxConsecFailures = 10 // Stop after this many consecutive failures + ) - for i, name := range stale { - // Circuit breaker: back off when server is overwhelmed - if consecutiveTimeouts >= timeoutThreshold { - fmt.Fprintf(os.Stderr, " ⚠ %d consecutive timeouts — backing off %s\n", - consecutiveTimeouts, backoffPause) - time.Sleep(backoffPause) - consecutiveTimeouts = 0 - } - - // Stop if too many consecutive failures — server is likely unhealthy - if failures >= maxConsecFailures { - fmt.Fprintf(os.Stderr, "\n✗ Aborting: %d consecutive failures suggest server is unhealthy.\n", failures) - fmt.Fprintf(os.Stderr, " Dropped %d/%d before stopping.\n", dropped, len(stale)) - return SilentExit() - } - - // Per-operation timeout: DROP DATABASE can be slow on Dolt - dropCtx, dropCancel := context.WithTimeout(context.Background(), perDropTimeout) - // Escape backticks in database name to prevent SQL injection (` → ``) - safeName := strings.ReplaceAll(name, "`", "``") - _, err := db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE `%s`", safeName)) //nolint:gosec // G201: identifier-escaped - dropCancel() - if err != nil { - fmt.Fprintf(os.Stderr, " FAIL: %s: %v\n", name, err) - failures++ - if isTimeoutError(err) { - consecutiveTimeouts++ + for i, name := range stale { + // Circuit breaker: back off when server is overwhelmed + if consecutiveTimeouts >= timeoutThreshold { + fmt.Fprintf(os.Stderr, " ⚠ %d consecutive timeouts — backing off %s\n", + consecutiveTimeouts, backoffPause) + time.Sleep(backoffPause) + consecutiveTimeouts = 0 } - } else { - fmt.Printf(" Dropped: %s\n", name) - dropped++ - failures = 0 - consecutiveTimeouts = 0 - } - // Rate limiting: pause between batches to let the server breathe - if (i+1)%batchSize == 0 && i+1 < len(stale) { - fmt.Printf(" [%d/%d] pausing %s...\n", i+1, len(stale), batchPause) - time.Sleep(batchPause) + // Stop if too many consecutive failures — server is likely unhealthy + if failures >= maxConsecFailures { + fmt.Fprintf(os.Stderr, "\n✗ Aborting: %d consecutive failures suggest server is unhealthy.\n", failures) + fmt.Fprintf(os.Stderr, " Dropped %d/%d before stopping.\n", dropped, len(stale)) + return SilentExit() + } + + // Per-operation timeout: DROP DATABASE can be slow on Dolt + dropCtx, dropCancel := context.WithTimeout(context.Background(), perDropTimeout) + // Escape backticks in database name to prevent SQL injection (` → ``) + safeName := strings.ReplaceAll(name, "`", "``") + _, err := db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE `%s`", safeName)) //nolint:gosec // G201: identifier-escaped + dropCancel() + if err != nil { + fmt.Fprintf(os.Stderr, " FAIL: %s: %v\n", name, err) + failures++ + if isTimeoutError(err) { + consecutiveTimeouts++ + } + } else { + fmt.Printf(" Dropped: %s\n", name) + dropped++ + failures = 0 + consecutiveTimeouts = 0 + } + + // Rate limiting: pause between batches to let the server breathe + if (i+1)%batchSize == 0 && i+1 < len(stale) { + fmt.Printf(" [%d/%d] pausing %s...\n", i+1, len(stale), batchPause) + time.Sleep(batchPause) + } } + fmt.Printf("\nDropped %d/%d stale databases.\n", dropped, len(stale)) + } + + // DROP DATABASE only marks a database as dropped; Dolt moves its + // directory under .dolt_dropped_databases/ so `dolt_undrop()` can + // restore it, but leaves the disk footprint in place until an + // explicit purge (be-pq5). + fmt.Println() + if shouldPurgeDroppedDatabases(purgeDropped, dropped) { + if err := purgeDroppedDatabases(context.Background(), db); err != nil { + fmt.Fprintf(os.Stderr, " WARN: PURGE_DROPPED_DATABASES failed: %v\n", err) + fmt.Fprintln(os.Stderr, " Try `dolt sql -q 'CALL DOLT_PURGE_DROPPED_DATABASES()'`.") + } else { + fmt.Println("Purged all dropped databases on this server (server-global, irreversible —") + fmt.Println("CALL DOLT_UNDROP is no longer available for any of them, not just this run's).") + } + } else { + fmt.Println("Dropped databases remain recoverable via `CALL DOLT_UNDROP(name)` until purged.") + fmt.Println("Pass --purge-dropped to permanently reclaim their disk. This purges ALL dropped") + fmt.Println("databases on the server (server-global), not just the ones from this run.") } - fmt.Printf("\nDropped %d/%d stale databases.\n", dropped, len(stale)) return nil }, } +// shouldPurgeDroppedDatabases reports whether clean-databases should invoke +// the (server-global, irreversible) purge. It gates purely on the +// --purge-dropped flag and deliberately ignores droppedCount: a prior +// clean-databases run may have left dropped-but-unpurged residue that this +// run's SHOW DATABASES scan never sees (the residue is already gone from +// SHOW DATABASES the moment it was dropped), so --purge-dropped must still +// fire the purge even when this run drops nothing. Extracted as a pure +// function so the gating contract itself — not just the SQL-level purge +// mechanism — has direct unit coverage. +func shouldPurgeDroppedDatabases(purgeDropped bool, droppedCount int) bool { + _ = droppedCount // deliberately unused — see doc comment above + return purgeDropped +} + +// purgeDroppedDatabases issues Dolt's DOLT_PURGE_DROPPED_DATABASES() stored +// procedure, which permanently deletes database directories that DROP +// DATABASE only moved into .dolt_dropped_databases/. This is server-global: +// Dolt has no way to scope it to a particular set of databases, so it +// purges every dropped-but-not-yet-purged database on the server, not just +// ones this process dropped. Extracted so tests can drive it directly +// against a live test server without going through the full +// clean-databases command wiring (config loading, SHOW DATABASES scan, +// batching/backoff). +func purgeDroppedDatabases(ctx context.Context, db *sql.DB) error { + purgeCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + _, err := db.ExecContext(purgeCtx, "CALL DOLT_PURGE_DROPPED_DATABASES()") + return err +} + // --- Dolt remote management commands --- type doltRemoteAddStore interface { @@ -1413,6 +1485,7 @@ func init() { doltPullCmd.Flags().String("remote", "", "Pull from a specific named remote instead of the default") doltCommitCmd.Flags().StringP("message", "m", "", "Commit message (default: auto-generated)") doltCleanDatabasesCmd.Flags().Bool("dry-run", false, "Show what would be dropped without dropping") + doltCleanDatabasesCmd.Flags().Bool("purge-dropped", false, "After dropping, also run CALL DOLT_PURGE_DROPPED_DATABASES() — server-global and irreversible, see --help") doltRemoteAddCmd.Flags().Bool("allow-git-origin", false, "Allow adding a Dolt remote whose URL matches the git origin (proceed with a warning instead of aborting)") doltRemoteCmd.AddCommand(doltRemoteAddCmd) doltRemoteCmd.AddCommand(doltRemoteListCmd) diff --git a/cmd/bd/dolt_clean_databases_purge_test.go b/cmd/bd/dolt_clean_databases_purge_test.go new file mode 100644 index 000000000..bd9bb71ba --- /dev/null +++ b/cmd/bd/dolt_clean_databases_purge_test.go @@ -0,0 +1,214 @@ +//go:build cgo + +package main + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/doltutil" +) + +// TestShouldPurgeDroppedDatabasesGatesOnFlagAlone pins the --purge-dropped +// gating contract at the pure-function level: purge fires if and only if +// the flag is set, regardless of how many databases this run dropped. In +// particular it fixes a P1 found in review of the initial version of this +// change: gating on "did this run drop anything" instead of the flag alone +// meant a run that found zero stale databases (stale == 0, e.g. because a +// prior run already dropped them but never purged) silently skipped the +// purge even with --purge-dropped set, leaving that residue unreclaimed +// forever. +func TestShouldPurgeDroppedDatabasesGatesOnFlagAlone(t *testing.T) { + cases := []struct { + name string + purgeDropped bool + droppedCount int + want bool + }{ + {"flag off, nothing dropped this run", false, 0, false}, + {"flag off, dropped this run", false, 3, false}, + {"flag on, nothing dropped this run (residue case)", true, 0, true}, + {"flag on, dropped this run", true, 3, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := shouldPurgeDroppedDatabases(tc.purgeDropped, tc.droppedCount); got != tc.want { + t.Errorf("shouldPurgeDroppedDatabases(%v, %d) = %v, want %v", + tc.purgeDropped, tc.droppedCount, got, tc.want) + } + }) + } +} + +// TestPurgeDroppedDatabasesRemovesUndropRecovery is the SQL-level regression +// gate for be-pq5's purge mechanism: purgeDroppedDatabases (called only when +// --purge-dropped is set — see TestShouldPurgeDroppedDatabasesGatesOnFlagAlone +// above and TestCleanDatabasesPurgeDroppedReclaimsResidue below for the CLI +// wiring) must actually reclaim a dropped database's directory, not just +// leave it recoverable. +// +// Dolt exposes no SQL view listing dropped-but-not-purged databases (the +// only on-disk signal is a directory inside the server's opaque data dir, +// which the shared testcontainer-based test server does not expose to the +// test process). Instead this uses Dolt's own `dolt_undrop()` stored +// procedure as the observable: a DROP DATABASE without a following PURGE +// leaves the database restorable via `CALL DOLT_UNDROP(name)`; once +// DOLT_PURGE_DROPPED_DATABASES() has run, the same call fails because +// there is nothing left to restore. +// +// purgeDroppedDatabases is server-global (see its doc comment), so this +// test assumes nothing else concurrently purges the shared test server +// between its own DROP and its own DOLT_UNDROP check. That holds here +// because neither subtest calls t.Parallel(): Go's testing package never +// runs two non-parallel top-level tests in the same package concurrently, +// so this test's DROP/(maybe PURGE)/UNDROP sequence for a given database +// name is never interleaved with another test's purge. +func TestPurgeDroppedDatabasesRemovesUndropRecovery(t *testing.T) { + if testDoltServerPort == 0 { + t.Skip("no test Dolt server running") + } + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: testDoltServerPort, User: "root"}.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("connect to test dolt server: %v", err) + } + t.Cleanup(func() { db.Close() }) + + ctx := context.Background() + + // Case (a): --purge-dropped's underlying mechanism. Purging after a DROP + // removes the DOLT_UNDROP recovery option. + t.Run("with purge, undrop fails", func(t *testing.T) { + name := fmt.Sprintf("benchdb_purgetest_%d", time.Now().UnixNano()) + mustExec(t, ctx, db, fmt.Sprintf("CREATE DATABASE `%s`", name)) + mustExec(t, ctx, db, fmt.Sprintf("DROP DATABASE `%s`", name)) + + if err := purgeDroppedDatabases(ctx, db); err != nil { + t.Fatalf("purgeDroppedDatabases: %v", err) + } + + undropCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err := db.ExecContext(undropCtx, fmt.Sprintf("CALL DOLT_UNDROP('%s')", name)) + if err == nil { + // Undrop succeeded, meaning the database directory was still + // present — the purge did not actually reclaim it. Clean up + // the restored database before failing. + mustExec(t, ctx, db, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", name)) + t.Fatalf("DOLT_UNDROP(%q) succeeded after purgeDroppedDatabases; database was not actually purged", name) + } + }) + + // Case (b): the default, no-flag behavior (shouldPurgeDroppedDatabases + // returns false, so purgeDroppedDatabases is never called). Without a + // purge, DROP DATABASE stays recoverable — this is what --purge-dropped + // being opt-in preserves for anyone else relying on DOLT_UNDROP on a + // shared server. It also pins the assertion methodology in case (a) + // above: if Dolt ever changes so that DOLT_UNDROP no longer + // distinguishes purged from merely-dropped databases, this control + // catches it as a false negative here rather than making case (a) + // silently vacuous. + t.Run("without purge, undrop succeeds", func(t *testing.T) { + name := fmt.Sprintf("benchdb_purgetest_%d", time.Now().UnixNano()) + mustExec(t, ctx, db, fmt.Sprintf("CREATE DATABASE `%s`", name)) + mustExec(t, ctx, db, fmt.Sprintf("DROP DATABASE `%s`", name)) + + undropCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := db.ExecContext(undropCtx, fmt.Sprintf("CALL DOLT_UNDROP('%s')", name)); err != nil { + t.Fatalf("DOLT_UNDROP(%q) failed without a purge in between: %v (assertion methodology in case (a) may be invalid)", name, err) + } + + // The undrop recreated the database; drop and purge it for real so + // the test doesn't leak a benchdb_* database into the shared server. + mustExec(t, ctx, db, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", name)) + if err := purgeDroppedDatabases(ctx, db); err != nil { + t.Logf("cleanup purge after control case: %v", err) + } + }) +} + +// TestCleanDatabasesPurgeDroppedReclaimsResidue is the CLI-level regression +// gate for the residue P1: it drives the real `bd dolt clean-databases +// --purge-dropped` command (not just the extracted helpers) against a +// database that was already dropped — but not purged — before the command +// runs, so this invocation's own SHOW DATABASES scan finds nothing stale +// (stale == 0). Before the round-2 fix, the purge call was gated on +// `dropped > 0`, so this exact scenario silently skipped the purge and the +// residue was never reclaimed despite --purge-dropped being passed. +// +// The shared test Dolt server container is itself bootstrapped with a +// "beads_test" database (internal/testutil/container_provider.go), which +// matches a stale prefix — so a single clean-databases invocation against a +// fresh container always finds at least one stale entry (dropped > 0), +// which would mask this exact bug under the old dropped>0 gate. This test +// primes the server with an unconditional --purge-dropped run first (drops +// and purges "beads_test" and any other ambient residue), then creates its +// own residue database and runs clean-databases --purge-dropped a second +// time, by which point SHOW DATABASES genuinely has nothing stale to drop. +// +// Reuses the buildBDUnderTest/writeServerRepo/runBDCommand harness from +// explicit_db_nodb_test.go, which already exercises `bd dolt ` +// against this package's shared test Dolt server the same way. +func TestCleanDatabasesPurgeDroppedReclaimsResidue(t *testing.T) { + if testDoltServerPort == 0 { + t.Skip("no test Dolt server running") + } + + binPath := buildBDUnderTest(t) + repoDir := t.TempDir() + writeServerRepo(t, repoDir, "purge_residue_test_db", "127.0.0.1", "purge-residue-test-origin", testDoltServerPort) + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: testDoltServerPort, User: "root"}.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("connect to test dolt server: %v", err) + } + t.Cleanup(func() { db.Close() }) + ctx := context.Background() + + // Prime: clear any ambient stale/residue databases (notably the + // container's own "beads_test" bootstrap DB) so the second, assertive + // run below starts from a genuinely clean SHOW DATABASES. + _ = runBDCommand(t, binPath, repoDir, nil, "dolt", "clean-databases", "--purge-dropped") + + // Simulate residue from a prior clean-databases run: drop a stale + // database directly, without purging. Once dropped it no longer shows + // up in SHOW DATABASES, so the upcoming clean-databases invocation's + // own scan will find zero stale databases — exactly the scenario that + // exposed the P1. + residue := fmt.Sprintf("benchdb_residue_%d", time.Now().UnixNano()) + mustExec(t, ctx, db, fmt.Sprintf("CREATE DATABASE `%s`", residue)) + mustExec(t, ctx, db, fmt.Sprintf("DROP DATABASE `%s`", residue)) + + out := string(runBDCommand(t, binPath, repoDir, nil, "dolt", "clean-databases", "--purge-dropped")) + + if !strings.Contains(out, "No stale databases found.") { + t.Fatalf("expected this run to find no stale databases (residue is already dropped, not in SHOW DATABASES), got:\n%s", out) + } + if !strings.Contains(out, "Purged all dropped databases") { + t.Fatalf("expected --purge-dropped to purge even with stale == 0, got:\n%s", out) + } + + undropCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, undropErr := db.ExecContext(undropCtx, fmt.Sprintf("CALL DOLT_UNDROP('%s')", residue)) + if undropErr == nil { + mustExec(t, ctx, db, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", residue)) + t.Fatalf("DOLT_UNDROP(%q) succeeded after `bd dolt clean-databases --purge-dropped` with stale == 0; residue was not purged", residue) + } +} + +func mustExec(t *testing.T, ctx context.Context, db *sql.DB, query string) { + t.Helper() + execCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := db.ExecContext(execCtx, query); err != nil { + t.Fatalf("exec %q: %v", query, err) + } +}