diff --git a/cmd/bd/compact_dolt.go b/cmd/bd/compact_dolt.go index 91e5c2781..9fa340355 100644 --- a/cmd/bd/compact_dolt.go +++ b/cmd/bd/compact_dolt.go @@ -35,7 +35,13 @@ How it works: 4. Swaps main branch to the compacted version 5. Prunes remote-tracking refs (they would keep the old history alive; the next push or fetch re-creates them at the new tip) - 6. Runs Dolt GC to reclaim space + 6. Runs a full Dolt GC (all storage generations) to reclaim the old history + +The GC pass is a full collection: Dolt storage is generational, and a default +GC never revisits data an earlier GC moved to the old generation. On any store +that has been GC'd before (bd gc, or a previous flatten or compact), only a +full collection reclaims the squashed history. A full GC can take minutes on +multi-gigabyte stores. Examples: bd compact --dry-run # Preview: show commit breakdown @@ -187,11 +193,7 @@ Examples: } // Reclaim disk space from orphaned old history - if gc, ok := storage.UnwrapStore(store).(storage.GarbageCollector); ok { - if err := gc.DoltGC(ctx); err != nil { - WarnError("dolt gc after compact failed: %v", err) - } - } + gcMode := runPostRewriteGC(ctx, "compact") sizeAfter := storeSizeBytes(ctx) elapsed := time.Since(start) @@ -208,6 +210,9 @@ Examples: "tags_anchoring": tags, "elapsed_ms": elapsed.Milliseconds(), } + if gcMode != "" { + result["gc_mode"] = gcMode + } addGCSizeJSON(result, sizeBefore, sizeAfter) return outputJSON(result) } diff --git a/cmd/bd/compact_proxied_server.go b/cmd/bd/compact_proxied_server.go index 5c5765609..e143d31fb 100644 --- a/cmd/bd/compact_proxied_server.go +++ b/cmd/bd/compact_proxied_server.go @@ -143,7 +143,13 @@ func runCompactProxiedServer(ctx context.Context) error { if tags, perr = versioncontrolops.ListTags(ctx, conn); perr != nil { WarnError("listing tags before GC: %v", perr) } - if perr := versioncontrolops.DoltGC(ctx, conn); perr != nil { + // Full pass: the squash orphans a commit chain that any earlier GC + // already promoted to the old generation, which a default pass never + // revisits. + if !jsonOutput { + fmt.Println(" Running full Dolt GC (all generations; can take minutes on large stores)...") + } + if perr := versioncontrolops.DoltGCFull(ctx, conn); perr != nil { WarnError("dolt gc after compact failed: %v", perr) } return nil @@ -167,6 +173,7 @@ func runCompactProxiedServer(ctx context.Context) error { "recent_kept": recentCommits, "remote_refs_pruned": pruned, "tags_anchoring": tags, + "gc_mode": gcModeFull, "elapsed_ms": elapsed.Milliseconds(), }) } diff --git a/cmd/bd/flatten.go b/cmd/bd/flatten.go index 4e7a26d7e..9261e1001 100644 --- a/cmd/bd/flatten.go +++ b/cmd/bd/flatten.go @@ -27,7 +27,13 @@ This uses the Tim Sehn recipe: 4. Swap main branch to the new flattened branch 5. Prune remote-tracking refs (they would keep the old history alive; the next push or fetch re-creates them at the new tip) - 6. Run Dolt GC to reclaim space from old history + 6. Run a full Dolt GC (all storage generations) to reclaim the old history + +The GC pass is a full collection: Dolt storage is generational, and a default +GC never revisits data an earlier GC moved to the old generation. On any store +that has been GC'd before (bd gc, or a previous flatten or compact), only a +full collection reclaims the squashed history. A full GC can take minutes on +multi-gigabyte stores. This is irreversible — all commit history is lost. The resulting database has exactly one commit containing all current data. @@ -143,11 +149,7 @@ Examples: printPruneReport(pruned, tags) } - if gc, ok := storage.UnwrapStore(store).(storage.GarbageCollector); ok { - if err := gc.DoltGC(ctx); err != nil { - WarnError("dolt gc after flatten failed: %v", err) - } - } + gcMode := runPostRewriteGC(ctx, "flatten") sizeAfter := storeSizeBytes(ctx) elapsed := time.Since(start) @@ -161,6 +163,9 @@ Examples: "tags_anchoring": tags, "elapsed_ms": elapsed.Milliseconds(), } + if gcMode != "" { + result["gc_mode"] = gcMode + } addGCSizeJSON(result, sizeBefore, sizeAfter) return outputJSON(result) } diff --git a/cmd/bd/gc.go b/cmd/bd/gc.go index 3a2a611f8..1914128c5 100644 --- a/cmd/bd/gc.go +++ b/cmd/bd/gc.go @@ -16,6 +16,7 @@ var ( gcOlderThan int gcSkipDecay bool gcSkipDolt bool + gcFull bool ) var gcCmd = &cobra.Command{ @@ -32,12 +33,19 @@ Runs three phases in sequence: Each phase can be skipped individually. Use --dry-run to preview all phases without making changes. +Phase 3 runs Dolt's default, generational GC: it only examines data written +since the last GC. Data that survived an earlier GC lives in the old generation +and is never revisited, so on a long-lived store the space freed by decay may +not be reclaimed. Use --full to collect all generations (slower on large +stores). + Examples: bd gc # Full GC with defaults (90 day decay) bd gc --dry-run # Preview what would happen bd gc --older-than 30 # Decay issues closed 30+ days ago bd gc --skip-decay # Skip issue deletion, just compact+GC bd gc --skip-dolt # Skip Dolt GC, just decay+compact + bd gc --full # Collect all storage generations bd gc --force # Skip confirmation prompt`, SilenceUsage: true, SilenceErrors: true, @@ -190,7 +198,11 @@ Examples: results = append(results, phaseResult{name: "Dolt GC", detail: "not supported"}) } else if gcDryRun { if !jsonOutput { - fmt.Println(" Would run DOLT_GC()") + if gcFull { + fmt.Println(" Would run a full DOLT_GC() (all storage generations)") + } else { + fmt.Println(" Would run DOLT_GC()") + } } results = append(results, phaseResult{name: "Dolt GC", detail: "dry-run"}) } else { @@ -200,7 +212,11 @@ Examples: // (bd-agctw). Sizes are reported so a no-op reclaim is visible. sizeBefore := storeSizeBytes(ctx) remoteRefs, tags := listRemoteRefsAndTags(ctx) - if err := gc.DoltGC(ctx); err != nil { + if gcFull && !jsonOutput { + fmt.Println(" Running full Dolt GC (all generations; can take minutes on large stores)...") + } + gcMode, err := runDoltGCPass(ctx, gc, gcFull) + if err != nil { WarnError("dolt gc failed: %v", err) results = append(results, phaseResult{name: "Dolt GC", detail: "failed"}) } else { @@ -211,6 +227,9 @@ Examples: } if !jsonOutput { fmt.Printf(" Done (%s)\n", detail) + if gcMode != gcModeFull && suggestFullGC(sizeBefore, sizeAfter) { + printFullGCHint() + } if len(remoteRefs)+len(tags) > 0 { fmt.Printf(" Note: %d remote-tracking ref(s) and %d tag(s) anchor history;\n", len(remoteRefs), len(tags)) fmt.Printf(" after a history squash, use bd flatten / bd compact so they are pruned first.\n") @@ -220,6 +239,7 @@ Examples: gcSizeInfo = map[string]interface{}{ "remote_refs": len(remoteRefs), "tags": len(tags), + "mode": gcMode, } addGCSizeJSON(gcSizeInfo, sizeBefore, sizeAfter) } @@ -275,6 +295,7 @@ func init() { gcCmd.Flags().IntVar(&gcOlderThan, "older-than", 90, "Delete closed issues older than N days") gcCmd.Flags().BoolVar(&gcSkipDecay, "skip-decay", false, "Skip issue deletion phase") gcCmd.Flags().BoolVar(&gcSkipDolt, "skip-dolt", false, "Skip Dolt garbage collection phase") + gcCmd.Flags().BoolVar(&gcFull, "full", false, "Run a full Dolt GC (all generations; slower, reclaims space default passes cannot)") rootCmd.AddCommand(gcCmd) } diff --git a/cmd/bd/gc_embedded_test.go b/cmd/bd/gc_embedded_test.go index 617cd5c2a..26f72588f 100644 --- a/cmd/bd/gc_embedded_test.go +++ b/cmd/bd/gc_embedded_test.go @@ -126,6 +126,34 @@ func TestEmbeddedGC(t *testing.T) { } }) + // ===== Full GC ===== + + t.Run("gc_full", func(t *testing.T) { + gcDir, _, _ := bdInit(t, bd, "--prefix", "gu") + for i := 0; i < 3; i++ { + bdCreate(t, bd, gcDir, fmt.Sprintf("Full GC issue %d", i), "--type", "task") + } + + // A default pass first, so the second one has an old generation to + // collect — the case a generational pass cannot reclaim (#5907). + bdGC(t, bd, gcDir, "--force", "--skip-decay") + + out := bdGC(t, bd, gcDir, "--full", "--force", "--skip-decay") + if !strings.Contains(out, "GC complete") { + t.Errorf("expected 'GC complete' from bd gc --full: %s", out) + } + if !strings.Contains(out, "full Dolt GC") { + t.Errorf("expected the full-GC notice: %s", out) + } + }) + + t.Run("gc_full_dry_run", func(t *testing.T) { + out := bdGC(t, bd, dir, "--full", "--dry-run") + if !strings.Contains(out, "full DOLT_GC()") { + t.Errorf("dry run does not say it would run a full GC: %s", out) + } + }) + // ===== No --force prompts ===== t.Run("gc_no_force_prompts", func(t *testing.T) { diff --git a/cmd/bd/gc_proxied_server.go b/cmd/bd/gc_proxied_server.go index d234dc5dc..ed4818373 100644 --- a/cmd/bd/gc_proxied_server.go +++ b/cmd/bd/gc_proxied_server.go @@ -155,12 +155,23 @@ func runGCProxiedServer(ctx context.Context) error { } if gcDryRun { if !jsonOutput { - fmt.Println(" Would run DOLT_GC() and DOLT_STATS_GC()") + if gcFull { + fmt.Println(" Would run a full DOLT_GC() (all storage generations) and DOLT_STATS_GC()") + } else { + fmt.Println(" Would run DOLT_GC() and DOLT_STATS_GC()") + } } results = append(results, phaseResult{name: "Dolt GC", detail: "dry-run"}) } else { + if gcFull && !jsonOutput { + fmt.Println(" Running full Dolt GC (all generations; can take minutes on large stores)...") + } err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { - if err := versioncontrolops.DoltGC(ctx, conn); err != nil { + doltGC := versioncontrolops.DoltGC + if gcFull { + doltGC = versioncontrolops.DoltGCFull + } + if err := doltGC(ctx, conn); err != nil { return err } if _, err := conn.ExecContext(ctx, "CALL DOLT_STATS_GC()"); err != nil { diff --git a/cmd/bd/gc_prune.go b/cmd/bd/gc_prune.go index 22831a730..a4030508e 100644 --- a/cmd/bd/gc_prune.go +++ b/cmd/bd/gc_prune.go @@ -74,6 +74,78 @@ func printPruneReport(pruned, tags []string) { } } +// Dolt GC pass kinds, reported as the "mode"/"gc_mode" JSON field. +const ( + gcModeGenerational = "generational" + gcModeFull = "full" +) + +// runDoltGCPass runs the requested Dolt GC pass and reports which pass actually +// ran. Dolt GC is generational: a default pass only visits data written since +// the last GC, while a full pass also collects the old generation. A store that +// does not implement FullGarbageCollector degrades to a generational pass with +// a warning. +func runDoltGCPass(ctx context.Context, gc storage.GarbageCollector, full bool) (string, error) { + if full { + if fullGC, ok := gc.(storage.FullGarbageCollector); ok { + return gcModeFull, fullGC.DoltGCFull(ctx) + } + WarnError("storage backend does not support a full GC; running a generational pass, " + + "which cannot reclaim data an earlier GC moved to the old generation") + } + return gcModeGenerational, gc.DoltGC(ctx) +} + +// runPostRewriteGC runs the GC that follows a history rewrite and reports which +// pass ran ("" when the backend has no GC at all). It is always a full pass: +// the rewrite orphans a commit chain that any earlier GC already promoted to +// the old generation, and a generational pass never revisits it. Failures are +// warnings — the rewrite itself already succeeded. +func runPostRewriteGC(ctx context.Context, op string) string { + gc, ok := storage.UnwrapStore(store).(storage.GarbageCollector) + if !ok { + return "" + } + if !jsonOutput { + fmt.Println(" Running full Dolt GC (all generations; can take minutes on large stores)...") + } + mode, err := runDoltGCPass(ctx, gc, true) + if err != nil { + WarnError("dolt gc after %s failed: %v", op, err) + } + return mode +} + +const ( + // fullGCHintMinStoreBytes keeps the hint off small stores, where a few + // reclaimed kilobytes are not worth a multi-minute full pass. + fullGCHintMinStoreBytes = 64 << 20 + // fullGCHintMaxFreedFraction is what counts as "reclaimed ~nothing". + fullGCHintMaxFreedFraction = 0.01 +) + +// suggestFullGC reports whether a completed generational GC pass freed so +// little of a non-trivial store that the caller should suggest --full. +// before/after are ActiveDatabaseSize measurements; negative means unmeasurable +// and disables the hint. +func suggestFullGC(before, after int64) bool { + if before < 0 || after < 0 || before < fullGCHintMinStoreBytes { + return false + } + freed := before - after + if freed < 0 { + freed = 0 + } + return float64(freed) < fullGCHintMaxFreedFraction*float64(before) +} + +// printFullGCHint explains why a default pass can reclaim nothing (text mode). +func printFullGCHint() { + fmt.Println(" Tip: little space was reclaimed. A default Dolt GC never revisits data an") + fmt.Println(" earlier GC moved to the old generation — run 'bd gc --full' to collect all") + fmt.Println(" generations.") +} + // gcSizeLine formats a before/after size pair as "X → Y (freed Z)", or "" // when either measurement failed. func gcSizeLine(before, after int64) string { diff --git a/cmd/bd/gc_prune_test.go b/cmd/bd/gc_prune_test.go index 534ab2db3..ee902eeff 100644 --- a/cmd/bd/gc_prune_test.go +++ b/cmd/bd/gc_prune_test.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "strings" "testing" "github.com/steveyegge/beads/internal/storage" @@ -95,3 +96,73 @@ func TestAddGCSizeJSONOmitsUnavailableMeasurements(t *testing.T) { t.Fatalf("freed_bytes was emitted without both measurements: %#v", result) } } + +func TestSuggestFullGC(t *testing.T) { + t.Parallel() + + const mib = 1 << 20 + tests := []struct { + name string + before, after int64 + want bool + }{ + {name: "unmeasurable before", before: -1, after: 100 * mib}, + {name: "unmeasurable after", before: 100 * mib, after: -1}, + {name: "both unmeasurable", before: -1, after: -1}, + {name: "small store freeing nothing is not worth a full pass", before: 8 * mib, after: 8 * mib}, + {name: "just below the store floor", before: 64*mib - 1, after: 64*mib - 1}, + {name: "at the store floor, freed nothing", before: 64 * mib, after: 64 * mib, want: true}, + {name: "large store freed nothing", before: 4096 * mib, after: 4096 * mib, want: true}, + {name: "freed 0.9% of the store", before: 1000 * mib, after: 991 * mib, want: true}, + {name: "freed exactly 1% is not ~nothing", before: 1000 * mib, after: 990 * mib}, + {name: "field report: 1276 MB -> 971 MB freed 24%", before: 1276 * mib, after: 971 * mib}, + {name: "store grew during the pass", before: 1000 * mib, after: 1001 * mib, want: true}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := suggestFullGC(tc.before, tc.after); got != tc.want { + t.Fatalf("suggestFullGC(%d, %d) = %v, want %v", tc.before, tc.after, got, tc.want) + } + }) + } +} + +// TestGCHelpDocumentsFullPass pins the help text explaining why a default Dolt +// GC can reclaim nothing. These Long strings are the source the CLI reference +// pages are generated from, so this wording is maintained only here. +func TestGCHelpDocumentsFullPass(t *testing.T) { + t.Parallel() + + full := gcCmd.Flags().Lookup("full") + if full == nil { + t.Fatal("bd gc has no --full flag") + } + if !strings.Contains(full.Usage, "all generations") { + t.Errorf("--full usage does not mention all generations: %q", full.Usage) + } + + for _, want := range []string{"--full", "old generation", "generational"} { + if !strings.Contains(gcCmd.Long, want) { + t.Errorf("gcCmd.Long does not mention %q", want) + } + } + + rewrites := []struct { + name string + long string + }{ + {"flatten", flattenCmd.Long}, + {"compact", compactDoltCmd.Long}, + } + for _, cmd := range rewrites { + if !strings.Contains(cmd.long, "full Dolt GC") { + t.Errorf("%s help does not say it runs a full Dolt GC", cmd.name) + } + if !strings.Contains(cmd.long, "old generation") { + t.Errorf("%s help does not explain the old generation", cmd.name) + } + } +} diff --git a/docs/architecture/dolt.md b/docs/architecture/dolt.md index 9e16ea6ea..53ed451d0 100644 --- a/docs/architecture/dolt.md +++ b/docs/architecture/dolt.md @@ -253,6 +253,25 @@ stop at that step, and an abandoned merge is left without its rollback. See [Which Dolt version to install](#which-dolt-version-to-install) for the check and the fix. +### Why a garbage collection can reclaim nothing + +Dolt's garbage collector is **generational**. Every pass moves the data +reachable at that moment into an *old generation*, and a default pass only +examines the *new generation* — data written since the last collection. So the +second collection on a store never revisits what the first one kept, however +much of it has since become unreachable. Squashing history and then running a +default `dolt gc` is the case where this bites: the orphaned commit chain +usually sits in the old generation, and the pass frees almost nothing. + +`dolt gc --full` collects both generations. It frees everything a default pass +frees, plus the old generation, so it is never worth running a default pass +first to save time. If a full collection frees nothing, the remaining bytes are +still referenced — by a branch, a tag, or a cached remote-tracking ref — and +the fix is to remove the reference, not to collect again. The +[History Bloat runbook](/recovery/history-squash) walks through that diagnosis; +[`bd flatten`](/cli-reference/flatten), [`bd compact`](/cli-reference/compact), +and [`bd gc`](/cli-reference/gc) document the collection each one runs. + ## Migrating Between Backends You can migrate data between embedded mode and server mode using `bd backup`. diff --git a/docs/recovery/history-squash.md b/docs/recovery/history-squash.md index ff37ed241..c96e96760 100644 --- a/docs/recovery/history-squash.md +++ b/docs/recovery/history-squash.md @@ -157,4 +157,6 @@ routine agent traffic does not mint history — claim leases and so bloat at this scale usually means something is writing versioned tables in a tight loop. Find and fix that writer, watch data-directory growth over time, and run `dolt gc` periodically so unreachable garbage never accumulates on top -of reachable history. +of reachable history — with an occasional `dolt gc --full`, since a default +pass is generational and never revisits data an earlier pass moved to the old +generation. diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 1ce328b32..0fbf42a36 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -301,6 +301,7 @@ var _ storage.ActiveDatabaseSizer = (*DoltStore)(nil) var _ storage.LifecycleManager = (*DoltStore)(nil) var _ storage.PendingCommitter = (*DoltStore)(nil) var _ storage.GarbageCollector = (*DoltStore)(nil) +var _ storage.FullGarbageCollector = (*DoltStore)(nil) var _ storage.Flattener = (*DoltStore)(nil) var _ storage.Compactor = (*DoltStore)(nil) var _ storage.SchemaMigrator = (*DoltStore)(nil) @@ -2869,8 +2870,8 @@ func (s *DoltStore) ActiveDatabaseSize(ctx context.Context) (int64, error) { return size, nil } -// DoltGC runs Dolt garbage collection to reclaim disk space. -// Pins a single connection to avoid session state loss on pooled *sql.DB. +// DoltGC runs Dolt's default, generational garbage collection to reclaim disk +// space. Pins a single connection to avoid session state loss on pooled *sql.DB. func (s *DoltStore) DoltGC(ctx context.Context) error { conn, err := s.db.Conn(ctx) if err != nil { @@ -2880,6 +2881,18 @@ func (s *DoltStore) DoltGC(ctx context.Context) error { return versioncontrolops.DoltGC(ctx, conn) } +// DoltGCFull runs a full Dolt garbage collection across all storage +// generations. Pins a single connection to avoid session state loss on pooled +// *sql.DB. +func (s *DoltStore) DoltGCFull(ctx context.Context) error { + conn, err := s.db.Conn(ctx) + if err != nil { + return fmt.Errorf("acquire connection for gc: %w", err) + } + defer conn.Close() + return versioncontrolops.DoltGCFull(ctx, conn) +} + // ListRemoteRefs returns the names of all cached remote-tracking refs. func (s *DoltStore) ListRemoteRefs(ctx context.Context) ([]string, error) { return versioncontrolops.ListRemoteRefs(ctx, s.db) diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index b7275fb48..4afae36eb 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -27,6 +27,7 @@ var _ storage.DoltStorage = (*EmbeddedDoltStore)(nil) var _ storage.StoreLocator = (*EmbeddedDoltStore)(nil) var _ storage.ActiveDatabaseSizer = (*EmbeddedDoltStore)(nil) var _ storage.GarbageCollector = (*EmbeddedDoltStore)(nil) +var _ storage.FullGarbageCollector = (*EmbeddedDoltStore)(nil) var _ storage.Flattener = (*EmbeddedDoltStore)(nil) var _ storage.Compactor = (*EmbeddedDoltStore)(nil) var _ storage.SchemaMigrator = (*EmbeddedDoltStore)(nil) @@ -681,13 +682,22 @@ func (s *EmbeddedDoltStore) Close() error { return nil } -// DoltGC runs Dolt garbage collection to reclaim disk space. +// DoltGC runs Dolt's default, generational garbage collection to reclaim disk +// space. func (s *EmbeddedDoltStore) DoltGC(ctx context.Context) error { return s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { return versioncontrolops.DoltGC(ctx, db) }) } +// DoltGCFull runs a full Dolt garbage collection across all storage +// generations. +func (s *EmbeddedDoltStore) DoltGCFull(ctx context.Context) error { + return s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { + return versioncontrolops.DoltGCFull(ctx, db) + }) +} + // ListRemoteRefs returns the names of all cached remote-tracking refs. func (s *EmbeddedDoltStore) ListRemoteRefs(ctx context.Context) ([]string, error) { var refs []string diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 953bf31f1..7754e1eaa 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -691,6 +691,19 @@ type GarbageCollector interface { DoltGC(ctx context.Context) error } +// FullGarbageCollector provides a full Dolt garbage collection, which collects +// the old generation as well as the new one. +// +// Dolt GC is generational: each pass promotes the chunks reachable at that +// moment into the old generation, and DoltGC only visits the new generation. +// After a history rewrite (Flatten, Compact) the orphaned commit chain usually +// lives in the old generation, so DoltGC reclaims nothing and callers must use +// DoltGCFull. Prefer DoltGC for periodic hygiene, where a full pass costs +// minutes on large stores for no extra reclaim. +type FullGarbageCollector interface { + DoltGCFull(ctx context.Context) error +} + // Flattener squashes all Dolt commit history into a single commit. // Callers should type-assert to this interface for history compaction. type Flattener interface { diff --git a/internal/storage/versioncontrolops/gc.go b/internal/storage/versioncontrolops/gc.go index f0885afa6..c0e584e25 100644 --- a/internal/storage/versioncontrolops/gc.go +++ b/internal/storage/versioncontrolops/gc.go @@ -5,12 +5,45 @@ import ( "fmt" ) -// DoltGC runs Dolt garbage collection to reclaim disk space. Archive level 0 -// writes classic Snappy table files instead of zstd archives. +// DoltGC runs Dolt's default, generational garbage collection to reclaim disk +// space. Archive level 0 writes classic Snappy table files instead of zstd +// archives. +// +// Dolt storage is generational: every GC promotes the chunks reachable at that +// moment into the old generation, and a default pass only visits the new +// generation. Data that survived an earlier GC is therefore never revisited, +// so this pass reclaims only what was written since the last one. Use +// DoltGCFull after a history rewrite, which orphans chunks an earlier GC +// already promoted. +// // conn must be a non-transactional database connection since // DOLT_GC cannot run inside an explicit transaction. func DoltGC(ctx context.Context, conn DBConn) error { - if _, err := conn.ExecContext(ctx, "CALL DOLT_GC('--archive-level', '0')"); err != nil { + return doltGC(ctx, conn, false) +} + +// DoltGCFull runs a full Dolt garbage collection, collecting both the old and +// the new generation. Its reclaim set is a strict superset of DoltGC's: it +// frees everything a default pass frees, plus data an earlier GC promoted to +// the old generation. If a full pass frees nothing, the remaining bytes are +// still referenced. +// +// Full collections rewrite the old generation, so they cost more than a +// default pass — minutes on multi-gigabyte stores. Callers should reserve it +// for history rewrites (flatten, compact) and explicit opt-in. +// +// conn must be a non-transactional database connection since +// DOLT_GC cannot run inside an explicit transaction. +func DoltGCFull(ctx context.Context, conn DBConn) error { + return doltGC(ctx, conn, true) +} + +func doltGC(ctx context.Context, conn DBConn, full bool) error { + stmt := "CALL DOLT_GC('--archive-level', '0')" + if full { + stmt = "CALL DOLT_GC('--full', '--archive-level', '0')" + } + if _, err := conn.ExecContext(ctx, stmt); err != nil { return fmt.Errorf("dolt gc: %w", err) } return nil diff --git a/internal/storage/versioncontrolops/gc_test.go b/internal/storage/versioncontrolops/gc_test.go index 803b92ecd..66fb9ce29 100644 --- a/internal/storage/versioncontrolops/gc_test.go +++ b/internal/storage/versioncontrolops/gc_test.go @@ -25,3 +25,21 @@ func TestDoltGCDisablesArchiveCompression(t *testing.T) { t.Fatal(err) } } + +func TestDoltGCFullCollectsAllGenerations(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_GC('--full', '--archive-level', '0')")). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := DoltGCFull(context.Background(), db); err != nil { + t.Fatalf("DoltGCFull: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +}