* feat(storage): add DoltGCFull for full, all-generation Dolt GC
Dolt GC is generational: every pass promotes the chunks reachable at that
moment into the old generation, and a default pass only visits the new
generation. Callers that rewrite history need a collection that revisits
the old generation too.
Adds versioncontrolops.DoltGCFull (CALL DOLT_GC('--full', '--archive-level',
'0')), the optional storage.FullGarbageCollector capability, and
implementations on both in-tree Dolt stores. No call sites change yet.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* fix(gc): run a full Dolt GC after flatten and compact
The post-rewrite GC was Dolt's default, generational pass. Any earlier GC —
bd gc, or a previous flatten or compact — has already promoted the commit
chain into the old generation, which a default pass never revisits. The
rewrite orphans that chain, so the follow-up GC freed ~nothing on every
mature store and orphaned --as-of addresses kept resolving.
flatten, compact, and the proxied-server compact now finish with a full
collection, which frees everything a default pass frees plus the old
generation. Text mode prints a notice first (a full pass can take minutes on
large stores) and the JSON result reports gc_mode.
Unchanged: bd gc's own phase 3, bd admin compact --dolt, and the
managed-server shutdown GC all stay generational.
Fixes #5907
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* feat(gc): add bd gc --full and hint when a default pass reclaims little
Periodic hygiene stays generational by default — a full pass costs minutes on
a large store and usually reclaims nothing extra. But phase 3 has flatten's
blind spot too: row data from decayed issues was promoted to the old
generation by an earlier GC, so a default pass cannot reclaim it. --full
selects a full collection on the embedded, server, and proxied paths.
When a default pass leaves a non-trivial store (>= 64 MiB) barely smaller
(< 1% freed), print a tip explaining the generational model and pointing at
--full. Unmeasurable sizes disable the hint.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* test(gc): cover suggestFullGC thresholds, GC help text, and bd gc --full
The threshold table includes the two field datapoints behind the
calibration: a large store that freed nothing hints, and the 1276 -> 971 MB
run that freed 24% does not.
The help-text assertions pin the wording in the three Cobra Long strings,
which are the source the CLI reference pages are generated from. One of them
also caught a line wrap that split "old generation" across a newline.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* docs: explain why a Dolt garbage collection can reclaim nothing
Adds a subsection to the Dolt backend page covering the generational model —
promotion to the old generation, why a default pass never revisits it, and
why a full collection is a strict superset — and extends the History Bloat
runbook's Prevention advice with an occasional full pass.
Both pages describe the pinned release (v1.2.2), so they stay on the dolt CLI
and do not name the bd behavior this branch adds; the generated CLI reference
picks that up when the pin is bumped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
168 lines
4.5 KiB
Go
168 lines
4.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/steveyegge/beads/internal/storage"
|
|
)
|
|
|
|
type gcSizeStoreStub struct {
|
|
storage.DoltStorage
|
|
size int64
|
|
err error
|
|
}
|
|
|
|
func (s *gcSizeStoreStub) ActiveDatabaseSize(context.Context) (int64, error) {
|
|
return s.size, s.err
|
|
}
|
|
|
|
type gcLocatorOnlyStoreStub struct {
|
|
storage.DoltStorage
|
|
path string
|
|
}
|
|
|
|
func (s *gcLocatorOnlyStoreStub) Path() string { return s.path }
|
|
func (s *gcLocatorOnlyStoreStub) CLIDir() string { return s.path }
|
|
|
|
func TestStoreSizeBytesForStoreUsesOnlyActiveDatabaseSizer(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
failure := errors.New("measurement failed")
|
|
tests := []struct {
|
|
name string
|
|
store storage.DoltStorage
|
|
want int64
|
|
}{
|
|
{
|
|
name: "active database available",
|
|
store: &gcSizeStoreStub{size: 42},
|
|
want: 42,
|
|
},
|
|
{
|
|
name: "unsupported active database",
|
|
store: &gcSizeStoreStub{err: &storage.ErrUnsupported{
|
|
Op: "ActiveDatabaseSize",
|
|
Backend: "external",
|
|
}},
|
|
want: -1,
|
|
},
|
|
{
|
|
name: "measurement failure",
|
|
store: &gcSizeStoreStub{err: failure},
|
|
want: -1,
|
|
},
|
|
{
|
|
name: "legacy locator is not a size fallback",
|
|
store: &gcLocatorOnlyStoreStub{path: t.TempDir()},
|
|
want: -1,
|
|
},
|
|
{
|
|
name: "nil store",
|
|
want: -1,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
if got := storeSizeBytesForStore(t.Context(), tc.store); got != tc.want {
|
|
t.Fatalf("storeSizeBytesForStore() = %d, want %d", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAddGCSizeJSONOmitsUnavailableMeasurements(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result := map[string]interface{}{}
|
|
addGCSizeJSON(result, -1, -1)
|
|
if len(result) != 0 {
|
|
t.Fatalf("unavailable measurements produced JSON fields: %#v", result)
|
|
}
|
|
|
|
addGCSizeJSON(result, 42, -1)
|
|
if got := result["size_before_bytes"]; got != int64(42) {
|
|
t.Fatalf("size_before_bytes = %#v, want 42", got)
|
|
}
|
|
if _, ok := result["size_after_bytes"]; ok {
|
|
t.Fatalf("unavailable size_after_bytes was emitted: %#v", result)
|
|
}
|
|
if _, ok := result["freed_bytes"]; ok {
|
|
t.Fatalf("freed_bytes was emitted without both measurements: %#v", result)
|
|
}
|
|
}
|
|
|
|
func TestSuggestFullGC(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const mib = 1 << 20
|
|
tests := []struct {
|
|
name string
|
|
before, after int64
|
|
want bool
|
|
}{
|
|
{name: "unmeasurable before", before: -1, after: 100 * mib},
|
|
{name: "unmeasurable after", before: 100 * mib, after: -1},
|
|
{name: "both unmeasurable", before: -1, after: -1},
|
|
{name: "small store freeing nothing is not worth a full pass", before: 8 * mib, after: 8 * mib},
|
|
{name: "just below the store floor", before: 64*mib - 1, after: 64*mib - 1},
|
|
{name: "at the store floor, freed nothing", before: 64 * mib, after: 64 * mib, want: true},
|
|
{name: "large store freed nothing", before: 4096 * mib, after: 4096 * mib, want: true},
|
|
{name: "freed 0.9% of the store", before: 1000 * mib, after: 991 * mib, want: true},
|
|
{name: "freed exactly 1% is not ~nothing", before: 1000 * mib, after: 990 * mib},
|
|
{name: "field report: 1276 MB -> 971 MB freed 24%", before: 1276 * mib, after: 971 * mib},
|
|
{name: "store grew during the pass", before: 1000 * mib, after: 1001 * mib, want: true},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
if got := suggestFullGC(tc.before, tc.after); got != tc.want {
|
|
t.Fatalf("suggestFullGC(%d, %d) = %v, want %v", tc.before, tc.after, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestGCHelpDocumentsFullPass pins the help text explaining why a default Dolt
|
|
// GC can reclaim nothing. These Long strings are the source the CLI reference
|
|
// pages are generated from, so this wording is maintained only here.
|
|
func TestGCHelpDocumentsFullPass(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
full := gcCmd.Flags().Lookup("full")
|
|
if full == nil {
|
|
t.Fatal("bd gc has no --full flag")
|
|
}
|
|
if !strings.Contains(full.Usage, "all generations") {
|
|
t.Errorf("--full usage does not mention all generations: %q", full.Usage)
|
|
}
|
|
|
|
for _, want := range []string{"--full", "old generation", "generational"} {
|
|
if !strings.Contains(gcCmd.Long, want) {
|
|
t.Errorf("gcCmd.Long does not mention %q", want)
|
|
}
|
|
}
|
|
|
|
rewrites := []struct {
|
|
name string
|
|
long string
|
|
}{
|
|
{"flatten", flattenCmd.Long},
|
|
{"compact", compactDoltCmd.Long},
|
|
}
|
|
for _, cmd := range rewrites {
|
|
if !strings.Contains(cmd.long, "full Dolt GC") {
|
|
t.Errorf("%s help does not say it runs a full Dolt GC", cmd.name)
|
|
}
|
|
if !strings.Contains(cmd.long, "old generation") {
|
|
t.Errorf("%s help does not explain the old generation", cmd.name)
|
|
}
|
|
}
|
|
}
|