diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index baeaa442d..25b7ac945 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -79,7 +79,20 @@ func saveBackupState(dir string, state *backupState) error { return atomicWriteFile(filepath.Join(dir, "backup_state.json"), data) } -// atomicWriteFile writes data to a temp file and renames it into place (crash-safe). +// atomicWriteFile writes data to a same-directory temp file, fsyncs the +// temp file's own contents, then renames it into place. This avoids a +// truncated/partial file at path if the process crashes mid-write. +// +// Two caveats this does NOT cover, narrowing the "crash-safe" claim rather +// than the implementation (existing callers' behavior is unchanged here): +// - Only the temp file's contents are fsynced, not the parent directory +// entry; a crash between the rename and a subsequent directory fsync +// can still lose the rename itself on some filesystems. +// - os.Rename's atomic-replace guarantee is a POSIX/Unix property; it is +// not guaranteed on Windows. It also does not follow a symlink at +// path — it replaces whatever is there, symlink or not — so a caller +// that must preserve a symlink's target should resolve path with +// filepath.EvalSymlinks first (see cmd/bd/proxied_server.go). func atomicWriteFile(path string, data []byte) error { dir := filepath.Dir(path) tmp, err := os.CreateTemp(dir, ".backup-tmp-*") diff --git a/cmd/bd/compact.go b/cmd/bd/compact.go index 3586357d9..b7d11cc54 100644 --- a/cmd/bd/compact.go +++ b/cmd/bd/compact.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/spf13/cobra" @@ -807,10 +808,25 @@ func runCompactDolt() error { fmt.Printf("Running Dolt garbage collection...\n") } - // Run dolt gc - cmd := exec.Command("dolt", "gc") // #nosec G204 -- fixed command, no user input + // Run dolt gc without archive compression. Level 0 writes classic Snappy + // table files instead of zstd archives, matching the in-process GC paths. + // The external `dolt` on PATH has no version guarantee (unlike the + // in-process paths, which are pinned by go.mod), so an older dolt that + // predates --archive-level would otherwise abort compact where plain + // `dolt gc` used to work. Detect that specific unknown-flag rejection + // and retry with plain `dolt gc` rather than fail outright; any other + // error (a genuine GC failure) is not swallowed. + cmd := exec.Command("dolt", "gc", "--archive-level", "0") // #nosec G204 -- fixed command, no user input cmd.Dir = doltPath output, err := cmd.CombinedOutput() + if err != nil && isUnknownArchiveLevelFlagError(string(output)) { + if !jsonOutput { + fmt.Fprintf(os.Stderr, "Notice: external dolt does not support --archive-level; falling back to plain 'dolt gc' (new table files may still use zstd archives)\n") + } + fallbackCmd := exec.Command("dolt", "gc") // #nosec G204 -- fixed command, no user input + fallbackCmd.Dir = doltPath + output, err = fallbackCmd.CombinedOutput() + } if err != nil { fmt.Fprintf(os.Stderr, "Error: dolt gc failed: %v\n", err) if len(output) > 0 { @@ -854,6 +870,29 @@ func runCompactDolt() error { return nil } +// isUnknownArchiveLevelFlagError reports whether output (the combined +// stdout+stderr of `dolt gc --archive-level 0`) indicates the external dolt +// binary rejected --archive-level as an unrecognized flag, rather than a +// genuine GC failure. Older Dolt releases that predate the flag report this +// via the pinned dolt module's argparser, e.g.: +// +// error: unknown option `archive-level' +// +// (see libraries/utils/argparser/errors.go in the pinned dolthub/dolt/go +// module). The check requires both an "unknown flag" phrasing AND the flag +// name to appear in the output, so a real GC failure that happens to +// contain the word "unknown" elsewhere is never misclassified as a missing +// flag and silently swallowed — genuine failures continue to fail. +func isUnknownArchiveLevelFlagError(output string) bool { + lower := strings.ToLower(output) + if !strings.Contains(lower, "archive-level") && !strings.Contains(lower, "archive_level") { + return false + } + return strings.Contains(lower, "unknown option") || + strings.Contains(lower, "unknown flag") || + strings.Contains(lower, "flag provided but not defined") +} + // getDirSize calculates the total size of a directory recursively func getDirSize(path string) (int64, error) { var size int64 diff --git a/cmd/bd/compact_gc_flag_test.go b/cmd/bd/compact_gc_flag_test.go new file mode 100644 index 000000000..dbc2629f7 --- /dev/null +++ b/cmd/bd/compact_gc_flag_test.go @@ -0,0 +1,71 @@ +package main + +import "testing" + +// TestIsUnknownArchiveLevelFlagError covers the classifier that decides +// whether runCompactDolt should retry `dolt gc` without --archive-level +// (gastownhall/beads#4986). It must recognize the real unknown-flag +// rejection from an older external dolt, and must NOT misclassify genuine +// GC failures — those should keep failing compact rather than being +// silently swallowed by a retry that then also fails. +func TestIsUnknownArchiveLevelFlagError(t *testing.T) { + tests := []struct { + name string + output string + want bool + }{ + { + name: "real dolt argparser unknown-option message", + output: "error: unknown option `archive-level'\n\nNAME\n\tdolt gc - Cleans up unreferenced data from the repository.\n", + want: true, + }, + { + name: "uppercase / mixed case tolerant", + output: "ERROR: Unknown Option `archive-level'", + want: true, + }, + { + name: "cobra-pflag style unknown flag message", + output: "unknown flag: --archive-level", + want: true, + }, + { + name: "go flag package style message", + output: "flag provided but not defined: -archive-level", + want: true, + }, + { + name: "underscore spelling still matches", + output: "error: unknown option `archive_level'", + want: true, + }, + { + name: "genuine GC failure must not be swallowed", + output: "error: could not acquire lock on repository; another process may be using it", + want: false, + }, + { + name: "unrelated unknown-option error for a different flag", + output: "error: unknown option `full'", + want: false, + }, + { + name: "mentions archive-level but is not an unknown-flag error", + output: "Specify the archive compression level garbage collection results. Default is 1, Disable with 0", + want: false, + }, + { + name: "empty output", + output: "", + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isUnknownArchiveLevelFlagError(tc.output); got != tc.want { + t.Errorf("isUnknownArchiveLevelFlagError(%q) = %v, want %v", tc.output, got, tc.want) + } + }) + } +} diff --git a/cmd/bd/proxied_server.go b/cmd/bd/proxied_server.go index dead1818e..dfec4b7dd 100644 --- a/cmd/bd/proxied_server.go +++ b/cmd/bd/proxied_server.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "os" "os/exec" @@ -17,6 +18,157 @@ import ( "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" ) +// archiveLevelSupportNoticeFormat is the one-time stderr notice emitted when +// generating a fresh managed-server config.yaml against an external dolt +// that predates auto_gc_behavior.archive_level support. Never fires for an +// existing config.yaml (ensureProxiedServerConfig only renders on first +// bootstrap), so this is not a per-invocation warning. +const archiveLevelSupportNoticeFormat = "Info: external dolt predates archive_level config support (need >= Dolt %s); " + + "this managed server's background auto-GC may still produce zstd archives.\n" + +// managedProxiedServerConfigMarker is written as a YAML comment header on +// every config.yaml Beads itself generates (renderProxiedServerConfig). +// YAML comments are ignored by Dolt's own parser, so this is invisible to +// dolt sql-server; Beads uses it to tell its own config.yaml apart from a +// user-supplied one (env var / sidecar custom path, or a default-path file +// pre-dating this marker) before deciding whether it is safe to +// automatically reconcile auto_gc_behavior.archive_level on load +// (reconcileManagedProxiedServerConfig) versus leaving the file untouched +// and only warning (warnUnmanagedProxiedServerConfig). See +// gastownhall/beads#4986. +const managedProxiedServerConfigMarker = "# Managed by Beads - safe to delete (will be regenerated). " + + "Hand edits may be lost on upgrade. See gastownhall/beads#4986.\n" + +// isManagedProxiedServerConfig reports whether body was generated by +// renderProxiedServerConfig (carries managedProxiedServerConfigMarker as +// its first line) rather than hand-written or supplied by a user. +func isManagedProxiedServerConfig(body []byte) bool { + return bytes.HasPrefix(body, []byte(managedProxiedServerConfigMarker)) +} + +// reconcileManagedProxiedServerConfig parse-modify-rewrites a Beads-managed +// config.yaml's auto_gc_behavior.archive_level so it matches the resolved +// external dolt's capability, preserving every other field (including +// port, and sibling auto_gc_behavior fields such as a user-set "enable") +// by round tripping through the servercfg YAML structs rather than string +// surgery: +// +// - archiveLevelSupported and archive_level is absent: set it to 0 +// (auto-GC stays enabled), creating the auto_gc_behavior block if +// necessary but preserving any existing sibling fields on it. +// - !archiveLevelSupported and archive_level is present: clear only +// archive_level, so a dolt downgrade cannot hard-fail on +// yaml.UnmarshalStrict seeing a key its own older YAMLConfig struct +// does not have. The block itself is only removed if doing so leaves +// it entirely empty (no enable, no incremental_file_size either). +// +// Deliberately inspects only ArchiveLevel_, not "the whole block is +// present" — an earlier version of this function tested +// cfg.BehaviorConfig.AutoGCBehavior != nil, which meant a block carrying +// only a user-set "enable" (no archive_level) both failed to gain +// archive_level when supported, and lost its "enable" setting entirely +// when the whole block was stripped on an unsupported binary. +// +// Returns the rewritten bytes (with the marker re-prepended) and whether a +// rewrite is actually needed; callers should skip the write when unchanged. +func reconcileManagedProxiedServerConfig(body []byte, archiveLevelSupported bool) ([]byte, bool, error) { + cfg, err := servercfg.NewYamlConfig(body) + if err != nil { + return nil, false, fmt.Errorf("parse: %w", err) + } + + gc := cfg.BehaviorConfig.AutoGCBehavior + hasArchiveLevel := gc != nil && gc.ArchiveLevel_ != nil + changed := false + + switch { + case archiveLevelSupported && !hasArchiveLevel: + if gc == nil { + gc = &servercfg.AutoGCBehaviorYAMLConfig{} + cfg.BehaviorConfig.AutoGCBehavior = gc + } + archiveLevel := 0 + gc.ArchiveLevel_ = &archiveLevel + changed = true + case !archiveLevelSupported && hasArchiveLevel: + gc.ArchiveLevel_ = nil + if isAutoGCBehaviorEmpty(gc) { + cfg.BehaviorConfig.AutoGCBehavior = nil + } + changed = true + } + + if !changed { + return body, false, nil + } + + rendered, err := yaml.Marshal(cfg) + if err != nil { + return nil, false, fmt.Errorf("render: %w", err) + } + return append([]byte(managedProxiedServerConfigMarker), rendered...), true, nil +} + +// isAutoGCBehaviorEmpty reports whether gc has no fields set at all, i.e. +// it is now safe to omit the whole auto_gc_behavior block rather than +// render an empty "{}" or "auto_gc_behavior:" with no children. +func isAutoGCBehaviorEmpty(gc *servercfg.AutoGCBehaviorYAMLConfig) bool { + return gc.Enable_ == nil && gc.ArchiveLevel_ == nil && gc.IncrementalFileSize_ == nil +} + +// warnUnmanagedProxiedServerConfig emits a one-line notice for a config.yaml +// Beads never rewrites (no managedProxiedServerConfigMarker: a user-custom +// path, or a default-path file predating this marker). It surfaces the same +// two risks reconcileManagedProxiedServerConfig fixes automatically for a +// Beads-owned config, without ever touching the file: +// - archiveLevelSupported and archive_level is absent: zstd auto-GC may +// still be active; tell the user how to opt in. +// - !archiveLevelSupported and archive_level is present: the resolved +// external dolt may refuse to start on this file (yaml.UnmarshalStrict +// on an unrecognized key). +// +// Deliberately checks only ArchiveLevel_ (via autoGCArchiveLevelIsSet), not +// "the whole auto_gc_behavior block is present" — the same field-vs-block +// distinction reconcileManagedProxiedServerConfig applies. A block carrying +// only a user-set "enable" (no archive_level) would otherwise both miss the +// zstd-risk warning on a capable binary, and get a false +// refuse-to-start warning on an incapable one. +// +// cfg must be the ALREADY-PARSED config (via servercfg.YamlConfigFromFile, +// not NewYamlConfig on raw bytes) — callers own an unmanaged file's parse, +// and that parse must be interpolation-aware (see the ensureProxiedServerConfig +// doc comment on why raw-bytes parsing is wrong for these files). +func warnUnmanagedProxiedServerConfig(path string, cfg servercfg.ServerConfig, archiveLevelSupported bool) { + hasArchiveLevel := autoGCArchiveLevelIsSet(cfg) + switch { + case archiveLevelSupported && !hasArchiveLevel: + fmt.Fprintf(os.Stderr, + "Warning: %s is not Beads-managed (no marker); zstd auto-GC may still be active. "+ + "To opt in, add 'auto_gc_behavior: {archive_level: 0}' under the top-level 'behavior:' key.\n", + path) + case !archiveLevelSupported && hasArchiveLevel: + fmt.Fprintf(os.Stderr, + "Warning: %s sets auto_gc_behavior.archive_level, but the resolved external dolt predates support "+ + "for it (need >= Dolt %s); the server may refuse to start. Remove the key or upgrade dolt.\n", + path, doltserver.MinDoltVersionForArchiveLevelConfig) + } +} + +// autoGCArchiveLevelIsSet reports whether cfg's auto_gc_behavior.archive_level +// key is explicitly set — not just "the auto_gc_behavior block is present" +// (a block can carry only a sibling field like "enable"). Both +// servercfg.YamlConfigFromFile and servercfg.NewYamlConfig always return a +// *servercfg.YAMLConfig underneath the ServerConfig interface in the +// pinned module, so the type assertion below is expected to succeed; if it +// ever doesn't, this falls back to the coarser "block is present" test +// rather than panicking. +func autoGCArchiveLevelIsSet(cfg servercfg.ServerConfig) bool { + if yc, ok := cfg.(*servercfg.YAMLConfig); ok { + return yc.BehaviorConfig.AutoGCBehavior != nil && yc.BehaviorConfig.AutoGCBehavior.ArchiveLevel_ != nil + } + return cfg.AutoGCBehavior() != nil +} + const ( proxiedServerConfigName = "config.yaml" proxiedServerLogName = "server.log" @@ -95,7 +247,21 @@ func resolveProxiedServerLogPath(beadsDir string) (path string, isCustom bool, e return filepath.Join(root, proxiedServerLogName), false, nil } -func ensureProxiedServerConfig(beadsDir string) (string, error) { +// ensureProxiedServerConfig resolves (creating if needed) the managed +// sql-server config.yaml. archiveLevelSupported gates whether a freshly +// generated config sets auto_gc_behavior.archive_level: 0 (see +// renderProxiedServerConfig). +// +// Parsing an EXISTING file (custom path, or a pre-marker default-path file +// we only warn about — never a Beads-managed one) MUST go through +// servercfg.YamlConfigFromFile, not servercfg.NewYamlConfig on raw bytes: +// YamlConfigFromFile expands ${VAR}-style environment placeholders before +// parsing (servercfg/env_interpolate.go), and a user's custom config is +// entitled to use that syntax the same way `dolt sql-server --config` +// itself would accept it. NewYamlConfig on raw bytes is only safe for a +// Beads-managed file (marker present), because Beads itself never writes +// interpolation syntax into a file it generates. +func ensureProxiedServerConfig(beadsDir string, archiveLevelSupported bool) (string, error) { path, isCustom, err := resolveProxiedServerConfigPath(beadsDir) if err != nil { return "", err @@ -109,9 +275,13 @@ func ensureProxiedServerConfig(beadsDir string) (string, error) { if !info.Mode().IsRegular() { return "", fmt.Errorf("ensureProxiedServerConfig: custom config %s: not a regular file", path) } - if _, err := servercfg.YamlConfigFromFile(filesys.LocalFS, path); err != nil { + cfg, err := servercfg.YamlConfigFromFile(filesys.LocalFS, path) + if err != nil { return "", fmt.Errorf("ensureProxiedServerConfig: custom config %s: parse: %w", path, err) } + // A custom config is always user-owned, whether or not it happens to + // carry the marker — never rewritten, only warned about. + warnUnmanagedProxiedServerConfig(path, cfg, archiveLevelSupported) return path, nil } @@ -122,8 +292,57 @@ func ensureProxiedServerConfig(beadsDir string) (string, error) { switch _, err := os.Stat(path); { case err == nil: - if _, err := servercfg.YamlConfigFromFile(filesys.LocalFS, path); err != nil { - return "", fmt.Errorf("ensureProxiedServerConfig: existing config %s: parse: %w", path, err) + // The marker is a literal comment line Beads itself writes and never + // interpolates — safe to check on raw bytes before deciding which + // (interpolation-aware or not) parse path applies below. + body, err := os.ReadFile(path) //nolint:gosec // G304: path is Beads' own default config.yaml location, not user-request input + if err != nil { + return "", fmt.Errorf("ensureProxiedServerConfig: read existing config %s: %w", path, err) + } + if isManagedProxiedServerConfig(body) { + // Beads-managed: never carries interpolation syntax, so raw-bytes + // parsing (inside reconcileManagedProxiedServerConfig) is safe and + // avoids a redundant interpolation pass. BUT the marker only + // proves Beads generated the file originally — an operator can + // still hand-edit a managed file afterward (e.g. add a ${VAR} + // placeholder), which the non-interpolating raw parse cannot + // handle. That must degrade gracefully rather than fail the + // whole command: dolt itself would still start fine against + // such a file once its own interpolation pass runs. + reconciled, changed, rerr := reconcileManagedProxiedServerConfig(body, archiveLevelSupported) + if rerr != nil { + fmt.Fprintf(os.Stderr, + "Warning: %s carries the Beads-managed marker but failed to parse (%v); treating as "+ + "unmanaged (not rewriting). If this file was hand-edited (e.g. with ${VAR} "+ + "placeholders), that's expected — dolt itself will interpolate it at startup.\n", + path, rerr) + cfg, cerr := servercfg.YamlConfigFromFile(filesys.LocalFS, path) + if cerr != nil { + return "", fmt.Errorf("ensureProxiedServerConfig: existing config %s: parse: %w", path, cerr) + } + warnUnmanagedProxiedServerConfig(path, cfg, archiveLevelSupported) + return path, nil + } + if changed { + // os.Rename (inside atomicWriteFile) does not follow a + // symlink at its destination — it replaces whatever is AT + // that path, symlink or not. Resolve to the physical file + // first so a symlinked managed config is rewritten through + // the symlink (the symlink itself survives) rather than + // having the symlink replaced by a regular file. + if werr := atomicWriteFile(resolveConfigWriteTarget(path), reconciled); werr != nil { + return "", fmt.Errorf("ensureProxiedServerConfig: rewrite managed config %s: %w", path, werr) + } + } + } else { + // Pre-marker default-path file (created by a Beads version before + // this fix) — treat like a custom config: interpolation-aware + // parse, never rewritten, only warned about. + cfg, err := servercfg.YamlConfigFromFile(filesys.LocalFS, path) + if err != nil { + return "", fmt.Errorf("ensureProxiedServerConfig: existing config %s: parse: %w", path, err) + } + warnUnmanagedProxiedServerConfig(path, cfg, archiveLevelSupported) } return path, nil case !os.IsNotExist(err): @@ -135,16 +354,41 @@ func ensureProxiedServerConfig(beadsDir string) (string, error) { return "", fmt.Errorf("ensureProxiedServerConfig: pick free port: %w", err) } - body, err := renderProxiedServerConfig(port) + if !archiveLevelSupported { + fmt.Fprintf(os.Stderr, archiveLevelSupportNoticeFormat, doltserver.MinDoltVersionForArchiveLevelConfig) + } + + body, err := renderProxiedServerConfig(port, archiveLevelSupported) if err != nil { return "", fmt.Errorf("ensureProxiedServerConfig: render YAML: %w", err) } - if err := os.WriteFile(path, body, 0o600); err != nil { + // See the reconcile call site above for why the write target is + // resolved through any symlink first: this branch only runs after + // os.Stat(path) reported IsNotExist, which is also true for a dangling + // symlink — resolveConfigWriteTarget falls back to path itself in that + // case, so a dangling symlink is simply replaced (self-healed) here. + if err := atomicWriteFile(resolveConfigWriteTarget(path), body); err != nil { return "", fmt.Errorf("ensureProxiedServerConfig: write %s: %w", path, err) } return path, nil } +// resolveConfigWriteTarget resolves path to its physical location before +// an atomic rewrite. os.Rename's destination argument does not follow +// symlinks — it unlinks and replaces whatever is AT that path, symlink or +// not — so writing straight to a symlinked config.yaml would silently +// replace the symlink itself with a regular file instead of updating the +// file it points at. Falls back to path unresolved when it does not exist +// yet (filepath.EvalSymlinks errors on a missing path), which covers both +// the ordinary "no config.yaml yet" case and a dangling symlink. +func resolveConfigWriteTarget(path string) string { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return path + } + return resolved +} + // Validators below emit source-neutral errors. Callers wrap with whichever // label is meaningful at their site: CLI callers prepend the flag name // (e.g. "--proxied-server-config-path"); runtime callers (uow factory, etc.) @@ -197,7 +441,17 @@ func validateProxiedServerLogPath(path string) error { return nil } -func renderProxiedServerConfig(port int) ([]byte, error) { +// renderProxiedServerConfig renders the managed sql-server config.yaml. When +// archiveLevelSupported is true it sets auto_gc_behavior.archive_level: 0, +// so this server's background auto-GC writes classic Snappy table files +// instead of zstd archives (gastownhall/beads#4986); auto-GC itself stays +// enabled (the sibling "enable" key is left unset, which Dolt defaults to +// true). When false, the key is omitted entirely — an older external dolt's +// own YAMLConfig struct may not have this field, and Dolt's config loader +// uses yaml.UnmarshalStrict, so an unrecognized key is a hard parse error at +// server startup rather than a silently-ignored one. Callers must gate +// archiveLevelSupported on doltserver.SupportsArchiveLevelConfig. +func renderProxiedServerConfig(port int, archiveLevelSupported bool) ([]byte, error) { host := proxiedServerListenerHost logLevel := string(servercfg.LogLevel_Info) yc := &servercfg.YAMLConfig{ @@ -207,7 +461,17 @@ func renderProxiedServerConfig(port int) ([]byte, error) { PortNumber: &port, }, } - return yaml.Marshal(yc) + if archiveLevelSupported { + archiveLevel := 0 + yc.BehaviorConfig.AutoGCBehavior = &servercfg.AutoGCBehaviorYAMLConfig{ + ArchiveLevel_: &archiveLevel, + } + } + body, err := yaml.Marshal(yc) + if err != nil { + return nil, err + } + return append([]byte(managedProxiedServerConfigMarker), body...), nil } const proxiedServerListenerHost = "127.0.0.1" diff --git a/cmd/bd/proxied_server_test.go b/cmd/bd/proxied_server_test.go index 5c1633d81..487a6bde7 100644 --- a/cmd/bd/proxied_server_test.go +++ b/cmd/bd/proxied_server_test.go @@ -12,10 +12,35 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) +// renderManagedConfigWithAutoGCBehavior builds a Beads-managed config.yaml +// (marker included) whose auto_gc_behavior block is exactly gc — unlike +// renderProxiedServerConfig, which only ever sets ArchiveLevel_. Used to +// seed the sibling-field-preservation tests below (a user-set "enable" +// alongside — or in place of — archive_level). +func renderManagedConfigWithAutoGCBehavior(t *testing.T, port int, gc *servercfg.AutoGCBehaviorYAMLConfig) []byte { + t.Helper() + host := proxiedServerListenerHost + logLevel := string(servercfg.LogLevel_Info) + yc := &servercfg.YAMLConfig{ + LogLevelStr: &logLevel, + ListenerConfig: servercfg.ListenerYAMLConfig{ + HostStr: &host, + PortNumber: &port, + }, + BehaviorConfig: servercfg.BehaviorYAMLConfig{ + AutoGCBehavior: gc, + }, + } + body, err := yaml.Marshal(yc) + require.NoError(t, err) + return append([]byte(managedProxiedServerConfigMarker), body...) +} + func TestRenderProxiedServerConfig_RoundTrips(t *testing.T) { - body, err := renderProxiedServerConfig(54321) + body, err := renderProxiedServerConfig(54321, true) require.NoError(t, err) cfg, err := servercfg.NewYamlConfig(body) @@ -26,10 +51,41 @@ func TestRenderProxiedServerConfig_RoundTrips(t *testing.T) { assert.Equal(t, servercfg.LogLevel_Info, cfg.LogLevel(), "LogLevel mismatch") } +// TestRenderProxiedServerConfig_ArchiveLevel covers the actual Snappy-GC fix +// (gastownhall/beads#4986): when archiveLevelSupported is true, the +// generated config sets auto_gc_behavior.archive_level: 0 with auto-GC left +// enabled; when false, the key is omitted entirely so an older external +// dolt's yaml.UnmarshalStrict loader never sees an unrecognized key. +func TestRenderProxiedServerConfig_ArchiveLevel(t *testing.T) { + t.Run("supported: archive_level 0, auto-GC enabled", func(t *testing.T) { + body, err := renderProxiedServerConfig(54321, true) + require.NoError(t, err) + assert.Contains(t, string(body), "archive_level: 0") + + cfg, err := servercfg.NewYamlConfig(body) + require.NoError(t, err) + gc := cfg.AutoGCBehavior() + require.NotNil(t, gc, "AutoGCBehavior must be set when archiveLevelSupported is true") + assert.Equal(t, 0, gc.ArchiveLevel()) + assert.True(t, gc.Enable(), "auto-GC must remain enabled") + }) + + t.Run("unsupported: no auto_gc_behavior key at all", func(t *testing.T) { + body, err := renderProxiedServerConfig(54321, false) + require.NoError(t, err) + assert.NotContains(t, string(body), "archive_level") + assert.NotContains(t, string(body), "auto_gc_behavior") + + cfg, err := servercfg.NewYamlConfig(body) + require.NoError(t, err) + assert.Nil(t, cfg.AutoGCBehavior()) + }) +} + func TestEnsureProxiedServerConfig_CreatesAndIsIdempotent(t *testing.T) { beadsDir := t.TempDir() - path1, err := ensureProxiedServerConfig(beadsDir) + path1, err := ensureProxiedServerConfig(beadsDir, true) require.NoError(t, err) assert.Equal(t, filepath.Join(beadsDir, "dolt", "config.yaml"), path1) @@ -39,7 +95,7 @@ func TestEnsureProxiedServerConfig_CreatesAndIsIdempotent(t *testing.T) { require.True(t, strings.Contains(string(body1), proxiedServerListenerHost)) // Second call must NOT rewrite — running daemon is bound to the existing port. - path2, err := ensureProxiedServerConfig(beadsDir) + path2, err := ensureProxiedServerConfig(beadsDir, true) require.NoError(t, err) assert.Equal(t, path1, path2) @@ -59,12 +115,12 @@ func TestEnsureProxiedServerConfig_ReusesExistingConfig(t *testing.T) { root := filepath.Join(beadsDir, "dolt") require.NoError(t, os.MkdirAll(root, 0o755)) - existing, err := renderProxiedServerConfig(45678) + existing, err := renderProxiedServerConfig(45678, true) require.NoError(t, err) cfgPath := filepath.Join(root, "config.yaml") require.NoError(t, os.WriteFile(cfgPath, existing, 0o600)) - path, err := ensureProxiedServerConfig(beadsDir) + path, err := ensureProxiedServerConfig(beadsDir, true) require.NoError(t, err) assert.Equal(t, cfgPath, path) @@ -73,6 +129,445 @@ func TestEnsureProxiedServerConfig_ReusesExistingConfig(t *testing.T) { assert.Equal(t, existing, got, "existing config.yaml must be reused unchanged") } +// TestIsManagedProxiedServerConfig covers the marker-detection helper in +// isolation from any file I/O. +func TestIsManagedProxiedServerConfig(t *testing.T) { + managed, err := renderProxiedServerConfig(1234, true) + require.NoError(t, err) + assert.True(t, isManagedProxiedServerConfig(managed)) + + assert.False(t, isManagedProxiedServerConfig([]byte("listener:\n port: 1234\n"))) + assert.False(t, isManagedProxiedServerConfig(nil)) +} + +// TestReconcileManagedProxiedServerConfig_AddsKey covers the (a) direction +// (gastownhall/beads#4986 round 2 major 2): a Beads-managed config that +// predates archive_level support gains the key once the resolved dolt +// supports it, with every other field (including port) preserved via the +// servercfg struct round trip. +func TestReconcileManagedProxiedServerConfig_AddsKey(t *testing.T) { + before, err := renderProxiedServerConfig(45678, false) + require.NoError(t, err) + require.NotContains(t, string(before), "archive_level") + + after, changed, err := reconcileManagedProxiedServerConfig(before, true) + require.NoError(t, err) + assert.True(t, changed) + assert.True(t, isManagedProxiedServerConfig(after), "marker must survive reconcile") + + cfg, err := servercfg.NewYamlConfig(after) + require.NoError(t, err) + assert.Equal(t, 45678, cfg.Port(), "port must be preserved") + gc := cfg.AutoGCBehavior() + require.NotNil(t, gc) + assert.Equal(t, 0, gc.ArchiveLevel()) + assert.True(t, gc.Enable()) +} + +// TestReconcileManagedProxiedServerConfig_StripsKey covers the (b) +// direction: a Beads-managed config written with archive_level set loses +// the key when the resolved dolt no longer supports it (e.g. a downgrade), +// so a later yaml.UnmarshalStrict on that older dolt cannot hard-fail. +func TestReconcileManagedProxiedServerConfig_StripsKey(t *testing.T) { + before, err := renderProxiedServerConfig(45678, true) + require.NoError(t, err) + require.Contains(t, string(before), "archive_level") + + after, changed, err := reconcileManagedProxiedServerConfig(before, false) + require.NoError(t, err) + assert.True(t, changed) + assert.NotContains(t, string(after), "archive_level") + assert.NotContains(t, string(after), "auto_gc_behavior") + + cfg, err := servercfg.NewYamlConfig(after) + require.NoError(t, err) + assert.Equal(t, 45678, cfg.Port(), "port must be preserved") + assert.Nil(t, cfg.AutoGCBehavior()) +} + +// TestReconcileManagedProxiedServerConfig_NoOpWhenAlreadyAligned asserts +// reconcile reports changed=false (and ensureProxiedServerConfig therefore +// skips the rewrite) when the config already matches the probe result — +// the steady-state case on every ordinary bd invocation. +func TestReconcileManagedProxiedServerConfig_NoOpWhenAlreadyAligned(t *testing.T) { + t.Run("already has key, still supported", func(t *testing.T) { + body, err := renderProxiedServerConfig(1, true) + require.NoError(t, err) + after, changed, err := reconcileManagedProxiedServerConfig(body, true) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, body, after) + }) + t.Run("already lacks key, still unsupported", func(t *testing.T) { + body, err := renderProxiedServerConfig(1, false) + require.NoError(t, err) + after, changed, err := reconcileManagedProxiedServerConfig(body, false) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, body, after) + }) +} + +// TestReconcileManagedProxiedServerConfig_PreservesSiblingFields covers the +// round-3 verification finding (gastownhall/beads#4986): reconcile must +// inspect and modify only ArchiveLevel_, not treat "the whole +// auto_gc_behavior block is present" as equivalent to "archive_level is +// set". A block carrying only a user-set "enable" (no archive_level) must +// both gain archive_level when supported, and keep its "enable" setting +// when the (now-absent-again) archive_level is later stripped on an +// unsupported binary — the earlier (buggy) whole-block check would have +// either left archive_level unset in the first case, or discarded "enable" +// entirely in the second. +func TestReconcileManagedProxiedServerConfig_PreservesSiblingFields(t *testing.T) { + enable := true + + t.Run("block with only enable gains archive_level; enable preserved", func(t *testing.T) { + before := renderManagedConfigWithAutoGCBehavior(t, 45678, &servercfg.AutoGCBehaviorYAMLConfig{ + Enable_: &enable, + }) + require.Contains(t, string(before), "enable: true") + require.NotContains(t, string(before), "archive_level") + + after, changed, err := reconcileManagedProxiedServerConfig(before, true) + require.NoError(t, err) + assert.True(t, changed) + + cfg, err := servercfg.NewYamlConfig(after) + require.NoError(t, err) + gc := cfg.AutoGCBehavior() + require.NotNil(t, gc) + assert.Equal(t, 0, gc.ArchiveLevel()) + assert.True(t, gc.Enable(), "enable must be preserved, not defaulted/lost") + }) + + t.Run("block with only enable on unsupported binary is untouched", func(t *testing.T) { + before := renderManagedConfigWithAutoGCBehavior(t, 45678, &servercfg.AutoGCBehaviorYAMLConfig{ + Enable_: &enable, + }) + + after, changed, err := reconcileManagedProxiedServerConfig(before, false) + require.NoError(t, err) + assert.False(t, changed, "nothing to strip when archive_level was never set") + assert.Equal(t, before, after) + }) + + t.Run("block with both on unsupported binary keeps enable, loses archive_level", func(t *testing.T) { + archiveLevel := 0 + before := renderManagedConfigWithAutoGCBehavior(t, 45678, &servercfg.AutoGCBehaviorYAMLConfig{ + Enable_: &enable, + ArchiveLevel_: &archiveLevel, + }) + require.Contains(t, string(before), "enable: true") + require.Contains(t, string(before), "archive_level: 0") + + after, changed, err := reconcileManagedProxiedServerConfig(before, false) + require.NoError(t, err) + assert.True(t, changed) + assert.NotContains(t, string(after), "archive_level") + + cfg, err := servercfg.NewYamlConfig(after) + require.NoError(t, err) + gc := cfg.AutoGCBehavior() + require.NotNil(t, gc, "block must survive since enable is still set") + assert.True(t, gc.Enable(), "enable must be preserved when only archive_level is stripped") + }) +} + +// TestEnsureProxiedServerConfig_ManagedWithoutKeyGainsIt is the +// ensureProxiedServerConfig-level counterpart to +// TestReconcileManagedProxiedServerConfig_AddsKey: an upgraded install +// whose config.yaml predates this fix gets archive_level added in place on +// the very next command, without a manual regenerate. +func TestEnsureProxiedServerConfig_ManagedWithoutKeyGainsIt(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + require.NoError(t, os.MkdirAll(root, 0o755)) + cfgPath := filepath.Join(root, "config.yaml") + + seed, err := renderProxiedServerConfig(45678, false) + require.NoError(t, err) + require.NoError(t, os.WriteFile(cfgPath, seed, 0o600)) + + path, err := ensureProxiedServerConfig(beadsDir, true) + require.NoError(t, err) + assert.Equal(t, cfgPath, path) + + got, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.Contains(t, string(got), "archive_level: 0") + + cfg, err := servercfg.NewYamlConfig(got) + require.NoError(t, err) + assert.Equal(t, 45678, cfg.Port(), "port must survive the rewrite") +} + +// TestEnsureProxiedServerConfig_ManagedWithKeyOnOldBinaryLosesIt is the +// ensureProxiedServerConfig-level counterpart to +// TestReconcileManagedProxiedServerConfig_StripsKey: a dolt downgrade after +// a managed config already set archive_level must not hard-fail server +// startup on the next command. +func TestEnsureProxiedServerConfig_ManagedWithKeyOnOldBinaryLosesIt(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + require.NoError(t, os.MkdirAll(root, 0o755)) + cfgPath := filepath.Join(root, "config.yaml") + + seed, err := renderProxiedServerConfig(45678, true) + require.NoError(t, err) + require.NoError(t, os.WriteFile(cfgPath, seed, 0o600)) + + path, err := ensureProxiedServerConfig(beadsDir, false) + require.NoError(t, err) + assert.Equal(t, cfgPath, path) + + got, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.NotContains(t, string(got), "archive_level") + + cfg, err := servercfg.NewYamlConfig(got) + require.NoError(t, err) + assert.Equal(t, 45678, cfg.Port(), "port must survive the rewrite") +} + +// TestEnsureProxiedServerConfig_CustomConfigUntouchedWithWarning asserts a +// genuinely user-owned config (env var / sidecar custom path) is NEVER +// rewritten, and instead gets a one-line stderr warning when it lacks +// archive_level on a capable binary. +func TestEnsureProxiedServerConfig_CustomConfigUntouchedWithWarning(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + bd := t.TempDir() + + customDir := t.TempDir() + customPath := filepath.Join(customDir, "my-server.yaml") + before, err := renderProxiedServerConfig(54321, false) + require.NoError(t, err) + // Strip the marker: this is meant to look like a genuinely hand-written + // custom config, not one Beads happened to generate elsewhere. + before = before[len(managedProxiedServerConfigMarker):] + require.NoError(t, os.WriteFile(customPath, before, 0o600)) + + writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: customPath}) + + var got string + var path string + got = captureStderr(t, func() { + path, err = ensureProxiedServerConfig(bd, true) + }) + require.NoError(t, err) + assert.Equal(t, customPath, path) + + after, err := os.ReadFile(customPath) + require.NoError(t, err) + assert.Equal(t, before, after, "custom config must never be rewritten") + assert.Contains(t, got, "not Beads-managed") + assert.Contains(t, got, customPath) +} + +// TestEnsureProxiedServerConfig_PreMarkerDefaultConfigUntouchedWithWarning +// covers the other unmanaged case: a default-path config.yaml written by a +// Beads version that predates the marker. Same contract as the custom-path +// case — never rewritten, only warned about — since Beads cannot prove it +// owns the file. +func TestEnsureProxiedServerConfig_PreMarkerDefaultConfigUntouchedWithWarning(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + require.NoError(t, os.MkdirAll(root, 0o755)) + cfgPath := filepath.Join(root, "config.yaml") + + seed, err := renderProxiedServerConfig(45678, false) + require.NoError(t, err) + seed = seed[len(managedProxiedServerConfigMarker):] // simulate a pre-marker file + require.NoError(t, os.WriteFile(cfgPath, seed, 0o600)) + + var path string + got := captureStderr(t, func() { + path, err = ensureProxiedServerConfig(beadsDir, true) + }) + require.NoError(t, err) + assert.Equal(t, cfgPath, path) + + after, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.Equal(t, seed, after, "pre-marker default config must never be rewritten") + assert.Contains(t, got, "not Beads-managed") +} + +// TestEnsureProxiedServerConfig_CustomConfigEnvVarInterpolation covers the +// round-3 verification finding (gastownhall/beads#4986): a custom config +// using Dolt's ${VAR}-style environment interpolation (see +// servercfg/env_interpolate.go, applied by YamlConfigFromFile before +// parsing) must be ACCEPTED, not rejected. An earlier version of this +// function validated custom configs via servercfg.NewYamlConfig on raw +// bytes, which skips interpolation entirely and would fail on a valid +// ${VAR} placeholder with a YAML/type error before the server ever starts. +func TestEnsureProxiedServerConfig_CustomConfigEnvVarInterpolation(t *testing.T) { + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + t.Setenv("BEADS_TEST_PROXIED_SERVER_PORT", "54321") + bd := t.TempDir() + + customDir := t.TempDir() + customPath := filepath.Join(customDir, "my-server.yaml") + // listener.port as a raw ${VAR} placeholder — only valid after + // interpolation; NewYamlConfig on the raw bytes would fail to unmarshal + // "${BEADS_TEST_PROXIED_SERVER_PORT}" into the int Port field. + const tmpl = "listener:\n host: 127.0.0.1\n port: ${BEADS_TEST_PROXIED_SERVER_PORT}\n" + require.NoError(t, os.WriteFile(customPath, []byte(tmpl), 0o600)) + + writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: customPath}) + + path, err := ensureProxiedServerConfig(bd, true) + require.NoError(t, err, "a valid ${VAR} custom config must not be rejected pre-start") + assert.Equal(t, customPath, path) + + // The file on disk must still carry the literal, un-interpolated + // placeholder — ensureProxiedServerConfig never rewrites custom configs. + after, err := os.ReadFile(customPath) + require.NoError(t, err) + assert.Equal(t, tmpl, string(after)) +} + +// TestEnsureProxiedServerConfig_ManagedFileWithPlaceholderDegradesGracefully +// covers the round-4 verification finding (gastownhall/beads#4986): the +// managedProxiedServerConfigMarker only proves Beads generated a file +// ORIGINALLY, not that an operator hasn't hand-edited it since (e.g. to add +// a ${VAR} placeholder). reconcileManagedProxiedServerConfig's raw-bytes +// parse (deliberately non-interpolating — see its doc comment) then fails. +// +// Verified before this fix: that parse error propagated as a hard error +// out of ensureProxiedServerConfig, which cmd/bd/uow_factory.go's +// newManagedProxiedServerUOWProvider returns directly — aborting the +// ENTIRE bd command before ever attempting to start the managed dolt +// server, even though a real dolt process would happily interpolate and +// start against the same file. This test asserts the fixed behavior: +// degrade to the unmanaged (warn-only, never-rewrite) treatment instead. +func TestEnsureProxiedServerConfig_ManagedFileWithPlaceholderDegradesGracefully(t *testing.T) { + t.Setenv("BEADS_TEST_PROXIED_SERVER_PORT2", "54321") + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + require.NoError(t, os.MkdirAll(root, 0o755)) + cfgPath := filepath.Join(root, "config.yaml") + + // Marker present (looks Beads-managed) but hand-edited afterward with a + // ${VAR} placeholder in a field NewYamlConfig cannot unmarshal without + // interpolation (an int field fed a string). + seed := managedProxiedServerConfigMarker + "listener:\n host: 127.0.0.1\n port: ${BEADS_TEST_PROXIED_SERVER_PORT2}\n" + require.NoError(t, os.WriteFile(cfgPath, []byte(seed), 0o600)) + + var path string + var err error + got := captureStderr(t, func() { + path, err = ensureProxiedServerConfig(beadsDir, true) + }) + require.NoError(t, err, "a managed file that fails the raw parse must not fail the whole command") + assert.Equal(t, cfgPath, path) + assert.Contains(t, got, "failed to parse", "must explain why it's degrading to unmanaged treatment") + + after, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.Equal(t, seed, string(after), "file must be left untouched when the managed parse fails") +} + +// TestWarnUnmanagedProxiedServerConfig_EnableOnlyFieldLevel covers the +// round-4 verification finding (gastownhall/beads#4986): the unmanaged +// warning must inspect ArchiveLevel_ specifically, not "the whole +// auto_gc_behavior block is present" — otherwise an enable-only custom +// config both misses the zstd-risk warning on a capable dolt, and gets a +// FALSE incompatibility warning on an incapable one (since the block is +// present even though archive_level itself was never set). +func TestWarnUnmanagedProxiedServerConfig_EnableOnlyFieldLevel(t *testing.T) { + enable := true + + newEnableOnlyCustomConfig := func(t *testing.T) string { + t.Helper() + t.Setenv("BEADS_PROXIED_SERVER_CONFIG", "") + body := renderManagedConfigWithAutoGCBehavior(t, 54321, &servercfg.AutoGCBehaviorYAMLConfig{ + Enable_: &enable, + }) + // Strip the marker: this must be evaluated as a genuinely + // hand-written custom config, not a Beads-managed one. + body = body[len(managedProxiedServerConfigMarker):] + require.NotContains(t, string(body), "archive_level") + + bd := t.TempDir() + customPath := filepath.Join(t.TempDir(), "my-server.yaml") + require.NoError(t, os.WriteFile(customPath, body, 0o600)) + writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: customPath}) + return bd + } + + t.Run("capable dolt: gets the zstd-risk warning", func(t *testing.T) { + bd := newEnableOnlyCustomConfig(t) + var err error + got := captureStderr(t, func() { + _, err = ensureProxiedServerConfig(bd, true) + }) + require.NoError(t, err) + assert.Contains(t, got, "zstd auto-GC may still be active", + "an enable-only block must still be treated as archive_level-absent") + }) + + t.Run("incapable dolt: no false incompatibility warning", func(t *testing.T) { + bd := newEnableOnlyCustomConfig(t) + var err error + got := captureStderr(t, func() { + _, err = ensureProxiedServerConfig(bd, false) + }) + require.NoError(t, err) + assert.Empty(t, got, + "an enable-only block (no archive_level) has nothing incompatible with an older dolt; "+ + "the old block-presence check would have warned here anyway") + }) +} + +// TestEnsureProxiedServerConfig_ManagedConfigBehindSymlinkPreservesSymlink +// covers the round-4 verification finding (gastownhall/beads#4986): +// os.Rename's destination argument does not follow a symlink — it +// replaces whatever is directly AT that path. Before this fix, reconciling +// a managed config.yaml that was itself a symlink (e.g. an operator +// centralizing config outside the per-project .beads/dolt directory) would +// silently replace the symlink with a regular file at the SAME location, +// losing the indirection and leaving whatever the symlink pointed at +// unmodified and now orphaned. +func TestEnsureProxiedServerConfig_ManagedConfigBehindSymlinkPreservesSymlink(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + require.NoError(t, os.MkdirAll(root, 0o755)) + + // The physical file lives elsewhere; config.yaml in the managed + // location is a symlink to it. + physicalPath := filepath.Join(t.TempDir(), "real-config.yaml") + seed, err := renderProxiedServerConfig(45678, false) // lacks archive_level: 0 + require.NoError(t, err) + require.NoError(t, os.WriteFile(physicalPath, seed, 0o600)) + + cfgPath := filepath.Join(root, "config.yaml") + if err := os.Symlink(physicalPath, cfgPath); err != nil { + t.Skipf("symlink not supported on this platform: %v", err) + } + + path, err := ensureProxiedServerConfig(beadsDir, true) // now supported -> should reconcile + require.NoError(t, err) + assert.Equal(t, cfgPath, path) + + // The symlink itself must survive, still pointing at the same physical file. + linkInfo, err := os.Lstat(cfgPath) + require.NoError(t, err) + require.True(t, linkInfo.Mode()&os.ModeSymlink != 0, "config.yaml must still be a symlink") + target, err := os.Readlink(cfgPath) + require.NoError(t, err) + assert.Equal(t, physicalPath, target) + + // The PHYSICAL file must carry the reconciled content. + physicalBody, err := os.ReadFile(physicalPath) + require.NoError(t, err) + assert.Contains(t, string(physicalBody), "archive_level: 0") + + // Reading through the symlink must observe the same reconciled content. + throughLink, err := os.ReadFile(cfgPath) + require.NoError(t, err) + assert.Equal(t, physicalBody, throughLink) +} + func TestProxiedServerPathHelpers(t *testing.T) { bd := "/tmp/some/.beads" assert.Equal(t, "/tmp/some/.beads/dolt", proxiedServerRoot(bd)) @@ -165,7 +660,7 @@ func TestResolveProxiedServerConfigPath(t *testing.T) { // and returns the path. Used to exercise the custom-config success path. func writeValidServerYAML(t *testing.T, path string) string { t.Helper() - body, err := renderProxiedServerConfig(54321) + body, err := renderProxiedServerConfig(54321, true) require.NoError(t, err) require.NoError(t, os.WriteFile(path, body, 0o600)) return path @@ -182,7 +677,7 @@ func TestEnsureProxiedServerConfig_CustomPathExists(t *testing.T) { customPath := writeValidServerYAML(t, filepath.Join(customDir, "my-server.yaml")) writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: customPath}) - got, err := ensureProxiedServerConfig(bd) + got, err := ensureProxiedServerConfig(bd, true) require.NoError(t, err) assert.Equal(t, customPath, got) @@ -199,7 +694,7 @@ func TestEnsureProxiedServerConfig_CustomPathMissing(t *testing.T) { missing := filepath.Join(t.TempDir(), "does-not-exist.yaml") writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: missing}) - _, err := ensureProxiedServerConfig(bd) + _, err := ensureProxiedServerConfig(bd, true) require.Error(t, err) assert.Contains(t, err.Error(), missing) } @@ -215,7 +710,7 @@ func TestEnsureProxiedServerConfig_CustomPathInvalidYAML(t *testing.T) { require.NoError(t, os.WriteFile(bad, []byte("listener: [host: 127.0.0.1\n"), 0o600)) writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: bad}) - _, err := ensureProxiedServerConfig(bd) + _, err := ensureProxiedServerConfig(bd, true) require.Error(t, err) assert.Contains(t, err.Error(), bad) assert.Contains(t, strings.ToLower(err.Error()), "parse") @@ -229,7 +724,7 @@ func TestEnsureProxiedServerConfig_CustomPathIsDirectory(t *testing.T) { dir := t.TempDir() writeProxiedClientInfo(t, bd, &configfile.ProxiedServerClientInfo{ConfigPath: dir}) - _, err := ensureProxiedServerConfig(bd) + _, err := ensureProxiedServerConfig(bd, true) require.Error(t, err) assert.Contains(t, err.Error(), dir) assert.Contains(t, err.Error(), "not a regular file") diff --git a/cmd/bd/uow_factory.go b/cmd/bd/uow_factory.go index 9881c2226..f09bf7d0c 100644 --- a/cmd/bd/uow_factory.go +++ b/cmd/bd/uow_factory.go @@ -9,6 +9,7 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/uow" ) @@ -99,7 +100,13 @@ func newManagedProxiedServerUOWProvider( return nil, fmt.Errorf("newProxiedServerUOWProvider: proxied server root (from env or %s): %w", configfile.ProxiedServerClientInfoFileName, err) } - configPath, err := ensureProxiedServerConfig(beadsDir) + // Gate auto_gc_behavior.archive_level: 0 on the resolved external dolt's + // version — Dolt's YAML config loader uses yaml.UnmarshalStrict, so an + // older dolt whose own YAMLConfig struct lacks this field would refuse + // to start rather than ignore the unknown key (gastownhall/beads#4986). + archiveLevelSupported := doltserver.SupportsArchiveLevelConfig(doltBin) + + configPath, err := ensureProxiedServerConfig(beadsDir, archiveLevelSupported) if err != nil { return nil, err } diff --git a/internal/doltserver/doltserver.go b/internal/doltserver/doltserver.go index 6f8abeec4..e43a715ea 100644 --- a/internal/doltserver/doltserver.go +++ b/internal/doltserver/doltserver.go @@ -31,6 +31,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "github.com/dolthub/dolt/go/libraries/utils/filesys" _ "github.com/go-sql-driver/mysql" + "gopkg.in/yaml.v3" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" @@ -767,6 +768,157 @@ func buildDoltServerArgs(host string, port int, debug bool, profDir string) []st return args } +// doltServerConfigFileName is the YAML config Start() writes when the +// resolved external dolt binary supports auto_gc_behavior.archive_level +// (see SupportsArchiveLevelConfig). It lives alongside the other gitignored +// server state files in beadsDir, not inside the Dolt data directory. +const doltServerConfigFileName = "dolt-server-config.yaml" + +func doltServerConfigPath(beadsDir string) string { + return filepath.Join(beadsDir, doltServerConfigFileName) +} + +// doltCfgDirName mirrors commands.DefaultCfgDirName in the pinned dolt +// module (cmd/dolt/commands/sql.go) — the on-disk directory name Dolt looks +// in for privileges.db (users/passwords) and branch_control.db. +const doltCfgDirName = ".doltcfg" + +// ErrMultipleDoltCfgDirs is returned by resolveCfgDir when both a parent +// and a data-directory .doltcfg exist. Mirrors Dolt's own ambiguous case +// (commands.ErrMultipleDoltCfgDirs in the pinned module) — guessing which +// one to use risks the same silent user/branch-control loss this function +// exists to prevent, so this is surfaced as a hard Start() failure instead. +var ErrMultipleDoltCfgDirs = errors.New("multiple .doltcfg directories detected") + +// resolveCfgDir replicates Dolt's own flag-mode .doltcfg discovery +// (setupDoltConfig in cmd/dolt/commands/sqlserver/sqlserver.go, pinned +// module) for our generated --config YAML. setupDoltConfig returns +// immediately when --config is passed, so a deployment that previously ran +// this launcher in CLI-flag mode — where dolt auto-discovers a parent +// ../.doltcfg holding privileges.db/branch_control.db — would otherwise +// silently fall back to a fresh $data_dir/.doltcfg under --config mode: +// existing users and branch controls abandoned, new passwordless root +// (gastownhall/beads#4986). +// +// doltDir is the server's data directory (== process cwd, since neither +// --data-dir nor --doltcfg-dir are ever passed). Mirrors Dolt exactly: +// - parent ../.doltcfg (relative to doltDir) if it exists and is a dir +// - else doltDir/.doltcfg +// - ErrMultipleDoltCfgDirs if BOTH exist — ambiguous, matches Dolt's own +// ErrMultipleDoltCfgDirs case rather than guessing. +// +// doltDir is resolved to its physical (symlink-free) location before +// computing the parent. filepath.Join/Clean strip ".." lexically without +// touching the filesystem: if doltDir is itself a symlink, a naive +// filepath.Join(doltDir, "..") resolves to the symlink's own lexical +// parent directory, not the physical parent of whatever it points at. On +// Linux, chdir into a symlink makes the kernel track the process's cwd +// physically, so the actual `dolt` child process's own "../.doltcfg" +// lookup (a bare relative path, resolved against its real, physically +// tracked cwd — see setupDoltConfig) sees the PHYSICAL parent. Using +// filepath.EvalSymlinks here keeps this resolution consistent with what +// the child process itself does; skipping it would let a symlinked data +// directory miss the very parent .doltcfg this function exists to find, +// re-abandoning existing privileges/branch-control (gastownhall/beads#4986 +// round 3). +// +// Returns an absolute path. +func resolveCfgDir(doltDir string) (string, error) { + // Fall back to the lexical doltDir on error (e.g. it does not exist + // yet) rather than failing outright — resolveCfgDir has no dependency + // on doltDir already existing beyond this best-effort symlink check, + // and ensureDoltInit already runs before this is called from Start(). + physicalDoltDir := doltDir + if resolved, err := filepath.EvalSymlinks(doltDir); err == nil { + physicalDoltDir = resolved + } + + parentDirCfg := filepath.Join(physicalDoltDir, "..", doltCfgDirName) + parentInfo, parentErr := os.Stat(parentDirCfg) + parentExists := parentErr == nil && parentInfo.IsDir() + + currDirCfg := filepath.Join(physicalDoltDir, doltCfgDirName) + currInfo, currErr := os.Stat(currDirCfg) + currExists := currErr == nil && currInfo.IsDir() + + if parentExists && currExists { + absParent, err := filepath.Abs(parentDirCfg) + if err != nil { + absParent = parentDirCfg + } + absCurr, err := filepath.Abs(currDirCfg) + if err != nil { + absCurr = currDirCfg + } + return "", fmt.Errorf("%w: %q and %q; remove or merge one before starting the managed dolt server "+ + "(matches Dolt's own ambiguous-.doltcfg case; see --doltcfg-dir in `dolt sql-server --help`)", + ErrMultipleDoltCfgDirs, absParent, absCurr) + } + + chosen := currDirCfg + if parentExists { + chosen = parentDirCfg + } + abs, err := filepath.Abs(chosen) + if err != nil { + return "", fmt.Errorf("resolving .doltcfg path %q: %w", chosen, err) + } + return abs, nil +} + +// buildDoltServerYAMLConfig renders a minimal sql-server YAML config +// equivalent to buildDoltServerArgs' CLI flags (host, port, log level), plus +// auto_gc_behavior.archive_level: 0 so this managed server's background +// auto-GC writes classic Snappy table files instead of zstd archives +// (gastownhall/beads#4986). Auto-GC itself is left enabled (archive_level's +// sibling "enable" key is omitted, which Dolt defaults to true). cfgDir must +// be pre-resolved via resolveCfgDir — --config mode skips Dolt's own +// flag-mode .doltcfg discovery entirely, so this is the one place that +// behavior must be replicated. +// +// Callers MUST only use this when SupportsArchiveLevelConfig(doltBin) +// reports true — an older external dolt's own YAMLConfig struct may lack +// this field, and Dolt's YAML loader uses yaml.UnmarshalStrict, so an +// unrecognized key is a hard parse error at server startup, not a +// silently-ignored one. +func buildDoltServerYAMLConfig(host string, port int, debug bool, cfgDir string) ([]byte, error) { + logLevel := doltServerLogLevel + if debug { + logLevel = "debug" + } + archiveLevel := 0 + yc := &servercfg.YAMLConfig{ + LogLevelStr: &logLevel, + ListenerConfig: servercfg.ListenerYAMLConfig{ + HostStr: &host, + PortNumber: &port, + }, + CfgDirStr: &cfgDir, + BehaviorConfig: servercfg.BehaviorYAMLConfig{ + AutoGCBehavior: &servercfg.AutoGCBehaviorYAMLConfig{ + ArchiveLevel_: &archiveLevel, + }, + }, + } + return yaml.Marshal(yc) +} + +// buildDoltServerArgsWithConfig is the --config counterpart to +// buildDoltServerArgs. Dolt's sql-server subcommand ignores all other +// command-line parameters when --config is present, so host/port/log-level +// must come from the YAML file at configPath (see buildDoltServerYAMLConfig) +// rather than from flags here. The top-level --prof/--prof-path flags are +// unaffected — they are consumed by the outer `dolt` command dispatcher +// before the sql-server subcommand's own arg parser ever sees --config. +func buildDoltServerArgsWithConfig(configPath string, debug bool, profDir string) []string { + var args []string + if debug { + args = append(args, "--prof", "cpu", "--prof-path", profDir) + } + args = append(args, "sql-server", "--config", configPath) + return args +} + // Start explicitly starts a dolt sql-server for the project. // Returns the State of the started server, or an error. func Start(beadsDir string) (*State, error) { @@ -824,6 +976,21 @@ func Start(beadsDir string) (*State, error) { return nil, fmt.Errorf("dolt is not installed (not found in PATH)\n\nInstall from: https://docs.dolthub.com/introduction/installation") } + // Prefer a generated YAML config (via --config) over plain CLI flags when + // the resolved external dolt is new enough to safely accept + // auto_gc_behavior.archive_level: 0. This is what keeps this managed + // server's background auto-GC producing Snappy table files instead of + // zstd archives (gastownhall/beads#4986). On an older/undetectable dolt + // we fail closed and keep the existing CLI-flag launch unchanged — never + // risk a refuse-to-start over an unrecognized YAML key. + useArchiveLevelConfig := SupportsArchiveLevelConfig(doltBin) + if !useArchiveLevelConfig { + fmt.Fprintf(os.Stderr, + "Info: external dolt at %s predates archive_level config support (need >= Dolt %s); "+ + "this managed server's background auto-GC may still produce zstd archives.\n", + doltBin, MinDoltVersionForArchiveLevelConfig) + } + // Ensure dolt identity is configured if err := ensureDoltIdentity(); err != nil { return nil, fmt.Errorf("configuring dolt identity: %w", err) @@ -858,6 +1025,18 @@ func Start(beadsDir string) (*State, error) { // only intervene between runs. See logrotate.go for the caveat discussion. maybeRotateLog(beadsDir) + // Resolve .doltcfg once, before the port retry loop — it does not + // depend on the port, and an ambiguous both-exist result must fail + // Start() outright rather than retry into a fresh unintended + // $data_dir/.doltcfg (see resolveCfgDir). + var cfgDir string + if useArchiveLevelConfig { + cfgDir, err = resolveCfgDir(doltDir) + if err != nil { + return nil, fmt.Errorf("resolving .doltcfg directory: %w", err) + } + } + // Open log file logFile, err := os.OpenFile(logPath(beadsDir), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) //nolint:gosec // G304: logPath derives from user-configured beadsDir if err != nil { @@ -903,7 +1082,41 @@ func Start(beadsDir string) (*State, error) { actualPort = p } - cmd := exec.Command(doltBin, buildDoltServerArgs(cfg.Host, actualPort, debug, profDir)...) //nolint:gosec // doltBin is resolved from PATH, not user input + var cmdArgs []string + if useArchiveLevelConfig { + cfgBody, cfgErr := buildDoltServerYAMLConfig(cfg.Host, actualPort, debug, cfgDir) + if cfgErr != nil { + lastErr = fmt.Errorf("rendering managed sql-server config: %w", cfgErr) + if !explicitPort { + continue + } + break + } + // filepath.Abs defensively: the child's cwd is doltDir (cmd.Dir + // below), so a relative configPath would resolve against the + // wrong directory (matches the sibling dbproxy/server path, + // which abs's its configPath in NewDoltServer). + absConfigPath, absErr := filepath.Abs(doltServerConfigPath(beadsDir)) + if absErr != nil { + lastErr = fmt.Errorf("resolving managed sql-server config path: %w", absErr) + if !explicitPort { + continue + } + break + } + if werr := os.WriteFile(absConfigPath, cfgBody, 0600); werr != nil { + lastErr = fmt.Errorf("writing managed sql-server config: %w", werr) + if !explicitPort { + continue + } + break + } + cmdArgs = buildDoltServerArgsWithConfig(absConfigPath, debug, profDir) + } else { + cmdArgs = buildDoltServerArgs(cfg.Host, actualPort, debug, profDir) + } + + cmd := exec.Command(doltBin, cmdArgs...) //nolint:gosec // doltBin is resolved from PATH, not user input cmd.Dir = doltDir cmd.Stdout = logFile cmd.Stderr = logFile @@ -1187,6 +1400,7 @@ func StateFilePaths(beadsDir string) []string { logPath(beadsDir), logPath(beadsDir) + ".1", DebugProfileDir(beadsDir), + doltServerConfigPath(beadsDir), } } diff --git a/internal/doltserver/doltserver_test.go b/internal/doltserver/doltserver_test.go index 1108ddb7e..73789022b 100644 --- a/internal/doltserver/doltserver_test.go +++ b/internal/doltserver/doltserver_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" ) @@ -1908,6 +1910,344 @@ func TestBuildDoltServerArgs_NoDebugFlagsWhenDisabled(t *testing.T) { } } +// TestBuildDoltServerYAMLConfig verifies the --config counterpart to +// buildDoltServerArgs: it must round-trip through Dolt's own YAML loader +// with the same host/port/log-level as the CLI-flag form, plus +// auto_gc_behavior.archive_level: 0 (the actual Snappy-GC fix, +// gastownhall/beads#4986) and auto-GC left enabled, plus cfg_dir set to the +// caller-resolved value (gastownhall/beads#4986 round 2: --config mode +// skips Dolt's own .doltcfg discovery, so this must be set explicitly). +func TestBuildDoltServerYAMLConfig(t *testing.T) { + body, err := buildDoltServerYAMLConfig("127.0.0.1", 54321, false, "/tmp/some/.doltcfg") + if err != nil { + t.Fatalf("buildDoltServerYAMLConfig: %v", err) + } + + cfg, err := servercfg.NewYamlConfig(body) + if err != nil { + t.Fatalf("servercfg.NewYamlConfig could not parse generated config: %v\nconfig:\n%s", err, body) + } + if got := cfg.Host(); got != "127.0.0.1" { + t.Errorf("Host = %q, want %q", got, "127.0.0.1") + } + if got := cfg.Port(); got != 54321 { + t.Errorf("Port = %d, want %d", got, 54321) + } + if got := string(cfg.LogLevel()); got != doltServerLogLevel { + t.Errorf("LogLevel = %q, want %q", got, doltServerLogLevel) + } + if got := cfg.CfgDir(); got != "/tmp/some/.doltcfg" { + t.Errorf("CfgDir = %q, want %q", got, "/tmp/some/.doltcfg") + } + + gc := cfg.AutoGCBehavior() + if gc == nil { + t.Fatal("AutoGCBehavior is nil; expected archive_level: 0 to be set") + } + if got := gc.ArchiveLevel(); got != 0 { + t.Errorf("ArchiveLevel = %d, want 0 (Snappy, not zstd)", got) + } + if !gc.Enable() { + t.Error("auto-GC must remain enabled; only the archive level should change") + } +} + +// TestBuildDoltServerYAMLConfig_DebugLogLevel asserts debug mode raises the +// YAML config's log level the same way buildDoltServerArgs does for the +// CLI-flag form. +func TestBuildDoltServerYAMLConfig_DebugLogLevel(t *testing.T) { + body, err := buildDoltServerYAMLConfig("127.0.0.1", 54321, true, "/tmp/some/.doltcfg") + if err != nil { + t.Fatalf("buildDoltServerYAMLConfig: %v", err) + } + cfg, err := servercfg.NewYamlConfig(body) + if err != nil { + t.Fatalf("servercfg.NewYamlConfig: %v", err) + } + if got := string(cfg.LogLevel()); got != "debug" { + t.Errorf("debug mode LogLevel = %q, want %q", got, "debug") + } +} + +// TestResolveCfgDir_NeitherExists is the common case: no parent or +// data-dir .doltcfg found on disk, so resolveCfgDir must default to +// dataDir/.doltcfg — matching Dolt's own flag-mode default ("Assign the one +// that exists, defaults to current if neither exist" in setupDoltConfig). +func TestResolveCfgDir_NeitherExists(t *testing.T) { + doltDir := filepath.Join(t.TempDir(), "dolt") + if err := os.MkdirAll(doltDir, 0o755); err != nil { + t.Fatalf("mkdir doltDir: %v", err) + } + + got, err := resolveCfgDir(doltDir) + if err != nil { + t.Fatalf("resolveCfgDir: %v", err) + } + want, err := filepath.Abs(filepath.Join(doltDir, doltCfgDirName)) + if err != nil { + t.Fatalf("filepath.Abs: %v", err) + } + if got != want { + t.Errorf("resolveCfgDir = %q, want %q", got, want) + } +} + +// TestResolveCfgDir_ParentExists is the actual bug this fix addresses: a +// legacy deployment previously run in CLI-flag mode discovered a parent +// ../.doltcfg (holding privileges.db/branch_control.db). --config mode +// would otherwise silently ignore it; resolveCfgDir must find it and point +// cfg_dir there instead of a fresh dataDir/.doltcfg. +func TestResolveCfgDir_ParentExists(t *testing.T) { + root := t.TempDir() + doltDir := filepath.Join(root, "dolt") + if err := os.MkdirAll(doltDir, 0o755); err != nil { + t.Fatalf("mkdir doltDir: %v", err) + } + parentCfg := filepath.Join(root, doltCfgDirName) + if err := os.MkdirAll(parentCfg, 0o755); err != nil { + t.Fatalf("mkdir parent .doltcfg: %v", err) + } + // Seed it so this test would fail loudly if resolution pointed elsewhere. + if err := os.WriteFile(filepath.Join(parentCfg, "privileges.db"), []byte("x"), 0o600); err != nil { + t.Fatalf("seed privileges.db: %v", err) + } + + got, err := resolveCfgDir(doltDir) + if err != nil { + t.Fatalf("resolveCfgDir: %v", err) + } + want, err := filepath.Abs(parentCfg) + if err != nil { + t.Fatalf("filepath.Abs: %v", err) + } + if got != want { + t.Errorf("resolveCfgDir = %q, want parent %q (existing users/branch-control must not be abandoned)", got, want) + } +} + +// TestResolveCfgDir_CurrentExists asserts a data-dir .doltcfg (no parent +// one) resolves to itself, matching flag-mode's "look in data directory" +// branch. +func TestResolveCfgDir_CurrentExists(t *testing.T) { + root := t.TempDir() + doltDir := filepath.Join(root, "dolt") + if err := os.MkdirAll(doltDir, 0o755); err != nil { + t.Fatalf("mkdir doltDir: %v", err) + } + currCfg := filepath.Join(doltDir, doltCfgDirName) + if err := os.MkdirAll(currCfg, 0o755); err != nil { + t.Fatalf("mkdir data-dir .doltcfg: %v", err) + } + + got, err := resolveCfgDir(doltDir) + if err != nil { + t.Fatalf("resolveCfgDir: %v", err) + } + want, err := filepath.Abs(currCfg) + if err != nil { + t.Fatalf("filepath.Abs: %v", err) + } + if got != want { + t.Errorf("resolveCfgDir = %q, want %q", got, want) + } +} + +// TestResolveCfgDir_BothExistIsAmbiguous mirrors Dolt's own +// ErrMultipleDoltCfgDirs case: resolveCfgDir must refuse to guess when both +// a parent and a data-dir .doltcfg exist, rather than silently picking one +// and risking the same class of silent data loss this fix exists to +// prevent. +func TestResolveCfgDir_BothExistIsAmbiguous(t *testing.T) { + root := t.TempDir() + doltDir := filepath.Join(root, "dolt") + if err := os.MkdirAll(doltDir, 0o755); err != nil { + t.Fatalf("mkdir doltDir: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, doltCfgDirName), 0o755); err != nil { + t.Fatalf("mkdir parent .doltcfg: %v", err) + } + if err := os.MkdirAll(filepath.Join(doltDir, doltCfgDirName), 0o755); err != nil { + t.Fatalf("mkdir data-dir .doltcfg: %v", err) + } + + _, err := resolveCfgDir(doltDir) + if err == nil { + t.Fatal("resolveCfgDir: expected an error when both parent and data-dir .doltcfg exist, got nil") + } + if !errors.Is(err, ErrMultipleDoltCfgDirs) { + t.Errorf("resolveCfgDir error = %v, want errors.Is(_, ErrMultipleDoltCfgDirs)", err) + } +} + +// TestResolveCfgDir_FileNotDirIgnored asserts a plain file named .doltcfg +// (not a directory) is not mistaken for a real .doltcfg dir — matches +// Dolt's own dEnv.FS.Exists(...) check, which also requires isDir. +func TestResolveCfgDir_FileNotDirIgnored(t *testing.T) { + root := t.TempDir() + doltDir := filepath.Join(root, "dolt") + if err := os.MkdirAll(doltDir, 0o755); err != nil { + t.Fatalf("mkdir doltDir: %v", err) + } + // A stray file (not a dir) named .doltcfg next to doltDir must be ignored. + if err := os.WriteFile(filepath.Join(root, doltCfgDirName), []byte("not a directory"), 0o600); err != nil { + t.Fatalf("write stray file: %v", err) + } + + got, err := resolveCfgDir(doltDir) + if err != nil { + t.Fatalf("resolveCfgDir: %v", err) + } + want, err := filepath.Abs(filepath.Join(doltDir, doltCfgDirName)) + if err != nil { + t.Fatalf("filepath.Abs: %v", err) + } + if got != want { + t.Errorf("resolveCfgDir = %q, want default %q (stray non-dir file must be ignored)", got, want) + } +} + +// TestResolveCfgDir_SymlinkedDataDirFindsPhysicalParent covers the round-3 +// verification finding (gastownhall/beads#4986): filepath.Join(doltDir, +// "..", ...) cleans ".." lexically, without touching the filesystem. If +// doltDir is itself a symlink, that yields the symlink's own lexical +// parent directory — not the physical parent of whatever directory the +// symlink actually points at. But the real `dolt` child process's own +// "../.doltcfg" lookup is a bare relative path resolved against its +// actual, kernel-tracked cwd, which chdir into a symlink resolves +// PHYSICALLY on Linux. A naive lexical join therefore misses a parent +// .doltcfg that sits next to the physical target directory, silently +// falling back to a fresh $data_dir/.doltcfg — the exact data-loss +// scenario this whole fix exists to prevent. +// +// Layout: +// +// /physical/parent/target/ (the real data directory) +// /physical/parent/.doltcfg/ (the REAL parent .doltcfg) +// /lexical/doltlink -> .../target (symlink passed as doltDir) +// +// A lexically-parent-of-the-symlink directory (/lexical/.doltcfg) is +// deliberately never created, so a buggy lexical resolution would find +// nothing there and fall through to the (also nonexistent, but still +// wrong) default $data_dir/.doltcfg instead of the real one. +func TestResolveCfgDir_SymlinkedDataDirFindsPhysicalParent(t *testing.T) { + root := t.TempDir() + + physicalParent := filepath.Join(root, "physical", "parent") + physicalTarget := filepath.Join(physicalParent, "target") + if err := os.MkdirAll(physicalTarget, 0o755); err != nil { + t.Fatalf("mkdir physical target: %v", err) + } + physicalCfgDir := filepath.Join(physicalParent, doltCfgDirName) + if err := os.MkdirAll(physicalCfgDir, 0o755); err != nil { + t.Fatalf("mkdir physical .doltcfg: %v", err) + } + // Seed it so this test would fail loudly if resolution missed it. + if err := os.WriteFile(filepath.Join(physicalCfgDir, "privileges.db"), []byte("x"), 0o600); err != nil { + t.Fatalf("seed privileges.db: %v", err) + } + + lexicalDir := filepath.Join(root, "lexical") + if err := os.MkdirAll(lexicalDir, 0o755); err != nil { + t.Fatalf("mkdir lexical dir: %v", err) + } + symlinkedDoltDir := filepath.Join(lexicalDir, "doltlink") + if err := os.Symlink(physicalTarget, symlinkedDoltDir); err != nil { + t.Skipf("symlink not supported on this platform: %v", err) + } + + got, err := resolveCfgDir(symlinkedDoltDir) + if err != nil { + t.Fatalf("resolveCfgDir: %v", err) + } + + wantPhysical, err := filepath.EvalSymlinks(physicalCfgDir) + if err != nil { + t.Fatalf("filepath.EvalSymlinks(physicalCfgDir): %v", err) + } + want, err := filepath.Abs(wantPhysical) + if err != nil { + t.Fatalf("filepath.Abs: %v", err) + } + + if got != want { + t.Errorf("resolveCfgDir(%q) = %q, want the PHYSICAL parent .doltcfg %q "+ + "(existing users/branch-control must not be abandoned behind a symlinked data dir)", + symlinkedDoltDir, got, want) + } + + // Sanity: the naive lexical join must NOT be what we got, or this test + // would pass vacuously against a regression. + lexicalWrong := filepath.Join(lexicalDir, doltCfgDirName) + if got == lexicalWrong { + t.Errorf("resolveCfgDir resolved to the lexical-parent-of-the-symlink %q, "+ + "not the physical parent — this is the exact bug this test guards against", lexicalWrong) + } +} + +// TestBuildDoltServerArgsWithConfig mirrors TestBuildDoltServerArgs_DebugMode +// for the --config launch form: --prof/--prof-path must still precede +// sql-server (top-level dolt flags), and --config must carry the path +// unmodified. Per Dolt's own sql-server docs, --config causes all other +// sql-server flags (host/port/log-level) to be ignored, so this form MUST +// NOT also pass -H/-P/--loglevel. +func TestBuildDoltServerArgsWithConfig(t *testing.T) { + t.Run("non-debug", func(t *testing.T) { + args := buildDoltServerArgsWithConfig("/tmp/dolt-server-config.yaml", false, "") + if len(args) == 0 || args[0] != "sql-server" { + t.Fatalf("args[0] = %q, want %q; full args: %v", firstOrEmpty(args), "sql-server", args) + } + cfgIdx := indexOf(args, "--config") + if cfgIdx < 0 || cfgIdx+1 >= len(args) { + t.Fatalf("missing --config in args: %v", args) + } + if got := args[cfgIdx+1]; got != "/tmp/dolt-server-config.yaml" { + t.Errorf("--config value = %q, want %q", got, "/tmp/dolt-server-config.yaml") + } + for _, flag := range []string{"-H", "-P", "--loglevel"} { + if indexOf(args, flag) >= 0 { + t.Errorf("--config mode must not also pass %s (Dolt ignores it, and its presence is misleading): %v", flag, args) + } + } + }) + + t.Run("debug mode places --prof before sql-server", func(t *testing.T) { + const profDir = "/tmp/test-pprof" + args := buildDoltServerArgsWithConfig("/tmp/dolt-server-config.yaml", true, profDir) + subIdx := indexOf(args, "sql-server") + if subIdx < 0 { + t.Fatalf("missing sql-server in args: %v", args) + } + profIdx := indexOf(args, "--prof") + if profIdx < 0 || profIdx >= subIdx { + t.Fatalf("--prof must precede sql-server; got: %v", args) + } + pathIdx := indexOf(args, "--prof-path") + if pathIdx < 0 || pathIdx >= subIdx { + t.Fatalf("--prof-path must precede sql-server; got: %v", args) + } + if got := args[pathIdx+1]; got != profDir { + t.Errorf("--prof-path value = %q, want %q", got, profDir) + } + }) +} + +func TestDoltServerConfigPath(t *testing.T) { + got := doltServerConfigPath("/tmp/some/.beads") + want := filepath.Join("/tmp/some/.beads", "dolt-server-config.yaml") + if got != want { + t.Errorf("doltServerConfigPath = %q, want %q", got, want) + } +} + +// TestStateFilePaths_IncludesGeneratedConfig guards against the generated +// sql-server config leaking as an untracked leftover after Stop/RemoveState. +func TestStateFilePaths_IncludesGeneratedConfig(t *testing.T) { + paths := StateFilePaths("/tmp/some/.beads") + if indexOf(paths, doltServerConfigPath("/tmp/some/.beads")) < 0 { + t.Errorf("StateFilePaths does not include the generated sql-server config; got: %v", paths) + } +} + func TestWaitForReady(t *testing.T) { // Allocate an ephemeral port, then release it so we can re-bind later. tmpListener, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/internal/doltserver/gc_config.go b/internal/doltserver/gc_config.go new file mode 100644 index 000000000..b3e7d81b5 --- /dev/null +++ b/internal/doltserver/gc_config.go @@ -0,0 +1,138 @@ +package doltserver + +import ( + "os" + "os/exec" + "strconv" + "strings" + "sync" +) + +// MinDoltVersionForArchiveLevelConfig is the earliest Dolt release known to +// accept the auto_gc_behavior.archive_level key in its YAML sql-server +// config (servercfg.AutoGCBehaviorYAMLConfig.ArchiveLevel_ carries +// minver:"1.52.1" in the pinned dolthub/dolt/go module). +// +// This matters because Dolt's YAML config loader parses with +// yaml.UnmarshalStrict (see servercfg/yaml_config.go), so an older external +// dolt binary whose own compiled-in YAMLConfig struct predates this field +// will fail to parse a config file that sets it — the unknown key is a hard +// parse error, not a silently-ignored one, and `dolt sql-server` refuses to +// start. See gastownhall/beads#4986. +const MinDoltVersionForArchiveLevelConfig = "1.52.1" + +// archiveLevelSupportCache memoizes SupportsArchiveLevelConfig, keyed by +// archiveLevelCacheKey (path + size + mtime, not path alone) for the +// lifetime of the process. `dolt version` does not change between two +// probes of the SAME on-disk binary, and Start()/the UOW provider can be +// invoked repeatedly against the same resolved binary (retries, multiple +// projects sharing one dolt, tests), so this avoids forking a +// `dolt version` subprocess on every call. Keying by file identity (not +// just the path string) lets an in-place dolt upgrade/downgrade at that +// same path invalidate the cache naturally on its next probe, rather than +// keeping a stale verdict for the rest of the process's lifetime. +var archiveLevelSupportCache sync.Map // map[archiveLevelCacheKey]bool + +// archiveLevelCacheKey identifies a specific on-disk binary, not just a +// path: two different binaries can share a path over time (a package +// manager upgrade replaces the file at the same location), and size+mtime +// is enough to distinguish them without hashing the whole binary. +type archiveLevelCacheKey struct { + path string + size int64 + modTime int64 // info.ModTime().UnixNano() +} + +// SupportsArchiveLevelConfig probes doltBin (an absolute path or a PATH +// lookup result) for a version new enough to safely accept +// auto_gc_behavior.archive_level in a YAML sql-server config. It fails +// closed: any error running or parsing `dolt version` returns false, so +// callers fall back to config generation that omits the key rather than +// risk a refuse-to-start on an older external dolt. +// +// Results are memoized per archiveLevelCacheKey (see doc comment) for the +// process lifetime. If os.Stat(doltBin) itself fails, the probe runs +// uncached — there is no stable identity to key on, and the exec below +// will fail closed anyway. Use ResetArchiveLevelSupportCacheForTest to +// force-clear the cache in a test that cannot otherwise change a stub's +// file identity between subtests. +func SupportsArchiveLevelConfig(doltBin string) bool { + key, hasKey := archiveLevelCacheKeyFor(doltBin) + if hasKey { + if cached, ok := archiveLevelSupportCache.Load(key); ok { + return cached.(bool) + } + } + + out, err := exec.Command(doltBin, "version").Output() //nolint:gosec // G204: doltBin is caller-resolved (PATH lookup or config), not user-request input + supported := err == nil && doltVersionAtLeast(string(out), MinDoltVersionForArchiveLevelConfig) + + if hasKey { + archiveLevelSupportCache.Store(key, supported) + } + return supported +} + +// archiveLevelCacheKeyFor stats doltBin and returns its cache key. The +// second return is false when doltBin cannot be stat'd (e.g. it does not +// exist), signaling the caller should not attempt to cache the result. +func archiveLevelCacheKeyFor(doltBin string) (archiveLevelCacheKey, bool) { + info, err := os.Stat(doltBin) + if err != nil { + return archiveLevelCacheKey{}, false + } + return archiveLevelCacheKey{ + path: doltBin, + size: info.Size(), + modTime: info.ModTime().UnixNano(), + }, true +} + +// ResetArchiveLevelSupportCacheForTest clears the memoization cache. Tests +// that probe SupportsArchiveLevelConfig against a stub binary path that +// gets rewritten between subtests (or reused across independent test +// binaries in the same package) must call this to avoid a stale result +// from a different test. +func ResetArchiveLevelSupportCacheForTest() { + archiveLevelSupportCache.Range(func(key, _ any) bool { + archiveLevelSupportCache.Delete(key) + return true + }) +} + +// doltVersionAtLeast parses the first line of `dolt version` output (e.g. +// "dolt version 1.52.3\n", possibly followed by extra lines such as +// "database storage format: ..." when run inside a Dolt repo) and reports +// whether the trailing version token is >= minVer, using numeric +// dotted-segment comparison. Returns false (fail closed) if the version +// cannot be parsed as a dotted sequence of non-negative integers. +func doltVersionAtLeast(versionOutput, minVer string) bool { + firstLine := versionOutput + if idx := strings.IndexByte(versionOutput, '\n'); idx >= 0 { + firstLine = versionOutput[:idx] + } + fields := strings.Fields(firstLine) + if len(fields) == 0 { + return false + } + + got := strings.Split(fields[len(fields)-1], ".") + want := strings.Split(minVer, ".") + + for i := range want { + var g, w int + var err error + if i < len(got) { + if g, err = strconv.Atoi(got[i]); err != nil { + return false + } + } + if w, err = strconv.Atoi(want[i]); err != nil { + return false + } + if g != w { + return g > w + } + } + return true +} diff --git a/internal/doltserver/gc_config_test.go b/internal/doltserver/gc_config_test.go new file mode 100644 index 000000000..4328701d6 --- /dev/null +++ b/internal/doltserver/gc_config_test.go @@ -0,0 +1,192 @@ +package doltserver + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestDoltVersionAtLeast(t *testing.T) { + tests := []struct { + name string + output string + minVer string + want bool + }{ + {"exact match", "dolt version 1.52.1\n", "1.52.1", true}, + {"newer patch", "dolt version 1.52.3\n", "1.52.1", true}, + {"newer minor", "dolt version 1.53.0\n", "1.52.1", true}, + {"newer major (sequential dolt 2.x)", "dolt version 2.2.2\n", "1.52.1", true}, + {"older patch", "dolt version 1.52.0\n", "1.52.1", false}, + {"older minor", "dolt version 1.51.9\n", "1.52.1", false}, + {"older major", "dolt version 0.99.9\n", "1.52.1", false}, + {"extra trailing lines ignored", "dolt version 1.52.1\ndatabase storage format: __DOLT__\n", "1.52.1", true}, + {"missing patch segment treated as 0", "dolt version 1.52\n", "1.52.1", false}, + {"missing patch segment, still newer", "dolt version 1.53\n", "1.52.1", true}, + {"empty output", "", "1.52.1", false}, + {"non-numeric version", "dolt version dev\n", "1.52.1", false}, + {"malformed no version token", "\n", "1.52.1", false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := doltVersionAtLeast(tc.output, tc.minVer); got != tc.want { + t.Errorf("doltVersionAtLeast(%q, %q) = %v, want %v", tc.output, tc.minVer, got, tc.want) + } + }) + } +} + +// TestSupportsArchiveLevelConfig_FailsClosed asserts that a doltBin which +// cannot be executed at all (nonexistent path) is treated as unsupported +// rather than panicking or erroring the caller. +func TestSupportsArchiveLevelConfig_FailsClosed(t *testing.T) { + if got := SupportsArchiveLevelConfig(filepath.Join(t.TempDir(), "no-such-dolt-binary")); got { + t.Errorf("SupportsArchiveLevelConfig with a nonexistent binary = true, want false (fail closed)") + } +} + +// TestSupportsArchiveLevelConfig_WithFakeBinary exercises the exec.Command +// wrapper end-to-end against a stub script that mimics `dolt version` +// output, rather than only the pure-parsing doltVersionAtLeast helper. +// Skipped on Windows: the stub is a POSIX shell script. +func TestSupportsArchiveLevelConfig_WithFakeBinary(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stub dolt binary is a POSIX shell script") + } + t.Cleanup(ResetArchiveLevelSupportCacheForTest) + + newStub := func(t *testing.T, versionLine string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "dolt") + script := fmt.Sprintf("#!/bin/sh\necho %q\n", versionLine) + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { //nolint:gosec // test fixture, intentionally executable + t.Fatalf("write stub dolt: %v", err) + } + return path + } + + t.Run("new enough", func(t *testing.T) { + bin := newStub(t, "dolt version 2.2.2") + if !SupportsArchiveLevelConfig(bin) { + t.Errorf("expected support for dolt version 2.2.2 (>= %s)", MinDoltVersionForArchiveLevelConfig) + } + }) + + t.Run("too old", func(t *testing.T) { + bin := newStub(t, "dolt version 1.40.0") + if SupportsArchiveLevelConfig(bin) { + t.Errorf("expected no support for dolt version 1.40.0 (< %s)", MinDoltVersionForArchiveLevelConfig) + } + }) +} + +// callCountingStub writes a stub "dolt" script at dir/dolt that appends one +// byte to countPath on every invocation (so probeCount below can observe +// exactly how many times it actually ran) before echoing versionLine, and +// returns (binPath, writeVersion, probeCount). +func callCountingStub(t *testing.T, dir, versionLine string) (bin string, writeVersion func(string), probeCount func() int) { + t.Helper() + bin = filepath.Join(dir, "dolt") + countPath := filepath.Join(dir, "calls") + + writeVersion = func(line string) { + t.Helper() + script := fmt.Sprintf("#!/bin/sh\nprintf x >> %q\necho %q\n", countPath, line) + if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { //nolint:gosec // test fixture, intentionally executable + t.Fatalf("write stub dolt: %v", err) + } + } + probeCount = func() int { + t.Helper() + data, err := os.ReadFile(countPath) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read call counter: %v", err) + } + return len(data) + } + + writeVersion(versionLine) + return bin, writeVersion, probeCount +} + +// TestSupportsArchiveLevelConfig_MemoizesUnchangedBinary covers the nit fix +// (gastownhall/beads#4986 round 2): repeated calls against the SAME, +// UNCHANGED on-disk binary must not re-fork `dolt version`. Proven directly +// via a call counter embedded in the stub script, rather than only +// inferring it from the returned bool. +func TestSupportsArchiveLevelConfig_MemoizesUnchangedBinary(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stub dolt binary is a POSIX shell script") + } + t.Cleanup(ResetArchiveLevelSupportCacheForTest) + + bin, _, probeCount := callCountingStub(t, t.TempDir(), "dolt version 2.2.2") + + if !SupportsArchiveLevelConfig(bin) { + t.Fatalf("expected support for dolt version 2.2.2 (>= %s)", MinDoltVersionForArchiveLevelConfig) + } + if got := probeCount(); got != 1 { + t.Fatalf("expected exactly 1 probe after the first call, got %d", got) + } + + if !SupportsArchiveLevelConfig(bin) { + t.Errorf("cached result changed on a second call against the unchanged binary") + } + if got := probeCount(); got != 1 { + t.Errorf("expected the second call against an unchanged binary to hit the cache (still 1 probe), got %d", got) + } +} + +// TestSupportsArchiveLevelConfig_InvalidatesOnFileIdentityChange covers the +// round-3 minor fix (gastownhall/beads#4986): the cache key includes +// size+mtime, not just the path string, so an in-place dolt +// upgrade/downgrade at the same path is re-probed on its very next call +// rather than serving a stale verdict for the rest of the process's +// lifetime. os.Chtimes forces a distinct mtime deterministically (no real +// sleep, robust to coarse filesystem mtime resolution); both stub scripts +// have identical byte length, so this specifically exercises mtime-based +// invalidation rather than accidentally relying on a size change. +func TestSupportsArchiveLevelConfig_InvalidatesOnFileIdentityChange(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stub dolt binary is a POSIX shell script") + } + t.Cleanup(ResetArchiveLevelSupportCacheForTest) + + bin, writeVersion, probeCount := callCountingStub(t, t.TempDir(), "dolt version 2.2.2") + + info1, err := os.Stat(bin) + if err != nil { + t.Fatalf("os.Stat: %v", err) + } + + if !SupportsArchiveLevelConfig(bin) { + t.Fatalf("expected support for dolt version 2.2.2 (>= %s)", MinDoltVersionForArchiveLevelConfig) + } + if got := probeCount(); got != 1 { + t.Fatalf("expected exactly 1 probe after the first call, got %d", got) + } + + // Rewrite the SAME path with an older version (same byte length as the + // first stub — this isolates the mtime-based half of the cache key from + // the size-based half), then force a distinct mtime. + writeVersion("dolt version 1.40.0") + newMtime := info1.ModTime().Add(time.Second) + if err := os.Chtimes(bin, newMtime, newMtime); err != nil { + t.Fatalf("os.Chtimes: %v", err) + } + + if SupportsArchiveLevelConfig(bin) { + t.Errorf("expected the rewritten (older) binary to invalidate the cache and report unsupported") + } + if got := probeCount(); got != 2 { + t.Errorf("expected a fresh probe (2 total) after the binary's file identity changed, got %d", got) + } +} diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index 9fe430ba4..8f786ec01 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -24,6 +24,7 @@ import ( "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" "github.com/steveyegge/beads/internal/storage/dbproxy/util" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) const defaultKeepAlivePeriod = 30 * time.Second @@ -373,8 +374,8 @@ func (s *DoltServer) runShutdownGC(ctx context.Context) (retErr error) { } defer func() { retErr = errors.Join(retErr, conn.Close()) }() - if _, err := conn.ExecContext(ctx, "CALL DOLT_GC()"); err != nil { - retErr = errors.Join(retErr, fmt.Errorf("dolt_gc: %w", err)) + if err := versioncontrolops.DoltGC(ctx, conn); err != nil { + retErr = errors.Join(retErr, err) } if _, err := conn.ExecContext(ctx, "CALL DOLT_STATS_GC()"); err != nil { retErr = errors.Join(retErr, fmt.Errorf("dolt_stats_gc: %w", err)) diff --git a/internal/storage/versioncontrolops/gc.go b/internal/storage/versioncontrolops/gc.go index 1bb22d22d..f0885afa6 100644 --- a/internal/storage/versioncontrolops/gc.go +++ b/internal/storage/versioncontrolops/gc.go @@ -5,11 +5,12 @@ import ( "fmt" ) -// DoltGC runs Dolt garbage collection to reclaim disk space. +// DoltGC runs Dolt garbage collection to reclaim disk space. Archive level 0 +// writes classic Snappy table files instead of zstd archives. // conn must be a non-transactional database connection since // DOLT_GC cannot run inside an explicit transaction. func DoltGC(ctx context.Context, conn DBConn) error { - if _, err := conn.ExecContext(ctx, "CALL DOLT_GC()"); err != nil { + if _, err := conn.ExecContext(ctx, "CALL DOLT_GC('--archive-level', '0')"); err != nil { return fmt.Errorf("dolt gc: %w", err) } return nil diff --git a/internal/storage/versioncontrolops/gc_test.go b/internal/storage/versioncontrolops/gc_test.go new file mode 100644 index 000000000..803b92ecd --- /dev/null +++ b/internal/storage/versioncontrolops/gc_test.go @@ -0,0 +1,27 @@ +package versioncontrolops + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestDoltGCDisablesArchiveCompression(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta("CALL DOLT_GC('--archive-level', '0')")). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := DoltGC(context.Background(), db); err != nil { + t.Fatalf("DoltGC: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +}